diff options
289 files changed, 11974 insertions, 6349 deletions
diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml index a6cfe9be97..7edb50b760 100644 --- a/.gitlab-ci.d/buildtest-template.yml +++ b/.gitlab-ci.d/buildtest-template.yml @@ -12,12 +12,12 @@ - mkdir build - cd build - ../configure --enable-werror --disable-docs --enable-fdt=system - ${LD_JOBS:+--meson=git} ${TARGETS:+--target-list="$TARGETS"} + ${TARGETS:+--target-list="$TARGETS"} $CONFIGURE_ARGS || { cat config.log meson-logs/meson-log.txt && exit 1; } - if test -n "$LD_JOBS"; then - ../meson/meson.py configure . -Dbackend_max_links="$LD_JOBS" ; + pyvenv/bin/meson configure . -Dbackend_max_links="$LD_JOBS" ; fi || exit 1; - make -j"$JOBS" - if test -n "$MAKE_CHECK_ARGS"; diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index bb3650a51c..307cba1aab 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -103,7 +103,7 @@ crash-test-debian: script: - cd build - make NINJA=":" check-venv - - tests/venv/bin/python3 scripts/device-crash-test -q --tcg-only ./qemu-system-i386 + - pyvenv/bin/python3 scripts/device-crash-test -q --tcg-only ./qemu-system-i386 build-system-fedora: extends: @@ -146,8 +146,8 @@ crash-test-fedora: script: - cd build - make NINJA=":" check-venv - - tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-ppc - - tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-riscv32 + - pyvenv/bin/python3 scripts/device-crash-test -q ./qemu-system-ppc + - pyvenv/bin/python3 scripts/device-crash-test -q ./qemu-system-riscv32 build-system-centos: extends: diff --git a/.gitlab-ci.d/cirrus/freebsd-13.vars b/.gitlab-ci.d/cirrus/freebsd-13.vars index 7622c849b2..facb649f5b 100644 --- a/.gitlab-ci.d/cirrus/freebsd-13.vars +++ b/.gitlab-ci.d/cirrus/freebsd-13.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio socat spice-protocol tesseract usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson mtools ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio socat spice-protocol tesseract usbredir virglrenderer vte3 xorriso zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/macos-12.vars b/.gitlab-ci.d/cirrus/macos-12.vars index da6aa6469b..ceb294e153 100644 --- a/.gitlab-ci.d/cirrus/macos-12.vars +++ b/.gitlab-ci.d/cirrus/macos-12.vars @@ -11,6 +11,6 @@ MAKE='/opt/homebrew/bin/gmake' NINJA='/opt/homebrew/bin/ninja' PACKAGING_COMMAND='brew' PIP3='/opt/homebrew/bin/pip3' -PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson ncurses nettle ninja pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy socat sparse spice-protocol tesseract usbredir vde vte3 zlib zstd' +PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson mtools ncurses nettle ninja pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy socat sparse spice-protocol tesseract usbredir vde vte3 xorriso zlib zstd' PYPI_PKGS='PyYAML numpy pillow sphinx sphinx-rtd-theme' PYTHON='/opt/homebrew/bin/python3' diff --git a/.gitmodules b/.gitmodules index 6ce5bf49c5..2a3a12033c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -49,9 +49,6 @@ [submodule "roms/qboot"] path = roms/qboot url = https://gitlab.com/qemu-project/qboot.git -[submodule "meson"] - path = meson - url = https://gitlab.com/qemu-project/meson.git [submodule "roms/vbootrom"] path = roms/vbootrom url = https://gitlab.com/qemu-project/vbootrom.git diff --git a/MAINTAINERS b/MAINTAINERS index 2e18c3cad4..1b6466496d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -246,6 +246,7 @@ M: Xiaojuan Yang <yangxiaojuan@loongson.cn> S: Maintained F: target/loongarch/ F: tests/tcg/loongarch64/ +F: tests/avocado/machine_loongarch.py M68K TCG CPUs M: Laurent Vivier <laurent@vivier.eu> @@ -940,6 +941,7 @@ SBSA-REF M: Radoslaw Biernacki <rad@semihalf.com> M: Peter Maydell <peter.maydell@linaro.org> R: Leif Lindholm <quic_llindhol@quicinc.com> +R: Marcin Juszkiewicz <marcin.juszkiewicz@linaro.org> L: qemu-arm@nongnu.org S: Maintained F: hw/arm/sbsa-ref.c @@ -2567,7 +2569,7 @@ Core Audio framework backend M: Gerd Hoffmann <kraxel@redhat.com> M: Philippe Mathieu-Daudé <philmd@linaro.org> R: Christian Schoenebeck <qemu_oss@crudebyte.com> -R: Akihiko Odaki <akihiko.odaki@gmail.com> +R: Akihiko Odaki <akihiko.odaki@daynix.com> S: Odd Fixes F: audio/coreaudio.c @@ -2851,7 +2853,7 @@ F: docs/devel/ui.rst Cocoa graphics M: Peter Maydell <peter.maydell@linaro.org> M: Philippe Mathieu-Daudé <philmd@linaro.org> -R: Akihiko Odaki <akihiko.odaki@gmail.com> +R: Akihiko Odaki <akihiko.odaki@daynix.com> S: Odd Fixes F: ui/cocoa.m diff --git a/Makefile b/Makefile index e421f8a1f4..3c7d67142f 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ quiet-command-run = $(if $(V),,$(if $2,printf " %-7s %s\n" $2 $3 && ))$1 quiet-@ = $(if $(V),,@) quiet-command = $(quiet-@)$(call quiet-command-run,$1,$2,$3) -UNCHECKED_GOALS := %clean TAGS cscope ctags dist \ +UNCHECKED_GOALS := TAGS gtags cscope ctags dist \ help check-help print-% \ docker docker-% vm-help vm-test vm-build-% @@ -176,10 +176,8 @@ plugins: endif # $(CONFIG_PLUGIN) else # config-host.mak does not exist -config-host.mak: ifneq ($(filter-out $(UNCHECKED_GOALS),$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail)) - @echo "Please call configure before running make!" - @exit 1 +$(error Please call configure before running make) endif endif # config-host.mak does not exist diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index cf3a88d90e..7679f397ae 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -1361,6 +1361,10 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml, */ if (kvm_state->kvm_dirty_ring_size) { kvm_dirty_ring_reap_locked(kvm_state, NULL); + if (kvm_state->kvm_dirty_ring_with_bitmap) { + kvm_slot_sync_dirty_pages(mem); + kvm_slot_get_dirty_log(kvm_state, mem); + } } else { kvm_slot_get_dirty_log(kvm_state, mem); } @@ -1458,6 +1462,69 @@ static int kvm_dirty_ring_reaper_init(KVMState *s) return 0; } +static int kvm_dirty_ring_init(KVMState *s) +{ + uint32_t ring_size = s->kvm_dirty_ring_size; + uint64_t ring_bytes = ring_size * sizeof(struct kvm_dirty_gfn); + unsigned int capability = KVM_CAP_DIRTY_LOG_RING; + int ret; + + s->kvm_dirty_ring_size = 0; + s->kvm_dirty_ring_bytes = 0; + + /* Bail if the dirty ring size isn't specified */ + if (!ring_size) { + return 0; + } + + /* + * Read the max supported pages. Fall back to dirty logging mode + * if the dirty ring isn't supported. + */ + ret = kvm_vm_check_extension(s, capability); + if (ret <= 0) { + capability = KVM_CAP_DIRTY_LOG_RING_ACQ_REL; + ret = kvm_vm_check_extension(s, capability); + } + + if (ret <= 0) { + warn_report("KVM dirty ring not available, using bitmap method"); + return 0; + } + + if (ring_bytes > ret) { + error_report("KVM dirty ring size %" PRIu32 " too big " + "(maximum is %ld). Please use a smaller value.", + ring_size, (long)ret / sizeof(struct kvm_dirty_gfn)); + return -EINVAL; + } + + ret = kvm_vm_enable_cap(s, capability, 0, ring_bytes); + if (ret) { + error_report("Enabling of KVM dirty ring failed: %s. " + "Suggested minimum value is 1024.", strerror(-ret)); + return -EIO; + } + + /* Enable the backup bitmap if it is supported */ + ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP); + if (ret > 0) { + ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP, 0); + if (ret) { + error_report("Enabling of KVM dirty ring's backup bitmap failed: " + "%s. ", strerror(-ret)); + return -EIO; + } + + s->kvm_dirty_ring_with_bitmap = true; + } + + s->kvm_dirty_ring_size = ring_size; + s->kvm_dirty_ring_bytes = ring_bytes; + + return 0; +} + static void kvm_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -1563,7 +1630,7 @@ static void kvm_log_sync(MemoryListener *listener, kvm_slots_unlock(); } -static void kvm_log_sync_global(MemoryListener *l) +static void kvm_log_sync_global(MemoryListener *l, bool last_stage) { KVMMemoryListener *kml = container_of(l, KVMMemoryListener, listener); KVMState *s = kvm_state; @@ -1582,6 +1649,12 @@ static void kvm_log_sync_global(MemoryListener *l) mem = &kml->slots[i]; if (mem->memory_size && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { kvm_slot_sync_dirty_pages(mem); + + if (s->kvm_dirty_ring_with_bitmap && last_stage && + kvm_slot_get_dirty_log(s, mem)) { + kvm_slot_sync_dirty_pages(mem); + } + /* * This is not needed by KVM_GET_DIRTY_LOG because the * ioctl will unconditionally overwrite the whole region. @@ -2521,35 +2594,9 @@ static int kvm_init(MachineState *ms) * Enable KVM dirty ring if supported, otherwise fall back to * dirty logging mode */ - if (s->kvm_dirty_ring_size > 0) { - uint64_t ring_bytes; - - ring_bytes = s->kvm_dirty_ring_size * sizeof(struct kvm_dirty_gfn); - - /* Read the max supported pages */ - ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING); - if (ret > 0) { - if (ring_bytes > ret) { - error_report("KVM dirty ring size %" PRIu32 " too big " - "(maximum is %ld). Please use a smaller value.", - s->kvm_dirty_ring_size, - (long)ret / sizeof(struct kvm_dirty_gfn)); - ret = -EINVAL; - goto err; - } - - ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING, 0, ring_bytes); - if (ret) { - error_report("Enabling of KVM dirty ring failed: %s. " - "Suggested minimum value is 1024.", strerror(-ret)); - goto err; - } - - s->kvm_dirty_ring_bytes = ring_bytes; - } else { - warn_report("KVM dirty ring not available, using bitmap method"); - s->kvm_dirty_ring_size = 0; - } + ret = kvm_dirty_ring_init(s); + if (ret < 0) { + goto err; } /* @@ -3710,6 +3757,7 @@ static void kvm_accel_instance_init(Object *obj) s->kernel_irqchip_split = ON_OFF_AUTO_AUTO; /* KVM dirty ring is by default off */ s->kvm_dirty_ring_size = 0; + s->kvm_dirty_ring_with_bitmap = false; s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN; s->notify_window = 0; s->xen_version = 0; diff --git a/accel/tcg/atomic_common.c.inc b/accel/tcg/atomic_common.c.inc index 8f2ce43ee6..fe0eea018f 100644 --- a/accel/tcg/atomic_common.c.inc +++ b/accel/tcg/atomic_common.c.inc @@ -13,20 +13,20 @@ * See the COPYING file in the top-level directory. */ -static void atomic_trace_rmw_post(CPUArchState *env, target_ulong addr, +static void atomic_trace_rmw_post(CPUArchState *env, uint64_t addr, MemOpIdx oi) { qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_RW); } #if HAVE_ATOMIC128 -static void atomic_trace_ld_post(CPUArchState *env, target_ulong addr, +static void atomic_trace_ld_post(CPUArchState *env, uint64_t addr, MemOpIdx oi) { qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); } -static void atomic_trace_st_post(CPUArchState *env, target_ulong addr, +static void atomic_trace_st_post(CPUArchState *env, uint64_t addr, MemOpIdx oi) { qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); @@ -40,7 +40,7 @@ static void atomic_trace_st_post(CPUArchState *env, target_ulong addr, */ #define CMPXCHG_HELPER(OP, TYPE) \ - TYPE HELPER(atomic_##OP)(CPUArchState *env, target_ulong addr, \ + TYPE HELPER(atomic_##OP)(CPUArchState *env, uint64_t addr, \ TYPE oldv, TYPE newv, uint32_t oi) \ { return cpu_atomic_##OP##_mmu(env, addr, oldv, newv, oi, GETPC()); } @@ -62,7 +62,7 @@ CMPXCHG_HELPER(cmpxchgo_le, Int128) #undef CMPXCHG_HELPER -Int128 HELPER(nonatomic_cmpxchgo_be)(CPUArchState *env, target_ulong addr, +Int128 HELPER(nonatomic_cmpxchgo_be)(CPUArchState *env, uint64_t addr, Int128 cmpv, Int128 newv, uint32_t oi) { #if TCG_TARGET_REG_BITS == 32 @@ -82,7 +82,7 @@ Int128 HELPER(nonatomic_cmpxchgo_be)(CPUArchState *env, target_ulong addr, #endif } -Int128 HELPER(nonatomic_cmpxchgo_le)(CPUArchState *env, target_ulong addr, +Int128 HELPER(nonatomic_cmpxchgo_le)(CPUArchState *env, uint64_t addr, Int128 cmpv, Int128 newv, uint32_t oi) { #if TCG_TARGET_REG_BITS == 32 @@ -103,7 +103,7 @@ Int128 HELPER(nonatomic_cmpxchgo_le)(CPUArchState *env, target_ulong addr, } #define ATOMIC_HELPER(OP, TYPE) \ - TYPE HELPER(glue(atomic_,OP))(CPUArchState *env, target_ulong addr, \ + TYPE HELPER(glue(atomic_,OP))(CPUArchState *env, uint64_t addr, \ TYPE val, uint32_t oi) \ { return glue(glue(cpu_atomic_,OP),_mmu)(env, addr, val, oi, GETPC()); } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 617777055a..ae0fbcdee2 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -40,6 +40,7 @@ #include "qemu/plugin-memory.h" #endif #include "tcg/tcg-ldst.h" +#include "exec/helper-proto.h" /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */ /* #define DEBUG_TLB */ @@ -1668,6 +1669,9 @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, return qemu_ram_addr_from_host_nofail(p); } +/* Load/store with atomicity primitives. */ +#include "ldst_atomicity.c.inc" + #ifdef CONFIG_PLUGIN /* * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure. @@ -2010,60 +2014,13 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, } /* - * Verify that we have passed the correct MemOp to the correct function. - * - * In the case of the helper_*_mmu functions, we will have done this by - * using the MemOp to look up the helper during code generation. - * - * In the case of the cpu_*_mmu functions, this is up to the caller. - * We could present one function to target code, and dispatch based on - * the MemOp, but so far we have worked hard to avoid an indirect function - * call along the memory path. - */ -static void validate_memop(MemOpIdx oi, MemOp expected) -{ -#ifdef CONFIG_DEBUG_TCG - MemOp have = get_memop(oi) & (MO_SIZE | MO_BSWAP); - assert(have == expected); -#endif -} - -/* * Load Helpers * * We support two different access types. SOFTMMU_CODE_ACCESS is * specifically for reading instructions from system memory. It is * called by the translation loop and in some helpers where the code * is disassembled. It shouldn't be called directly by guest code. - */ - -typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); - -static inline uint64_t QEMU_ALWAYS_INLINE -load_memop(const void *haddr, MemOp op) -{ - switch (op) { - case MO_UB: - return ldub_p(haddr); - case MO_BEUW: - return lduw_be_p(haddr); - case MO_LEUW: - return lduw_le_p(haddr); - case MO_BEUL: - return (uint32_t)ldl_be_p(haddr); - case MO_LEUL: - return (uint32_t)ldl_le_p(haddr); - case MO_BEUQ: - return ldq_be_p(haddr); - case MO_LEUQ: - return ldq_le_p(haddr); - default: - qemu_build_not_reached(); - } -} - -/* + * * For the benefit of TCG generated code, we want to avoid the * complication of ABI-specific return type promotion and always * return a value extended to the register size of the host. This is @@ -2119,20 +2076,224 @@ static uint64_t do_ld_bytes_beN(MMULookupPageData *p, uint64_t ret_be) return ret_be; } +/** + * do_ld_parts_beN + * @p: translation parameters + * @ret_be: accumulated data + * + * As do_ld_bytes_beN, but atomically on each aligned part. + */ +static uint64_t do_ld_parts_beN(MMULookupPageData *p, uint64_t ret_be) +{ + void *haddr = p->haddr; + int size = p->size; + + do { + uint64_t x; + int n; + + /* + * Find minimum of alignment and size. + * This is slightly stronger than required by MO_ATOM_SUBALIGN, which + * would have only checked the low bits of addr|size once at the start, + * but is just as easy. + */ + switch (((uintptr_t)haddr | size) & 7) { + case 4: + x = cpu_to_be32(load_atomic4(haddr)); + ret_be = (ret_be << 32) | x; + n = 4; + break; + case 2: + case 6: + x = cpu_to_be16(load_atomic2(haddr)); + ret_be = (ret_be << 16) | x; + n = 2; + break; + default: + x = *(uint8_t *)haddr; + ret_be = (ret_be << 8) | x; + n = 1; + break; + case 0: + g_assert_not_reached(); + } + haddr += n; + size -= n; + } while (size != 0); + return ret_be; +} + +/** + * do_ld_parts_be4 + * @p: translation parameters + * @ret_be: accumulated data + * + * As do_ld_bytes_beN, but with one atomic load. + * Four aligned bytes are guaranteed to cover the load. + */ +static uint64_t do_ld_whole_be4(MMULookupPageData *p, uint64_t ret_be) +{ + int o = p->addr & 3; + uint32_t x = load_atomic4(p->haddr - o); + + x = cpu_to_be32(x); + x <<= o * 8; + x >>= (4 - p->size) * 8; + return (ret_be << (p->size * 8)) | x; +} + +/** + * do_ld_parts_be8 + * @p: translation parameters + * @ret_be: accumulated data + * + * As do_ld_bytes_beN, but with one atomic load. + * Eight aligned bytes are guaranteed to cover the load. + */ +static uint64_t do_ld_whole_be8(CPUArchState *env, uintptr_t ra, + MMULookupPageData *p, uint64_t ret_be) +{ + int o = p->addr & 7; + uint64_t x = load_atomic8_or_exit(env, ra, p->haddr - o); + + x = cpu_to_be64(x); + x <<= o * 8; + x >>= (8 - p->size) * 8; + return (ret_be << (p->size * 8)) | x; +} + +/** + * do_ld_parts_be16 + * @p: translation parameters + * @ret_be: accumulated data + * + * As do_ld_bytes_beN, but with one atomic load. + * 16 aligned bytes are guaranteed to cover the load. + */ +static Int128 do_ld_whole_be16(CPUArchState *env, uintptr_t ra, + MMULookupPageData *p, uint64_t ret_be) +{ + int o = p->addr & 15; + Int128 x, y = load_atomic16_or_exit(env, ra, p->haddr - o); + int size = p->size; + + if (!HOST_BIG_ENDIAN) { + y = bswap128(y); + } + y = int128_lshift(y, o * 8); + y = int128_urshift(y, (16 - size) * 8); + x = int128_make64(ret_be); + x = int128_lshift(x, size * 8); + return int128_or(x, y); +} + /* * Wrapper for the above. */ static uint64_t do_ld_beN(CPUArchState *env, MMULookupPageData *p, - uint64_t ret_be, int mmu_idx, - MMUAccessType type, uintptr_t ra) + uint64_t ret_be, int mmu_idx, MMUAccessType type, + MemOp mop, uintptr_t ra) { + MemOp atom; + unsigned tmp, half_size; + if (unlikely(p->flags & TLB_MMIO)) { return do_ld_mmio_beN(env, p, ret_be, mmu_idx, type, ra); - } else { + } + + /* + * It is a given that we cross a page and therefore there is no + * atomicity for the load as a whole, but subobjects may need attention. + */ + atom = mop & MO_ATOM_MASK; + switch (atom) { + case MO_ATOM_SUBALIGN: + return do_ld_parts_beN(p, ret_be); + + case MO_ATOM_IFALIGN_PAIR: + case MO_ATOM_WITHIN16_PAIR: + tmp = mop & MO_SIZE; + tmp = tmp ? tmp - 1 : 0; + half_size = 1 << tmp; + if (atom == MO_ATOM_IFALIGN_PAIR + ? p->size == half_size + : p->size >= half_size) { + if (!HAVE_al8_fast && p->size < 4) { + return do_ld_whole_be4(p, ret_be); + } else { + return do_ld_whole_be8(env, ra, p, ret_be); + } + } + /* fall through */ + + case MO_ATOM_IFALIGN: + case MO_ATOM_WITHIN16: + case MO_ATOM_NONE: return do_ld_bytes_beN(p, ret_be); + + default: + g_assert_not_reached(); } } +/* + * Wrapper for the above, for 8 < size < 16. + */ +static Int128 do_ld16_beN(CPUArchState *env, MMULookupPageData *p, + uint64_t a, int mmu_idx, MemOp mop, uintptr_t ra) +{ + int size = p->size; + uint64_t b; + MemOp atom; + + if (unlikely(p->flags & TLB_MMIO)) { + p->size = size - 8; + a = do_ld_mmio_beN(env, p, a, mmu_idx, MMU_DATA_LOAD, ra); + p->addr += p->size; + p->size = 8; + b = do_ld_mmio_beN(env, p, 0, mmu_idx, MMU_DATA_LOAD, ra); + return int128_make128(b, a); + } + + /* + * It is a given that we cross a page and therefore there is no + * atomicity for the load as a whole, but subobjects may need attention. + */ + atom = mop & MO_ATOM_MASK; + switch (atom) { + case MO_ATOM_SUBALIGN: + p->size = size - 8; + a = do_ld_parts_beN(p, a); + p->haddr += size - 8; + p->size = 8; + b = do_ld_parts_beN(p, 0); + break; + + case MO_ATOM_WITHIN16_PAIR: + /* Since size > 8, this is the half that must be atomic. */ + return do_ld_whole_be16(env, ra, p, a); + + case MO_ATOM_IFALIGN_PAIR: + /* + * Since size > 8, both halves are misaligned, + * and so neither is atomic. + */ + case MO_ATOM_IFALIGN: + case MO_ATOM_WITHIN16: + case MO_ATOM_NONE: + p->size = size - 8; + a = do_ld_bytes_beN(p, a); + b = ldq_be_p(p->haddr + size - 8); + break; + + default: + g_assert_not_reached(); + } + + return int128_make128(b, a); +} + static uint8_t do_ld_1(CPUArchState *env, MMULookupPageData *p, int mmu_idx, MMUAccessType type, uintptr_t ra) { @@ -2153,7 +2314,7 @@ static uint16_t do_ld_2(CPUArchState *env, MMULookupPageData *p, int mmu_idx, } /* Perform the load host endian, then swap if necessary. */ - ret = load_memop(p->haddr, MO_UW); + ret = load_atom_2(env, ra, p->haddr, memop); if (memop & MO_BSWAP) { ret = bswap16(ret); } @@ -2170,7 +2331,7 @@ static uint32_t do_ld_4(CPUArchState *env, MMULookupPageData *p, int mmu_idx, } /* Perform the load host endian. */ - ret = load_memop(p->haddr, MO_UL); + ret = load_atom_4(env, ra, p->haddr, memop); if (memop & MO_BSWAP) { ret = bswap32(ret); } @@ -2187,7 +2348,7 @@ static uint64_t do_ld_8(CPUArchState *env, MMULookupPageData *p, int mmu_idx, } /* Perform the load host endian. */ - ret = load_memop(p->haddr, MO_UQ); + ret = load_atom_8(env, ra, p->haddr, memop); if (memop & MO_BSWAP) { ret = bswap64(ret); } @@ -2206,10 +2367,10 @@ static uint8_t do_ld1_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi, return do_ld_1(env, &l.page[0], l.mmu_idx, access_type, ra); } -tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) +tcg_target_ulong helper_ldub_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr) { - validate_memop(oi, MO_UB); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_8); return do_ld1_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD); } @@ -2237,17 +2398,10 @@ static uint16_t do_ld2_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi, return ret; } -tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) +tcg_target_ulong helper_lduw_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr) { - validate_memop(oi, MO_LEUW); - return do_ld2_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD); -} - -tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) -{ - validate_memop(oi, MO_BEUW); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16); return do_ld2_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD); } @@ -2263,25 +2417,18 @@ static uint32_t do_ld4_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi, return do_ld_4(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra); } - ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, ra); - ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, ra); + ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra); + ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra); if ((l.memop & MO_BSWAP) == MO_LE) { ret = bswap32(ret); } return ret; } -tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) +tcg_target_ulong helper_ldul_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr) { - validate_memop(oi, MO_LEUL); - return do_ld4_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD); -} - -tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) -{ - validate_memop(oi, MO_BEUL); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32); return do_ld4_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD); } @@ -2297,25 +2444,18 @@ static uint64_t do_ld8_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi, return do_ld_8(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra); } - ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, ra); - ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, ra); + ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra); + ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra); if ((l.memop & MO_BSWAP) == MO_LE) { ret = bswap64(ret); } return ret; } -uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) -{ - validate_memop(oi, MO_LEUQ); - return do_ld8_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD); -} - -uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) +uint64_t helper_ldq_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr) { - validate_memop(oi, MO_BEUQ); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64); return do_ld8_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD); } @@ -2324,35 +2464,96 @@ uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr, * avoid this for 64-bit data, or for 32-bit data on 32-bit host. */ +tcg_target_ulong helper_ldsb_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr) +{ + return (int8_t)helper_ldub_mmu(env, addr, oi, retaddr); +} -tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) +tcg_target_ulong helper_ldsw_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr) { - return (int8_t)helper_ret_ldub_mmu(env, addr, oi, retaddr); + return (int16_t)helper_lduw_mmu(env, addr, oi, retaddr); } -tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) +tcg_target_ulong helper_ldsl_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr) { - return (int16_t)helper_le_lduw_mmu(env, addr, oi, retaddr); + return (int32_t)helper_ldul_mmu(env, addr, oi, retaddr); } -tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) +static Int128 do_ld16_mmu(CPUArchState *env, target_ulong addr, + MemOpIdx oi, uintptr_t ra) { - return (int16_t)helper_be_lduw_mmu(env, addr, oi, retaddr); + MMULookupLocals l; + bool crosspage; + uint64_t a, b; + Int128 ret; + int first; + + crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD, &l); + if (likely(!crosspage)) { + /* Perform the load host endian. */ + if (unlikely(l.page[0].flags & TLB_MMIO)) { + QEMU_IOTHREAD_LOCK_GUARD(); + a = io_readx(env, l.page[0].full, l.mmu_idx, addr, + ra, MMU_DATA_LOAD, MO_64); + b = io_readx(env, l.page[0].full, l.mmu_idx, addr + 8, + ra, MMU_DATA_LOAD, MO_64); + ret = int128_make128(HOST_BIG_ENDIAN ? b : a, + HOST_BIG_ENDIAN ? a : b); + } else { + ret = load_atom_16(env, ra, l.page[0].haddr, l.memop); + } + if (l.memop & MO_BSWAP) { + ret = bswap128(ret); + } + return ret; + } + + first = l.page[0].size; + if (first == 8) { + MemOp mop8 = (l.memop & ~MO_SIZE) | MO_64; + + a = do_ld_8(env, &l.page[0], l.mmu_idx, MMU_DATA_LOAD, mop8, ra); + b = do_ld_8(env, &l.page[1], l.mmu_idx, MMU_DATA_LOAD, mop8, ra); + if ((mop8 & MO_BSWAP) == MO_LE) { + ret = int128_make128(a, b); + } else { + ret = int128_make128(b, a); + } + return ret; + } + + if (first < 8) { + a = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, + MMU_DATA_LOAD, l.memop, ra); + ret = do_ld16_beN(env, &l.page[1], a, l.mmu_idx, l.memop, ra); + } else { + ret = do_ld16_beN(env, &l.page[0], 0, l.mmu_idx, l.memop, ra); + b = int128_getlo(ret); + ret = int128_lshift(ret, l.page[1].size * 8); + a = int128_gethi(ret); + b = do_ld_beN(env, &l.page[1], b, l.mmu_idx, + MMU_DATA_LOAD, l.memop, ra); + ret = int128_make128(b, a); + } + if ((l.memop & MO_BSWAP) == MO_LE) { + ret = bswap128(ret); + } + return ret; } -tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) +Int128 helper_ld16_mmu(CPUArchState *env, uint64_t addr, + uint32_t oi, uintptr_t retaddr) { - return (int32_t)helper_le_ldul_mmu(env, addr, oi, retaddr); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128); + return do_ld16_mmu(env, addr, oi, retaddr); } -tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) +Int128 helper_ld_i128(CPUArchState *env, uint64_t addr, uint32_t oi) { - return (int32_t)helper_be_ldul_mmu(env, addr, oi, retaddr); + return helper_ld16_mmu(env, addr, oi, GETPC()); } /* @@ -2368,7 +2569,7 @@ uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) { uint8_t ret; - validate_memop(oi, MO_UB); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_UB); ret = do_ld1_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; @@ -2379,7 +2580,7 @@ uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr, { uint16_t ret; - validate_memop(oi, MO_BEUW); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUW); ret = do_ld2_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; @@ -2390,7 +2591,7 @@ uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr, { uint32_t ret; - validate_memop(oi, MO_BEUL); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUL); ret = do_ld4_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; @@ -2401,7 +2602,7 @@ uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr, { uint64_t ret; - validate_memop(oi, MO_BEUQ); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUQ); ret = do_ld8_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; @@ -2412,7 +2613,7 @@ uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr, { uint16_t ret; - validate_memop(oi, MO_LEUW); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUW); ret = do_ld2_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; @@ -2423,7 +2624,7 @@ uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr, { uint32_t ret; - validate_memop(oi, MO_LEUL); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUL); ret = do_ld4_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; @@ -2434,7 +2635,7 @@ uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr, { uint64_t ret; - validate_memop(oi, MO_LEUQ); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUQ); ret = do_ld8_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; @@ -2443,95 +2644,29 @@ uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr, Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - int mmu_idx = get_mmuidx(oi); - MemOpIdx new_oi; - unsigned a_bits; - uint64_t h, l; - - tcg_debug_assert((mop & (MO_BSWAP|MO_SSIZE)) == (MO_BE|MO_128)); - a_bits = get_alignment_bits(mop); - - /* Handle CPU specific unaligned behaviour */ - if (addr & ((1 << a_bits) - 1)) { - cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_LOAD, - mmu_idx, ra); - } + Int128 ret; - /* Construct an unaligned 64-bit replacement MemOpIdx. */ - mop = (mop & ~(MO_SIZE | MO_AMASK)) | MO_64 | MO_UNALN; - new_oi = make_memop_idx(mop, mmu_idx); - - h = helper_be_ldq_mmu(env, addr, new_oi, ra); - l = helper_be_ldq_mmu(env, addr + 8, new_oi, ra); - - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - return int128_make128(l, h); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_BE|MO_128)); + ret = do_ld16_mmu(env, addr, oi, ra); + plugin_load_cb(env, addr, oi); + return ret; } Int128 cpu_ld16_le_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - int mmu_idx = get_mmuidx(oi); - MemOpIdx new_oi; - unsigned a_bits; - uint64_t h, l; - - tcg_debug_assert((mop & (MO_BSWAP|MO_SSIZE)) == (MO_LE|MO_128)); - a_bits = get_alignment_bits(mop); + Int128 ret; - /* Handle CPU specific unaligned behaviour */ - if (addr & ((1 << a_bits) - 1)) { - cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_LOAD, - mmu_idx, ra); - } - - /* Construct an unaligned 64-bit replacement MemOpIdx. */ - mop = (mop & ~(MO_SIZE | MO_AMASK)) | MO_64 | MO_UNALN; - new_oi = make_memop_idx(mop, mmu_idx); - - l = helper_le_ldq_mmu(env, addr, new_oi, ra); - h = helper_le_ldq_mmu(env, addr + 8, new_oi, ra); - - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - return int128_make128(l, h); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_LE|MO_128)); + ret = do_ld16_mmu(env, addr, oi, ra); + plugin_load_cb(env, addr, oi); + return ret; } /* * Store Helpers */ -static inline void QEMU_ALWAYS_INLINE -store_memop(void *haddr, uint64_t val, MemOp op) -{ - switch (op) { - case MO_UB: - stb_p(haddr, val); - break; - case MO_BEUW: - stw_be_p(haddr, val); - break; - case MO_LEUW: - stw_le_p(haddr, val); - break; - case MO_BEUL: - stl_be_p(haddr, val); - break; - case MO_LEUL: - stl_le_p(haddr, val); - break; - case MO_BEUQ: - stq_be_p(haddr, val); - break; - case MO_LEUQ: - stq_le_p(haddr, val); - break; - default: - qemu_build_not_reached(); - } -} - /** * do_st_mmio_leN: * @env: cpu context @@ -2558,38 +2693,110 @@ static uint64_t do_st_mmio_leN(CPUArchState *env, MMULookupPageData *p, return val_le; } -/** - * do_st_bytes_leN: - * @p: translation parameters - * @val_le: data to store - * - * Store @p->size bytes at @p->haddr, which is RAM. - * The bytes to store are extracted in little-endian order from @val_le; - * return the bytes of @val_le beyond @p->size that have not been stored. +/* + * Wrapper for the above. */ -static uint64_t do_st_bytes_leN(MMULookupPageData *p, uint64_t val_le) +static uint64_t do_st_leN(CPUArchState *env, MMULookupPageData *p, + uint64_t val_le, int mmu_idx, + MemOp mop, uintptr_t ra) { - uint8_t *haddr = p->haddr; - int i, size = p->size; + MemOp atom; + unsigned tmp, half_size; - for (i = 0; i < size; i++, val_le >>= 8) { - haddr[i] = val_le; + if (unlikely(p->flags & TLB_MMIO)) { + return do_st_mmio_leN(env, p, val_le, mmu_idx, ra); + } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { + return val_le >> (p->size * 8); + } + + /* + * It is a given that we cross a page and therefore there is no atomicity + * for the store as a whole, but subobjects may need attention. + */ + atom = mop & MO_ATOM_MASK; + switch (atom) { + case MO_ATOM_SUBALIGN: + return store_parts_leN(p->haddr, p->size, val_le); + + case MO_ATOM_IFALIGN_PAIR: + case MO_ATOM_WITHIN16_PAIR: + tmp = mop & MO_SIZE; + tmp = tmp ? tmp - 1 : 0; + half_size = 1 << tmp; + if (atom == MO_ATOM_IFALIGN_PAIR + ? p->size == half_size + : p->size >= half_size) { + if (!HAVE_al8_fast && p->size <= 4) { + return store_whole_le4(p->haddr, p->size, val_le); + } else if (HAVE_al8) { + return store_whole_le8(p->haddr, p->size, val_le); + } else { + cpu_loop_exit_atomic(env_cpu(env), ra); + } + } + /* fall through */ + + case MO_ATOM_IFALIGN: + case MO_ATOM_WITHIN16: + case MO_ATOM_NONE: + return store_bytes_leN(p->haddr, p->size, val_le); + + default: + g_assert_not_reached(); } - return val_le; } /* - * Wrapper for the above. + * Wrapper for the above, for 8 < size < 16. */ -static uint64_t do_st_leN(CPUArchState *env, MMULookupPageData *p, - uint64_t val_le, int mmu_idx, uintptr_t ra) +static uint64_t do_st16_leN(CPUArchState *env, MMULookupPageData *p, + Int128 val_le, int mmu_idx, + MemOp mop, uintptr_t ra) { + int size = p->size; + MemOp atom; + if (unlikely(p->flags & TLB_MMIO)) { - return do_st_mmio_leN(env, p, val_le, mmu_idx, ra); + p->size = 8; + do_st_mmio_leN(env, p, int128_getlo(val_le), mmu_idx, ra); + p->size = size - 8; + p->addr += 8; + return do_st_mmio_leN(env, p, int128_gethi(val_le), mmu_idx, ra); } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { - return val_le >> (p->size * 8); - } else { - return do_st_bytes_leN(p, val_le); + return int128_gethi(val_le) >> ((size - 8) * 8); + } + + /* + * It is a given that we cross a page and therefore there is no atomicity + * for the store as a whole, but subobjects may need attention. + */ + atom = mop & MO_ATOM_MASK; + switch (atom) { + case MO_ATOM_SUBALIGN: + store_parts_leN(p->haddr, 8, int128_getlo(val_le)); + return store_parts_leN(p->haddr + 8, p->size - 8, + int128_gethi(val_le)); + + case MO_ATOM_WITHIN16_PAIR: + /* Since size > 8, this is the half that must be atomic. */ + if (!HAVE_al16) { + cpu_loop_exit_atomic(env_cpu(env), ra); + } + return store_whole_le16(p->haddr, p->size, val_le); + + case MO_ATOM_IFALIGN_PAIR: + /* + * Since size > 8, both halves are misaligned, + * and so neither is atomic. + */ + case MO_ATOM_IFALIGN: + case MO_ATOM_NONE: + stq_le_p(p->haddr, int128_getlo(val_le)); + return store_bytes_leN(p->haddr + 8, p->size - 8, + int128_gethi(val_le)); + + default: + g_assert_not_reached(); } } @@ -2617,7 +2824,7 @@ static void do_st_2(CPUArchState *env, MMULookupPageData *p, uint16_t val, if (memop & MO_BSWAP) { val = bswap16(val); } - store_memop(p->haddr, val, MO_UW); + store_atom_2(env, ra, p->haddr, memop, val); } } @@ -2633,7 +2840,7 @@ static void do_st_4(CPUArchState *env, MMULookupPageData *p, uint32_t val, if (memop & MO_BSWAP) { val = bswap32(val); } - store_memop(p->haddr, val, MO_UL); + store_atom_4(env, ra, p->haddr, memop, val); } } @@ -2649,17 +2856,17 @@ static void do_st_8(CPUArchState *env, MMULookupPageData *p, uint64_t val, if (memop & MO_BSWAP) { val = bswap64(val); } - store_memop(p->haddr, val, MO_UQ); + store_atom_8(env, ra, p->haddr, memop, val); } } -void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t ra) +void helper_stb_mmu(CPUArchState *env, uint64_t addr, uint32_t val, + MemOpIdx oi, uintptr_t ra) { MMULookupLocals l; bool crosspage; - validate_memop(oi, MO_UB); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_8); crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l); tcg_debug_assert(!crosspage); @@ -2688,17 +2895,10 @@ static void do_st2_mmu(CPUArchState *env, target_ulong addr, uint16_t val, do_st_1(env, &l.page[1], b, l.mmu_idx, ra); } -void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr) -{ - validate_memop(oi, MO_LEUW); - do_st2_mmu(env, addr, val, oi, retaddr); -} - -void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr) +void helper_stw_mmu(CPUArchState *env, uint64_t addr, uint32_t val, + MemOpIdx oi, uintptr_t retaddr) { - validate_memop(oi, MO_BEUW); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16); do_st2_mmu(env, addr, val, oi, retaddr); } @@ -2718,21 +2918,14 @@ static void do_st4_mmu(CPUArchState *env, target_ulong addr, uint32_t val, if ((l.memop & MO_BSWAP) != MO_LE) { val = bswap32(val); } - val = do_st_leN(env, &l.page[0], val, l.mmu_idx, ra); - (void) do_st_leN(env, &l.page[1], val, l.mmu_idx, ra); + val = do_st_leN(env, &l.page[0], val, l.mmu_idx, l.memop, ra); + (void) do_st_leN(env, &l.page[1], val, l.mmu_idx, l.memop, ra); } -void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr) -{ - validate_memop(oi, MO_LEUL); - do_st4_mmu(env, addr, val, oi, retaddr); -} - -void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr) +void helper_stl_mmu(CPUArchState *env, uint64_t addr, uint32_t val, + MemOpIdx oi, uintptr_t retaddr) { - validate_memop(oi, MO_BEUL); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32); do_st4_mmu(env, addr, val, oi, retaddr); } @@ -2752,22 +2945,88 @@ static void do_st8_mmu(CPUArchState *env, target_ulong addr, uint64_t val, if ((l.memop & MO_BSWAP) != MO_LE) { val = bswap64(val); } - val = do_st_leN(env, &l.page[0], val, l.mmu_idx, ra); - (void) do_st_leN(env, &l.page[1], val, l.mmu_idx, ra); + val = do_st_leN(env, &l.page[0], val, l.mmu_idx, l.memop, ra); + (void) do_st_leN(env, &l.page[1], val, l.mmu_idx, l.memop, ra); } -void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - MemOpIdx oi, uintptr_t retaddr) +void helper_stq_mmu(CPUArchState *env, uint64_t addr, uint64_t val, + MemOpIdx oi, uintptr_t retaddr) { - validate_memop(oi, MO_LEUQ); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64); do_st8_mmu(env, addr, val, oi, retaddr); } -void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - MemOpIdx oi, uintptr_t retaddr) +static void do_st16_mmu(CPUArchState *env, target_ulong addr, Int128 val, + MemOpIdx oi, uintptr_t ra) { - validate_memop(oi, MO_BEUQ); - do_st8_mmu(env, addr, val, oi, retaddr); + MMULookupLocals l; + bool crosspage; + uint64_t a, b; + int first; + + crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l); + if (likely(!crosspage)) { + /* Swap to host endian if necessary, then store. */ + if (l.memop & MO_BSWAP) { + val = bswap128(val); + } + if (unlikely(l.page[0].flags & TLB_MMIO)) { + QEMU_IOTHREAD_LOCK_GUARD(); + if (HOST_BIG_ENDIAN) { + b = int128_getlo(val), a = int128_gethi(val); + } else { + a = int128_getlo(val), b = int128_gethi(val); + } + io_writex(env, l.page[0].full, l.mmu_idx, a, addr, ra, MO_64); + io_writex(env, l.page[0].full, l.mmu_idx, b, addr + 8, ra, MO_64); + } else if (unlikely(l.page[0].flags & TLB_DISCARD_WRITE)) { + /* nothing */ + } else { + store_atom_16(env, ra, l.page[0].haddr, l.memop, val); + } + return; + } + + first = l.page[0].size; + if (first == 8) { + MemOp mop8 = (l.memop & ~(MO_SIZE | MO_BSWAP)) | MO_64; + + if (l.memop & MO_BSWAP) { + val = bswap128(val); + } + if (HOST_BIG_ENDIAN) { + b = int128_getlo(val), a = int128_gethi(val); + } else { + a = int128_getlo(val), b = int128_gethi(val); + } + do_st_8(env, &l.page[0], a, l.mmu_idx, mop8, ra); + do_st_8(env, &l.page[1], b, l.mmu_idx, mop8, ra); + return; + } + + if ((l.memop & MO_BSWAP) != MO_LE) { + val = bswap128(val); + } + if (first < 8) { + do_st_leN(env, &l.page[0], int128_getlo(val), l.mmu_idx, l.memop, ra); + val = int128_urshift(val, first * 8); + do_st16_leN(env, &l.page[1], val, l.mmu_idx, l.memop, ra); + } else { + b = do_st16_leN(env, &l.page[0], val, l.mmu_idx, l.memop, ra); + do_st_leN(env, &l.page[1], b, l.mmu_idx, l.memop, ra); + } +} + +void helper_st16_mmu(CPUArchState *env, uint64_t addr, Int128 val, + MemOpIdx oi, uintptr_t retaddr) +{ + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128); + do_st16_mmu(env, addr, val, oi, retaddr); +} + +void helper_st_i128(CPUArchState *env, uint64_t addr, Int128 val, MemOpIdx oi) +{ + helper_st16_mmu(env, addr, val, oi, GETPC()); } /* @@ -2782,104 +3041,72 @@ static void plugin_store_cb(CPUArchState *env, abi_ptr addr, MemOpIdx oi) void cpu_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val, MemOpIdx oi, uintptr_t retaddr) { - helper_ret_stb_mmu(env, addr, val, oi, retaddr); + helper_stb_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } void cpu_stw_be_mmu(CPUArchState *env, target_ulong addr, uint16_t val, MemOpIdx oi, uintptr_t retaddr) { - helper_be_stw_mmu(env, addr, val, oi, retaddr); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUW); + do_st2_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } void cpu_stl_be_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr) { - helper_be_stl_mmu(env, addr, val, oi, retaddr); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUL); + do_st4_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } void cpu_stq_be_mmu(CPUArchState *env, target_ulong addr, uint64_t val, MemOpIdx oi, uintptr_t retaddr) { - helper_be_stq_mmu(env, addr, val, oi, retaddr); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUQ); + do_st8_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } void cpu_stw_le_mmu(CPUArchState *env, target_ulong addr, uint16_t val, MemOpIdx oi, uintptr_t retaddr) { - helper_le_stw_mmu(env, addr, val, oi, retaddr); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUW); + do_st2_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } void cpu_stl_le_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr) { - helper_le_stl_mmu(env, addr, val, oi, retaddr); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUL); + do_st4_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } void cpu_stq_le_mmu(CPUArchState *env, target_ulong addr, uint64_t val, MemOpIdx oi, uintptr_t retaddr) { - helper_le_stq_mmu(env, addr, val, oi, retaddr); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUQ); + do_st8_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } -void cpu_st16_be_mmu(CPUArchState *env, abi_ptr addr, Int128 val, - MemOpIdx oi, uintptr_t ra) +void cpu_st16_be_mmu(CPUArchState *env, target_ulong addr, Int128 val, + MemOpIdx oi, uintptr_t retaddr) { - MemOp mop = get_memop(oi); - int mmu_idx = get_mmuidx(oi); - MemOpIdx new_oi; - unsigned a_bits; - - tcg_debug_assert((mop & (MO_BSWAP|MO_SSIZE)) == (MO_BE|MO_128)); - a_bits = get_alignment_bits(mop); - - /* Handle CPU specific unaligned behaviour */ - if (addr & ((1 << a_bits) - 1)) { - cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE, - mmu_idx, ra); - } - - /* Construct an unaligned 64-bit replacement MemOpIdx. */ - mop = (mop & ~(MO_SIZE | MO_AMASK)) | MO_64 | MO_UNALN; - new_oi = make_memop_idx(mop, mmu_idx); - - helper_be_stq_mmu(env, addr, int128_gethi(val), new_oi, ra); - helper_be_stq_mmu(env, addr + 8, int128_getlo(val), new_oi, ra); - - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_BE|MO_128)); + do_st16_mmu(env, addr, val, oi, retaddr); + plugin_store_cb(env, addr, oi); } -void cpu_st16_le_mmu(CPUArchState *env, abi_ptr addr, Int128 val, - MemOpIdx oi, uintptr_t ra) +void cpu_st16_le_mmu(CPUArchState *env, target_ulong addr, Int128 val, + MemOpIdx oi, uintptr_t retaddr) { - MemOp mop = get_memop(oi); - int mmu_idx = get_mmuidx(oi); - MemOpIdx new_oi; - unsigned a_bits; - - tcg_debug_assert((mop & (MO_BSWAP|MO_SSIZE)) == (MO_LE|MO_128)); - a_bits = get_alignment_bits(mop); - - /* Handle CPU specific unaligned behaviour */ - if (addr & ((1 << a_bits) - 1)) { - cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE, - mmu_idx, ra); - } - - /* Construct an unaligned 64-bit replacement MemOpIdx. */ - mop = (mop & ~(MO_SIZE | MO_AMASK)) | MO_64 | MO_UNALN; - new_oi = make_memop_idx(mop, mmu_idx); - - helper_le_stq_mmu(env, addr, int128_getlo(val), new_oi, ra); - helper_le_stq_mmu(env, addr + 8, int128_gethi(val), new_oi, ra); - - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); + tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_LE|MO_128)); + do_st16_mmu(env, addr, val, oi, retaddr); + plugin_store_cb(env, addr, oi); } #include "ldst_common.c.inc" diff --git a/accel/tcg/ldst_atomicity.c.inc b/accel/tcg/ldst_atomicity.c.inc new file mode 100644 index 0000000000..ba5db7c366 --- /dev/null +++ b/accel/tcg/ldst_atomicity.c.inc @@ -0,0 +1,1262 @@ +/* + * Routines common to user and system emulation of load/store. + * + * Copyright (c) 2022 Linaro, Ltd. + * + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifdef CONFIG_ATOMIC64 +# define HAVE_al8 true +#else +# define HAVE_al8 false +#endif +#define HAVE_al8_fast (ATOMIC_REG_SIZE >= 8) + +/* + * If __alignof(unsigned __int128) < 16, GCC may refuse to inline atomics + * that are supported by the host, e.g. s390x. We can force the pointer to + * have our known alignment with __builtin_assume_aligned, however prior to + * GCC 13 that was only reliable with optimization enabled. See + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107389 + */ +#if defined(CONFIG_ATOMIC128_OPT) +# if !defined(__OPTIMIZE__) +# define ATTRIBUTE_ATOMIC128_OPT __attribute__((optimize("O1"))) +# endif +# define CONFIG_ATOMIC128 +#endif +#ifndef ATTRIBUTE_ATOMIC128_OPT +# define ATTRIBUTE_ATOMIC128_OPT +#endif + +#if defined(CONFIG_ATOMIC128) +# define HAVE_al16_fast true +#else +# define HAVE_al16_fast false +#endif +#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128) +# define HAVE_al16 true +#else +# define HAVE_al16 false +#endif + + +/** + * required_atomicity: + * + * Return the lg2 bytes of atomicity required by @memop for @p. + * If the operation must be split into two operations to be + * examined separately for atomicity, return -lg2. + */ +static int required_atomicity(CPUArchState *env, uintptr_t p, MemOp memop) +{ + MemOp atom = memop & MO_ATOM_MASK; + MemOp size = memop & MO_SIZE; + MemOp half = size ? size - 1 : 0; + unsigned tmp; + int atmax; + + switch (atom) { + case MO_ATOM_NONE: + atmax = MO_8; + break; + + case MO_ATOM_IFALIGN_PAIR: + size = half; + /* fall through */ + + case MO_ATOM_IFALIGN: + tmp = (1 << size) - 1; + atmax = p & tmp ? MO_8 : size; + break; + + case MO_ATOM_WITHIN16: + tmp = p & 15; + atmax = (tmp + (1 << size) <= 16 ? size : MO_8); + break; + + case MO_ATOM_WITHIN16_PAIR: + tmp = p & 15; + if (tmp + (1 << size) <= 16) { + atmax = size; + } else if (tmp + (1 << half) == 16) { + /* + * The pair exactly straddles the boundary. + * Both halves are naturally aligned and atomic. + */ + atmax = half; + } else { + /* + * One of the pair crosses the boundary, and is non-atomic. + * The other of the pair does not cross, and is atomic. + */ + atmax = -half; + } + break; + + case MO_ATOM_SUBALIGN: + /* + * Examine the alignment of p to determine if there are subobjects + * that must be aligned. Note that we only really need ctz4() -- + * any more sigificant bits are discarded by the immediately + * following comparison. + */ + tmp = ctz32(p); + atmax = MIN(size, tmp); + break; + + default: + g_assert_not_reached(); + } + + /* + * Here we have the architectural atomicity of the operation. + * However, when executing in a serial context, we need no extra + * host atomicity in order to avoid racing. This reduction + * avoids looping with cpu_loop_exit_atomic. + */ + if (cpu_in_serial_context(env_cpu(env))) { + return MO_8; + } + return atmax; +} + +/** + * load_atomic2: + * @pv: host address + * + * Atomically load 2 aligned bytes from @pv. + */ +static inline uint16_t load_atomic2(void *pv) +{ + uint16_t *p = __builtin_assume_aligned(pv, 2); + return qatomic_read(p); +} + +/** + * load_atomic4: + * @pv: host address + * + * Atomically load 4 aligned bytes from @pv. + */ +static inline uint32_t load_atomic4(void *pv) +{ + uint32_t *p = __builtin_assume_aligned(pv, 4); + return qatomic_read(p); +} + +/** + * load_atomic8: + * @pv: host address + * + * Atomically load 8 aligned bytes from @pv. + */ +static inline uint64_t load_atomic8(void *pv) +{ + uint64_t *p = __builtin_assume_aligned(pv, 8); + + qemu_build_assert(HAVE_al8); + return qatomic_read__nocheck(p); +} + +/** + * load_atomic16: + * @pv: host address + * + * Atomically load 16 aligned bytes from @pv. + */ +static inline Int128 ATTRIBUTE_ATOMIC128_OPT +load_atomic16(void *pv) +{ +#ifdef CONFIG_ATOMIC128 + __uint128_t *p = __builtin_assume_aligned(pv, 16); + Int128Alias r; + + r.u = qatomic_read__nocheck(p); + return r.s; +#else + qemu_build_not_reached(); +#endif +} + +/** + * load_atomic8_or_exit: + * @env: cpu context + * @ra: host unwind address + * @pv: host address + * + * Atomically load 8 aligned bytes from @pv. + * If this is not possible, longjmp out to restart serially. + */ +static uint64_t load_atomic8_or_exit(CPUArchState *env, uintptr_t ra, void *pv) +{ + if (HAVE_al8) { + return load_atomic8(pv); + } + +#ifdef CONFIG_USER_ONLY + /* + * If the page is not writable, then assume the value is immutable + * and requires no locking. This ignores the case of MAP_SHARED with + * another process, because the fallback start_exclusive solution + * provides no protection across processes. + */ + if (!page_check_range(h2g(pv), 8, PAGE_WRITE)) { + uint64_t *p = __builtin_assume_aligned(pv, 8); + return *p; + } +#endif + + /* Ultimate fallback: re-execute in serial context. */ + cpu_loop_exit_atomic(env_cpu(env), ra); +} + +/** + * load_atomic16_or_exit: + * @env: cpu context + * @ra: host unwind address + * @pv: host address + * + * Atomically load 16 aligned bytes from @pv. + * If this is not possible, longjmp out to restart serially. + */ +static Int128 load_atomic16_or_exit(CPUArchState *env, uintptr_t ra, void *pv) +{ + Int128 *p = __builtin_assume_aligned(pv, 16); + + if (HAVE_al16_fast) { + return load_atomic16(p); + } + +#ifdef CONFIG_USER_ONLY + /* + * We can only use cmpxchg to emulate a load if the page is writable. + * If the page is not writable, then assume the value is immutable + * and requires no locking. This ignores the case of MAP_SHARED with + * another process, because the fallback start_exclusive solution + * provides no protection across processes. + */ + if (!page_check_range(h2g(p), 16, PAGE_WRITE)) { + return *p; + } +#endif + + /* + * In system mode all guest pages are writable, and for user-only + * we have just checked writability. Try cmpxchg. + */ +#if defined(CONFIG_CMPXCHG128) + /* Swap 0 with 0, with the side-effect of returning the old value. */ + { + Int128Alias r; + r.u = __sync_val_compare_and_swap_16((__uint128_t *)p, 0, 0); + return r.s; + } +#endif + + /* Ultimate fallback: re-execute in serial context. */ + cpu_loop_exit_atomic(env_cpu(env), ra); +} + +/** + * load_atom_extract_al4x2: + * @pv: host address + * + * Load 4 bytes from @p, from two sequential atomic 4-byte loads. + */ +static uint32_t load_atom_extract_al4x2(void *pv) +{ + uintptr_t pi = (uintptr_t)pv; + int sh = (pi & 3) * 8; + uint32_t a, b; + + pv = (void *)(pi & ~3); + a = load_atomic4(pv); + b = load_atomic4(pv + 4); + + if (HOST_BIG_ENDIAN) { + return (a << sh) | (b >> (-sh & 31)); + } else { + return (a >> sh) | (b << (-sh & 31)); + } +} + +/** + * load_atom_extract_al8x2: + * @pv: host address + * + * Load 8 bytes from @p, from two sequential atomic 8-byte loads. + */ +static uint64_t load_atom_extract_al8x2(void *pv) +{ + uintptr_t pi = (uintptr_t)pv; + int sh = (pi & 7) * 8; + uint64_t a, b; + + pv = (void *)(pi & ~7); + a = load_atomic8(pv); + b = load_atomic8(pv + 8); + + if (HOST_BIG_ENDIAN) { + return (a << sh) | (b >> (-sh & 63)); + } else { + return (a >> sh) | (b << (-sh & 63)); + } +} + +/** + * load_atom_extract_al8_or_exit: + * @env: cpu context + * @ra: host unwind address + * @pv: host address + * @s: object size in bytes, @s <= 4. + * + * Atomically load @s bytes from @p, when p % s != 0, and [p, p+s-1] does + * not cross an 8-byte boundary. This means that we can perform an atomic + * 8-byte load and extract. + * The value is returned in the low bits of a uint32_t. + */ +static uint32_t load_atom_extract_al8_or_exit(CPUArchState *env, uintptr_t ra, + void *pv, int s) +{ + uintptr_t pi = (uintptr_t)pv; + int o = pi & 7; + int shr = (HOST_BIG_ENDIAN ? 8 - s - o : o) * 8; + + pv = (void *)(pi & ~7); + return load_atomic8_or_exit(env, ra, pv) >> shr; +} + +/** + * load_atom_extract_al16_or_exit: + * @env: cpu context + * @ra: host unwind address + * @p: host address + * @s: object size in bytes, @s <= 8. + * + * Atomically load @s bytes from @p, when p % 16 < 8 + * and p % 16 + s > 8. I.e. does not cross a 16-byte + * boundary, but *does* cross an 8-byte boundary. + * This is the slow version, so we must have eliminated + * any faster load_atom_extract_al8_or_exit case. + * + * If this is not possible, longjmp out to restart serially. + */ +static uint64_t load_atom_extract_al16_or_exit(CPUArchState *env, uintptr_t ra, + void *pv, int s) +{ + uintptr_t pi = (uintptr_t)pv; + int o = pi & 7; + int shr = (HOST_BIG_ENDIAN ? 16 - s - o : o) * 8; + Int128 r; + + /* + * Note constraints above: p & 8 must be clear. + * Provoke SIGBUS if possible otherwise. + */ + pv = (void *)(pi & ~7); + r = load_atomic16_or_exit(env, ra, pv); + + r = int128_urshift(r, shr); + return int128_getlo(r); +} + +/** + * load_atom_extract_al16_or_al8: + * @p: host address + * @s: object size in bytes, @s <= 8. + * + * Load @s bytes from @p, when p % s != 0. If [p, p+s-1] does not + * cross an 16-byte boundary then the access must be 16-byte atomic, + * otherwise the access must be 8-byte atomic. + */ +static inline uint64_t ATTRIBUTE_ATOMIC128_OPT +load_atom_extract_al16_or_al8(void *pv, int s) +{ +#if defined(CONFIG_ATOMIC128) + uintptr_t pi = (uintptr_t)pv; + int o = pi & 7; + int shr = (HOST_BIG_ENDIAN ? 16 - s - o : o) * 8; + __uint128_t r; + + pv = (void *)(pi & ~7); + if (pi & 8) { + uint64_t *p8 = __builtin_assume_aligned(pv, 16, 8); + uint64_t a = qatomic_read__nocheck(p8); + uint64_t b = qatomic_read__nocheck(p8 + 1); + + if (HOST_BIG_ENDIAN) { + r = ((__uint128_t)a << 64) | b; + } else { + r = ((__uint128_t)b << 64) | a; + } + } else { + __uint128_t *p16 = __builtin_assume_aligned(pv, 16, 0); + r = qatomic_read__nocheck(p16); + } + return r >> shr; +#else + qemu_build_not_reached(); +#endif +} + +/** + * load_atom_4_by_2: + * @pv: host address + * + * Load 4 bytes from @pv, with two 2-byte atomic loads. + */ +static inline uint32_t load_atom_4_by_2(void *pv) +{ + uint32_t a = load_atomic2(pv); + uint32_t b = load_atomic2(pv + 2); + + if (HOST_BIG_ENDIAN) { + return (a << 16) | b; + } else { + return (b << 16) | a; + } +} + +/** + * load_atom_8_by_2: + * @pv: host address + * + * Load 8 bytes from @pv, with four 2-byte atomic loads. + */ +static inline uint64_t load_atom_8_by_2(void *pv) +{ + uint32_t a = load_atom_4_by_2(pv); + uint32_t b = load_atom_4_by_2(pv + 4); + + if (HOST_BIG_ENDIAN) { + return ((uint64_t)a << 32) | b; + } else { + return ((uint64_t)b << 32) | a; + } +} + +/** + * load_atom_8_by_4: + * @pv: host address + * + * Load 8 bytes from @pv, with two 4-byte atomic loads. + */ +static inline uint64_t load_atom_8_by_4(void *pv) +{ + uint32_t a = load_atomic4(pv); + uint32_t b = load_atomic4(pv + 4); + + if (HOST_BIG_ENDIAN) { + return ((uint64_t)a << 32) | b; + } else { + return ((uint64_t)b << 32) | a; + } +} + +/** + * load_atom_8_by_8_or_4: + * @pv: host address + * + * Load 8 bytes from aligned @pv, with at least 4-byte atomicity. + */ +static inline uint64_t load_atom_8_by_8_or_4(void *pv) +{ + if (HAVE_al8_fast) { + return load_atomic8(pv); + } else { + return load_atom_8_by_4(pv); + } +} + +/** + * load_atom_2: + * @p: host address + * @memop: the full memory op + * + * Load 2 bytes from @p, honoring the atomicity of @memop. + */ +static uint16_t load_atom_2(CPUArchState *env, uintptr_t ra, + void *pv, MemOp memop) +{ + uintptr_t pi = (uintptr_t)pv; + int atmax; + + if (likely((pi & 1) == 0)) { + return load_atomic2(pv); + } + if (HAVE_al16_fast) { + return load_atom_extract_al16_or_al8(pv, 2); + } + + atmax = required_atomicity(env, pi, memop); + switch (atmax) { + case MO_8: + return lduw_he_p(pv); + case MO_16: + /* The only case remaining is MO_ATOM_WITHIN16. */ + if (!HAVE_al8_fast && (pi & 3) == 1) { + /* Big or little endian, we want the middle two bytes. */ + return load_atomic4(pv - 1) >> 8; + } + if ((pi & 15) != 7) { + return load_atom_extract_al8_or_exit(env, ra, pv, 2); + } + return load_atom_extract_al16_or_exit(env, ra, pv, 2); + default: + g_assert_not_reached(); + } +} + +/** + * load_atom_4: + * @p: host address + * @memop: the full memory op + * + * Load 4 bytes from @p, honoring the atomicity of @memop. + */ +static uint32_t load_atom_4(CPUArchState *env, uintptr_t ra, + void *pv, MemOp memop) +{ + uintptr_t pi = (uintptr_t)pv; + int atmax; + + if (likely((pi & 3) == 0)) { + return load_atomic4(pv); + } + if (HAVE_al16_fast) { + return load_atom_extract_al16_or_al8(pv, 4); + } + + atmax = required_atomicity(env, pi, memop); + switch (atmax) { + case MO_8: + case MO_16: + case -MO_16: + /* + * For MO_ATOM_IFALIGN, this is more atomicity than required, + * but it's trivially supported on all hosts, better than 4 + * individual byte loads (when the host requires alignment), + * and overlaps with the MO_ATOM_SUBALIGN case of p % 2 == 0. + */ + return load_atom_extract_al4x2(pv); + case MO_32: + if (!(pi & 4)) { + return load_atom_extract_al8_or_exit(env, ra, pv, 4); + } + return load_atom_extract_al16_or_exit(env, ra, pv, 4); + default: + g_assert_not_reached(); + } +} + +/** + * load_atom_8: + * @p: host address + * @memop: the full memory op + * + * Load 8 bytes from @p, honoring the atomicity of @memop. + */ +static uint64_t load_atom_8(CPUArchState *env, uintptr_t ra, + void *pv, MemOp memop) +{ + uintptr_t pi = (uintptr_t)pv; + int atmax; + + /* + * If the host does not support 8-byte atomics, wait until we have + * examined the atomicity parameters below. + */ + if (HAVE_al8 && likely((pi & 7) == 0)) { + return load_atomic8(pv); + } + if (HAVE_al16_fast) { + return load_atom_extract_al16_or_al8(pv, 8); + } + + atmax = required_atomicity(env, pi, memop); + if (atmax == MO_64) { + if (!HAVE_al8 && (pi & 7) == 0) { + load_atomic8_or_exit(env, ra, pv); + } + return load_atom_extract_al16_or_exit(env, ra, pv, 8); + } + if (HAVE_al8_fast) { + return load_atom_extract_al8x2(pv); + } + switch (atmax) { + case MO_8: + return ldq_he_p(pv); + case MO_16: + return load_atom_8_by_2(pv); + case MO_32: + return load_atom_8_by_4(pv); + case -MO_32: + if (HAVE_al8) { + return load_atom_extract_al8x2(pv); + } + cpu_loop_exit_atomic(env_cpu(env), ra); + default: + g_assert_not_reached(); + } +} + +/** + * load_atom_16: + * @p: host address + * @memop: the full memory op + * + * Load 16 bytes from @p, honoring the atomicity of @memop. + */ +static Int128 load_atom_16(CPUArchState *env, uintptr_t ra, + void *pv, MemOp memop) +{ + uintptr_t pi = (uintptr_t)pv; + int atmax; + Int128 r; + uint64_t a, b; + + /* + * If the host does not support 16-byte atomics, wait until we have + * examined the atomicity parameters below. + */ + if (HAVE_al16_fast && likely((pi & 15) == 0)) { + return load_atomic16(pv); + } + + atmax = required_atomicity(env, pi, memop); + switch (atmax) { + case MO_8: + memcpy(&r, pv, 16); + return r; + case MO_16: + a = load_atom_8_by_2(pv); + b = load_atom_8_by_2(pv + 8); + break; + case MO_32: + a = load_atom_8_by_4(pv); + b = load_atom_8_by_4(pv + 8); + break; + case MO_64: + if (!HAVE_al8) { + cpu_loop_exit_atomic(env_cpu(env), ra); + } + a = load_atomic8(pv); + b = load_atomic8(pv + 8); + break; + case -MO_64: + if (!HAVE_al8) { + cpu_loop_exit_atomic(env_cpu(env), ra); + } + a = load_atom_extract_al8x2(pv); + b = load_atom_extract_al8x2(pv + 8); + break; + case MO_128: + return load_atomic16_or_exit(env, ra, pv); + default: + g_assert_not_reached(); + } + return int128_make128(HOST_BIG_ENDIAN ? b : a, HOST_BIG_ENDIAN ? a : b); +} + +/** + * store_atomic2: + * @pv: host address + * @val: value to store + * + * Atomically store 2 aligned bytes to @pv. + */ +static inline void store_atomic2(void *pv, uint16_t val) +{ + uint16_t *p = __builtin_assume_aligned(pv, 2); + qatomic_set(p, val); +} + +/** + * store_atomic4: + * @pv: host address + * @val: value to store + * + * Atomically store 4 aligned bytes to @pv. + */ +static inline void store_atomic4(void *pv, uint32_t val) +{ + uint32_t *p = __builtin_assume_aligned(pv, 4); + qatomic_set(p, val); +} + +/** + * store_atomic8: + * @pv: host address + * @val: value to store + * + * Atomically store 8 aligned bytes to @pv. + */ +static inline void store_atomic8(void *pv, uint64_t val) +{ + uint64_t *p = __builtin_assume_aligned(pv, 8); + + qemu_build_assert(HAVE_al8); + qatomic_set__nocheck(p, val); +} + +/** + * store_atomic16: + * @pv: host address + * @val: value to store + * + * Atomically store 16 aligned bytes to @pv. + */ +static inline void ATTRIBUTE_ATOMIC128_OPT +store_atomic16(void *pv, Int128Alias val) +{ +#if defined(CONFIG_ATOMIC128) + __uint128_t *pu = __builtin_assume_aligned(pv, 16); + qatomic_set__nocheck(pu, val.u); +#elif defined(CONFIG_CMPXCHG128) + __uint128_t *pu = __builtin_assume_aligned(pv, 16); + __uint128_t o; + + /* + * Without CONFIG_ATOMIC128, __atomic_compare_exchange_n will always + * defer to libatomic, so we must use __sync_*_compare_and_swap_16 + * and accept the sequential consistency that comes with it. + */ + do { + o = *pu; + } while (!__sync_bool_compare_and_swap_16(pu, o, val.u)); +#else + qemu_build_not_reached(); +#endif +} + +/** + * store_atom_4x2 + */ +static inline void store_atom_4_by_2(void *pv, uint32_t val) +{ + store_atomic2(pv, val >> (HOST_BIG_ENDIAN ? 16 : 0)); + store_atomic2(pv + 2, val >> (HOST_BIG_ENDIAN ? 0 : 16)); +} + +/** + * store_atom_8_by_2 + */ +static inline void store_atom_8_by_2(void *pv, uint64_t val) +{ + store_atom_4_by_2(pv, val >> (HOST_BIG_ENDIAN ? 32 : 0)); + store_atom_4_by_2(pv + 4, val >> (HOST_BIG_ENDIAN ? 0 : 32)); +} + +/** + * store_atom_8_by_4 + */ +static inline void store_atom_8_by_4(void *pv, uint64_t val) +{ + store_atomic4(pv, val >> (HOST_BIG_ENDIAN ? 32 : 0)); + store_atomic4(pv + 4, val >> (HOST_BIG_ENDIAN ? 0 : 32)); +} + +/** + * store_atom_insert_al4: + * @p: host address + * @val: shifted value to store + * @msk: mask for value to store + * + * Atomically store @val to @p, masked by @msk. + */ +static void store_atom_insert_al4(uint32_t *p, uint32_t val, uint32_t msk) +{ + uint32_t old, new; + + p = __builtin_assume_aligned(p, 4); + old = qatomic_read(p); + do { + new = (old & ~msk) | val; + } while (!__atomic_compare_exchange_n(p, &old, new, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)); +} + +/** + * store_atom_insert_al8: + * @p: host address + * @val: shifted value to store + * @msk: mask for value to store + * + * Atomically store @val to @p masked by @msk. + */ +static void store_atom_insert_al8(uint64_t *p, uint64_t val, uint64_t msk) +{ + uint64_t old, new; + + qemu_build_assert(HAVE_al8); + p = __builtin_assume_aligned(p, 8); + old = qatomic_read__nocheck(p); + do { + new = (old & ~msk) | val; + } while (!__atomic_compare_exchange_n(p, &old, new, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)); +} + +/** + * store_atom_insert_al16: + * @p: host address + * @val: shifted value to store + * @msk: mask for value to store + * + * Atomically store @val to @p masked by @msk. + */ +static void ATTRIBUTE_ATOMIC128_OPT +store_atom_insert_al16(Int128 *ps, Int128Alias val, Int128Alias msk) +{ +#if defined(CONFIG_ATOMIC128) + __uint128_t *pu, old, new; + + /* With CONFIG_ATOMIC128, we can avoid the memory barriers. */ + pu = __builtin_assume_aligned(ps, 16); + old = *pu; + do { + new = (old & ~msk.u) | val.u; + } while (!__atomic_compare_exchange_n(pu, &old, new, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)); +#elif defined(CONFIG_CMPXCHG128) + __uint128_t *pu, old, new; + + /* + * Without CONFIG_ATOMIC128, __atomic_compare_exchange_n will always + * defer to libatomic, so we must use __sync_*_compare_and_swap_16 + * and accept the sequential consistency that comes with it. + */ + pu = __builtin_assume_aligned(ps, 16); + do { + old = *pu; + new = (old & ~msk.u) | val.u; + } while (!__sync_bool_compare_and_swap_16(pu, old, new)); +#else + qemu_build_not_reached(); +#endif +} + +/** + * store_bytes_leN: + * @pv: host address + * @size: number of bytes to store + * @val_le: data to store + * + * Store @size bytes at @p. The bytes to store are extracted in little-endian order + * from @val_le; return the bytes of @val_le beyond @size that have not been stored. + */ +static uint64_t store_bytes_leN(void *pv, int size, uint64_t val_le) +{ + uint8_t *p = pv; + for (int i = 0; i < size; i++, val_le >>= 8) { + p[i] = val_le; + } + return val_le; +} + +/** + * store_parts_leN + * @pv: host address + * @size: number of bytes to store + * @val_le: data to store + * + * As store_bytes_leN, but atomically on each aligned part. + */ +G_GNUC_UNUSED +static uint64_t store_parts_leN(void *pv, int size, uint64_t val_le) +{ + do { + int n; + + /* Find minimum of alignment and size */ + switch (((uintptr_t)pv | size) & 7) { + case 4: + store_atomic4(pv, le32_to_cpu(val_le)); + val_le >>= 32; + n = 4; + break; + case 2: + case 6: + store_atomic2(pv, le16_to_cpu(val_le)); + val_le >>= 16; + n = 2; + break; + default: + *(uint8_t *)pv = val_le; + val_le >>= 8; + n = 1; + break; + case 0: + g_assert_not_reached(); + } + pv += n; + size -= n; + } while (size != 0); + + return val_le; +} + +/** + * store_whole_le4 + * @pv: host address + * @size: number of bytes to store + * @val_le: data to store + * + * As store_bytes_leN, but atomically as a whole. + * Four aligned bytes are guaranteed to cover the store. + */ +static uint64_t store_whole_le4(void *pv, int size, uint64_t val_le) +{ + int sz = size * 8; + int o = (uintptr_t)pv & 3; + int sh = o * 8; + uint32_t m = MAKE_64BIT_MASK(0, sz); + uint32_t v; + + if (HOST_BIG_ENDIAN) { + v = bswap32(val_le) >> sh; + m = bswap32(m) >> sh; + } else { + v = val_le << sh; + m <<= sh; + } + store_atom_insert_al4(pv - o, v, m); + return val_le >> sz; +} + +/** + * store_whole_le8 + * @pv: host address + * @size: number of bytes to store + * @val_le: data to store + * + * As store_bytes_leN, but atomically as a whole. + * Eight aligned bytes are guaranteed to cover the store. + */ +static uint64_t store_whole_le8(void *pv, int size, uint64_t val_le) +{ + int sz = size * 8; + int o = (uintptr_t)pv & 7; + int sh = o * 8; + uint64_t m = MAKE_64BIT_MASK(0, sz); + uint64_t v; + + qemu_build_assert(HAVE_al8); + if (HOST_BIG_ENDIAN) { + v = bswap64(val_le) >> sh; + m = bswap64(m) >> sh; + } else { + v = val_le << sh; + m <<= sh; + } + store_atom_insert_al8(pv - o, v, m); + return val_le >> sz; +} + +/** + * store_whole_le16 + * @pv: host address + * @size: number of bytes to store + * @val_le: data to store + * + * As store_bytes_leN, but atomically as a whole. + * 16 aligned bytes are guaranteed to cover the store. + */ +static uint64_t store_whole_le16(void *pv, int size, Int128 val_le) +{ + int sz = size * 8; + int o = (uintptr_t)pv & 15; + int sh = o * 8; + Int128 m, v; + + qemu_build_assert(HAVE_al16); + + /* Like MAKE_64BIT_MASK(0, sz), but larger. */ + if (sz <= 64) { + m = int128_make64(MAKE_64BIT_MASK(0, sz)); + } else { + m = int128_make128(-1, MAKE_64BIT_MASK(0, sz - 64)); + } + + if (HOST_BIG_ENDIAN) { + v = int128_urshift(bswap128(val_le), sh); + m = int128_urshift(bswap128(m), sh); + } else { + v = int128_lshift(val_le, sh); + m = int128_lshift(m, sh); + } + store_atom_insert_al16(pv - o, v, m); + + /* Unused if sz <= 64. */ + return int128_gethi(val_le) >> (sz - 64); +} + +/** + * store_atom_2: + * @p: host address + * @val: the value to store + * @memop: the full memory op + * + * Store 2 bytes to @p, honoring the atomicity of @memop. + */ +static void store_atom_2(CPUArchState *env, uintptr_t ra, + void *pv, MemOp memop, uint16_t val) +{ + uintptr_t pi = (uintptr_t)pv; + int atmax; + + if (likely((pi & 1) == 0)) { + store_atomic2(pv, val); + return; + } + + atmax = required_atomicity(env, pi, memop); + if (atmax == MO_8) { + stw_he_p(pv, val); + return; + } + + /* + * The only case remaining is MO_ATOM_WITHIN16. + * Big or little endian, we want the middle two bytes in each test. + */ + if ((pi & 3) == 1) { + store_atom_insert_al4(pv - 1, (uint32_t)val << 8, MAKE_64BIT_MASK(8, 16)); + return; + } else if ((pi & 7) == 3) { + if (HAVE_al8) { + store_atom_insert_al8(pv - 3, (uint64_t)val << 24, MAKE_64BIT_MASK(24, 16)); + return; + } + } else if ((pi & 15) == 7) { + if (HAVE_al16) { + Int128 v = int128_lshift(int128_make64(val), 56); + Int128 m = int128_lshift(int128_make64(0xffff), 56); + store_atom_insert_al16(pv - 7, v, m); + return; + } + } else { + g_assert_not_reached(); + } + + cpu_loop_exit_atomic(env_cpu(env), ra); +} + +/** + * store_atom_4: + * @p: host address + * @val: the value to store + * @memop: the full memory op + * + * Store 4 bytes to @p, honoring the atomicity of @memop. + */ +static void store_atom_4(CPUArchState *env, uintptr_t ra, + void *pv, MemOp memop, uint32_t val) +{ + uintptr_t pi = (uintptr_t)pv; + int atmax; + + if (likely((pi & 3) == 0)) { + store_atomic4(pv, val); + return; + } + + atmax = required_atomicity(env, pi, memop); + switch (atmax) { + case MO_8: + stl_he_p(pv, val); + return; + case MO_16: + store_atom_4_by_2(pv, val); + return; + case -MO_16: + { + uint32_t val_le = cpu_to_le32(val); + int s2 = pi & 3; + int s1 = 4 - s2; + + switch (s2) { + case 1: + val_le = store_whole_le4(pv, s1, val_le); + *(uint8_t *)(pv + 3) = val_le; + break; + case 3: + *(uint8_t *)pv = val_le; + store_whole_le4(pv + 1, s2, val_le >> 8); + break; + case 0: /* aligned */ + case 2: /* atmax MO_16 */ + default: + g_assert_not_reached(); + } + } + return; + case MO_32: + if ((pi & 7) < 4) { + if (HAVE_al8) { + store_whole_le8(pv, 4, cpu_to_le32(val)); + return; + } + } else { + if (HAVE_al16) { + store_whole_le16(pv, 4, int128_make64(cpu_to_le32(val))); + return; + } + } + cpu_loop_exit_atomic(env_cpu(env), ra); + default: + g_assert_not_reached(); + } +} + +/** + * store_atom_8: + * @p: host address + * @val: the value to store + * @memop: the full memory op + * + * Store 8 bytes to @p, honoring the atomicity of @memop. + */ +static void store_atom_8(CPUArchState *env, uintptr_t ra, + void *pv, MemOp memop, uint64_t val) +{ + uintptr_t pi = (uintptr_t)pv; + int atmax; + + if (HAVE_al8 && likely((pi & 7) == 0)) { + store_atomic8(pv, val); + return; + } + + atmax = required_atomicity(env, pi, memop); + switch (atmax) { + case MO_8: + stq_he_p(pv, val); + return; + case MO_16: + store_atom_8_by_2(pv, val); + return; + case MO_32: + store_atom_8_by_4(pv, val); + return; + case -MO_32: + if (HAVE_al8) { + uint64_t val_le = cpu_to_le64(val); + int s2 = pi & 7; + int s1 = 8 - s2; + + switch (s2) { + case 1 ... 3: + val_le = store_whole_le8(pv, s1, val_le); + store_bytes_leN(pv + s1, s2, val_le); + break; + case 5 ... 7: + val_le = store_bytes_leN(pv, s1, val_le); + store_whole_le8(pv + s1, s2, val_le); + break; + case 0: /* aligned */ + case 4: /* atmax MO_32 */ + default: + g_assert_not_reached(); + } + return; + } + break; + case MO_64: + if (HAVE_al16) { + store_whole_le16(pv, 8, int128_make64(cpu_to_le64(val))); + return; + } + break; + default: + g_assert_not_reached(); + } + cpu_loop_exit_atomic(env_cpu(env), ra); +} + +/** + * store_atom_16: + * @p: host address + * @val: the value to store + * @memop: the full memory op + * + * Store 16 bytes to @p, honoring the atomicity of @memop. + */ +static void store_atom_16(CPUArchState *env, uintptr_t ra, + void *pv, MemOp memop, Int128 val) +{ + uintptr_t pi = (uintptr_t)pv; + uint64_t a, b; + int atmax; + + if (HAVE_al16_fast && likely((pi & 15) == 0)) { + store_atomic16(pv, val); + return; + } + + atmax = required_atomicity(env, pi, memop); + + a = HOST_BIG_ENDIAN ? int128_gethi(val) : int128_getlo(val); + b = HOST_BIG_ENDIAN ? int128_getlo(val) : int128_gethi(val); + switch (atmax) { + case MO_8: + memcpy(pv, &val, 16); + return; + case MO_16: + store_atom_8_by_2(pv, a); + store_atom_8_by_2(pv + 8, b); + return; + case MO_32: + store_atom_8_by_4(pv, a); + store_atom_8_by_4(pv + 8, b); + return; + case MO_64: + if (HAVE_al8) { + store_atomic8(pv, a); + store_atomic8(pv + 8, b); + return; + } + break; + case -MO_64: + if (HAVE_al16) { + uint64_t val_le; + int s2 = pi & 15; + int s1 = 16 - s2; + + if (HOST_BIG_ENDIAN) { + val = bswap128(val); + } + switch (s2) { + case 1 ... 7: + val_le = store_whole_le16(pv, s1, val); + store_bytes_leN(pv + s1, s2, val_le); + break; + case 9 ... 15: + store_bytes_leN(pv, s1, int128_getlo(val)); + val = int128_urshift(val, s1 * 8); + store_whole_le16(pv + s1, s2, val); + break; + case 0: /* aligned */ + case 8: /* atmax MO_64 */ + default: + g_assert_not_reached(); + } + return; + } + break; + case MO_128: + if (HAVE_al16) { + store_atomic16(pv, val); + return; + } + break; + default: + g_assert_not_reached(); + } + cpu_loop_exit_atomic(env_cpu(env), ra); +} diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index 5efb8db258..5b73a39ce5 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -92,27 +92,6 @@ void HELPER(plugin_vcpu_mem_cb)(unsigned int vcpu_index, void *userdata) { } -static void do_gen_mem_cb(TCGv vaddr, uint32_t info) -{ - TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); - TCGv_i32 meminfo = tcg_temp_ebb_new_i32(); - TCGv_i64 vaddr64 = tcg_temp_ebb_new_i64(); - TCGv_ptr udata = tcg_temp_ebb_new_ptr(); - - tcg_gen_movi_i32(meminfo, info); - tcg_gen_movi_ptr(udata, 0); - tcg_gen_ld_i32(cpu_index, cpu_env, - -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); - tcg_gen_extu_tl_i64(vaddr64, vaddr); - - gen_helper_plugin_vcpu_mem_cb(cpu_index, meminfo, vaddr64, udata); - - tcg_temp_free_ptr(udata); - tcg_temp_free_i64(vaddr64); - tcg_temp_free_i32(meminfo); - tcg_temp_free_i32(cpu_index); -} - static void gen_empty_udata_cb(void) { TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); @@ -145,9 +124,22 @@ static void gen_empty_inline_cb(void) tcg_temp_free_i64(val); } -static void gen_empty_mem_cb(TCGv addr, uint32_t info) +static void gen_empty_mem_cb(TCGv_i64 addr, uint32_t info) { - do_gen_mem_cb(addr, info); + TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); + TCGv_i32 meminfo = tcg_temp_ebb_new_i32(); + TCGv_ptr udata = tcg_temp_ebb_new_ptr(); + + tcg_gen_movi_i32(meminfo, info); + tcg_gen_movi_ptr(udata, 0); + tcg_gen_ld_i32(cpu_index, cpu_env, + -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); + + gen_helper_plugin_vcpu_mem_cb(cpu_index, meminfo, addr, udata); + + tcg_temp_free_ptr(udata); + tcg_temp_free_i32(meminfo); + tcg_temp_free_i32(cpu_index); } /* @@ -202,35 +194,17 @@ static void plugin_gen_empty_callback(enum plugin_gen_from from) } } -union mem_gen_fn { - void (*mem_fn)(TCGv, uint32_t); - void (*inline_fn)(void); -}; - -static void gen_mem_wrapped(enum plugin_gen_cb type, - const union mem_gen_fn *f, TCGv addr, - uint32_t info, bool is_mem) +void plugin_gen_empty_mem_callback(TCGv_i64 addr, uint32_t info) { enum qemu_plugin_mem_rw rw = get_plugin_meminfo_rw(info); - gen_plugin_cb_start(PLUGIN_GEN_FROM_MEM, type, rw); - if (is_mem) { - f->mem_fn(addr, info); - } else { - f->inline_fn(); - } + gen_plugin_cb_start(PLUGIN_GEN_FROM_MEM, PLUGIN_GEN_CB_MEM, rw); + gen_empty_mem_cb(addr, info); tcg_gen_plugin_cb_end(); -} -void plugin_gen_empty_mem_callback(TCGv addr, uint32_t info) -{ - union mem_gen_fn fn; - - fn.mem_fn = gen_empty_mem_cb; - gen_mem_wrapped(PLUGIN_GEN_CB_MEM, &fn, addr, info, true); - - fn.inline_fn = gen_empty_inline_cb; - gen_mem_wrapped(PLUGIN_GEN_CB_INLINE, &fn, 0, info, false); + gen_plugin_cb_start(PLUGIN_GEN_FROM_MEM, PLUGIN_GEN_CB_INLINE, rw); + gen_empty_inline_cb(); + tcg_gen_plugin_cb_end(); } static TCGOp *find_op(TCGOp *op, TCGOpcode opc) @@ -280,33 +254,6 @@ static TCGOp *copy_op(TCGOp **begin_op, TCGOp *op, TCGOpcode opc) return op; } -static TCGOp *copy_extu_i32_i64(TCGOp **begin_op, TCGOp *op) -{ - if (TCG_TARGET_REG_BITS == 32) { - /* mov_i32 */ - op = copy_op(begin_op, op, INDEX_op_mov_i32); - /* mov_i32 w/ $0 */ - op = copy_op(begin_op, op, INDEX_op_mov_i32); - } else { - /* extu_i32_i64 */ - op = copy_op(begin_op, op, INDEX_op_extu_i32_i64); - } - return op; -} - -static TCGOp *copy_mov_i64(TCGOp **begin_op, TCGOp *op) -{ - if (TCG_TARGET_REG_BITS == 32) { - /* 2x mov_i32 */ - op = copy_op(begin_op, op, INDEX_op_mov_i32); - op = copy_op(begin_op, op, INDEX_op_mov_i32); - } else { - /* mov_i64 */ - op = copy_op(begin_op, op, INDEX_op_mov_i64); - } - return op; -} - static TCGOp *copy_const_ptr(TCGOp **begin_op, TCGOp *op, void *ptr) { if (UINTPTR_MAX == UINT32_MAX) { @@ -321,18 +268,6 @@ static TCGOp *copy_const_ptr(TCGOp **begin_op, TCGOp *op, void *ptr) return op; } -static TCGOp *copy_extu_tl_i64(TCGOp **begin_op, TCGOp *op) -{ - if (TARGET_LONG_BITS == 32) { - /* extu_i32_i64 */ - op = copy_extu_i32_i64(begin_op, op); - } else { - /* mov_i64 */ - op = copy_mov_i64(begin_op, op); - } - return op; -} - static TCGOp *copy_ld_i64(TCGOp **begin_op, TCGOp *op) { if (TCG_TARGET_REG_BITS == 32) { @@ -477,9 +412,6 @@ static TCGOp *append_mem_cb(const struct qemu_plugin_dyn_cb *cb, tcg_debug_assert(begin_op && begin_op->opc == INDEX_op_ld_i32); } - /* extu_tl_i64 */ - op = copy_extu_tl_i64(&begin_op, op); - if (type == PLUGIN_GEN_CB_MEM) { /* call */ op = copy_call(&begin_op, op, HELPER(plugin_vcpu_mem_cb), diff --git a/accel/tcg/tcg-accel-ops-rr.c b/accel/tcg/tcg-accel-ops-rr.c index 5788efa5ff..b6d10fa9a2 100644 --- a/accel/tcg/tcg-accel-ops-rr.c +++ b/accel/tcg/tcg-accel-ops-rr.c @@ -72,11 +72,13 @@ static void rr_kick_next_cpu(void) { CPUState *cpu; do { - cpu = qatomic_mb_read(&rr_current_cpu); + cpu = qatomic_read(&rr_current_cpu); if (cpu) { cpu_exit(cpu); } - } while (cpu != qatomic_mb_read(&rr_current_cpu)); + /* Finish kicking this cpu before reading again. */ + smp_mb(); + } while (cpu != qatomic_read(&rr_current_cpu)); } static void rr_kick_thread(void *opaque) @@ -241,8 +243,9 @@ static void *rr_cpu_thread_fn(void *arg) } while (cpu && cpu_work_list_empty(cpu) && !cpu->exit_request) { - + /* Store rr_current_cpu before evaluating cpu_can_run(). */ qatomic_mb_set(&rr_current_cpu, cpu); + current_cpu = cpu; qemu_clock_enable(QEMU_CLOCK_VIRTUAL, @@ -280,7 +283,7 @@ static void *rr_cpu_thread_fn(void *arg) cpu = CPU_NEXT(cpu); } /* while (cpu && !cpu->exit_request).. */ - /* Does not need qatomic_mb_set because a spurious wakeup is okay. */ + /* Does not need a memory barrier because a spurious wakeup is okay. */ qatomic_set(&rr_current_cpu, NULL); if (cpu && cpu->exit_request) { diff --git a/accel/tcg/tcg-runtime.h b/accel/tcg/tcg-runtime.h index b8e6421c8a..6f8c2061d0 100644 --- a/accel/tcg/tcg-runtime.h +++ b/accel/tcg/tcg-runtime.h @@ -39,62 +39,65 @@ DEF_HELPER_FLAGS_1(exit_atomic, TCG_CALL_NO_WG, noreturn, env) DEF_HELPER_FLAGS_3(memset, TCG_CALL_NO_RWG, ptr, ptr, int, ptr) #endif /* IN_HELPER_PROTO */ +DEF_HELPER_FLAGS_3(ld_i128, TCG_CALL_NO_WG, i128, env, i64, i32) +DEF_HELPER_FLAGS_4(st_i128, TCG_CALL_NO_WG, void, env, i64, i128, i32) + DEF_HELPER_FLAGS_5(atomic_cmpxchgb, TCG_CALL_NO_WG, - i32, env, tl, i32, i32, i32) + i32, env, i64, i32, i32, i32) DEF_HELPER_FLAGS_5(atomic_cmpxchgw_be, TCG_CALL_NO_WG, - i32, env, tl, i32, i32, i32) + i32, env, i64, i32, i32, i32) DEF_HELPER_FLAGS_5(atomic_cmpxchgw_le, TCG_CALL_NO_WG, - i32, env, tl, i32, i32, i32) + i32, env, i64, i32, i32, i32) DEF_HELPER_FLAGS_5(atomic_cmpxchgl_be, TCG_CALL_NO_WG, - i32, env, tl, i32, i32, i32) + i32, env, i64, i32, i32, i32) DEF_HELPER_FLAGS_5(atomic_cmpxchgl_le, TCG_CALL_NO_WG, - i32, env, tl, i32, i32, i32) + i32, env, i64, i32, i32, i32) #ifdef CONFIG_ATOMIC64 DEF_HELPER_FLAGS_5(atomic_cmpxchgq_be, TCG_CALL_NO_WG, - i64, env, tl, i64, i64, i32) + i64, env, i64, i64, i64, i32) DEF_HELPER_FLAGS_5(atomic_cmpxchgq_le, TCG_CALL_NO_WG, - i64, env, tl, i64, i64, i32) + i64, env, i64, i64, i64, i32) #endif #ifdef CONFIG_CMPXCHG128 DEF_HELPER_FLAGS_5(atomic_cmpxchgo_be, TCG_CALL_NO_WG, - i128, env, tl, i128, i128, i32) + i128, env, i64, i128, i128, i32) DEF_HELPER_FLAGS_5(atomic_cmpxchgo_le, TCG_CALL_NO_WG, - i128, env, tl, i128, i128, i32) + i128, env, i64, i128, i128, i32) #endif DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo_be, TCG_CALL_NO_WG, - i128, env, tl, i128, i128, i32) + i128, env, i64, i128, i128, i32) DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo_le, TCG_CALL_NO_WG, - i128, env, tl, i128, i128, i32) + i128, env, i64, i128, i128, i32) #ifdef CONFIG_ATOMIC64 #define GEN_ATOMIC_HELPERS(NAME) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), b), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) \ + TCG_CALL_NO_WG, i32, env, i64, i32, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), w_le), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) \ + TCG_CALL_NO_WG, i32, env, i64, i32, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), w_be), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) \ + TCG_CALL_NO_WG, i32, env, i64, i32, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), l_le), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) \ + TCG_CALL_NO_WG, i32, env, i64, i32, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), l_be), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) \ + TCG_CALL_NO_WG, i32, env, i64, i32, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), q_le), \ - TCG_CALL_NO_WG, i64, env, tl, i64, i32) \ + TCG_CALL_NO_WG, i64, env, i64, i64, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), q_be), \ - TCG_CALL_NO_WG, i64, env, tl, i64, i32) + TCG_CALL_NO_WG, i64, env, i64, i64, i32) #else #define GEN_ATOMIC_HELPERS(NAME) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), b), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) \ + TCG_CALL_NO_WG, i32, env, i64, i32, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), w_le), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) \ + TCG_CALL_NO_WG, i32, env, i64, i32, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), w_be), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) \ + TCG_CALL_NO_WG, i32, env, i64, i32, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), l_le), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) \ + TCG_CALL_NO_WG, i32, env, i64, i32, i32) \ DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), l_be), \ - TCG_CALL_NO_WG, i32, env, tl, i32, i32) + TCG_CALL_NO_WG, i32, env, i64, i32, i32) #endif /* CONFIG_ATOMIC64 */ GEN_ATOMIC_HELPERS(fetch_add) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 5b13281119..353849ca6d 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -72,9 +72,11 @@ QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > TBContext tb_ctx; -/* Encode VAL as a signed leb128 sequence at P. - Return P incremented past the encoded value. */ -static uint8_t *encode_sleb128(uint8_t *p, target_long val) +/* + * Encode VAL as a signed leb128 sequence at P. + * Return P incremented past the encoded value. + */ +static uint8_t *encode_sleb128(uint8_t *p, int64_t val) { int more, byte; @@ -92,21 +94,23 @@ static uint8_t *encode_sleb128(uint8_t *p, target_long val) return p; } -/* Decode a signed leb128 sequence at *PP; increment *PP past the - decoded value. Return the decoded value. */ -static target_long decode_sleb128(const uint8_t **pp) +/* + * Decode a signed leb128 sequence at *PP; increment *PP past the + * decoded value. Return the decoded value. + */ +static int64_t decode_sleb128(const uint8_t **pp) { const uint8_t *p = *pp; - target_long val = 0; + int64_t val = 0; int byte, shift = 0; do { byte = *p++; - val |= (target_ulong)(byte & 0x7f) << shift; + val |= (int64_t)(byte & 0x7f) << shift; shift += 7; } while (byte & 0x80); if (shift < TARGET_LONG_BITS && (byte & 0x40)) { - val |= -(target_ulong)1 << shift; + val |= -(int64_t)1 << shift; } *pp = p; @@ -132,7 +136,7 @@ static int encode_search(TranslationBlock *tb, uint8_t *block) int i, j, n; for (i = 0, n = tb->icount; i < n; ++i) { - target_ulong prev; + uint64_t prev; for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { if (i == 0) { @@ -352,6 +356,13 @@ TranslationBlock *tb_gen_code(CPUState *cpu, tb_set_page_addr0(tb, phys_pc); tb_set_page_addr1(tb, -1); tcg_ctx->gen_tb = tb; + tcg_ctx->addr_type = TCG_TYPE_TL; +#ifdef CONFIG_SOFTMMU + tcg_ctx->page_bits = TARGET_PAGE_BITS; + tcg_ctx->page_mask = TARGET_PAGE_MASK; + tcg_ctx->tlb_dyn_max_bits = CPU_TLB_DYN_MAX_BITS; +#endif + tb_overflow: #ifdef CONFIG_PROFILER @@ -444,7 +455,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, /* Dump header and the first instruction */ fprintf(logfile, "OUT: [size=%d]\n", gen_code_size); fprintf(logfile, - " -- guest addr 0x" TARGET_FMT_lx " + tb prologue\n", + " -- guest addr 0x%016" PRIx64 " + tb prologue\n", tcg_ctx->gen_insn_data[insn][0]); chunk_start = tcg_ctx->gen_insn_end_off[insn]; disas(logfile, tb->tc.ptr, chunk_start); @@ -457,7 +468,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, while (insn < tb->icount) { size_t chunk_end = tcg_ctx->gen_insn_end_off[insn]; if (chunk_end > chunk_start) { - fprintf(logfile, " -- guest addr 0x" TARGET_FMT_lx "\n", + fprintf(logfile, " -- guest addr 0x%016" PRIx64 "\n", tcg_ctx->gen_insn_data[insn][0]); disas(logfile, tb->tc.ptr + chunk_start, chunk_end - chunk_start); diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index fc597a010d..36ad8284a5 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -889,35 +889,9 @@ void page_reset_target_data(target_ulong start, target_ulong last) { } /* The softmmu versions of these helpers are in cputlb.c. */ -/* - * Verify that we have passed the correct MemOp to the correct function. - * - * We could present one function to target code, and dispatch based on - * the MemOp, but so far we have worked hard to avoid an indirect function - * call along the memory path. - */ -static void validate_memop(MemOpIdx oi, MemOp expected) -{ -#ifdef CONFIG_DEBUG_TCG - MemOp have = get_memop(oi) & (MO_SIZE | MO_BSWAP); - assert(have == expected); -#endif -} - -void helper_unaligned_ld(CPUArchState *env, target_ulong addr) -{ - cpu_loop_exit_sigbus(env_cpu(env), addr, MMU_DATA_LOAD, GETPC()); -} - -void helper_unaligned_st(CPUArchState *env, target_ulong addr) -{ - cpu_loop_exit_sigbus(env_cpu(env), addr, MMU_DATA_STORE, GETPC()); -} - -static void *cpu_mmu_lookup(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t ra, MMUAccessType type) +static void *cpu_mmu_lookup(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra, MMUAccessType type) { - MemOp mop = get_memop(oi); int a_bits = get_alignment_bits(mop); void *ret; @@ -931,116 +905,251 @@ static void *cpu_mmu_lookup(CPUArchState *env, target_ulong addr, return ret; } -uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +#include "ldst_atomicity.c.inc" + +static uint8_t do_ld1_mmu(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra) { void *haddr; uint8_t ret; - validate_memop(oi, MO_UB); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); + tcg_debug_assert((mop & MO_SIZE) == MO_8); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD); ret = ldub_p(haddr); clear_helper_retaddr(); + return ret; +} + +tcg_target_ulong helper_ldub_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) +{ + return do_ld1_mmu(env, addr, get_memop(oi), ra); +} + +tcg_target_ulong helper_ldsb_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) +{ + return (int8_t)do_ld1_mmu(env, addr, get_memop(oi), ra); +} + +uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) +{ + uint8_t ret = do_ld1_mmu(env, addr, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); return ret; } -uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +static uint16_t do_ld2_he_mmu(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra) { void *haddr; uint16_t ret; - validate_memop(oi, MO_BEUW); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); - ret = lduw_be_p(haddr); + tcg_debug_assert((mop & MO_SIZE) == MO_16); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD); + ret = load_atom_2(env, ra, haddr, mop); clear_helper_retaddr(); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); return ret; } -uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr, +tcg_target_ulong helper_lduw_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + uint16_t ret = do_ld2_he_mmu(env, addr, mop, ra); + + if (mop & MO_BSWAP) { + ret = bswap16(ret); + } + return ret; +} + +tcg_target_ulong helper_ldsw_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + int16_t ret = do_ld2_he_mmu(env, addr, mop, ra); + + if (mop & MO_BSWAP) { + ret = bswap16(ret); + } + return ret; +} + +uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) { + MemOp mop = get_memop(oi); + uint16_t ret; + + tcg_debug_assert((mop & MO_BSWAP) == MO_BE); + ret = do_ld2_he_mmu(env, addr, mop, ra); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); + return cpu_to_be16(ret); +} + +uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + uint16_t ret; + + tcg_debug_assert((mop & MO_BSWAP) == MO_LE); + ret = do_ld2_he_mmu(env, addr, mop, ra); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); + return cpu_to_le16(ret); +} + +static uint32_t do_ld4_he_mmu(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra) +{ void *haddr; uint32_t ret; - validate_memop(oi, MO_BEUL); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); - ret = ldl_be_p(haddr); + tcg_debug_assert((mop & MO_SIZE) == MO_32); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD); + ret = load_atom_4(env, ra, haddr, mop); clear_helper_retaddr(); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); return ret; } -uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr, +tcg_target_ulong helper_ldul_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + uint32_t ret = do_ld4_he_mmu(env, addr, mop, ra); + + if (mop & MO_BSWAP) { + ret = bswap32(ret); + } + return ret; +} + +tcg_target_ulong helper_ldsl_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + int32_t ret = do_ld4_he_mmu(env, addr, mop, ra); + + if (mop & MO_BSWAP) { + ret = bswap32(ret); + } + return ret; +} + +uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) { + MemOp mop = get_memop(oi); + uint32_t ret; + + tcg_debug_assert((mop & MO_BSWAP) == MO_BE); + ret = do_ld4_he_mmu(env, addr, mop, ra); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); + return cpu_to_be32(ret); +} + +uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + uint32_t ret; + + tcg_debug_assert((mop & MO_BSWAP) == MO_LE); + ret = do_ld4_he_mmu(env, addr, mop, ra); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); + return cpu_to_le32(ret); +} + +static uint64_t do_ld8_he_mmu(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra) +{ void *haddr; uint64_t ret; - validate_memop(oi, MO_BEUQ); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); - ret = ldq_be_p(haddr); + tcg_debug_assert((mop & MO_SIZE) == MO_64); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD); + ret = load_atom_8(env, ra, haddr, mop); clear_helper_retaddr(); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); return ret; } -uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr, +uint64_t helper_ldq_mmu(CPUArchState *env, uint64_t addr, MemOpIdx oi, uintptr_t ra) { - void *haddr; - uint16_t ret; + MemOp mop = get_memop(oi); + uint64_t ret = do_ld8_he_mmu(env, addr, mop, ra); - validate_memop(oi, MO_LEUW); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); - ret = lduw_le_p(haddr); - clear_helper_retaddr(); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); + if (mop & MO_BSWAP) { + ret = bswap64(ret); + } return ret; } -uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr, +uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) { - void *haddr; - uint32_t ret; + MemOp mop = get_memop(oi); + uint64_t ret; - validate_memop(oi, MO_LEUL); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); - ret = ldl_le_p(haddr); - clear_helper_retaddr(); + tcg_debug_assert((mop & MO_BSWAP) == MO_BE); + ret = do_ld8_he_mmu(env, addr, mop, ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - return ret; + return cpu_to_be64(ret); } uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) { - void *haddr; + MemOp mop = get_memop(oi); uint64_t ret; - validate_memop(oi, MO_LEUQ); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); - ret = ldq_le_p(haddr); - clear_helper_retaddr(); + tcg_debug_assert((mop & MO_BSWAP) == MO_LE); + ret = do_ld8_he_mmu(env, addr, mop, ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); + return cpu_to_le64(ret); +} + +static Int128 do_ld16_he_mmu(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra) +{ + void *haddr; + Int128 ret; + + tcg_debug_assert((mop & MO_SIZE) == MO_128); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD); + ret = load_atom_16(env, ra, haddr, mop); + clear_helper_retaddr(); return ret; } +Int128 helper_ld16_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + Int128 ret = do_ld16_he_mmu(env, addr, mop, ra); + + if (mop & MO_BSWAP) { + ret = bswap128(ret); + } + return ret; +} + +Int128 helper_ld_i128(CPUArchState *env, uint64_t addr, MemOpIdx oi) +{ + return helper_ld16_mmu(env, addr, oi, GETPC()); +} + Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) { - void *haddr; + MemOp mop = get_memop(oi); Int128 ret; - validate_memop(oi, MO_128 | MO_BE); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); - memcpy(&ret, haddr, 16); - clear_helper_retaddr(); + tcg_debug_assert((mop & MO_BSWAP) == MO_BE); + ret = do_ld16_he_mmu(env, addr, mop, ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - if (!HOST_BIG_ENDIAN) { ret = bswap128(ret); } @@ -1050,132 +1159,218 @@ Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr, Int128 cpu_ld16_le_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) { - void *haddr; + MemOp mop = get_memop(oi); Int128 ret; - validate_memop(oi, MO_128 | MO_LE); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); - memcpy(&ret, haddr, 16); - clear_helper_retaddr(); + tcg_debug_assert((mop & MO_BSWAP) == MO_LE); + ret = do_ld16_he_mmu(env, addr, mop, ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - if (HOST_BIG_ENDIAN) { ret = bswap128(ret); } return ret; } -void cpu_stb_mmu(CPUArchState *env, abi_ptr addr, uint8_t val, - MemOpIdx oi, uintptr_t ra) +static void do_st1_mmu(CPUArchState *env, abi_ptr addr, uint8_t val, + MemOp mop, uintptr_t ra) { void *haddr; - validate_memop(oi, MO_UB); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE); + tcg_debug_assert((mop & MO_SIZE) == MO_8); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE); stb_p(haddr, val); clear_helper_retaddr(); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } -void cpu_stw_be_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, +void helper_stb_mmu(CPUArchState *env, uint64_t addr, uint32_t val, MemOpIdx oi, uintptr_t ra) { - void *haddr; + do_st1_mmu(env, addr, val, get_memop(oi), ra); +} - validate_memop(oi, MO_BEUW); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE); - stw_be_p(haddr, val); - clear_helper_retaddr(); +void cpu_stb_mmu(CPUArchState *env, abi_ptr addr, uint8_t val, + MemOpIdx oi, uintptr_t ra) +{ + do_st1_mmu(env, addr, val, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } -void cpu_stl_be_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, - MemOpIdx oi, uintptr_t ra) +static void do_st2_he_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, + MemOp mop, uintptr_t ra) { void *haddr; - validate_memop(oi, MO_BEUL); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE); - stl_be_p(haddr, val); + tcg_debug_assert((mop & MO_SIZE) == MO_16); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE); + store_atom_2(env, ra, haddr, mop, val); clear_helper_retaddr(); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } -void cpu_stq_be_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, +void helper_stw_mmu(CPUArchState *env, uint64_t addr, uint32_t val, MemOpIdx oi, uintptr_t ra) { - void *haddr; + MemOp mop = get_memop(oi); - validate_memop(oi, MO_BEUQ); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE); - stq_be_p(haddr, val); - clear_helper_retaddr(); + if (mop & MO_BSWAP) { + val = bswap16(val); + } + do_st2_he_mmu(env, addr, val, mop, ra); +} + +void cpu_stw_be_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + + tcg_debug_assert((mop & MO_BSWAP) == MO_BE); + do_st2_he_mmu(env, addr, be16_to_cpu(val), mop, ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stw_le_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, MemOpIdx oi, uintptr_t ra) { + MemOp mop = get_memop(oi); + + tcg_debug_assert((mop & MO_BSWAP) == MO_LE); + do_st2_he_mmu(env, addr, le16_to_cpu(val), mop, ra); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); +} + +static void do_st4_he_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, + MemOp mop, uintptr_t ra) +{ void *haddr; - validate_memop(oi, MO_LEUW); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE); - stw_le_p(haddr, val); + tcg_debug_assert((mop & MO_SIZE) == MO_32); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE); + store_atom_4(env, ra, haddr, mop, val); clear_helper_retaddr(); +} + +void helper_stl_mmu(CPUArchState *env, uint64_t addr, uint32_t val, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + + if (mop & MO_BSWAP) { + val = bswap32(val); + } + do_st4_he_mmu(env, addr, val, mop, ra); +} + +void cpu_stl_be_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + + tcg_debug_assert((mop & MO_BSWAP) == MO_BE); + do_st4_he_mmu(env, addr, be32_to_cpu(val), mop, ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stl_le_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, MemOpIdx oi, uintptr_t ra) { + MemOp mop = get_memop(oi); + + tcg_debug_assert((mop & MO_BSWAP) == MO_LE); + do_st4_he_mmu(env, addr, le32_to_cpu(val), mop, ra); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); +} + +static void do_st8_he_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, + MemOp mop, uintptr_t ra) +{ void *haddr; - validate_memop(oi, MO_LEUL); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE); - stl_le_p(haddr, val); + tcg_debug_assert((mop & MO_SIZE) == MO_64); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE); + store_atom_8(env, ra, haddr, mop, val); clear_helper_retaddr(); +} + +void helper_stq_mmu(CPUArchState *env, uint64_t addr, uint64_t val, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + + if (mop & MO_BSWAP) { + val = bswap64(val); + } + do_st8_he_mmu(env, addr, val, mop, ra); +} + +void cpu_stq_be_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + + tcg_debug_assert((mop & MO_BSWAP) == MO_BE); + do_st8_he_mmu(env, addr, cpu_to_be64(val), mop, ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stq_le_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, MemOpIdx oi, uintptr_t ra) { + MemOp mop = get_memop(oi); + + tcg_debug_assert((mop & MO_BSWAP) == MO_LE); + do_st8_he_mmu(env, addr, cpu_to_le64(val), mop, ra); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); +} + +static void do_st16_he_mmu(CPUArchState *env, abi_ptr addr, Int128 val, + MemOp mop, uintptr_t ra) +{ void *haddr; - validate_memop(oi, MO_LEUQ); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE); - stq_le_p(haddr, val); + tcg_debug_assert((mop & MO_SIZE) == MO_128); + haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE); + store_atom_16(env, ra, haddr, mop, val); clear_helper_retaddr(); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); +} + +void helper_st16_mmu(CPUArchState *env, uint64_t addr, Int128 val, + MemOpIdx oi, uintptr_t ra) +{ + MemOp mop = get_memop(oi); + + if (mop & MO_BSWAP) { + val = bswap128(val); + } + do_st16_he_mmu(env, addr, val, mop, ra); +} + +void helper_st_i128(CPUArchState *env, uint64_t addr, Int128 val, MemOpIdx oi) +{ + helper_st16_mmu(env, addr, val, oi, GETPC()); } void cpu_st16_be_mmu(CPUArchState *env, abi_ptr addr, Int128 val, MemOpIdx oi, uintptr_t ra) { - void *haddr; + MemOp mop = get_memop(oi); - validate_memop(oi, MO_128 | MO_BE); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE); + tcg_debug_assert((mop & MO_BSWAP) == MO_BE); if (!HOST_BIG_ENDIAN) { val = bswap128(val); } - memcpy(haddr, &val, 16); - clear_helper_retaddr(); + do_st16_he_mmu(env, addr, val, mop, ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } void cpu_st16_le_mmu(CPUArchState *env, abi_ptr addr, Int128 val, MemOpIdx oi, uintptr_t ra) { - void *haddr; + MemOp mop = get_memop(oi); - validate_memop(oi, MO_128 | MO_LE); - haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE); + tcg_debug_assert((mop & MO_BSWAP) == MO_LE); if (HOST_BIG_ENDIAN) { val = bswap128(val); } - memcpy(haddr, &val, 16); - clear_helper_retaddr(); + do_st16_he_mmu(env, addr, val, mop, ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } @@ -1267,7 +1462,6 @@ uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr, void *haddr; uint64_t ret; - validate_memop(oi, MO_BEUQ); haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD); ret = ldq_p(haddr); clear_helper_retaddr(); diff --git a/block.c b/block.c index dad9a4fa43..f04a6ad4e8 100644 --- a/block.c +++ b/block.c @@ -7982,6 +7982,25 @@ void bdrv_add_child(BlockDriverState *parent_bs, BlockDriverState *child_bs, return; } + /* + * Non-zoned block drivers do not follow zoned storage constraints + * (i.e. sequential writes to zones). Refuse mixing zoned and non-zoned + * drivers in a graph. + */ + if (!parent_bs->drv->supports_zoned_children && + child_bs->bl.zoned == BLK_Z_HM) { + /* + * The host-aware model allows zoned storage constraints and random + * write. Allow mixing host-aware and non-zoned drivers. Using + * host-aware device as a regular device. + */ + error_setg(errp, "Cannot add a %s child to a %s parent", + child_bs->bl.zoned == BLK_Z_HM ? "zoned" : "non-zoned", + parent_bs->drv->supports_zoned_children ? + "support zoned children" : "not support zoned children"); + return; + } + if (!QLIST_EMPTY(&child_bs->parents)) { error_setg(errp, "The node %s already has a parent", child_bs->node_name); diff --git a/block/block-backend.c b/block/block-backend.c index e37d55d3e9..ca537cd0ad 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -1845,6 +1845,204 @@ int coroutine_fn blk_co_flush(BlockBackend *blk) return ret; } +static void coroutine_fn blk_aio_zone_report_entry(void *opaque) +{ + BlkAioEmAIOCB *acb = opaque; + BlkRwCo *rwco = &acb->rwco; + + rwco->ret = blk_co_zone_report(rwco->blk, rwco->offset, + (unsigned int*)(uintptr_t)acb->bytes, + rwco->iobuf); + blk_aio_complete(acb); +} + +BlockAIOCB *blk_aio_zone_report(BlockBackend *blk, int64_t offset, + unsigned int *nr_zones, + BlockZoneDescriptor *zones, + BlockCompletionFunc *cb, void *opaque) +{ + BlkAioEmAIOCB *acb; + Coroutine *co; + IO_CODE(); + + blk_inc_in_flight(blk); + acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque); + acb->rwco = (BlkRwCo) { + .blk = blk, + .offset = offset, + .iobuf = zones, + .ret = NOT_DONE, + }; + acb->bytes = (int64_t)(uintptr_t)nr_zones, + acb->has_returned = false; + + co = qemu_coroutine_create(blk_aio_zone_report_entry, acb); + aio_co_enter(blk_get_aio_context(blk), co); + + acb->has_returned = true; + if (acb->rwco.ret != NOT_DONE) { + replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + blk_aio_complete_bh, acb); + } + + return &acb->common; +} + +static void coroutine_fn blk_aio_zone_mgmt_entry(void *opaque) +{ + BlkAioEmAIOCB *acb = opaque; + BlkRwCo *rwco = &acb->rwco; + + rwco->ret = blk_co_zone_mgmt(rwco->blk, + (BlockZoneOp)(uintptr_t)rwco->iobuf, + rwco->offset, acb->bytes); + blk_aio_complete(acb); +} + +BlockAIOCB *blk_aio_zone_mgmt(BlockBackend *blk, BlockZoneOp op, + int64_t offset, int64_t len, + BlockCompletionFunc *cb, void *opaque) { + BlkAioEmAIOCB *acb; + Coroutine *co; + IO_CODE(); + + blk_inc_in_flight(blk); + acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque); + acb->rwco = (BlkRwCo) { + .blk = blk, + .offset = offset, + .iobuf = (void *)(uintptr_t)op, + .ret = NOT_DONE, + }; + acb->bytes = len; + acb->has_returned = false; + + co = qemu_coroutine_create(blk_aio_zone_mgmt_entry, acb); + aio_co_enter(blk_get_aio_context(blk), co); + + acb->has_returned = true; + if (acb->rwco.ret != NOT_DONE) { + replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + blk_aio_complete_bh, acb); + } + + return &acb->common; +} + +static void coroutine_fn blk_aio_zone_append_entry(void *opaque) +{ + BlkAioEmAIOCB *acb = opaque; + BlkRwCo *rwco = &acb->rwco; + + rwco->ret = blk_co_zone_append(rwco->blk, (int64_t *)(uintptr_t)acb->bytes, + rwco->iobuf, rwco->flags); + blk_aio_complete(acb); +} + +BlockAIOCB *blk_aio_zone_append(BlockBackend *blk, int64_t *offset, + QEMUIOVector *qiov, BdrvRequestFlags flags, + BlockCompletionFunc *cb, void *opaque) { + BlkAioEmAIOCB *acb; + Coroutine *co; + IO_CODE(); + + blk_inc_in_flight(blk); + acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque); + acb->rwco = (BlkRwCo) { + .blk = blk, + .ret = NOT_DONE, + .flags = flags, + .iobuf = qiov, + }; + acb->bytes = (int64_t)(uintptr_t)offset; + acb->has_returned = false; + + co = qemu_coroutine_create(blk_aio_zone_append_entry, acb); + aio_co_enter(blk_get_aio_context(blk), co); + acb->has_returned = true; + if (acb->rwco.ret != NOT_DONE) { + replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + blk_aio_complete_bh, acb); + } + + return &acb->common; +} + +/* + * Send a zone_report command. + * offset is a byte offset from the start of the device. No alignment + * required for offset. + * nr_zones represents IN maximum and OUT actual. + */ +int coroutine_fn blk_co_zone_report(BlockBackend *blk, int64_t offset, + unsigned int *nr_zones, + BlockZoneDescriptor *zones) +{ + int ret; + IO_CODE(); + + blk_inc_in_flight(blk); /* increase before waiting */ + blk_wait_while_drained(blk); + GRAPH_RDLOCK_GUARD(); + if (!blk_is_available(blk)) { + blk_dec_in_flight(blk); + return -ENOMEDIUM; + } + ret = bdrv_co_zone_report(blk_bs(blk), offset, nr_zones, zones); + blk_dec_in_flight(blk); + return ret; +} + +/* + * Send a zone_management command. + * op is the zone operation; + * offset is the byte offset from the start of the zoned device; + * len is the maximum number of bytes the command should operate on. It + * should be aligned with the device zone size. + */ +int coroutine_fn blk_co_zone_mgmt(BlockBackend *blk, BlockZoneOp op, + int64_t offset, int64_t len) +{ + int ret; + IO_CODE(); + + blk_inc_in_flight(blk); + blk_wait_while_drained(blk); + GRAPH_RDLOCK_GUARD(); + + ret = blk_check_byte_request(blk, offset, len); + if (ret < 0) { + blk_dec_in_flight(blk); + return ret; + } + + ret = bdrv_co_zone_mgmt(blk_bs(blk), op, offset, len); + blk_dec_in_flight(blk); + return ret; +} + +/* + * Send a zone_append command. + */ +int coroutine_fn blk_co_zone_append(BlockBackend *blk, int64_t *offset, + QEMUIOVector *qiov, BdrvRequestFlags flags) +{ + int ret; + IO_CODE(); + + blk_inc_in_flight(blk); + blk_wait_while_drained(blk); + GRAPH_RDLOCK_GUARD(); + if (!blk_is_available(blk)) { + blk_dec_in_flight(blk); + return -ENOMEDIUM; + } + + ret = bdrv_co_zone_append(blk_bs(blk), offset, qiov, flags); + blk_dec_in_flight(blk); + return ret; +} + void blk_drain(BlockBackend *blk) { BlockDriverState *bs = blk_bs(blk); diff --git a/block/file-posix.c b/block/file-posix.c index c7b723368e..0ab158efba 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -68,6 +68,9 @@ #include <sys/param.h> #include <sys/syscall.h> #include <sys/vfs.h> +#if defined(CONFIG_BLKZONED) +#include <linux/blkzoned.h> +#endif #include <linux/cdrom.h> #include <linux/fd.h> #include <linux/fs.h> @@ -157,6 +160,7 @@ typedef struct BDRVRawState { bool has_write_zeroes:1; bool use_linux_aio:1; bool use_linux_io_uring:1; + int64_t *offset; /* offset of zone append operation */ int page_cache_inconsistent; /* errno from fdatasync failure */ bool has_fallocate; bool needs_alignment; @@ -216,6 +220,13 @@ typedef struct RawPosixAIOData { PreallocMode prealloc; Error **errp; } truncate; + struct { + unsigned int *nr_zones; + BlockZoneDescriptor *zones; + } zone_report; + struct { + unsigned long op; + } zone_mgmt; }; } RawPosixAIOData; @@ -766,6 +777,18 @@ static int raw_open_common(BlockDriverState *bs, QDict *options, goto fail; } } +#ifdef CONFIG_BLKZONED + /* + * The kernel page cache does not reliably work for writes to SWR zones + * of zoned block device because it can not guarantee the order of writes. + */ + if ((bs->bl.zoned != BLK_Z_NONE) && + (!(s->open_flags & O_DIRECT))) { + error_setg(errp, "The driver supports zoned devices, and it requires " + "cache.direct=on, which was not specified."); + return -EINVAL; /* No host kernel page cache */ + } +#endif if (S_ISBLK(st.st_mode)) { #ifdef __linux__ @@ -1202,15 +1225,91 @@ static int hdev_get_max_hw_transfer(int fd, struct stat *st) #endif } -static int hdev_get_max_segments(int fd, struct stat *st) +/* + * Get a sysfs attribute value as character string. + */ +#ifdef CONFIG_LINUX +static int get_sysfs_str_val(struct stat *st, const char *attribute, + char **val) { + g_autofree char *sysfspath = NULL; + int ret; + size_t len; + + if (!S_ISBLK(st->st_mode)) { + return -ENOTSUP; + } + + sysfspath = g_strdup_printf("/sys/dev/block/%u:%u/queue/%s", + major(st->st_rdev), minor(st->st_rdev), + attribute); + ret = g_file_get_contents(sysfspath, val, &len, NULL); + if (ret == -1) { + return -ENOENT; + } + + /* The file is ended with '\n' */ + char *p; + p = *val; + if (*(p + len - 1) == '\n') { + *(p + len - 1) = '\0'; + } + return ret; +} +#endif + +#if defined(CONFIG_BLKZONED) +static int get_sysfs_zoned_model(struct stat *st, BlockZoneModel *zoned) { + g_autofree char *val = NULL; + int ret; + + ret = get_sysfs_str_val(st, "zoned", &val); + if (ret < 0) { + return ret; + } + + if (strcmp(val, "host-managed") == 0) { + *zoned = BLK_Z_HM; + } else if (strcmp(val, "host-aware") == 0) { + *zoned = BLK_Z_HA; + } else if (strcmp(val, "none") == 0) { + *zoned = BLK_Z_NONE; + } else { + return -ENOTSUP; + } + return 0; +} +#endif /* defined(CONFIG_BLKZONED) */ + +/* + * Get a sysfs attribute value as a long integer. + */ #ifdef CONFIG_LINUX - char buf[32]; +static long get_sysfs_long_val(struct stat *st, const char *attribute) +{ + g_autofree char *str = NULL; const char *end; - char *sysfspath = NULL; + long val; + int ret; + + ret = get_sysfs_str_val(st, attribute, &str); + if (ret < 0) { + return ret; + } + + /* The file is ended with '\n', pass 'end' to accept that. */ + ret = qemu_strtol(str, &end, 10, &val); + if (ret == 0 && end && *end == '\0') { + ret = val; + } + return ret; +} +#endif + +static int hdev_get_max_segments(int fd, struct stat *st) +{ +#ifdef CONFIG_LINUX int ret; - int sysfd = -1; - long max_segments; if (S_ISCHR(st->st_mode)) { if (ioctl(fd, SG_GET_SG_TABLESIZE, &ret) == 0) { @@ -1218,43 +1317,175 @@ static int hdev_get_max_segments(int fd, struct stat *st) } return -ENOTSUP; } + return get_sysfs_long_val(st, "max_segments"); +#else + return -ENOTSUP; +#endif +} - if (!S_ISBLK(st->st_mode)) { - return -ENOTSUP; +#if defined(CONFIG_BLKZONED) +/* + * If the reset_all flag is true, then the wps of zone whose state is + * not readonly or offline should be all reset to the start sector. + * Else, take the real wp of the device. + */ +static int get_zones_wp(BlockDriverState *bs, int fd, int64_t offset, + unsigned int nrz, bool reset_all) +{ + struct blk_zone *blkz; + size_t rep_size; + uint64_t sector = offset >> BDRV_SECTOR_BITS; + BlockZoneWps *wps = bs->wps; + unsigned int j = offset / bs->bl.zone_size; + unsigned int n = 0, i = 0; + int ret; + rep_size = sizeof(struct blk_zone_report) + nrz * sizeof(struct blk_zone); + g_autofree struct blk_zone_report *rep = NULL; + + rep = g_malloc(rep_size); + blkz = (struct blk_zone *)(rep + 1); + while (n < nrz) { + memset(rep, 0, rep_size); + rep->sector = sector; + rep->nr_zones = nrz - n; + + do { + ret = ioctl(fd, BLKREPORTZONE, rep); + } while (ret != 0 && errno == EINTR); + if (ret != 0) { + error_report("%d: ioctl BLKREPORTZONE at %" PRId64 " failed %d", + fd, offset, errno); + return -errno; + } + + if (!rep->nr_zones) { + break; + } + + for (i = 0; i < rep->nr_zones; ++i, ++n, ++j) { + /* + * The wp tracking cares only about sequential writes required and + * sequential write preferred zones so that the wp can advance to + * the right location. + * Use the most significant bit of the wp location to indicate the + * zone type: 0 for SWR/SWP zones and 1 for conventional zones. + */ + if (blkz[i].type == BLK_ZONE_TYPE_CONVENTIONAL) { + wps->wp[j] |= 1ULL << 63; + } else { + switch(blkz[i].cond) { + case BLK_ZONE_COND_FULL: + case BLK_ZONE_COND_READONLY: + /* Zone not writable */ + wps->wp[j] = (blkz[i].start + blkz[i].len) << BDRV_SECTOR_BITS; + break; + case BLK_ZONE_COND_OFFLINE: + /* Zone not writable nor readable */ + wps->wp[j] = (blkz[i].start) << BDRV_SECTOR_BITS; + break; + default: + if (reset_all) { + wps->wp[j] = blkz[i].start << BDRV_SECTOR_BITS; + } else { + wps->wp[j] = blkz[i].wp << BDRV_SECTOR_BITS; + } + break; + } + } + } + sector = blkz[i - 1].start + blkz[i - 1].len; } - sysfspath = g_strdup_printf("/sys/dev/block/%u:%u/queue/max_segments", - major(st->st_rdev), minor(st->st_rdev)); - sysfd = open(sysfspath, O_RDONLY); - if (sysfd == -1) { - ret = -errno; - goto out; + return 0; +} + +static void update_zones_wp(BlockDriverState *bs, int fd, int64_t offset, + unsigned int nrz) +{ + if (get_zones_wp(bs, fd, offset, nrz, 0) < 0) { + error_report("update zone wp failed"); + } +} + +static void raw_refresh_zoned_limits(BlockDriverState *bs, struct stat *st, + Error **errp) +{ + BDRVRawState *s = bs->opaque; + BlockZoneModel zoned; + int ret; + + bs->bl.zoned = BLK_Z_NONE; + + ret = get_sysfs_zoned_model(st, &zoned); + if (ret < 0 || zoned == BLK_Z_NONE) { + return; + } + bs->bl.zoned = zoned; + + ret = get_sysfs_long_val(st, "max_open_zones"); + if (ret >= 0) { + bs->bl.max_open_zones = ret; + } + + ret = get_sysfs_long_val(st, "max_active_zones"); + if (ret >= 0) { + bs->bl.max_active_zones = ret; } - ret = RETRY_ON_EINTR(read(sysfd, buf, sizeof(buf) - 1)); + + /* + * The zoned device must at least have zone size and nr_zones fields. + */ + ret = get_sysfs_long_val(st, "chunk_sectors"); if (ret < 0) { - ret = -errno; - goto out; - } else if (ret == 0) { - ret = -EIO; - goto out; + error_setg_errno(errp, -ret, "Unable to read chunk_sectors " + "sysfs attribute"); + return; + } else if (!ret) { + error_setg(errp, "Read 0 from chunk_sectors sysfs attribute"); + return; } - buf[ret] = 0; - /* The file is ended with '\n', pass 'end' to accept that. */ - ret = qemu_strtol(buf, &end, 10, &max_segments); - if (ret == 0 && end && *end == '\n') { - ret = max_segments; + bs->bl.zone_size = ret << BDRV_SECTOR_BITS; + + ret = get_sysfs_long_val(st, "nr_zones"); + if (ret < 0) { + error_setg_errno(errp, -ret, "Unable to read nr_zones " + "sysfs attribute"); + return; + } else if (!ret) { + error_setg(errp, "Read 0 from nr_zones sysfs attribute"); + return; } + bs->bl.nr_zones = ret; -out: - if (sysfd != -1) { - close(sysfd); + ret = get_sysfs_long_val(st, "zone_append_max_bytes"); + if (ret > 0) { + bs->bl.max_append_sectors = ret >> BDRV_SECTOR_BITS; } - g_free(sysfspath); - return ret; -#else - return -ENOTSUP; -#endif + + ret = get_sysfs_long_val(st, "physical_block_size"); + if (ret >= 0) { + bs->bl.write_granularity = ret; + } + + /* The refresh_limits() function can be called multiple times. */ + g_free(bs->wps); + bs->wps = g_malloc(sizeof(BlockZoneWps) + + sizeof(int64_t) * bs->bl.nr_zones); + ret = get_zones_wp(bs, s->fd, 0, bs->bl.nr_zones, 0); + if (ret < 0) { + error_setg_errno(errp, -ret, "report wps failed"); + bs->wps = NULL; + return; + } + qemu_co_mutex_init(&bs->wps->colock); } +#else /* !defined(CONFIG_BLKZONED) */ +static void raw_refresh_zoned_limits(BlockDriverState *bs, struct stat *st, + Error **errp) +{ + bs->bl.zoned = BLK_Z_NONE; +} +#endif /* !defined(CONFIG_BLKZONED) */ static void raw_refresh_limits(BlockDriverState *bs, Error **errp) { @@ -1297,6 +1528,8 @@ static void raw_refresh_limits(BlockDriverState *bs, Error **errp) bs->bl.max_hw_iov = ret; } } + + raw_refresh_zoned_limits(bs, &st, errp); } static int check_for_dasd(int fd) @@ -1320,9 +1553,12 @@ static int hdev_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz) BDRVRawState *s = bs->opaque; int ret; - /* If DASD, get blocksizes */ + /* If DASD or zoned devices, get blocksizes */ if (check_for_dasd(s->fd) < 0) { - return -ENOTSUP; + /* zoned devices are not DASD */ + if (bs->bl.zoned == BLK_Z_NONE) { + return -ENOTSUP; + } } ret = probe_logical_blocksize(s->fd, &bsz->log); if (ret < 0) { @@ -1463,7 +1699,7 @@ static ssize_t handle_aiocb_rw_vector(RawPosixAIOData *aiocb) ssize_t len; len = RETRY_ON_EINTR( - (aiocb->aio_type & QEMU_AIO_WRITE) ? + (aiocb->aio_type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) ? qemu_pwritev(aiocb->aio_fildes, aiocb->io.iov, aiocb->io.niov, @@ -1492,7 +1728,7 @@ static ssize_t handle_aiocb_rw_linear(RawPosixAIOData *aiocb, char *buf) ssize_t len; while (offset < aiocb->aio_nbytes) { - if (aiocb->aio_type & QEMU_AIO_WRITE) { + if (aiocb->aio_type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) { len = pwrite(aiocb->aio_fildes, (const char *)buf + offset, aiocb->aio_nbytes - offset, @@ -1585,7 +1821,7 @@ static int handle_aiocb_rw(void *opaque) } nbytes = handle_aiocb_rw_linear(aiocb, buf); - if (!(aiocb->aio_type & QEMU_AIO_WRITE)) { + if (!(aiocb->aio_type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND))) { char *p = buf; size_t count = aiocb->aio_nbytes, copy; int i; @@ -1790,6 +2026,147 @@ static off_t copy_file_range(int in_fd, off_t *in_off, int out_fd, } #endif +/* + * parse_zone - Fill a zone descriptor + */ +#if defined(CONFIG_BLKZONED) +static inline int parse_zone(struct BlockZoneDescriptor *zone, + const struct blk_zone *blkz) { + zone->start = blkz->start << BDRV_SECTOR_BITS; + zone->length = blkz->len << BDRV_SECTOR_BITS; + zone->wp = blkz->wp << BDRV_SECTOR_BITS; + +#ifdef HAVE_BLK_ZONE_REP_CAPACITY + zone->cap = blkz->capacity << BDRV_SECTOR_BITS; +#else + zone->cap = blkz->len << BDRV_SECTOR_BITS; +#endif + + switch (blkz->type) { + case BLK_ZONE_TYPE_SEQWRITE_REQ: + zone->type = BLK_ZT_SWR; + break; + case BLK_ZONE_TYPE_SEQWRITE_PREF: + zone->type = BLK_ZT_SWP; + break; + case BLK_ZONE_TYPE_CONVENTIONAL: + zone->type = BLK_ZT_CONV; + break; + default: + error_report("Unsupported zone type: 0x%x", blkz->type); + return -ENOTSUP; + } + + switch (blkz->cond) { + case BLK_ZONE_COND_NOT_WP: + zone->state = BLK_ZS_NOT_WP; + break; + case BLK_ZONE_COND_EMPTY: + zone->state = BLK_ZS_EMPTY; + break; + case BLK_ZONE_COND_IMP_OPEN: + zone->state = BLK_ZS_IOPEN; + break; + case BLK_ZONE_COND_EXP_OPEN: + zone->state = BLK_ZS_EOPEN; + break; + case BLK_ZONE_COND_CLOSED: + zone->state = BLK_ZS_CLOSED; + break; + case BLK_ZONE_COND_READONLY: + zone->state = BLK_ZS_RDONLY; + break; + case BLK_ZONE_COND_FULL: + zone->state = BLK_ZS_FULL; + break; + case BLK_ZONE_COND_OFFLINE: + zone->state = BLK_ZS_OFFLINE; + break; + default: + error_report("Unsupported zone state: 0x%x", blkz->cond); + return -ENOTSUP; + } + return 0; +} +#endif + +#if defined(CONFIG_BLKZONED) +static int handle_aiocb_zone_report(void *opaque) +{ + RawPosixAIOData *aiocb = opaque; + int fd = aiocb->aio_fildes; + unsigned int *nr_zones = aiocb->zone_report.nr_zones; + BlockZoneDescriptor *zones = aiocb->zone_report.zones; + /* zoned block devices use 512-byte sectors */ + uint64_t sector = aiocb->aio_offset / 512; + + struct blk_zone *blkz; + size_t rep_size; + unsigned int nrz; + int ret; + unsigned int n = 0, i = 0; + + nrz = *nr_zones; + rep_size = sizeof(struct blk_zone_report) + nrz * sizeof(struct blk_zone); + g_autofree struct blk_zone_report *rep = NULL; + rep = g_malloc(rep_size); + + blkz = (struct blk_zone *)(rep + 1); + while (n < nrz) { + memset(rep, 0, rep_size); + rep->sector = sector; + rep->nr_zones = nrz - n; + + do { + ret = ioctl(fd, BLKREPORTZONE, rep); + } while (ret != 0 && errno == EINTR); + if (ret != 0) { + error_report("%d: ioctl BLKREPORTZONE at %" PRId64 " failed %d", + fd, sector, errno); + return -errno; + } + + if (!rep->nr_zones) { + break; + } + + for (i = 0; i < rep->nr_zones; i++, n++) { + ret = parse_zone(&zones[n], &blkz[i]); + if (ret != 0) { + return ret; + } + + /* The next report should start after the last zone reported */ + sector = blkz[i].start + blkz[i].len; + } + } + + *nr_zones = n; + return 0; +} +#endif + +#if defined(CONFIG_BLKZONED) +static int handle_aiocb_zone_mgmt(void *opaque) +{ + RawPosixAIOData *aiocb = opaque; + int fd = aiocb->aio_fildes; + uint64_t sector = aiocb->aio_offset / 512; + int64_t nr_sectors = aiocb->aio_nbytes / 512; + struct blk_zone_range range; + int ret; + + /* Execute the operation */ + range.sector = sector; + range.nr_sectors = nr_sectors; + do { + ret = ioctl(fd, aiocb->zone_mgmt.op, &range); + } while (ret != 0 && errno == EINTR); + + return ret < 0 ? -errno : ret; +} +#endif + static int handle_aiocb_copy_range(void *opaque) { RawPosixAIOData *aiocb = opaque; @@ -2072,9 +2449,19 @@ static int coroutine_fn raw_co_prw(BlockDriverState *bs, uint64_t offset, { BDRVRawState *s = bs->opaque; RawPosixAIOData acb; + int ret; if (fd_open(bs) < 0) return -EIO; +#if defined(CONFIG_BLKZONED) + if ((type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) && bs->wps) { + qemu_co_mutex_lock(&bs->wps->colock); + if (type & QEMU_AIO_ZONE_APPEND && bs->bl.zone_size) { + int index = offset / bs->bl.zone_size; + offset = bs->wps->wp[index]; + } + } +#endif /* * When using O_DIRECT, the request must be aligned to be able to use @@ -2087,12 +2474,15 @@ static int coroutine_fn raw_co_prw(BlockDriverState *bs, uint64_t offset, #ifdef CONFIG_LINUX_IO_URING } else if (s->use_linux_io_uring) { assert(qiov->size == bytes); - return luring_co_submit(bs, s->fd, offset, qiov, type); + ret = luring_co_submit(bs, s->fd, offset, qiov, type); + goto out; #endif #ifdef CONFIG_LINUX_AIO } else if (s->use_linux_aio) { assert(qiov->size == bytes); - return laio_co_submit(s->fd, offset, qiov, type, s->aio_max_batch); + ret = laio_co_submit(s->fd, offset, qiov, type, + s->aio_max_batch); + goto out; #endif } @@ -2109,7 +2499,41 @@ static int coroutine_fn raw_co_prw(BlockDriverState *bs, uint64_t offset, }; assert(qiov->size == bytes); - return raw_thread_pool_submit(handle_aiocb_rw, &acb); + ret = raw_thread_pool_submit(handle_aiocb_rw, &acb); + goto out; /* Avoid the compiler err of unused label */ + +out: +#if defined(CONFIG_BLKZONED) +{ + BlockZoneWps *wps = bs->wps; + if (ret == 0) { + if ((type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) + && wps && bs->bl.zone_size) { + uint64_t *wp = &wps->wp[offset / bs->bl.zone_size]; + if (!BDRV_ZT_IS_CONV(*wp)) { + if (type & QEMU_AIO_ZONE_APPEND) { + *s->offset = *wp; + trace_zbd_zone_append_complete(bs, *s->offset + >> BDRV_SECTOR_BITS); + } + /* Advance the wp if needed */ + if (offset + bytes > *wp) { + *wp = offset + bytes; + } + } + } + } else { + if (type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) { + update_zones_wp(bs, s->fd, 0, 1); + } + } + + if ((type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) && wps) { + qemu_co_mutex_unlock(&wps->colock); + } +} +#endif + return ret; } static int coroutine_fn raw_co_preadv(BlockDriverState *bs, int64_t offset, @@ -2212,6 +2636,9 @@ static void raw_close(BlockDriverState *bs) BDRVRawState *s = bs->opaque; if (s->fd >= 0) { +#if defined(CONFIG_BLKZONED) + g_free(bs->wps); +#endif qemu_close(s->fd); s->fd = -1; } @@ -2969,6 +3396,171 @@ static void raw_account_discard(BDRVRawState *s, uint64_t nbytes, int ret) } } +/* + * zone report - Get a zone block device's information in the form + * of an array of zone descriptors. + * zones is an array of zone descriptors to hold zone information on reply; + * offset can be any byte within the entire size of the device; + * nr_zones is the maxium number of sectors the command should operate on. + */ +#if defined(CONFIG_BLKZONED) +static int coroutine_fn raw_co_zone_report(BlockDriverState *bs, int64_t offset, + unsigned int *nr_zones, + BlockZoneDescriptor *zones) { + BDRVRawState *s = bs->opaque; + RawPosixAIOData acb = (RawPosixAIOData) { + .bs = bs, + .aio_fildes = s->fd, + .aio_type = QEMU_AIO_ZONE_REPORT, + .aio_offset = offset, + .zone_report = { + .nr_zones = nr_zones, + .zones = zones, + }, + }; + + trace_zbd_zone_report(bs, *nr_zones, offset >> BDRV_SECTOR_BITS); + return raw_thread_pool_submit(handle_aiocb_zone_report, &acb); +} +#endif + +/* + * zone management operations - Execute an operation on a zone + */ +#if defined(CONFIG_BLKZONED) +static int coroutine_fn raw_co_zone_mgmt(BlockDriverState *bs, BlockZoneOp op, + int64_t offset, int64_t len) { + BDRVRawState *s = bs->opaque; + RawPosixAIOData acb; + int64_t zone_size, zone_size_mask; + const char *op_name; + unsigned long zo; + int ret; + BlockZoneWps *wps = bs->wps; + int64_t capacity = bs->total_sectors << BDRV_SECTOR_BITS; + + zone_size = bs->bl.zone_size; + zone_size_mask = zone_size - 1; + if (offset & zone_size_mask) { + error_report("sector offset %" PRId64 " is not aligned to zone size " + "%" PRId64 "", offset / 512, zone_size / 512); + return -EINVAL; + } + + if (((offset + len) < capacity && len & zone_size_mask) || + offset + len > capacity) { + error_report("number of sectors %" PRId64 " is not aligned to zone size" + " %" PRId64 "", len / 512, zone_size / 512); + return -EINVAL; + } + + uint32_t i = offset / bs->bl.zone_size; + uint32_t nrz = len / bs->bl.zone_size; + uint64_t *wp = &wps->wp[i]; + if (BDRV_ZT_IS_CONV(*wp) && len != capacity) { + error_report("zone mgmt operations are not allowed for conventional zones"); + return -EIO; + } + + switch (op) { + case BLK_ZO_OPEN: + op_name = "BLKOPENZONE"; + zo = BLKOPENZONE; + break; + case BLK_ZO_CLOSE: + op_name = "BLKCLOSEZONE"; + zo = BLKCLOSEZONE; + break; + case BLK_ZO_FINISH: + op_name = "BLKFINISHZONE"; + zo = BLKFINISHZONE; + break; + case BLK_ZO_RESET: + op_name = "BLKRESETZONE"; + zo = BLKRESETZONE; + break; + default: + error_report("Unsupported zone op: 0x%x", op); + return -ENOTSUP; + } + + acb = (RawPosixAIOData) { + .bs = bs, + .aio_fildes = s->fd, + .aio_type = QEMU_AIO_ZONE_MGMT, + .aio_offset = offset, + .aio_nbytes = len, + .zone_mgmt = { + .op = zo, + }, + }; + + trace_zbd_zone_mgmt(bs, op_name, offset >> BDRV_SECTOR_BITS, + len >> BDRV_SECTOR_BITS); + ret = raw_thread_pool_submit(handle_aiocb_zone_mgmt, &acb); + if (ret != 0) { + update_zones_wp(bs, s->fd, offset, i); + error_report("ioctl %s failed %d", op_name, ret); + return ret; + } + + if (zo == BLKRESETZONE && len == capacity) { + ret = get_zones_wp(bs, s->fd, 0, bs->bl.nr_zones, 1); + if (ret < 0) { + error_report("reporting single wp failed"); + return ret; + } + } else if (zo == BLKRESETZONE) { + for (unsigned int j = 0; j < nrz; ++j) { + wp[j] = offset + j * zone_size; + } + } else if (zo == BLKFINISHZONE) { + for (unsigned int j = 0; j < nrz; ++j) { + /* The zoned device allows the last zone smaller that the + * zone size. */ + wp[j] = MIN(offset + (j + 1) * zone_size, offset + len); + } + } + + return ret; +} +#endif + +#if defined(CONFIG_BLKZONED) +static int coroutine_fn raw_co_zone_append(BlockDriverState *bs, + int64_t *offset, + QEMUIOVector *qiov, + BdrvRequestFlags flags) { + assert(flags == 0); + int64_t zone_size_mask = bs->bl.zone_size - 1; + int64_t iov_len = 0; + int64_t len = 0; + BDRVRawState *s = bs->opaque; + s->offset = offset; + + if (*offset & zone_size_mask) { + error_report("sector offset %" PRId64 " is not aligned to zone size " + "%" PRId32 "", *offset / 512, bs->bl.zone_size / 512); + return -EINVAL; + } + + int64_t wg = bs->bl.write_granularity; + int64_t wg_mask = wg - 1; + for (int i = 0; i < qiov->niov; i++) { + iov_len = qiov->iov[i].iov_len; + if (iov_len & wg_mask) { + error_report("len of IOVector[%d] %" PRId64 " is not aligned to " + "block size %" PRId64 "", i, iov_len, wg); + return -EINVAL; + } + len += iov_len; + } + + trace_zbd_zone_append(bs, *offset >> BDRV_SECTOR_BITS); + return raw_co_prw(bs, *offset, len, qiov, QEMU_AIO_ZONE_APPEND); +} +#endif + static coroutine_fn int raw_do_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes, bool blkdev) @@ -3724,6 +4316,14 @@ static BlockDriver bdrv_host_device = { #ifdef __linux__ .bdrv_co_ioctl = hdev_co_ioctl, #endif + + /* zoned device */ +#if defined(CONFIG_BLKZONED) + /* zone management operations */ + .bdrv_co_zone_report = raw_co_zone_report, + .bdrv_co_zone_mgmt = raw_co_zone_mgmt, + .bdrv_co_zone_append = raw_co_zone_append, +#endif }; #if defined(__linux__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) diff --git a/block/io.c b/block/io.c index 58557f2f96..4d54fda593 100644 --- a/block/io.c +++ b/block/io.c @@ -3113,6 +3113,74 @@ out: return co.ret; } +int coroutine_fn bdrv_co_zone_report(BlockDriverState *bs, int64_t offset, + unsigned int *nr_zones, + BlockZoneDescriptor *zones) +{ + BlockDriver *drv = bs->drv; + CoroutineIOCompletion co = { + .coroutine = qemu_coroutine_self(), + }; + IO_CODE(); + + bdrv_inc_in_flight(bs); + if (!drv || !drv->bdrv_co_zone_report || bs->bl.zoned == BLK_Z_NONE) { + co.ret = -ENOTSUP; + goto out; + } + co.ret = drv->bdrv_co_zone_report(bs, offset, nr_zones, zones); +out: + bdrv_dec_in_flight(bs); + return co.ret; +} + +int coroutine_fn bdrv_co_zone_mgmt(BlockDriverState *bs, BlockZoneOp op, + int64_t offset, int64_t len) +{ + BlockDriver *drv = bs->drv; + CoroutineIOCompletion co = { + .coroutine = qemu_coroutine_self(), + }; + IO_CODE(); + + bdrv_inc_in_flight(bs); + if (!drv || !drv->bdrv_co_zone_mgmt || bs->bl.zoned == BLK_Z_NONE) { + co.ret = -ENOTSUP; + goto out; + } + co.ret = drv->bdrv_co_zone_mgmt(bs, op, offset, len); +out: + bdrv_dec_in_flight(bs); + return co.ret; +} + +int coroutine_fn bdrv_co_zone_append(BlockDriverState *bs, int64_t *offset, + QEMUIOVector *qiov, + BdrvRequestFlags flags) +{ + int ret; + BlockDriver *drv = bs->drv; + CoroutineIOCompletion co = { + .coroutine = qemu_coroutine_self(), + }; + IO_CODE(); + + ret = bdrv_check_qiov_request(*offset, qiov->size, qiov, 0, NULL); + if (ret < 0) { + return ret; + } + + bdrv_inc_in_flight(bs); + if (!drv || !drv->bdrv_co_zone_append || bs->bl.zoned == BLK_Z_NONE) { + co.ret = -ENOTSUP; + goto out; + } + co.ret = drv->bdrv_co_zone_append(bs, offset, qiov, flags); +out: + bdrv_dec_in_flight(bs); + return co.ret; +} + void *qemu_blockalign(BlockDriverState *bs, size_t size) { IO_CODE(); diff --git a/block/io_uring.c b/block/io_uring.c index 989f9a99ed..82cab6a5bd 100644 --- a/block/io_uring.c +++ b/block/io_uring.c @@ -350,6 +350,10 @@ static int luring_do_submit(int fd, LuringAIOCB *luringcb, LuringState *s, io_uring_prep_writev(sqes, fd, luringcb->qiov->iov, luringcb->qiov->niov, offset); break; + case QEMU_AIO_ZONE_APPEND: + io_uring_prep_writev(sqes, fd, luringcb->qiov->iov, + luringcb->qiov->niov, offset); + break; case QEMU_AIO_READ: io_uring_prep_readv(sqes, fd, luringcb->qiov->iov, luringcb->qiov->niov, offset); diff --git a/block/linux-aio.c b/block/linux-aio.c index fc50cdd1bf..442c86209b 100644 --- a/block/linux-aio.c +++ b/block/linux-aio.c @@ -394,6 +394,9 @@ static int laio_do_submit(int fd, struct qemu_laiocb *laiocb, off_t offset, case QEMU_AIO_WRITE: io_prep_pwritev(iocbs, fd, qiov->iov, qiov->niov, offset); break; + case QEMU_AIO_ZONE_APPEND: + io_prep_pwritev(iocbs, fd, qiov->iov, qiov->niov, offset); + break; case QEMU_AIO_READ: io_prep_preadv(iocbs, fd, qiov->iov, qiov->niov, offset); break; diff --git a/block/qapi-sysemu.c b/block/qapi-sysemu.c index 7bd7554150..cec3c1afb4 100644 --- a/block/qapi-sysemu.c +++ b/block/qapi-sysemu.c @@ -517,6 +517,7 @@ void qmp_block_latency_histogram_set( bool has_boundaries, uint64List *boundaries, bool has_boundaries_read, uint64List *boundaries_read, bool has_boundaries_write, uint64List *boundaries_write, + bool has_boundaries_append, uint64List *boundaries_append, bool has_boundaries_flush, uint64List *boundaries_flush, Error **errp) { @@ -557,6 +558,16 @@ void qmp_block_latency_histogram_set( } } + if (has_boundaries || has_boundaries_append) { + ret = block_latency_histogram_set( + stats, BLOCK_ACCT_ZONE_APPEND, + has_boundaries_append ? boundaries_append : boundaries); + if (ret) { + error_setg(errp, "Device '%s' set append write boundaries fail", id); + return; + } + } + if (has_boundaries || has_boundaries_flush) { ret = block_latency_histogram_set( stats, BLOCK_ACCT_FLUSH, diff --git a/block/qapi.c b/block/qapi.c index 71f2751257..f34f95e0ef 100644 --- a/block/qapi.c +++ b/block/qapi.c @@ -533,27 +533,36 @@ static void bdrv_query_blk_stats(BlockDeviceStats *ds, BlockBackend *blk) ds->rd_bytes = stats->nr_bytes[BLOCK_ACCT_READ]; ds->wr_bytes = stats->nr_bytes[BLOCK_ACCT_WRITE]; + ds->zone_append_bytes = stats->nr_bytes[BLOCK_ACCT_ZONE_APPEND]; ds->unmap_bytes = stats->nr_bytes[BLOCK_ACCT_UNMAP]; ds->rd_operations = stats->nr_ops[BLOCK_ACCT_READ]; ds->wr_operations = stats->nr_ops[BLOCK_ACCT_WRITE]; + ds->zone_append_operations = stats->nr_ops[BLOCK_ACCT_ZONE_APPEND]; ds->unmap_operations = stats->nr_ops[BLOCK_ACCT_UNMAP]; ds->failed_rd_operations = stats->failed_ops[BLOCK_ACCT_READ]; ds->failed_wr_operations = stats->failed_ops[BLOCK_ACCT_WRITE]; + ds->failed_zone_append_operations = + stats->failed_ops[BLOCK_ACCT_ZONE_APPEND]; ds->failed_flush_operations = stats->failed_ops[BLOCK_ACCT_FLUSH]; ds->failed_unmap_operations = stats->failed_ops[BLOCK_ACCT_UNMAP]; ds->invalid_rd_operations = stats->invalid_ops[BLOCK_ACCT_READ]; ds->invalid_wr_operations = stats->invalid_ops[BLOCK_ACCT_WRITE]; + ds->invalid_zone_append_operations = + stats->invalid_ops[BLOCK_ACCT_ZONE_APPEND]; ds->invalid_flush_operations = stats->invalid_ops[BLOCK_ACCT_FLUSH]; ds->invalid_unmap_operations = stats->invalid_ops[BLOCK_ACCT_UNMAP]; ds->rd_merged = stats->merged[BLOCK_ACCT_READ]; ds->wr_merged = stats->merged[BLOCK_ACCT_WRITE]; + ds->zone_append_merged = stats->merged[BLOCK_ACCT_ZONE_APPEND]; ds->unmap_merged = stats->merged[BLOCK_ACCT_UNMAP]; ds->flush_operations = stats->nr_ops[BLOCK_ACCT_FLUSH]; ds->wr_total_time_ns = stats->total_time_ns[BLOCK_ACCT_WRITE]; + ds->zone_append_total_time_ns = + stats->total_time_ns[BLOCK_ACCT_ZONE_APPEND]; ds->rd_total_time_ns = stats->total_time_ns[BLOCK_ACCT_READ]; ds->flush_total_time_ns = stats->total_time_ns[BLOCK_ACCT_FLUSH]; ds->unmap_total_time_ns = stats->total_time_ns[BLOCK_ACCT_UNMAP]; @@ -571,6 +580,7 @@ static void bdrv_query_blk_stats(BlockDeviceStats *ds, BlockBackend *blk) TimedAverage *rd = &ts->latency[BLOCK_ACCT_READ]; TimedAverage *wr = &ts->latency[BLOCK_ACCT_WRITE]; + TimedAverage *zap = &ts->latency[BLOCK_ACCT_ZONE_APPEND]; TimedAverage *fl = &ts->latency[BLOCK_ACCT_FLUSH]; dev_stats->interval_length = ts->interval_length; @@ -583,6 +593,10 @@ static void bdrv_query_blk_stats(BlockDeviceStats *ds, BlockBackend *blk) dev_stats->max_wr_latency_ns = timed_average_max(wr); dev_stats->avg_wr_latency_ns = timed_average_avg(wr); + dev_stats->min_zone_append_latency_ns = timed_average_min(zap); + dev_stats->max_zone_append_latency_ns = timed_average_max(zap); + dev_stats->avg_zone_append_latency_ns = timed_average_avg(zap); + dev_stats->min_flush_latency_ns = timed_average_min(fl); dev_stats->max_flush_latency_ns = timed_average_max(fl); dev_stats->avg_flush_latency_ns = timed_average_avg(fl); @@ -591,6 +605,8 @@ static void bdrv_query_blk_stats(BlockDeviceStats *ds, BlockBackend *blk) block_acct_queue_depth(ts, BLOCK_ACCT_READ); dev_stats->avg_wr_queue_depth = block_acct_queue_depth(ts, BLOCK_ACCT_WRITE); + dev_stats->avg_zone_append_queue_depth = + block_acct_queue_depth(ts, BLOCK_ACCT_ZONE_APPEND); QAPI_LIST_PREPEND(ds->timed_stats, dev_stats); } @@ -600,6 +616,8 @@ static void bdrv_query_blk_stats(BlockDeviceStats *ds, BlockBackend *blk) = bdrv_latency_histogram_stats(&hgram[BLOCK_ACCT_READ]); ds->wr_latency_histogram = bdrv_latency_histogram_stats(&hgram[BLOCK_ACCT_WRITE]); + ds->zone_append_latency_histogram + = bdrv_latency_histogram_stats(&hgram[BLOCK_ACCT_ZONE_APPEND]); ds->flush_latency_histogram = bdrv_latency_histogram_stats(&hgram[BLOCK_ACCT_FLUSH]); } diff --git a/block/raw-format.c b/block/raw-format.c index fd9e61f58e..3a3946213f 100644 --- a/block/raw-format.c +++ b/block/raw-format.c @@ -317,6 +317,28 @@ raw_co_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes) return bdrv_co_pdiscard(bs->file, offset, bytes); } +static int coroutine_fn GRAPH_RDLOCK +raw_co_zone_report(BlockDriverState *bs, int64_t offset, + unsigned int *nr_zones, + BlockZoneDescriptor *zones) +{ + return bdrv_co_zone_report(bs->file->bs, offset, nr_zones, zones); +} + +static int coroutine_fn GRAPH_RDLOCK +raw_co_zone_mgmt(BlockDriverState *bs, BlockZoneOp op, + int64_t offset, int64_t len) +{ + return bdrv_co_zone_mgmt(bs->file->bs, op, offset, len); +} + +static int coroutine_fn GRAPH_RDLOCK +raw_co_zone_append(BlockDriverState *bs,int64_t *offset, QEMUIOVector *qiov, + BdrvRequestFlags flags) +{ + return bdrv_co_zone_append(bs->file->bs, offset, qiov, flags); +} + static int64_t coroutine_fn GRAPH_RDLOCK raw_co_getlength(BlockDriverState *bs) { @@ -608,6 +630,7 @@ static void raw_child_perm(BlockDriverState *bs, BdrvChild *c, BlockDriver bdrv_raw = { .format_name = "raw", .instance_size = sizeof(BDRVRawState), + .supports_zoned_children = true, .bdrv_probe = &raw_probe, .bdrv_reopen_prepare = &raw_reopen_prepare, .bdrv_reopen_commit = &raw_reopen_commit, @@ -619,6 +642,9 @@ BlockDriver bdrv_raw = { .bdrv_co_pwritev = &raw_co_pwritev, .bdrv_co_pwrite_zeroes = &raw_co_pwrite_zeroes, .bdrv_co_pdiscard = &raw_co_pdiscard, + .bdrv_co_zone_report = &raw_co_zone_report, + .bdrv_co_zone_mgmt = &raw_co_zone_mgmt, + .bdrv_co_zone_append = &raw_co_zone_append, .bdrv_co_block_status = &raw_co_block_status, .bdrv_co_copy_range_from = &raw_co_copy_range_from, .bdrv_co_copy_range_to = &raw_co_copy_range_to, diff --git a/block/trace-events b/block/trace-events index 48dbf10c66..32665158d6 100644 --- a/block/trace-events +++ b/block/trace-events @@ -209,6 +209,10 @@ file_FindEjectableOpticalMedia(const char *media) "Matching using %s" file_setup_cdrom(const char *partition) "Using %s as optical disc" file_hdev_is_sg(int type, int version) "SG device found: type=%d, version=%d" file_flush_fdatasync_failed(int err) "errno %d" +zbd_zone_report(void *bs, unsigned int nr_zones, int64_t sector) "bs %p report %d zones starting at sector offset 0x%" PRIx64 "" +zbd_zone_mgmt(void *bs, const char *op_name, int64_t sector, int64_t len) "bs %p %s starts at sector offset 0x%" PRIx64 " over a range of 0x%" PRIx64 " sectors" +zbd_zone_append(void *bs, int64_t sector) "bs %p append at sector offset 0x%" PRIx64 "" +zbd_zone_append_complete(void *bs, int64_t sector) "bs %p returns append sector 0x%" PRIx64 "" # ssh.c sftp_error(const char *op, const char *ssh_err, int ssh_err_code, int sftp_err_code) "%s failed: %s (libssh error code: %d, sftp error code: %d)" diff --git a/configure b/configure index 0c3f7ba62f..2a556d14c9 100755 --- a/configure +++ b/configure @@ -79,7 +79,6 @@ fi TMPB="qemu-conf" TMPC="${TMPDIR1}/${TMPB}.c" TMPO="${TMPDIR1}/${TMPB}.o" -TMPM="${TMPDIR1}/${TMPB}.m" TMPE="${TMPDIR1}/${TMPB}.exe" rm -f config.log @@ -125,62 +124,20 @@ lines: ${BASH_LINENO[*]}" $compiler "$@" >> config.log 2>&1 || return $? } -do_compiler_werror() { - # Run the compiler, capturing its output to the log. First argument - # is compiler binary to execute. - compiler="$1" - shift - if test -n "$BASH_VERSION"; then eval ' - echo >>config.log " -funcs: ${FUNCNAME[*]} -lines: ${BASH_LINENO[*]}" - '; fi - echo $compiler "$@" >> config.log - $compiler "$@" >> config.log 2>&1 || return $? - # Test passed. If this is an --enable-werror build, rerun - # the test with -Werror and bail out if it fails. This - # makes warning-generating-errors in configure test code - # obvious to developers. - if test "$werror" != "yes"; then - return 0 - fi - # Don't bother rerunning the compile if we were already using -Werror - case "$*" in - *-Werror*) - return 0 - ;; - esac - echo $compiler -Werror "$@" >> config.log - $compiler -Werror "$@" >> config.log 2>&1 && return $? - error_exit "configure test passed without -Werror but failed with -Werror." \ - "This is probably a bug in the configure script. The failing command" \ - "will be at the bottom of config.log." \ - "You can run configure with --disable-werror to bypass this check." -} - do_cc() { - do_compiler_werror "$cc" $CPU_CFLAGS "$@" -} - -do_objc() { - do_compiler_werror "$objcc" $CPU_CFLAGS "$@" -} - -# Append $2 to the variable named $1, with space separation -add_to() { - eval $1=\${$1:+\"\$$1 \"}\$2 + do_compiler "$cc" $CPU_CFLAGS "$@" } compile_object() { local_cflags="$1" - do_cc $CFLAGS $EXTRA_CFLAGS $CONFIGURE_CFLAGS $QEMU_CFLAGS $local_cflags -c -o $TMPO $TMPC + do_cc $CFLAGS $EXTRA_CFLAGS $local_cflags -c -o $TMPO $TMPC } compile_prog() { local_cflags="$1" local_ldflags="$2" - do_cc $CFLAGS $EXTRA_CFLAGS $CONFIGURE_CFLAGS $QEMU_CFLAGS $local_cflags -o $TMPE $TMPC \ - $LDFLAGS $EXTRA_LDFLAGS $CONFIGURE_LDFLAGS $QEMU_LDFLAGS $local_ldflags + do_cc $CFLAGS $EXTRA_CFLAGS $local_cflags -o $TMPE $TMPC \ + $LDFLAGS $EXTRA_LDFLAGS $local_ldflags } # symbolically link $1 to $2. Portable version of "ln -sf". @@ -220,32 +177,18 @@ then error_exit "main directory cannot contain spaces nor colons" fi +# parse CC options first; some compiler tests are used to establish +# some defaults, based on the host environment + # default parameters cpu="" -static="no" cross_compile="no" cross_prefix="" host_cc="cc" -stack_protector="" -safe_stack="" -use_containers="yes" -gdb_bin=$(command -v "gdb-multiarch" || command -v "gdb") -gdb_arches="" -glib_has_gslice="no" - -if test -e "$source_path/.git" -then - git_submodules_action="update" -else - git_submodules_action="ignore" -fi - -git_submodules="ui/keycodemapdb" -git="git" - -# Don't accept a target_list environment variable. -unset target_list -unset target_list_exclude +EXTRA_CFLAGS="" +EXTRA_CXXFLAGS="" +EXTRA_OBJCFLAGS="" +EXTRA_LDFLAGS="" # Default value for a variable defining feature "foo". # * foo="no" feature will only be used if --enable-foo arg is given @@ -258,54 +201,8 @@ unset target_list_exclude # Always add --enable-foo and --disable-foo command line args. # Distributions want to ensure that several features are compiled in, and it # is impossible without a --enable-foo that exits if a feature is not found. - default_feature="" -# parse CC options second -for opt do - optarg=$(expr "x$opt" : 'x[^=]*=\(.*\)') - case "$opt" in - --without-default-features) - default_feature="no" - ;; - esac -done - -EXTRA_CFLAGS="" -EXTRA_CXXFLAGS="" -EXTRA_OBJCFLAGS="" -EXTRA_LDFLAGS="" - -debug_tcg="no" -sanitizers="no" -tsan="no" -fortify_source="yes" -EXESUF="" -modules="no" -prefix="/usr/local" -qemu_suffix="qemu" -softmmu="yes" -linux_user="" -bsd_user="" -pie="" -coroutine="" -plugins="$default_feature" -meson="" -ninja="" -bindir="bin" -skip_meson=no -vfio_user_server="disabled" - -# The following Meson options are handled manually (still they -# are included in the automatically generated help message) - -# 1. Track which submodules are needed -fdt="auto" - -# 2. Automatically enable/disable other options -tcg="auto" -cfi="false" -# parse CC options second for opt do optarg=$(expr "x$opt" : 'x[^=]*=\(.*\)') case "$opt" in @@ -344,9 +241,60 @@ for opt do --cross-prefix-*) cc_arch=${opt#--cross-prefix-}; cc_arch=${cc_arch%%=*} eval "cross_prefix_${cc_arch}=\$optarg" ;; + --without-default-features) default_feature="no" + ;; esac done + +if test -e "$source_path/.git" +then + git_submodules_action="update" +else + git_submodules_action="ignore" +fi + +git_submodules="ui/keycodemapdb" +git="git" +debug_tcg="no" +docs="auto" +EXESUF="" +prefix="/usr/local" +qemu_suffix="qemu" +softmmu="yes" +linux_user="" +bsd_user="" +plugins="$default_feature" +ninja="" +python= +pypi="enabled" +bindir="bin" +skip_meson=no +vfio_user_server="disabled" +use_containers="yes" +gdb_bin=$(command -v "gdb-multiarch" || command -v "gdb") +gdb_arches="" +werror="" + +# Don't accept a target_list environment variable. +unset target_list +unset target_list_exclude + +# The following Meson options are handled manually (still they +# are included in the automatically generated help message) + +# 1. Track which submodules are needed +fdt="auto" + +# 2. Automatically enable/disable other options +tcg="auto" +cfi="false" + +# 3. Need to check for -static-pie before Meson runs. Also, +# Meson has PIE as a boolean rather than enabled/disabled/auto. +pie="" +static="no" + # Preferred compiler: # ${CC} (if set) # ${cross_prefix}gcc (if cross-prefix specified) @@ -390,28 +338,9 @@ strip="${STRIP-${cross_prefix}strip}" widl="${WIDL-${cross_prefix}widl}" windres="${WINDRES-${cross_prefix}windres}" windmc="${WINDMC-${cross_prefix}windmc}" -pkg_config_exe="${PKG_CONFIG-${cross_prefix}pkg-config}" -query_pkg_config() { - "${pkg_config_exe}" ${QEMU_PKG_CONFIG_FLAGS} "$@" -} -pkg_config=query_pkg_config +pkg_config="${PKG_CONFIG-${cross_prefix}pkg-config}" sdl2_config="${SDL2_CONFIG-${cross_prefix}sdl2-config}" -# default flags for all hosts -# We use -fwrapv to tell the compiler that we require a C dialect where -# left shift of signed integers is well defined and has the expected -# 2s-complement style results. (Both clang and gcc agree that it -# provides these semantics.) -QEMU_CFLAGS="-fno-strict-aliasing -fno-common -fwrapv" -QEMU_CFLAGS="-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE $QEMU_CFLAGS" - -QEMU_LDFLAGS= - -# Flags that are needed during configure but later taken care of by Meson -CONFIGURE_CFLAGS="-std=gnu11 -Wall" -CONFIGURE_LDFLAGS= - - check_define() { cat > $TMPC <<EOF #if !defined($1) @@ -422,14 +351,6 @@ EOF compile_object } -check_include() { -cat > $TMPC <<EOF -#include <$1> -int main(void) { return 0; } -EOF - compile_object -} - write_c_skeleton() { cat > $TMPC <<EOF int main(void) { return 0; } @@ -498,21 +419,13 @@ openbsd) darwin) bsd="yes" darwin="yes" - # Disable attempts to use ObjectiveC features in os/object.h since they - # won't work when we're compiling with gcc as a C compiler. - QEMU_CFLAGS="-DOS_OBJECT_USE_OBJC=0 $QEMU_CFLAGS" ;; sunos) solaris="yes" make="${MAKE-gmake}" -# needed for CMSG_ macros in sys/socket.h - QEMU_CFLAGS="-D_XOPEN_SOURCE=600 $QEMU_CFLAGS" -# needed for TIOCWIN* defines in termios.h - QEMU_CFLAGS="-D__EXTENSIONS__ $QEMU_CFLAGS" ;; haiku) pie="no" - QEMU_CFLAGS="-DB_USE_POSITIVE_POSIX_ERRORS -D_BSD_SOURCE -fPIC $QEMU_CFLAGS" ;; linux) linux="yes" @@ -617,18 +530,16 @@ esac check_py_version() { - # We require python >= 3.6. + # We require python >= 3.7. # NB: a True python conditional creates a non-zero return code (Failure) - "$1" -c 'import sys; sys.exit(sys.version_info < (3,6))' + "$1" -c 'import sys; sys.exit(sys.version_info < (3,7))' } -python= first_python= if test -z "${PYTHON}"; then - explicit_python=no # A bare 'python' is traditionally python 2.x, but some distros # have it as python 3.x, so check in both places. - for binary in python3 python python3.11 python3.10 python3.9 python3.8 python3.7 python3.6; do + for binary in python3 python python3.11 python3.10 python3.9 python3.8 python3.7; do if has "$binary"; then python=$(command -v "$binary") if check_py_version "$python"; then @@ -644,7 +555,6 @@ else # Same as above, but only check the environment variable. has "${PYTHON}" || error_exit "The PYTHON environment variable does not point to an executable" python=$(command -v "$PYTHON") - explicit_python=yes if check_py_version "$python"; then # This one is good. first_python= @@ -666,15 +576,11 @@ done if test "$mingw32" = "yes" ; then EXESUF=".exe" - # MinGW needs -mthreads for TLS and macro _MT. - CONFIGURE_CFLAGS="-mthreads $CONFIGURE_CFLAGS" prefix="/qemu" bindir="" qemu_suffix="" fi -werror="" - meson_option_build_array() { printf '[' (if test "$targetos" = windows; then @@ -729,12 +635,10 @@ for opt do ;; --install=*) ;; - --python=*) python="$optarg" ; explicit_python=yes + --python=*) python="$optarg" ;; --skip-meson) skip_meson=yes ;; - --meson=*) meson="$optarg" - ;; --ninja=*) ninja="$optarg" ;; --smbd=*) smbd="$optarg" @@ -751,15 +655,9 @@ for opt do ;; --cross-prefix-*) ;; - --enable-debug-info) meson_option_add -Ddebug=true - ;; - --disable-debug-info) meson_option_add -Ddebug=false - ;; - --enable-modules) - modules="yes" + --enable-docs) docs=enabled ;; - --disable-modules) - modules="no" + --disable-docs) docs=disabled ;; --cpu=*) ;; @@ -791,9 +689,7 @@ for opt do ;; --without-default-features) # processed above ;; - --static) - static="yes" - QEMU_PKG_CONFIG_FLAGS="--static $QEMU_PKG_CONFIG_FLAGS" + --static) static="yes" ;; --bindir=*) bindir="$optarg" ;; @@ -819,15 +715,6 @@ for opt do meson_option_parse --enable-debug-graph-lock "" meson_option_parse --enable-debug-mutex "" meson_option_add -Doptimization=0 - fortify_source="no" - ;; - --enable-sanitizers) sanitizers="yes" - ;; - --disable-sanitizers) sanitizers="no" - ;; - --enable-tsan) tsan="yes" - ;; - --disable-tsan) tsan="no" ;; --disable-tcg) tcg="disabled" plugins="no" @@ -859,14 +746,6 @@ for opt do ;; --disable-werror) werror="no" ;; - --enable-stack-protector) stack_protector="yes" - ;; - --disable-stack-protector) stack_protector="no" - ;; - --enable-safe-stack) safe_stack="yes" - ;; - --disable-safe-stack) safe_stack="no" - ;; --enable-cfi) cfi="true"; meson_option_add -Db_lto=true @@ -881,13 +760,15 @@ for opt do ;; --enable-fdt=*) fdt="$optarg" ;; - --with-coroutine=*) coroutine="$optarg" - ;; --with-git=*) git="$optarg" ;; --with-git-submodules=*) git_submodules_action="$optarg" ;; + --disable-pypi) pypi="disabled" + ;; + --enable-pypi) pypi="enabled" + ;; --enable-plugins) if test "$mingw32" = "yes"; then error_exit "TCG plugins not currently supported on Windows platforms" else @@ -1019,7 +900,6 @@ Advanced options (experts only): --cross-prefix-ARCH=PREFIX cross compiler prefix when building ARCH guest test cases --make=MAKE use specified make [$make] --python=PYTHON use specified python [$python] - --meson=MESON use specified meson [$meson] --ninja=NINJA use specified ninja [$ninja] --smbd=SMBD use specified smbd [$smbd] --with-git=GIT use specified git [$git] @@ -1035,13 +915,8 @@ Advanced options (experts only): desired devices in configs/devices/) --with-devices-ARCH=NAME override default configs/devices --enable-debug enable common debug build options - --enable-sanitizers enable default sanitizers - --enable-tsan enable thread sanitizer --disable-werror disable compilation abort on warning - --disable-stack-protector disable compiler-provided stack protection --cpu=CPU Build for host CPU [$cpu] - --with-coroutine=BACKEND coroutine backend. Supported options: - ucontext, sigaltstack, windows --enable-plugins enable plugins via shared library loading --disable-containers don't use containers for cross-building @@ -1054,11 +929,7 @@ cat << EOF linux-user all linux usermode emulation targets bsd-user all BSD usermode emulation targets pie Position Independent Executables - modules modules support (non-Windows) debug-tcg TCG debugging (default is disabled) - debug-info debugging information - safe-stack SafeStack Stack Smash Protection. Depends on - clang/llvm and requires coroutine backend ucontext. NOTE: The object files are built at the place where configure is launched EOF @@ -1073,7 +944,7 @@ then # If first_python is set, there was a binary somewhere even though # it was not suitable. Use it for the error message. if test -n "$first_python"; then - error_exit "Cannot use '$first_python', Python >= 3.6 is required." \ + error_exit "Cannot use '$first_python', Python >= 3.7 is required." \ "Use --python=/path/to/python to specify a supported Python." else error_exit "Python not found. Use --python=/path/to/python" @@ -1086,65 +957,86 @@ then fi if ! check_py_version "$python"; then - error_exit "Cannot use '$python', Python >= 3.6 is required." \ - "Use --python=/path/to/python to specify a supported Python." + error_exit "Cannot use '$python', Python >= 3.7 is required." \ + "Use --python=/path/to/python to specify a supported Python." \ + "Maybe try:" \ + " openSUSE Leap 15.3+: zypper install python39" \ + " CentOS 8: dnf install python38" fi -# Resolve PATH + suppress writing compiled files -python="$(command -v "$python") -B" - -has_meson() { - local python_dir=$(dirname "$python") - # PEP405: pyvenv.cfg is either adjacent to the Python executable - # or one directory above - if test -f $python_dir/pyvenv.cfg || test -f $python_dir/../pyvenv.cfg; then - # Ensure that Meson and Python come from the same virtual environment - test -x "$python_dir/meson" && - test "$(command -v meson)" -ef "$python_dir/meson" - else - has meson - fi -} +# Resolve PATH +python="$(command -v "$python")" -if test -z "$meson"; then - if test "$explicit_python" = no && has_meson && version_ge "$(meson --version)" 0.61.5; then - meson=meson - elif test "$git_submodules_action" != 'ignore' ; then - meson=git - elif test -e "${source_path}/meson/meson.py" ; then - meson=internal - else - if test "$explicit_python" = yes; then - error_exit "--python requires using QEMU's embedded Meson distribution, but it was not found." - else - error_exit "Meson not found. Use --meson=/path/to/meson" - fi - fi -else - # Meson uses its own Python interpreter to invoke other Python scripts, - # but the user wants to use the one they specified with --python. - # - # We do not want to override the distro Python interpreter (and sometimes - # cannot: for example in Homebrew /usr/bin/meson is a bash script), so - # just require --meson=git|internal together with --python. - if test "$explicit_python" = yes; then - case "$meson" in - git | internal) ;; - *) error_exit "--python requires using QEMU's embedded Meson distribution." ;; - esac - fi +# Create a Python virtual environment using our configured python. +# The stdout of this script will be the location of a symlink that +# points to the configured Python. +# Entry point scripts for pip, meson, and sphinx are generated if those +# packages are present. + +# Defaults assumed for now: +# - venv is cleared if it exists already; +# - venv is allowed to use system packages; +# - all setup can be performed offline; +# - missing packages may be fetched from PyPI, +# unless --disable-pypi is passed. +# - pip is not installed into the venv when possible, +# but ensurepip is called as a fallback when necessary. + +echo "python determined to be '$python'" +echo "python version: $($python --version)" + +python="$($python -B "${source_path}/python/scripts/mkvenv.py" create pyvenv)" +if test "$?" -ne 0 ; then + error_exit "python venv creation failed" fi -if test "$meson" = git; then - git_submodules="${git_submodules} meson" +# Suppress writing compiled files +python="$python -B" +mkvenv="$python ${source_path}/python/scripts/mkvenv.py" + +mkvenv_flags="" +if test "$pypi" = "enabled" ; then + mkvenv_flags="--online" fi -case "$meson" in - git | internal) - meson="$python ${source_path}/meson/meson.py" - ;; - *) meson=$(command -v "$meson") ;; -esac +if ! $mkvenv ensure \ + $mkvenv_flags \ + --dir "${source_path}/python/wheels" \ + --diagnose "meson" \ + "meson>=0.63.0" ; +then + exit 1 +fi + +# At this point, we expect Meson to be installed and available. +# We expect mkvenv or pip to have created pyvenv/bin/meson for us. +# We ignore PATH completely here: we want to use the venv's Meson +# *exclusively*. + +meson="$(cd pyvenv/bin; pwd)/meson" + +# Conditionally ensure Sphinx is installed. + +mkvenv_flags="" +if test "$pypi" = "enabled" -a "$docs" = "enabled" ; then + mkvenv_flags="--online" +fi + +if test "$docs" != "disabled" ; then + if ! $mkvenv ensure \ + $mkvenv_flags \ + --diagnose "sphinx-build" \ + "sphinx>=1.6.0" "sphinx-rtd-theme>=0.5.0"; + then + if test "$docs" = "enabled" ; then + exit 1 + fi + echo "Sphinx not found/usable, disabling docs." + docs=disabled + else + docs=enabled + fi +fi # Probe for ninja @@ -1160,20 +1052,6 @@ if test -z "$ninja"; then fi fi -# Check that the C compiler works. Doing this here before testing -# the host CPU ensures that we had a valid CC to autodetect the -# $cpu var (and we should bail right here if that's not the case). -# It also allows the help message to be printed without a CC. -write_c_skeleton; -if compile_object ; then - : C compiler works ok -else - error_exit "\"$cc\" either does not exist or does not work" -fi -if ! compile_prog ; then - error_exit "\"$cc\" cannot build an executable (is your linker broken?)" -fi - # Consult white-list to determine whether to enable werror # by default. Only enable by default for git builds if test -z "$werror" ; then @@ -1193,147 +1071,7 @@ if test "$targetos" = "bogus"; then error_exit "Unrecognized host OS (uname -s reports '$(uname -s)')" fi -# Check whether the compiler matches our minimum requirements: -cat > $TMPC << EOF -#if defined(__clang_major__) && defined(__clang_minor__) -# ifdef __apple_build_version__ -# if __clang_major__ < 12 || (__clang_major__ == 12 && __clang_minor__ < 0) -# error You need at least XCode Clang v12.0 to compile QEMU -# endif -# else -# if __clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 0) -# error You need at least Clang v10.0 to compile QEMU -# endif -# endif -#elif defined(__GNUC__) && defined(__GNUC_MINOR__) -# if __GNUC__ < 7 || (__GNUC__ == 7 && __GNUC_MINOR__ < 4) -# error You need at least GCC v7.4.0 to compile QEMU -# endif -#else -# error You either need GCC or Clang to compiler QEMU -#endif -int main (void) { return 0; } -EOF -if ! compile_prog "" "" ; then - error_exit "You need at least GCC v7.4 or Clang v10.0 (or XCode Clang v12.0)" -fi - -# Accumulate -Wfoo and -Wno-bar separately. -# We will list all of the enable flags first, and the disable flags second. -# Note that we do not add -Werror, because that would enable it for all -# configure tests. If a configure test failed due to -Werror this would -# just silently disable some features, so it's too error prone. - -warn_flags= -add_to warn_flags -Wundef -add_to warn_flags -Wwrite-strings -add_to warn_flags -Wmissing-prototypes -add_to warn_flags -Wstrict-prototypes -add_to warn_flags -Wredundant-decls -add_to warn_flags -Wold-style-declaration -add_to warn_flags -Wold-style-definition -add_to warn_flags -Wtype-limits -add_to warn_flags -Wformat-security -add_to warn_flags -Wformat-y2k -add_to warn_flags -Winit-self -add_to warn_flags -Wignored-qualifiers -add_to warn_flags -Wempty-body -add_to warn_flags -Wnested-externs -add_to warn_flags -Wendif-labels -add_to warn_flags -Wexpansion-to-defined -add_to warn_flags -Wimplicit-fallthrough=2 -add_to warn_flags -Wmissing-format-attribute - -if test "$targetos" != "darwin"; then - add_to warn_flags -Wthread-safety -fi - -nowarn_flags= -add_to nowarn_flags -Wno-initializer-overrides -add_to nowarn_flags -Wno-missing-include-dirs -add_to nowarn_flags -Wno-shift-negative-value -add_to nowarn_flags -Wno-string-plus-int -add_to nowarn_flags -Wno-typedef-redefinition -add_to nowarn_flags -Wno-tautological-type-limit-compare -add_to nowarn_flags -Wno-psabi -add_to nowarn_flags -Wno-gnu-variable-sized-type-not-at-end - -gcc_flags="$warn_flags $nowarn_flags" - -cc_has_warning_flag() { - write_c_skeleton; - - # Use the positive sense of the flag when testing for -Wno-wombat - # support (gcc will happily accept the -Wno- form of unknown - # warning options). - optflag="$(echo $1 | sed -e 's/^-Wno-/-W/')" - compile_prog "-Werror $optflag" "" -} - -objcc_has_warning_flag() { - cat > $TMPM <<EOF -int main(void) { return 0; } -EOF - - # Use the positive sense of the flag when testing for -Wno-wombat - # support (gcc will happily accept the -Wno- form of unknown - # warning options). - optflag="$(echo $1 | sed -e 's/^-Wno-/-W/')" - do_objc -Werror $optflag \ - $OBJCFLAGS $EXTRA_OBJCFLAGS $CONFIGURE_OBJCFLAGS $QEMU_OBJCFLAGS \ - -o $TMPE $TMPM $QEMU_LDFLAGS -} - -for flag in $gcc_flags; do - if cc_has_warning_flag $flag ; then - QEMU_CFLAGS="$QEMU_CFLAGS $flag" - fi - if objcc_has_warning_flag $flag ; then - QEMU_OBJCFLAGS="$QEMU_OBJCFLAGS $flag" - fi -done - -if test "$stack_protector" != "no"; then - cat > $TMPC << EOF -int main(int argc, char *argv[]) -{ - char arr[64], *p = arr, *c = argv[argc - 1]; - while (*c) { - *p++ = *c++; - } - return 0; -} -EOF - gcc_flags="-fstack-protector-strong -fstack-protector-all" - sp_on=0 - for flag in $gcc_flags; do - # We need to check both a compile and a link, since some compiler - # setups fail only on a .c->.o compile and some only at link time - if compile_object "-Werror $flag" && - compile_prog "-Werror $flag" ""; then - QEMU_CFLAGS="$QEMU_CFLAGS $flag" - QEMU_LDFLAGS="$QEMU_LDFLAGS $flag" - sp_on=1 - break - fi - done - if test "$stack_protector" = yes; then - if test $sp_on = 0; then - error_exit "Stack protector not supported" - fi - fi -fi - -# Our module code doesn't support Windows -if test "$modules" = "yes" && test "$mingw32" = "yes" ; then - error_exit "Modules are not available for Windows" -fi - -# Static linking is not possible with plugins, modules or PIE if test "$static" = "yes" ; then - if test "$modules" = "yes" ; then - error_exit "static and modules are mutually incompatible" - fi if test "$plugins" = "yes"; then error_exit "static and plugins are mutually incompatible" else @@ -1353,59 +1091,26 @@ static THREAD int tls_var; int main(void) { return tls_var; } EOF -# Meson currently only handles pie as a boolean for now so if we have -# explicitly disabled PIE we need to extend our cflags because it wont. if test "$static" = "yes"; then if test "$pie" != "no" && compile_prog "-Werror -fPIE -DPIE" "-static-pie"; then - CONFIGURE_CFLAGS="-fPIE -DPIE $CONFIGURE_CFLAGS" pie="yes" elif test "$pie" = "yes"; then error_exit "-static-pie not available due to missing toolchain support" else pie="no" - QEMU_CFLAGS="-fno-pie $QEMU_CFLAGS" fi -elif test "$pie" = "no"; then - if compile_prog "-Werror -fno-pie" "-no-pie"; then - CONFIGURE_CFLAGS="-fno-pie $CONFIGURE_CFLAGS" - CONFIGURE_LDFLAGS="-no-pie $CONFIGURE_LDFLAGS" - QEMU_CFLAGS="-fno-pie -no-pie $QEMU_CFLAGS" +elif test "$pie" != "no"; then + if compile_prog "-Werror -fPIE -DPIE" "-pie"; then + pie="yes" + elif test "$pie" = "yes"; then + error_exit "PIE not available due to missing toolchain support" + else + echo "Disabling PIE due to missing toolchain support" + pie="no" fi -elif compile_prog "-Werror -fPIE -DPIE" "-pie"; then - CONFIGURE_CFLAGS="-fPIE -DPIE $CONFIGURE_CFLAGS" - CONFIGURE_LDFLAGS="-pie $CONFIGURE_LDFLAGS" - pie="yes" -elif test "$pie" = "yes"; then - error_exit "PIE not available due to missing toolchain support" -else - echo "Disabling PIE due to missing toolchain support" - pie="no" fi ########################################## -# __sync_fetch_and_and requires at least -march=i486. Many toolchains -# use i686 as default anyway, but for those that don't, an explicit -# specification is necessary - -if test "$cpu" = "i386"; then - cat > $TMPC << EOF -static int sfaa(int *ptr) -{ - return __sync_fetch_and_and(ptr, 0); -} - -int main(void) -{ - int val = 42; - val = __sync_val_compare_and_swap(&val, 0, 1); - sfaa(&val); - return val; -} -EOF - if ! compile_prog "" "" ; then - QEMU_CFLAGS="-march=i486 $QEMU_CFLAGS" - fi -fi if test -z "${target_list+xxx}" ; then default_targets=yes @@ -1478,102 +1183,6 @@ EOF fi ########################################## -# pkg-config probe - -if ! has "$pkg_config_exe"; then - error_exit "pkg-config binary '$pkg_config_exe' not found" -fi - -########################################## -# glib support probe - -# When bumping glib_req_ver, please check also whether we should increase -# the _WIN32_WINNT setting in osdep.h according to the value from glib -glib_req_ver=2.56 -glib_modules=gthread-2.0 -if test "$modules" = yes; then - glib_modules="$glib_modules gmodule-export-2.0" -elif test "$plugins" = "yes"; then - glib_modules="$glib_modules gmodule-no-export-2.0" -fi - -for i in $glib_modules; do - if $pkg_config --atleast-version=$glib_req_ver $i; then - glib_cflags=$($pkg_config --cflags $i) - glib_libs=$($pkg_config --libs $i) - else - error_exit "glib-$glib_req_ver $i is required to compile QEMU" - fi -done - -# Check whether glib has gslice, which we have to avoid for correctness. -# TODO: remove this check and the corresponding workaround (qtree) when -# the minimum supported glib is >= $glib_dropped_gslice_version. -glib_dropped_gslice_version=2.75.3 -for i in $glib_modules; do - if ! $pkg_config --atleast-version=$glib_dropped_gslice_version $i; then - glib_has_gslice="yes" - break - fi -done - -glib_bindir="$($pkg_config --variable=bindir glib-2.0)" -if test -z "$glib_bindir" ; then - glib_bindir="$($pkg_config --variable=prefix glib-2.0)"/bin -fi - -# This workaround is required due to a bug in pkg-config file for glib as it -# doesn't define GLIB_STATIC_COMPILATION for pkg-config --static - -if test "$static" = yes && test "$mingw32" = yes; then - glib_cflags="-DGLIB_STATIC_COMPILATION $glib_cflags" -fi - -# Sanity check that the current size_t matches the -# size that glib thinks it should be. This catches -# problems on multi-arch where people try to build -# 32-bit QEMU while pointing at 64-bit glib headers -cat > $TMPC <<EOF -#include <glib.h> -#include <unistd.h> - -#define QEMU_BUILD_BUG_ON(x) \ - typedef char qemu_build_bug_on[(x)?-1:1] __attribute__((unused)); - -int main(void) { - QEMU_BUILD_BUG_ON(sizeof(size_t) != GLIB_SIZEOF_SIZE_T); - return 0; -} -EOF - -if ! compile_prog "$glib_cflags" "$glib_libs" ; then - error_exit "sizeof(size_t) doesn't match GLIB_SIZEOF_SIZE_T."\ - "You probably need to set PKG_CONFIG_LIBDIR"\ - "to point to the right pkg-config files for your"\ - "build target" -fi - -# Silence clang warnings triggered by glib < 2.57.2 -cat > $TMPC << EOF -#include <glib.h> -typedef struct Foo { - int i; -} Foo; -static void foo_free(Foo *f) -{ - g_free(f); -} -G_DEFINE_AUTOPTR_CLEANUP_FUNC(Foo, foo_free) -int main(void) { return 0; } -EOF -if ! compile_prog "$glib_cflags -Werror" "$glib_libs" ; then - if cc_has_warning_flag "-Wno-unused-function"; then - glib_cflags="$glib_cflags -Wno-unused-function" - CONFIGURE_CFLAGS="$CONFIGURE_CFLAGS -Wno-unused-function" - fi -fi - -########################################## # fdt probe case "$fdt" in @@ -1583,123 +1192,6 @@ case "$fdt" in ;; esac -########################################## -# check and set a backend for coroutine - -# We prefer ucontext, but it's not always possible. The fallback -# is sigcontext. On Windows the only valid backend is the Windows -# specific one. - -ucontext_works=no -if test "$darwin" != "yes"; then - cat > $TMPC << EOF -#include <ucontext.h> -#ifdef __stub_makecontext -#error Ignoring glibc stub makecontext which will always fail -#endif -int main(void) { makecontext(0, 0, 0); return 0; } -EOF - if compile_prog "" "" ; then - ucontext_works=yes - fi -fi - -if test "$coroutine" = ""; then - if test "$mingw32" = "yes"; then - coroutine=win32 - elif test "$ucontext_works" = "yes"; then - coroutine=ucontext - else - coroutine=sigaltstack - fi -else - case $coroutine in - windows) - if test "$mingw32" != "yes"; then - error_exit "'windows' coroutine backend only valid for Windows" - fi - # Unfortunately the user visible backend name doesn't match the - # coroutine-*.c filename for this case, so we have to adjust it here. - coroutine=win32 - ;; - ucontext) - if test "$ucontext_works" != "yes"; then - error_exit "'ucontext' backend requested but makecontext not available" - fi - ;; - sigaltstack) - if test "$mingw32" = "yes"; then - error_exit "only the 'windows' coroutine backend is valid for Windows" - fi - ;; - *) - error_exit "unknown coroutine backend $coroutine" - ;; - esac -fi - -################################################## -# SafeStack - - -if test "$safe_stack" = "yes"; then -cat > $TMPC << EOF -int main(void) -{ -#if ! __has_feature(safe_stack) -#error SafeStack Disabled -#endif - return 0; -} -EOF - flag="-fsanitize=safe-stack" - # Check that safe-stack is supported and enabled. - if compile_prog "-Werror $flag" "$flag"; then - # Flag needed both at compilation and at linking - QEMU_CFLAGS="$QEMU_CFLAGS $flag" - QEMU_LDFLAGS="$QEMU_LDFLAGS $flag" - else - error_exit "SafeStack not supported by your compiler" - fi - if test "$coroutine" != "ucontext"; then - error_exit "SafeStack is only supported by the coroutine backend ucontext" - fi -else -cat > $TMPC << EOF -int main(void) -{ -#if defined(__has_feature) -#if __has_feature(safe_stack) -#error SafeStack Enabled -#endif -#endif - return 0; -} -EOF -if test "$safe_stack" = "no"; then - # Make sure that safe-stack is disabled - if ! compile_prog "-Werror" ""; then - # SafeStack was already enabled, try to explicitly remove the feature - flag="-fno-sanitize=safe-stack" - if ! compile_prog "-Werror $flag" "$flag"; then - error_exit "Configure cannot disable SafeStack" - fi - QEMU_CFLAGS="$QEMU_CFLAGS $flag" - QEMU_LDFLAGS="$QEMU_LDFLAGS $flag" - fi -else # "$safe_stack" = "" - # Set safe_stack to yes or no based on pre-existing flags - if compile_prog "-Werror" ""; then - safe_stack="no" - else - safe_stack="yes" - if test "$coroutine" != "ucontext"; then - error_exit "SafeStack is only supported by the coroutine backend ucontext" - fi - fi -fi -fi - ######################################## # check if ccache is interfering with # semantic analysis of macros @@ -1728,87 +1220,6 @@ if ! compile_object "-Werror"; then ccache_cpp2=yes fi -################################################# -# clang does not support glibc + FORTIFY_SOURCE. - -if test "$fortify_source" != "no"; then - if echo | $cc -dM -E - | grep __clang__ > /dev/null 2>&1 ; then - fortify_source="no"; - elif test -n "$cxx" && has $cxx && - echo | $cxx -dM -E - | grep __clang__ >/dev/null 2>&1 ; then - fortify_source="no"; - else - fortify_source="yes" - fi -fi - -########################################## -# checks for sanitizers - -have_asan=no -have_ubsan=no -have_asan_iface_h=no -have_asan_iface_fiber=no - -if test "$sanitizers" = "yes" ; then - write_c_skeleton - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=address" ""; then - have_asan=yes - fi - - # we could use a simple skeleton for flags checks, but this also - # detect the static linking issue of ubsan, see also: - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84285 - cat > $TMPC << EOF -int main(int argc, char **argv) -{ - return argc + 1; -} -EOF - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=undefined" ""; then - have_ubsan=yes - fi - - if check_include "sanitizer/asan_interface.h" ; then - have_asan_iface_h=yes - fi - - cat > $TMPC << EOF -#include <sanitizer/asan_interface.h> -int main(void) { - __sanitizer_start_switch_fiber(0, 0, 0); - return 0; -} -EOF - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=address" "" ; then - have_asan_iface_fiber=yes - fi -fi - -# Thread sanitizer is, for now, much noisier than the other sanitizers; -# keep it separate until that is not the case. -if test "$tsan" = "yes" && test "$sanitizers" = "yes"; then - error_exit "TSAN is not supported with other sanitiziers." -fi -have_tsan=no -have_tsan_iface_fiber=no -if test "$tsan" = "yes" ; then - write_c_skeleton - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then - have_tsan=yes - fi - cat > $TMPC << EOF -#include <sanitizer/tsan_interface.h> -int main(void) { - __tsan_create_fiber(0); - return 0; -} -EOF - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then - have_tsan_iface_fiber=yes - fi -fi - ########################################## # functions to probe cross compilers @@ -1913,13 +1324,6 @@ probe_target_compiler() { container_cross_ranlib= container_cross_strip= - # We shall skip configuring the target compiler if the user didn't - # bother enabling an appropriate guest. This avoids building - # extraneous firmware images and tests. - if test "${target_list#*$1}" = "$1"; then - return 1 - fi - target_arch=${1%%-*} case $target_arch in aarch64) container_hosts="x86_64 aarch64" ;; @@ -2234,42 +1638,6 @@ case "$vfio_user_server" in ;; esac -########################################## -# End of CC checks -# After here, no more $cc or $ld runs - -write_c_skeleton - -if test "$fortify_source" = "yes" ; then - QEMU_CFLAGS="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $QEMU_CFLAGS" -fi - -if test "$have_asan" = "yes"; then - QEMU_CFLAGS="-fsanitize=address $QEMU_CFLAGS" - QEMU_LDFLAGS="-fsanitize=address $QEMU_LDFLAGS" - if test "$have_asan_iface_h" = "no" ; then - echo "ASAN build enabled, but ASAN header missing." \ - "Without code annotation, the report may be inferior." - elif test "$have_asan_iface_fiber" = "no" ; then - echo "ASAN build enabled, but ASAN header is too old." \ - "Without code annotation, the report may be inferior." - fi -fi -if test "$have_tsan" = "yes" ; then - if test "$have_tsan_iface_fiber" = "yes" ; then - QEMU_CFLAGS="-fsanitize=thread $QEMU_CFLAGS" - QEMU_LDFLAGS="-fsanitize=thread $QEMU_LDFLAGS" - else - error_exit "Cannot enable TSAN due to missing fiber annotation interface." - fi -elif test "$tsan" = "yes" ; then - error_exit "Cannot enable TSAN due to missing sanitize thread interface." -fi -if test "$have_ubsan" = "yes"; then - QEMU_CFLAGS="-fsanitize=undefined $QEMU_CFLAGS" - QEMU_LDFLAGS="-fsanitize=undefined $QEMU_LDFLAGS" -fi - ####################################### # cross-compiled firmware targets @@ -2293,7 +1661,6 @@ LINKS="$LINKS python" LINKS="$LINKS contrib/plugins/Makefile " for f in $LINKS ; do if [ -e "$source_path/$f" ]; then - mkdir -p "$(dirname ./"$f")" symlink "$source_path/$f" "$f" fi done @@ -2384,30 +1751,14 @@ fi if test "$solaris" = "yes" ; then echo "CONFIG_SOLARIS=y" >> $config_host_mak fi -if test "$static" = "yes" ; then - echo "CONFIG_STATIC=y" >> $config_host_mak -fi echo "SRC_PATH=$source_path" >> $config_host_mak echo "TARGET_DIRS=$target_list" >> $config_host_mak -if test "$modules" = "yes"; then - echo "CONFIG_MODULES=y" >> $config_host_mak -fi # XXX: suppress that if [ "$bsd" = "yes" ] ; then echo "CONFIG_BSD=y" >> $config_host_mak fi -echo "CONFIG_COROUTINE_BACKEND=$coroutine" >> $config_host_mak - -if test "$have_asan_iface_fiber" = "yes" ; then - echo "CONFIG_ASAN_IFACE_FIBER=y" >> $config_host_mak -fi - -if test "$have_tsan" = "yes" && test "$have_tsan_iface_fiber" = "yes" ; then - echo "CONFIG_TSAN=y" >> $config_host_mak -fi - if test "$plugins" = "yes" ; then echo "CONFIG_PLUGIN=y" >> $config_host_mak fi @@ -2432,17 +1783,8 @@ echo "PYTHON=$python" >> $config_host_mak echo "GENISOIMAGE=$genisoimage" >> $config_host_mak echo "MESON=$meson" >> $config_host_mak echo "NINJA=$ninja" >> $config_host_mak +echo "PKG_CONFIG=${pkg_config}" >> $config_host_mak echo "CC=$cc" >> $config_host_mak -echo "QEMU_CFLAGS=$QEMU_CFLAGS" >> $config_host_mak -echo "QEMU_OBJCFLAGS=$QEMU_OBJCFLAGS" >> $config_host_mak -echo "GLIB_CFLAGS=$glib_cflags" >> $config_host_mak -echo "GLIB_LIBS=$glib_libs" >> $config_host_mak -echo "GLIB_BINDIR=$glib_bindir" >> $config_host_mak -echo "GLIB_VERSION=$($pkg_config --modversion glib-2.0)" >> $config_host_mak -if test "$glib_has_gslice" = "yes" ; then - echo "HAVE_GLIB_WITH_SLICE_ALLOCATOR=y" >> $config_host_mak -fi -echo "QEMU_LDFLAGS=$QEMU_LDFLAGS" >> $config_host_mak echo "EXESUF=$EXESUF" >> $config_host_mak # use included Linux headers @@ -2496,10 +1838,6 @@ if test "$ccache_cpp2" = "yes"; then echo "export CCACHE_CPP2=y" >> $config_host_mak fi -if test "$safe_stack" = "yes"; then - echo "CONFIG_SAFESTACK=y" >> $config_host_mak -fi - # tests/tcg configuration (config_host_mak=tests/tcg/config-host.mak mkdir -p tests/tcg @@ -2588,7 +1926,7 @@ if test "$skip_meson" = no; then test -n "$objcc" && echo "objc = [$(meson_quote $objcc $CPU_CFLAGS)]" >> $cross echo "ar = [$(meson_quote $ar)]" >> $cross echo "nm = [$(meson_quote $nm)]" >> $cross - echo "pkgconfig = [$(meson_quote $pkg_config_exe)]" >> $cross + echo "pkgconfig = [$(meson_quote $pkg_config)]" >> $cross echo "ranlib = [$(meson_quote $ranlib)]" >> $cross if has $sdl2_config; then echo "sdl2-config = [$(meson_quote $sdl2_config)]" >> $cross @@ -2629,11 +1967,13 @@ if test "$skip_meson" = no; then # Built-in options test "$bindir" != "bin" && meson_option_add "-Dbindir=$bindir" test "$default_feature" = no && meson_option_add -Dauto_features=disabled + test "$static" = yes && meson_option_add -Dprefer_static=true test "$pie" = no && meson_option_add -Db_pie=false test "$werror" = yes && meson_option_add -Dwerror=true # QEMU options test "$cfi" != false && meson_option_add "-Dcfi=$cfi" + test "$docs" != auto && meson_option_add "-Ddocs=$docs" test "$fdt" != auto && meson_option_add "-Dfdt=$fdt" test -n "${LIB_FUZZING_ENGINE+xxx}" && meson_option_add "-Dfuzzing_engine=$LIB_FUZZING_ENGINE" test "$qemu_suffix" != qemu && meson_option_add "-Dqemu_suffix=$qemu_suffix" diff --git a/contrib/plugins/Makefile b/contrib/plugins/Makefile index 23e0396687..b2b9db9f51 100644 --- a/contrib/plugins/Makefile +++ b/contrib/plugins/Makefile @@ -3,7 +3,7 @@ # This Makefile example is fairly independent from the main makefile # so users can take and adapt it for their build. We only really # include config-host.mak so we don't have to repeat probing for -# cflags that the main configure has already done for us. +# programs that the main configure has already done for us. # BUILD_DIR := $(CURDIR)/../.. @@ -26,9 +26,8 @@ SONAMES := $(addsuffix .so,$(addprefix lib,$(NAMES))) # The main QEMU uses Glib extensively so it's perfectly fine to use it # in plugins (which many example do). -CFLAGS = $(GLIB_CFLAGS) -CFLAGS += -fPIC -Wall $(filter -W%, $(QEMU_CFLAGS)) -CFLAGS += $(if $(findstring no-psabi,$(QEMU_CFLAGS)),-Wpsabi) +CFLAGS := $(shell $(PKG_CONFIG) --cflags glib-2.0) +CFLAGS += -fPIC -Wall CFLAGS += $(if $(CONFIG_DEBUG_TCG), -ggdb -O0) CFLAGS += -I$(SRC_PATH)/include/qemu diff --git a/cpu.c b/cpu.c index 9105c85404..65ebaf8159 100644 --- a/cpu.c +++ b/cpu.c @@ -427,6 +427,11 @@ bool target_words_bigendian(void) #endif } +const char *target_name(void) +{ + return TARGET_NAME; +} + void page_size_init(void) { /* NOTE: we can always suppose that qemu_host_page_size >= diff --git a/docs/about/build-platforms.rst b/docs/about/build-platforms.rst index 89cae5a6bb..0e2cb9e770 100644 --- a/docs/about/build-platforms.rst +++ b/docs/about/build-platforms.rst @@ -98,7 +98,7 @@ Python runtime option of the ``configure`` script to point QEMU to a supported version of the Python runtime. - As of QEMU |version|, the minimum supported version of Python is 3.6. + As of QEMU |version|, the minimum supported version of Python is 3.7. Python build dependencies Some of QEMU's build dependencies are written in Python. Usually these diff --git a/docs/about/deprecated.rst b/docs/about/deprecated.rst index 4c7f08803e..7bb4d2f4f6 100644 --- a/docs/about/deprecated.rst +++ b/docs/about/deprecated.rst @@ -111,6 +111,11 @@ Use ``-machine acpi=off`` instead. The HAXM project has been retired (see https://github.com/intel/haxm#status). Use "whpx" (on Windows) or "hvf" (on macOS) instead. +``-async-teardown`` (since 8.1) +''''''''''''''''''''''''''''''' + +Use ``-run-with async-teardown=on`` instead. + ``-singlestep`` (since 8.1) ''''''''''''''''''''''''''' diff --git a/docs/about/emulation.rst b/docs/about/emulation.rst index b510a54418..0ad0b86f0d 100644 --- a/docs/about/emulation.rst +++ b/docs/about/emulation.rst @@ -99,7 +99,7 @@ depending on the guest architecture. - Yes - A configurable 32 bit soft core now owned by Cadence -A number of features are are only available when running under +A number of features are only available when running under emulation including :ref:`Record/Replay<replay>` and :ref:`TCG Plugins`. .. _Semihosting: diff --git a/docs/conf.py b/docs/conf.py index 00767b0e24..c687ff2663 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,15 +32,6 @@ import sphinx from distutils.version import LooseVersion from sphinx.errors import ConfigError -# Make Sphinx fail cleanly if using an old Python, rather than obscurely -# failing because some code in one of our extensions doesn't work there. -# In newer versions of Sphinx this will display nicely; in older versions -# Sphinx will also produce a Python backtrace but at least the information -# gets printed... -if sys.version_info < (3,6): - raise ConfigError( - "QEMU requires a Sphinx that uses Python 3.6 or better\n") - # The per-manual conf.py will set qemu_docdir for a single-manual build; # otherwise set it here if this is an entire-manual-set build. # This is always the absolute path of the docs/ directory in the source tree. diff --git a/docs/devel/acpi-bits.rst b/docs/devel/acpi-bits.rst index 22e2580200..9677b0098f 100644 --- a/docs/devel/acpi-bits.rst +++ b/docs/devel/acpi-bits.rst @@ -61,19 +61,19 @@ Under ``tests/avocado/`` as the root we have: :: $ make check-venv (needed only the first time to create the venv) - $ ./tests/venv/bin/avocado run -t acpi tests/avocado + $ ./pyvenv/bin/avocado run -t acpi tests/avocado The above will run all acpi avocado tests including this one. In order to run the individual tests, perform the following: :: - $ ./tests/venv/bin/avocado run tests/avocado/acpi-bits.py --tap - + $ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py --tap - The above will produce output in tap format. You can omit "--tap -" in the end and it will produce output like the following: :: - $ ./tests/venv/bin/avocado run tests/avocado/acpi-bits.py + $ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py Fetching asset from tests/avocado/acpi-bits.py:AcpiBitsTest.test_acpi_smbios_bits JOB ID : eab225724da7b64c012c65705dc2fa14ab1defef JOB LOG : /home/anisinha/avocado/job-results/job-2022-10-10T17.58-eab2257/job.log diff --git a/docs/devel/build-system.rst b/docs/devel/build-system.rst index 1894721743..551c5a5ac0 100644 --- a/docs/devel/build-system.rst +++ b/docs/devel/build-system.rst @@ -4,30 +4,14 @@ The QEMU build system architecture This document aims to help developers understand the architecture of the QEMU build system. As with projects using GNU autotools, the QEMU build -system has two stages, first the developer runs the "configure" script +system has two stages; first the developer runs the "configure" script to determine the local build environment characteristics, then they run -"make" to build the project. There is about where the similarities with +"make" to build the project. This is about where the similarities with GNU autotools end, so try to forget what you know about them. +The two general ways to perform a build are as follows: -Stage 1: configure -================== - -The QEMU configure script is written directly in shell, and should be -compatible with any POSIX shell, hence it uses #!/bin/sh. An important -implication of this is that it is important to avoid using bash-isms on -development platforms where bash is the primary host. - -In contrast to autoconf scripts, QEMU's configure is expected to be -silent while it is checking for features. It will only display output -when an error occurs, or to show the final feature enablement summary -on completion. - -Because QEMU uses the Meson build system under the hood, only VPATH -builds are supported. There are two general ways to invoke configure & -perform a build: - - - VPATH, build artifacts outside of QEMU source tree entirely:: + - build artifacts outside of QEMU source tree entirely:: cd ../ mkdir build @@ -35,88 +19,122 @@ perform a build: ../qemu/configure make - - VPATH, build artifacts in a subdir of QEMU source tree:: + - build artifacts in a subdir of QEMU source tree:: mkdir build cd build ../configure make -The configure script automatically recognizes -command line options for which a same-named Meson option exists; -dashes in the command line are replaced with underscores. +Most of the actual build process uses Meson under the hood, therefore +build artifacts cannot be placed in the source tree itself. + + +Stage 1: configure +================== + +The configure script has five tasks: + + - detect the host architecture -Many checks on the compilation environment are still found in configure -rather than ``meson.build``, but new checks should be added directly to -``meson.build``. + - list the targets for which to build emulators; the list of + targets also affects which firmware binaries and tests to build + + - find the compilers (native and cross) used to build executables, + firmware and tests. The results are written as either Makefile + fragments (``config-host.mak``) or a Meson machine file + (``config-meson.cross``) + + - create a virtual environment in which all Python code runs during + the build, and possibly install packages into it from PyPI + + - invoke Meson in the virtual environment, to perform the actual + configuration step for the emulator build + +The configure script automatically recognizes command line options for +which a same-named Meson option exists; dashes in the command line are +replaced with underscores. + +Almost all QEMU developers that need to modify the build system will +only be concerned with Meson, and therefore can skip the rest of this +section. + + +Modifying ``configure`` +----------------------- + +``configure`` is a shell script; it uses ``#!/bin/sh`` and therefore +should be compatible with any POSIX shell. It is important to avoid +using bash-isms to avoid breaking development platforms where bash is +the primary host. + +The configure script provides a variety of functions to help writing +portable shell code and providing consistent behavior across architectures +and operating systems: + +``error_exit $MESSAGE $MORE...`` + Print $MESSAGE to stderr, followed by $MORE... and then exit from the + configure script with non-zero status. -Patches are also welcome to move existing checks from the configure -phase to ``meson.build``. When doing so, ensure that ``meson.build`` does -not use anymore the keys that you have removed from ``config-host.mak``. -Typically these will be replaced in ``meson.build`` by boolean variables, -``get_option('optname')`` invocations, or ``dep.found()`` expressions. -In general, the remaining checks have little or no interdependencies, -so they can be moved one by one. +``has $COMMAND`` + Determine if $COMMAND exists in the current environment, either as a + shell builtin, or executable binary, returning 0 on success. The + replacement in Meson is ``find_program()``. -Helper functions ----------------- +``probe_target_compiler $TARGET`` + Detect a cross compiler and cross tools for the QEMU target $TARGET (e.g., + ``$CPU-softmmu``, ``$CPU-linux-user``, ``$CPU-bsd-user``). If a working + compiler is present, return success and set variables ``$target_cc``, + ``$target_ar``, etc. to non-empty values. -The configure script provides a variety of helper functions to assist -developers in checking for system features: +``write_target_makefile`` + Write a Makefile fragment to stdout, exposing the result of the most + ``probe_target_compiler`` call as the usual Make variables (``CC``, + ``AR``, ``LD``, etc.). -``do_cc $ARGS...`` - Attempt to run the system C compiler passing it $ARGS... -``do_cxx $ARGS...`` - Attempt to run the system C++ compiler passing it $ARGS... +Configure does not generally perform tests for compiler options beyond +basic checks to detect the host platform and ensure the compiler is +functioning. These are performed using a few more helper functions: ``compile_object $CFLAGS`` Attempt to compile a test program with the system C compiler using $CFLAGS. The test program must have been previously written to a file - called $TMPC. The replacement in Meson is the compiler object ``cc``, - which has methods such as ``cc.compiles()``, - ``cc.check_header()``, ``cc.has_function()``. + called $TMPC. ``compile_prog $CFLAGS $LDFLAGS`` Attempt to compile a test program with the system C compiler using $CFLAGS and link it with the system linker using $LDFLAGS. The test program must have been previously written to a file called $TMPC. - The replacement in Meson is ``cc.find_library()`` and ``cc.links()``. - -``has $COMMAND`` - Determine if $COMMAND exists in the current environment, either as a - shell builtin, or executable binary, returning 0 on success. The - replacement in Meson is ``find_program()``. ``check_define $NAME`` - Determine if the macro $NAME is defined by the system C compiler + Determine if the macro $NAME is defined by the system C compiler. -``check_include $NAME`` - Determine if the include $NAME file is available to the system C - compiler. The replacement in Meson is ``cc.has_header()``. +``do_compiler $CC $ARGS...`` + Attempt to run the C compiler $CC, passing it $ARGS... This function + does not use flags passed via options such as ``--extra-cflags``, and + therefore can be used to check for cross compilers. However, most + such checks are done at ``make`` time instead (see for example the + ``cc-option`` macro in ``pc-bios/option-rom/Makefile``). ``write_c_skeleton`` Write a minimal C program main() function to the temporary file - indicated by $TMPC + indicated by $TMPC. -``error_exit $MESSAGE $MORE...`` - Print $MESSAGE to stderr, followed by $MORE... and then exit from the - configure script with non-zero status -``query_pkg_config $ARGS...`` - Run pkg-config passing it $ARGS. If QEMU is doing a static build, - then --static will be automatically added to $ARGS +Python virtual environments and the QEMU build system +----------------------------------------------------- +TBD Stage 2: Meson ============== -The Meson build system is currently used to describe the build -process for: +The Meson build system describes the build and install process for: 1) executables, which include: - - Tools - ``qemu-img``, ``qemu-nbd``, ``qga`` (guest agent), etc + - Tools - ``qemu-img``, ``qemu-nbd``, ``qemu-ga`` (guest agent), etc - System emulators - ``qemu-system-$ARCH`` @@ -126,7 +144,8 @@ process for: 2) documentation -3) ROMs, which can be either installed as binary blobs or compiled +3) ROMs, whether provided as binary blobs in the QEMU distributions + or cross compiled under the direction of the configure script 4) other data files, such as icons or desktop files @@ -281,8 +300,7 @@ system/userspace emulation target Adding checks ------------- -New checks should be added to Meson. Compiler checks can be as simple as -the following:: +Compiler checks can be as simple as the following:: config_host_data.set('HAVE_BTRFS_H', cc.has_header('linux/btrfs.h')) @@ -311,8 +329,7 @@ dependency will be used:: sdl_image = not_found if not get_option('sdl_image').auto() or have_system sdl_image = dependency('SDL2_image', required: get_option('sdl_image'), - method: 'pkg-config', - static: enable_static) + method: 'pkg-config') endif This avoids warnings on static builds of user-mode emulators, for example. @@ -360,22 +377,30 @@ script, which may point to something other than the first python3 binary on the path. -Stage 3: makefiles -================== +Stage 3: Make +============= -The use of GNU make is required with the QEMU build system. +The next step in building QEMU is to invoke make. GNU Make is required +to build QEMU, and may be installed as ``gmake`` on some hosts. -The output of Meson is a build.ninja file, which is used with the Ninja -build system. QEMU uses a different approach, where Makefile rules are -synthesized from the build.ninja file. The main Makefile includes these -rules and wraps them so that e.g. submodules are built before QEMU. -The resulting build system is largely non-recursive in nature, in -contrast to common practices seen with automake. +The output of Meson is a ``build.ninja`` file, which is used with the +Ninja build tool. However, QEMU's build comprises other components than +just the emulators (namely firmware and the tests in ``tests/tcg``) which +need different cross compilers. The QEMU Makefile wraps both Ninja and +the smaller build systems for firmware and tests; it also takes care of +running ``configure`` again when the script changes. Apart from invoking +these sub-Makefiles, the resulting build is largely non-recursive. -Tests are also ran by the Makefile with the traditional ``make check`` -phony target, while benchmarks are run with ``make bench``. Meson test -suites such as ``unit`` can be ran with ``make check-unit`` too. It is also -possible to run tests defined in meson.build with ``meson test``. +Tests, whether defined in ``meson.build`` or not, are also ran by the +Makefile with the traditional ``make check`` phony target, while benchmarks +are run with ``make bench``. Meson test suites such as ``unit`` can be ran +with ``make check-unit``, and ``make check-tcg`` builds and runs "non-Meson" +tests for all targets. + +If desired, it is also possible to use ``ninja`` and ``meson test``, +respectively to build emulators and run tests defined in meson.build. +The main difference is that ``make`` needs the ``-jN`` flag in order to +enable parallel builds or tests. Useful make targets ------------------- @@ -387,6 +412,7 @@ Useful make targets Print the value of the variable VAR. Useful for debugging the build system. + Important files for the build system ==================================== @@ -400,8 +426,7 @@ number of dynamically created files listed later. ``Makefile`` The main entry point used when invoking make to build all the components of QEMU. The default 'all' target will naturally result in the build of - every component. Makefile takes care of recursively building submodules - directly via a non-recursive set of rules. + every component. ``*/meson.build`` The meson.build file in the root directory is the main entry point for the @@ -410,59 +435,92 @@ number of dynamically created files listed later. other meson.build files spread throughout the QEMU source tree. ``tests/Makefile.include`` - Rules for external test harnesses. These include the TCG tests, - ``qemu-iotests`` and the Avocado-based integration tests. + Rules for external test harnesses. These include the TCG tests + and the Avocado-based integration tests. ``tests/docker/Makefile.include`` - Rules for Docker tests. Like tests/Makefile, this file is included - directly by the top level Makefile, anything defined in this file will - influence the entire build system. + Rules for Docker tests. Like ``tests/Makefile.include``, this file is + included directly by the top level Makefile, anything defined in this + file will influence the entire build system. ``tests/vm/Makefile.include`` - Rules for VM-based tests. Like tests/Makefile, this file is included - directly by the top level Makefile, anything defined in this file will - influence the entire build system. + Rules for VM-based tests. Like ``tests/Makefile.include``, this file is + included directly by the top level Makefile, anything defined in this + file will influence the entire build system. Dynamically created files ------------------------- -The following files are generated dynamically by configure in order to -control the behaviour of the statically defined makefiles. This avoids -the need for QEMU makefiles to go through any pre-processing as seen -with autotools, where Makefile.am generates Makefile.in which generates -Makefile. +The following files are generated at run-time in order to control the +behaviour of the Makefiles. This avoids the need for QEMU makefiles to +go through any pre-processing as seen with autotools, where configure +generates ``Makefile`` from ``Makefile.in``. Built by configure: ``config-host.mak`` When configure has determined the characteristics of the build host it - will write a long list of variables to config-host.mak file. This - provides the various install directories, compiler / linker flags and a + will write them to this file for use in ``Makefile`` and to a smaller + extent ``meson.build``. These include the paths to various tools and a variety of ``CONFIG_*`` variables related to optionally enabled features. - This is imported by the top level Makefile and meson.build in order to - tailor the build output. - config-host.mak is also used as a dependency checking mechanism. If make + ``config-host.mak`` is also used as a dependency checking mechanism. If make sees that the modification timestamp on configure is newer than that on - config-host.mak, then configure will be re-run. + ``config-host.mak``, then configure will be re-run. + + The variables defined here apply to all QEMU + build outputs. + +``config-meson.cross`` + + A Meson "cross file" (or native file) used to communicate the paths to + the toolchain and other configuration options. + +``config.status`` - The variables defined here are those which are applicable to all QEMU - build outputs. Variables which are potentially different for each - emulator target are defined by the next file... + A small shell script that will invoke configure again with the same + environment variables that were set during the first run. It's used to + rerun configure after changes to the source code, but it can also be + inspected manually to check the contents of the environment. +``Makefile.prereqs`` + + A set of Makefile dependencies that order the build and execution of + firmware and tests after the container images and emulators that they + need. + +``pc-bios/*/config.mak``, ``tests/tcg/config-host.mak``, ``tests/tcg/*/config-target.mak`` + + Configuration variables used to build the firmware and TCG tests, + including paths to cross compilation toolchains. + +``pyvenv`` + + A Python virtual environment that is used for all Python code running + during the build. Using a virtual environment ensures that even code + that is run via ``sphinx-build``, ``meson`` etc. uses the same interpreter + and packages. Built by Meson: +``config-host.h`` + Used by C code to determine the properties of the build environment + and the set of enabled features for the entire build. + ``${TARGET-NAME}-config-devices.mak`` - TARGET-NAME is again the name of a system or userspace emulator. The - config-devices.mak file is automatically generated by make using the - scripts/make_device_config.sh program, feeding it the - default-configs/$TARGET-NAME file as input. - -``config-host.h``, ``$TARGET_NAME-config-target.h``, ``$TARGET_NAME-config-devices.h`` - These files are used by source code to determine what features are - enabled. They are generated from the contents of the corresponding - ``*.mak`` files using Meson's ``configure_file()`` function. + TARGET-NAME is the name of a system emulator. The file is + generated by Meson using files under ``configs/devices`` as input. + +``${TARGET-NAME}-config-target.mak`` + TARGET-NAME is the name of a system or usermode emulator. The file is + generated by Meson using files under ``configs/targets`` as input. + +``$TARGET_NAME-config-target.h``, ``$TARGET_NAME-config-devices.h`` + Used by C code to determine the properties and enabled + features for each target. enabled. They are generated from + the contents of the corresponding ``*.mak`` files using Meson's + ``configure_file()`` function; each target can include them using + the ``CONFIG_TARGET`` and ``CONFIG_DEVICES`` macro respectively. ``build.ninja`` The build rules. diff --git a/docs/devel/index-api.rst b/docs/devel/index-api.rst index 60c0d7459d..7108821746 100644 --- a/docs/devel/index-api.rst +++ b/docs/devel/index-api.rst @@ -12,3 +12,4 @@ generated from in-code annotations to function prototypes. memory modules ui + zoned-storage diff --git a/docs/devel/loads-stores.rst b/docs/devel/loads-stores.rst index ad5dfe133e..d2cefc77a2 100644 --- a/docs/devel/loads-stores.rst +++ b/docs/devel/loads-stores.rst @@ -297,31 +297,20 @@ swap: ``translator_ld{sign}{size}_swap(env, ptr, swap)`` Regexes for git grep - ``\<translator_ld[us]\?[bwlq]\(_swap\)\?\>`` -``helper_*_{ld,st}*_mmu`` +``helper_{ld,st}*_mmu`` ~~~~~~~~~~~~~~~~~~~~~~~~~ These functions are intended primarily to be called by the code -generated by the TCG backend. They may also be called by target -CPU helper function code. Like the ``cpu_{ld,st}_mmuidx_ra`` functions -they perform accesses by guest virtual address, with a given ``mmuidx``. +generated by the TCG backend. Like the ``cpu_{ld,st}_mmu`` functions +they perform accesses by guest virtual address, with a given ``MemOpIdx``. -These functions specify an ``opindex`` parameter which encodes -(among other things) the mmu index to use for the access. This parameter -should be created by calling ``make_memop_idx()``. +They differ from ``cpu_{ld,st}_mmu`` in that they take the endianness +of the operation only from the MemOpIdx, and loads extend the return +value to the size of a host general register (``tcg_target_ulong``). -The ``retaddr`` parameter should be the result of GETPC() called directly -from the top level HELPER(foo) function (or 0 if no guest CPU state -unwinding is required). +load: ``helper_ld{sign}{size}_mmu(env, addr, opindex, retaddr)`` -**TODO** The names of these functions are a bit odd for historical -reasons because they were originally expected to be called only from -within generated code. We should rename them to bring them more in -line with the other memory access functions. The explicit endianness -is the only feature they have beyond ``*_mmuidx_ra``. - -load: ``helper_{endian}_ld{sign}{size}_mmu(env, addr, opindex, retaddr)`` - -store: ``helper_{endian}_st{size}_mmu(env, addr, val, opindex, retaddr)`` +store: ``helper_{size}_mmu(env, addr, val, opindex, retaddr)`` ``sign`` - (empty) : for 32 or 64 bit sizes @@ -334,14 +323,9 @@ store: ``helper_{endian}_st{size}_mmu(env, addr, val, opindex, retaddr)`` - ``l`` : 32 bits - ``q`` : 64 bits -``endian`` - - ``le`` : little endian - - ``be`` : big endian - - ``ret`` : target endianness - Regexes for git grep - - ``\<helper_\(le\|be\|ret\)_ld[us]\?[bwlq]_mmu\>`` - - ``\<helper_\(le\|be\|ret\)_st[bwlq]_mmu\>`` + - ``\<helper_ld[us]\?[bwlq]_mmu\>`` + - ``\<helper_st[bwlq]_mmu\>`` ``address_space_*`` ~~~~~~~~~~~~~~~~~~~ diff --git a/docs/devel/tcg-ops.rst b/docs/devel/tcg-ops.rst index f3f451b77f..6a166c5665 100644 --- a/docs/devel/tcg-ops.rst +++ b/docs/devel/tcg-ops.rst @@ -672,19 +672,20 @@ QEMU specific operations | This operation is optional. If the TCG backend does not implement the goto_ptr opcode, emitting this op is equivalent to emitting exit_tb(0). - * - qemu_ld_i32/i64 *t0*, *t1*, *flags*, *memidx* + * - qemu_ld_i32/i64/i128 *t0*, *t1*, *flags*, *memidx* - qemu_st_i32/i64 *t0*, *t1*, *flags*, *memidx* + qemu_st_i32/i64/i128 *t0*, *t1*, *flags*, *memidx* qemu_st8_i32 *t0*, *t1*, *flags*, *memidx* - | Load data at the guest address *t1* into *t0*, or store data in *t0* at guest - address *t1*. The _i32/_i64 size applies to the size of the input/output + address *t1*. The _i32/_i64/_i128 size applies to the size of the input/output register *t0* only. The address *t1* is always sized according to the guest, and the width of the memory operation is controlled by *flags*. | | Both *t0* and *t1* may be split into little-endian ordered pairs of registers - if dealing with 64-bit quantities on a 32-bit host. + if dealing with 64-bit quantities on a 32-bit host, or 128-bit quantities on + a 64-bit host. | | The *memidx* selects the qemu tlb index to use (e.g. user or kernel access). The flags are the MemOp bits, selecting the sign, width, and endianness @@ -693,6 +694,8 @@ QEMU specific operations | For a 32-bit host, qemu_ld/st_i64 is guaranteed to only be used with a 64-bit memory access specified in *flags*. | + | For qemu_ld/st_i128, these are only supported for a 64-bit host. + | | For i386, qemu_st8_i32 is exactly like qemu_st_i32, except the size of the memory operation is known to be 8-bit. This allows the backend to provide a different set of register constraints. diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst index 4071e72710..2cafec4178 100644 --- a/docs/devel/testing.rst +++ b/docs/devel/testing.rst @@ -479,6 +479,12 @@ first to contribute the mapping to the ``libvirt-ci`` project: contains the ``mappings.yml`` update. Then add the prerequisite and run ``make lcitool-refresh``. + * Please also trigger gitlab container generation pipelines on your change + for as many OS distros as practical to make sure that there are no + obvious breakages when adding the new pre-requisite. Please see + `CI <https://www.qemu.org/docs/master/devel/ci.html>`__ documentation + page on how to trigger gitlab CI pipelines on your change. + For enterprise distros that default to old, end-of-life versions of the Python runtime, QEMU uses a separate set of mappings that work with more recent versions. These can be found in ``tests/lcitool/mappings.yml``. @@ -882,9 +888,9 @@ You can run the avocado tests simply by executing: make check-avocado -This involves the automatic creation of Python virtual environment -within the build tree (at ``tests/venv``) which will have all the -right dependencies, and will save tests results also within the +This involves the automatic installation, from PyPI, of all the +necessary avocado-framework dependencies into the QEMU venv within the +build tree (at ``./pyvenv``). Test results are also saved within the build tree (at ``tests/results``). Note: the build environment must be using a Python 3 stack, and have @@ -941,7 +947,7 @@ may be invoked by running: .. code:: - tests/venv/bin/avocado run $OPTION1 $OPTION2 tests/avocado/ + pyvenv/bin/avocado run $OPTION1 $OPTION2 tests/avocado/ Note that if ``make check-avocado`` was not executed before, it is possible to create the Python virtual environment with the dependencies @@ -956,20 +962,20 @@ a test file. To run tests from a single file within the build tree, use: .. code:: - tests/venv/bin/avocado run tests/avocado/$TESTFILE + pyvenv/bin/avocado run tests/avocado/$TESTFILE To run a single test within a test file, use: .. code:: - tests/venv/bin/avocado run tests/avocado/$TESTFILE:$TESTCLASS.$TESTNAME + pyvenv/bin/avocado run tests/avocado/$TESTFILE:$TESTCLASS.$TESTNAME Valid test names are visible in the output from any previous execution of Avocado or ``make check-avocado``, and can also be queried using: .. code:: - tests/venv/bin/avocado list tests/avocado + pyvenv/bin/avocado list tests/avocado Manual Installation ~~~~~~~~~~~~~~~~~~~ diff --git a/docs/devel/zoned-storage.rst b/docs/devel/zoned-storage.rst new file mode 100644 index 0000000000..30296d3c85 --- /dev/null +++ b/docs/devel/zoned-storage.rst @@ -0,0 +1,62 @@ +============= +zoned-storage +============= + +Zoned Block Devices (ZBDs) divide the LBA space into block regions called zones +that are larger than the LBA size. They can only allow sequential writes, which +can reduce write amplification in SSDs, and potentially lead to higher +throughput and increased capacity. More details about ZBDs can be found at: + +https://zonedstorage.io/docs/introduction/zoned-storage + +1. Block layer APIs for zoned storage +------------------------------------- +QEMU block layer supports three zoned storage models: +- BLK_Z_HM: The host-managed zoned model only allows sequential writes access +to zones. It supports ZBD-specific I/O commands that can be used by a host to +manage the zones of a device. +- BLK_Z_HA: The host-aware zoned model allows random write operations in +zones, making it backward compatible with regular block devices. +- BLK_Z_NONE: The non-zoned model has no zones support. It includes both +regular and drive-managed ZBD devices. ZBD-specific I/O commands are not +supported. + +The block device information resides inside BlockDriverState. QEMU uses +BlockLimits struct(BlockDriverState::bl) that is continuously accessed by the +block layer while processing I/O requests. A BlockBackend has a root pointer to +a BlockDriverState graph(for example, raw format on top of file-posix). The +zoned storage information can be propagated from the leaf BlockDriverState all +the way up to the BlockBackend. If the zoned storage model in file-posix is +set to BLK_Z_HM, then block drivers will declare support for zoned host device. + +The block layer APIs support commands needed for zoned storage devices, +including report zones, four zone operations, and zone append. + +2. Emulating zoned storage controllers +-------------------------------------- +When the BlockBackend's BlockLimits model reports a zoned storage device, users +like the virtio-blk emulation or the qemu-io-cmds.c utility can use block layer +APIs for zoned storage emulation or testing. + +For example, to test zone_report on a null_blk device using qemu-io is:: + + $ path/to/qemu-io --image-opts -n driver=host_device,filename=/dev/nullb0 -c "zrp offset nr_zones" + +To expose the host's zoned block device through virtio-blk, the command line +can be (includes the -device parameter):: + + -blockdev node-name=drive0,driver=host_device,filename=/dev/nullb0,cache.direct=on \ + -device virtio-blk-pci,drive=drive0 + +Or only use the -drive parameter:: + + -driver driver=host_device,file=/dev/nullb0,if=virtio,cache.direct=on + +Additionally, QEMU has several ways of supporting zoned storage, including: +(1) Using virtio-scsi: --device scsi-block allows for the passing through of +SCSI ZBC devices, enabling the attachment of ZBC or ZAC HDDs to QEMU. +(2) PCI device pass-through: While NVMe ZNS emulation is available for testing +purposes, it cannot yet pass through a zoned device from the host. To pass on +the NVMe ZNS device to the guest, use VFIO PCI pass the entire NVMe PCI adapter +through to the guest. Likewise, an HDD HBA can be passed on to QEMU all HDDs +attached to the HBA. diff --git a/docs/meson.build b/docs/meson.build index f220800e3e..9040f860ae 100644 --- a/docs/meson.build +++ b/docs/meson.build @@ -1,4 +1,4 @@ -sphinx_build = find_program(get_option('sphinx_build'), +sphinx_build = find_program(fs.parent(python.full_path()) / 'sphinx-build', required: get_option('docs')) # Check if tools are available to build documentation. @@ -10,6 +10,18 @@ if sphinx_build.found() SPHINX_ARGS += [ '-W', '-Dkerneldoc_werror=1' ] endif + sphinx_version = run_command(SPHINX_ARGS + ['--version'], + check: true).stdout().split()[1] + if sphinx_version.version_compare('>=1.7.0') + SPHINX_ARGS += ['-j', 'auto'] + else + nproc = find_program('nproc') + if nproc.found() + jobs = run_command(nproc, check: true).stdout() + SPHINX_ARGS += ['-j', jobs] + endif + endif + # This is a bit awkward but works: create a trivial document and # try to run it with our configuration file (which enforces a # version requirement). This will fail if sphinx-build is too old. diff --git a/docs/sphinx/dbusdomain.py b/docs/sphinx/dbusdomain.py index 2ea95af623..9872fd5bf6 100644 --- a/docs/sphinx/dbusdomain.py +++ b/docs/sphinx/dbusdomain.py @@ -400,6 +400,10 @@ class DBusDomain(Domain): for refname, obj in self.objects.items(): yield (refname, refname, obj.objtype, obj.docname, obj.node_id, 1) + def merge_domaindata(self, docnames, otherdata): + for name, obj in otherdata['objects'].items(): + if obj.docname in docnames: + self.data['objects'][name] = obj def setup(app): app.add_domain(DBusDomain) diff --git a/docs/sphinx/fakedbusdoc.py b/docs/sphinx/fakedbusdoc.py index d2c5079046..2d2e6ef640 100644 --- a/docs/sphinx/fakedbusdoc.py +++ b/docs/sphinx/fakedbusdoc.py @@ -23,3 +23,8 @@ class FakeDBusDocDirective(Directive): def setup(app: Sphinx) -> Dict[str, Any]: """Register a fake dbus-doc directive with Sphinx""" app.add_directive("dbus-doc", FakeDBusDocDirective) + + return dict( + parallel_read_safe = True, + parallel_write_safe = True + ) diff --git a/docs/sphinx/qmp_lexer.py b/docs/sphinx/qmp_lexer.py index f7e4c0e198..a59de8a079 100644 --- a/docs/sphinx/qmp_lexer.py +++ b/docs/sphinx/qmp_lexer.py @@ -41,3 +41,8 @@ def setup(sphinx): sphinx.add_lexer('QMP', QMPExampleLexer) except errors.VersionRequirementError: sphinx.add_lexer('QMP', QMPExampleLexer()) + + return dict( + parallel_read_safe = True, + parallel_write_safe = True + ) diff --git a/docs/system/device-emulation.rst b/docs/system/device-emulation.rst index c1b1934e3d..8d4a1821fa 100644 --- a/docs/system/device-emulation.rst +++ b/docs/system/device-emulation.rst @@ -93,4 +93,5 @@ Emulated Devices devices/virtio-pmem.rst devices/vhost-user-rng.rst devices/canokey.rst + devices/usb-u2f.rst devices/igb.rst diff --git a/docs/system/devices/igb.rst b/docs/system/devices/igb.rst index 70edadd574..0bcdd85747 100644 --- a/docs/system/devices/igb.rst +++ b/docs/system/devices/igb.rst @@ -29,7 +29,7 @@ Using igb ========= Using igb should be nothing different from using another network device. See -:ref:`pcsys_005fnetwork` in general. +:ref:`Network_emulation` in general. However, you may also need to perform additional steps to activate SR-IOV feature on your guest. For Linux, refer to [4]_. diff --git a/docs/system/devices/ivshmem.rst b/docs/system/devices/ivshmem.rst index b03a48afa3..e7aaf34c20 100644 --- a/docs/system/devices/ivshmem.rst +++ b/docs/system/devices/ivshmem.rst @@ -1,5 +1,3 @@ -.. _pcsys_005fivshmem: - Inter-VM Shared Memory device ----------------------------- diff --git a/docs/system/devices/net.rst b/docs/system/devices/net.rst index 4b2640c448..2ab516d4b0 100644 --- a/docs/system/devices/net.rst +++ b/docs/system/devices/net.rst @@ -1,4 +1,4 @@ -.. _pcsys_005fnetwork: +.. _Network_Emulation: Network emulation ----------------- diff --git a/docs/system/devices/usb-u2f.rst b/docs/system/devices/usb-u2f.rst new file mode 100644 index 0000000000..4f57d5c8c3 --- /dev/null +++ b/docs/system/devices/usb-u2f.rst @@ -0,0 +1,93 @@ +Universal Second Factor (U2F) USB Key Device +============================================ + +U2F is an open authentication standard that enables relying parties +exposed to the internet to offer a strong second factor option for end +user authentication. + +The second factor is provided by a device implementing the U2F +protocol. In case of a USB U2F security key, it is a USB HID device +that implements the U2F protocol. + +QEMU supports both pass-through of a host U2F key device to a VM, +and software emulation of a U2F key. + +``u2f-passthru`` +---------------- + +The ``u2f-passthru`` device allows you to connect a real hardware +U2F key on your host to a guest VM. All requests made from the guest +are passed through to the physical security key connected to the +host machine and vice versa. + +In addition, the dedicated pass-through allows you to share a single +U2F security key with several guest VMs, which is not possible with a +simple host device assignment pass-through. + +You can specify the host U2F key to use with the ``hidraw`` +option, which takes the host path to a Linux ``/dev/hidrawN`` device: + +.. parsed-literal:: + |qemu_system| -usb -device u2f-passthru,hidraw=/dev/hidraw0 + +If you don't specify the device, the ``u2f-passthru`` device will +autoscan to take the first U2F device it finds on the host (this +requires a working libudev): + +.. parsed-literal:: + |qemu_system| -usb -device u2f-passthru + +``u2f-emulated`` +---------------- + +``u2f-emulated`` is a completely software emulated U2F device. +It uses `libu2f-emu <https://github.com/MattGorko/libu2f-emu>`__ +for the U2F key emulation. libu2f-emu +provides a complete implementation of the U2F protocol device part for +all specified transports given by the FIDO Alliance. + +To work, an emulated U2F device must have four elements: + + * ec x509 certificate + * ec private key + * counter (four bytes value) + * 48 bytes of entropy (random bits) + +To use this type of device, these have to be configured, and these +four elements must be passed one way or another. + +Assuming that you have a working libu2f-emu installed on the host, +there are three possible ways to configure the ``u2f-emulated`` device: + + * ephemeral + * setup directory + * manual + +Ephemeral is the simplest way to configure; it lets the device generate +all the elements it needs for a single use of the lifetime of the device. +It is the default if you do not pass any other options to the device. + +.. parsed-literal:: + |qemu_system| -usb -device u2f-emulated + +You can pass the device the path of a setup directory on the host +using the ``dir`` option; the directory must contain these four files: + + * ``certificate.pem``: ec x509 certificate + * ``private-key.pem``: ec private key + * ``counter``: counter value + * ``entropy``: 48 bytes of entropy + +.. parsed-literal:: + |qemu_system| -usb -device u2f-emulated,dir=$dir + +You can also manually pass the device the paths to each of these files, +if you don't want them all to be in the same directory, using the options + + * ``cert`` + * ``priv`` + * ``counter`` + * ``entropy`` + +.. parsed-literal:: + |qemu_system| -usb -device u2f-emulated,cert=$DIR1/$FILE1,priv=$DIR2/$FILE2,counter=$DIR3/$FILE3,entropy=$DIR4/$FILE4 diff --git a/docs/system/devices/usb.rst b/docs/system/devices/usb.rst index 37cb9b33ae..a6ca7b0c37 100644 --- a/docs/system/devices/usb.rst +++ b/docs/system/devices/usb.rst @@ -1,5 +1,3 @@ -.. _pcsys_005fusb: - USB emulation ------------- @@ -209,7 +207,7 @@ option or the ``device_add`` monitor command. Available devices are: USB audio device ``u2f-{emulated,passthru}`` - Universal Second Factor device + :doc:`usb-u2f` ``canokey`` An Open-source Secure Key implementing FIDO2, OpenPGP, PIV and more. diff --git a/docs/system/keys.rst b/docs/system/keys.rst index e596ae6c4e..0fc17b994d 100644 --- a/docs/system/keys.rst +++ b/docs/system/keys.rst @@ -1,4 +1,4 @@ -.. _pcsys_005fkeys: +.. _GUI_keys: Keys in the graphical frontends ------------------------------- diff --git a/docs/system/linuxboot.rst b/docs/system/linuxboot.rst index 228650abc5..5db2e560dc 100644 --- a/docs/system/linuxboot.rst +++ b/docs/system/linuxboot.rst @@ -27,4 +27,4 @@ virtual serial port and the QEMU monitor to the console with the -append "root=/dev/hda console=ttyS0" -nographic Use Ctrl-a c to switch between the serial console and the monitor (see -:ref:`pcsys_005fkeys`). +:ref:`GUI_keys`). diff --git a/docs/system/qemu-block-drivers.rst.inc b/docs/system/qemu-block-drivers.rst.inc index dfe5d2293d..105cb9679c 100644 --- a/docs/system/qemu-block-drivers.rst.inc +++ b/docs/system/qemu-block-drivers.rst.inc @@ -430,6 +430,12 @@ Hard disks you may corrupt your host data (use the ``-snapshot`` command line option or modify the device permissions accordingly). +Zoned block devices + Zoned block devices can be passed through to the guest if the emulated storage + controller supports zoned storage. Use ``--blockdev host_device, + node-name=drive0,filename=/dev/nullb0,cache.direct=on`` to pass through + ``/dev/nullb0`` as ``drive0``. + Windows ^^^^^^^ diff --git a/docs/system/target-i386.rst b/docs/system/target-i386.rst index 77c2f3b979..1b8a1f248a 100644 --- a/docs/system/target-i386.rst +++ b/docs/system/target-i386.rst @@ -3,8 +3,6 @@ x86 System emulator ------------------- -.. _pcsys_005fdevices: - Board-specific documentation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -32,8 +30,6 @@ Architectural features i386/sgx i386/amd-memory-encryption -.. _pcsys_005freq: - OS requirements ~~~~~~~~~~~~~~~ diff --git a/docs/u2f.txt b/docs/u2f.txt deleted file mode 100644 index 7f5813a0b7..0000000000 --- a/docs/u2f.txt +++ /dev/null @@ -1,110 +0,0 @@ -QEMU U2F Key Device Documentation. - -Contents -1. USB U2F key device -2. Building -3. Using u2f-emulated -4. Using u2f-passthru -5. Libu2f-emu - -1. USB U2F key device - -U2F is an open authentication standard that enables relying parties -exposed to the internet to offer a strong second factor option for end -user authentication. - -The standard brings many advantages to both parties, client and server, -allowing to reduce over-reliance on passwords, it increases authentication -security and simplifies passwords. - -The second factor is materialized by a device implementing the U2F -protocol. In case of a USB U2F security key, it is a USB HID device -that implements the U2F protocol. - -In QEMU, the USB U2F key device offers a dedicated support of U2F, allowing -guest USB FIDO/U2F security keys operating in two possible modes: -pass-through and emulated. - -The pass-through mode consists of passing all requests made from the guest -to the physical security key connected to the host machine and vice versa. -In addition, the dedicated pass-through allows to have a U2F security key -shared on several guests which is not possible with a simple host device -assignment pass-through. - -The emulated mode consists of completely emulating the behavior of an -U2F device through software part. Libu2f-emu is used for that. - - -2. Building - -To ensure the build of the u2f-emulated device variant which depends -on libu2f-emu: configuring and building: - - ./configure --enable-u2f && make - -The pass-through mode is built by default on Linux. To take advantage -of the autoscan option it provides, make sure you have a working libudev -installed on the host. - - -3. Using u2f-emulated - -To work, an emulated U2F device must have four elements: - * ec x509 certificate - * ec private key - * counter (four bytes value) - * 48 bytes of entropy (random bits) - -To use this type of device, this one has to be configured, and these -four elements must be passed one way or another. - -Assuming that you have a working libu2f-emu installed on the host. -There are three possible ways of configurations: - * ephemeral - * setup directory - * manual - -Ephemeral is the simplest way to configure, it lets the device generate -all the elements it needs for a single use of the lifetime of the device. - - qemu -usb -device u2f-emulated - -Setup directory allows to configure the device from a directory containing -four files: - * certificate.pem: ec x509 certificate - * private-key.pem: ec private key - * counter: counter value - * entropy: 48 bytes of entropy - - qemu -usb -device u2f-emulated,dir=$dir - -Manual allows to configure the device more finely by specifying each -of the elements necessary for the device: - * cert - * priv - * counter - * entropy - - qemu -usb -device u2f-emulated,cert=$DIR1/$FILE1,priv=$DIR2/$FILE2,counter=$DIR3/$FILE3,entropy=$DIR4/$FILE4 - - -4. Using u2f-passthru - -On the host specify the u2f-passthru device with a suitable hidraw: - - qemu -usb -device u2f-passthru,hidraw=/dev/hidraw0 - -Alternately, the u2f-passthru device can autoscan to take the first -U2F device it finds on the host (this requires a working libudev): - - qemu -usb -device u2f-passthru - - -5. Libu2f-emu - -The u2f-emulated device uses libu2f-emu for the U2F key emulation. Libu2f-emu -implements completely the U2F protocol device part for all specified -transport given by the FIDO Alliance. - -For more information about libu2f-emu see this page: -https://github.com/MattGorko/libu2f-emu. diff --git a/hw/9pfs/trace-events b/hw/9pfs/trace-events index 6c77966c0b..a12e55c165 100644 --- a/hw/9pfs/trace-events +++ b/hw/9pfs/trace-events @@ -48,3 +48,9 @@ v9fs_readlink(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d" v9fs_readlink_return(uint16_t tag, uint8_t id, char* target) "tag %d id %d name %s" v9fs_setattr(uint16_t tag, uint8_t id, int32_t fid, int32_t valid, int32_t mode, int32_t uid, int32_t gid, int64_t size, int64_t atime_sec, int64_t mtime_sec) "tag %u id %u fid %d iattr={valid %d mode %d uid %d gid %d size %"PRId64" atime=%"PRId64" mtime=%"PRId64" }" v9fs_setattr_return(uint16_t tag, uint8_t id) "tag %u id %u" + +# xen-9p-backend.c +xen_9pfs_alloc(char *name) "name %s" +xen_9pfs_connect(char *name) "name %s" +xen_9pfs_disconnect(char *name) "name %s" +xen_9pfs_free(char *name) "name %s" diff --git a/hw/9pfs/xen-9p-backend.c b/hw/9pfs/xen-9p-backend.c index 0e266c552b..4aa9c8c736 100644 --- a/hw/9pfs/xen-9p-backend.c +++ b/hw/9pfs/xen-9p-backend.c @@ -25,6 +25,8 @@ #include "qemu/iov.h" #include "fsdev/qemu-fsdev.h" +#include "trace.h" + #define VERSIONS "1" #define MAX_RINGS 8 #define MAX_RING_ORDER 9 @@ -337,6 +339,8 @@ static void xen_9pfs_disconnect(struct XenLegacyDevice *xendev) Xen9pfsDev *xen_9pdev = container_of(xendev, Xen9pfsDev, xendev); int i; + trace_xen_9pfs_disconnect(xendev->name); + for (i = 0; i < xen_9pdev->num_rings; i++) { if (xen_9pdev->rings[i].evtchndev != NULL) { qemu_set_fd_handler(qemu_xen_evtchn_fd(xen_9pdev->rings[i].evtchndev), @@ -345,40 +349,41 @@ static void xen_9pfs_disconnect(struct XenLegacyDevice *xendev) xen_9pdev->rings[i].local_port); xen_9pdev->rings[i].evtchndev = NULL; } - } -} - -static int xen_9pfs_free(struct XenLegacyDevice *xendev) -{ - Xen9pfsDev *xen_9pdev = container_of(xendev, Xen9pfsDev, xendev); - int i; - - if (xen_9pdev->rings[0].evtchndev != NULL) { - xen_9pfs_disconnect(xendev); - } - - for (i = 0; i < xen_9pdev->num_rings; i++) { if (xen_9pdev->rings[i].data != NULL) { xen_be_unmap_grant_refs(&xen_9pdev->xendev, xen_9pdev->rings[i].data, xen_9pdev->rings[i].intf->ref, (1 << xen_9pdev->rings[i].ring_order)); + xen_9pdev->rings[i].data = NULL; } if (xen_9pdev->rings[i].intf != NULL) { xen_be_unmap_grant_ref(&xen_9pdev->xendev, xen_9pdev->rings[i].intf, xen_9pdev->rings[i].ref); + xen_9pdev->rings[i].intf = NULL; } if (xen_9pdev->rings[i].bh != NULL) { qemu_bh_delete(xen_9pdev->rings[i].bh); + xen_9pdev->rings[i].bh = NULL; } } g_free(xen_9pdev->id); + xen_9pdev->id = NULL; g_free(xen_9pdev->tag); + xen_9pdev->tag = NULL; g_free(xen_9pdev->path); + xen_9pdev->path = NULL; g_free(xen_9pdev->security_model); + xen_9pdev->security_model = NULL; g_free(xen_9pdev->rings); + xen_9pdev->rings = NULL; +} + +static int xen_9pfs_free(struct XenLegacyDevice *xendev) +{ + trace_xen_9pfs_free(xendev->name); + return 0; } @@ -390,6 +395,8 @@ static int xen_9pfs_connect(struct XenLegacyDevice *xendev) V9fsState *s = &xen_9pdev->state; QemuOpts *fsdev; + trace_xen_9pfs_connect(xendev->name); + if (xenstore_read_fe_int(&xen_9pdev->xendev, "num-rings", &xen_9pdev->num_rings) == -1 || xen_9pdev->num_rings > MAX_RINGS || xen_9pdev->num_rings < 1) { @@ -499,6 +506,8 @@ out: static void xen_9pfs_alloc(struct XenLegacyDevice *xendev) { + trace_xen_9pfs_alloc(xendev->name); + xenstore_write_be_str(xendev, "versions", VERSIONS); xenstore_write_be_int(xendev, "max-rings", MAX_RINGS); xenstore_write_be_int(xendev, "max-ring-page-order", MAX_RING_ORDER); diff --git a/hw/arm/sbsa-ref.c b/hw/arm/sbsa-ref.c index 0b93558dde..792371fdce 100644 --- a/hw/arm/sbsa-ref.c +++ b/hw/arm/sbsa-ref.c @@ -648,7 +648,7 @@ static void create_pcie(SBSAMachineState *sms) } } - pci_create_simple(pci->bus, -1, "VGA"); + pci_create_simple(pci->bus, -1, "bochs-display"); create_smmu(sms, pci->bus); } @@ -852,7 +852,7 @@ static void sbsa_ref_class_init(ObjectClass *oc, void *data) mc->init = sbsa_ref_init; mc->desc = "QEMU 'SBSA Reference' ARM Virtual Machine"; - mc->default_cpu_type = ARM_CPU_TYPE_NAME("cortex-a57"); + mc->default_cpu_type = ARM_CPU_TYPE_NAME("neoverse-n1"); mc->max_cpus = 512; mc->pci_allow_0_address = true; mc->minimum_page_bits = 12; diff --git a/hw/arm/vexpress.c b/hw/arm/vexpress.c index 34b012b528..56abadd9b8 100644 --- a/hw/arm/vexpress.c +++ b/hw/arm/vexpress.c @@ -173,6 +173,11 @@ struct VexpressMachineClass { struct VexpressMachineState { MachineState parent; + MemoryRegion vram; + MemoryRegion sram; + MemoryRegion flashalias; + MemoryRegion lowram; + MemoryRegion a15sram; bool secure; bool virt; }; @@ -182,7 +187,7 @@ struct VexpressMachineState { #define TYPE_VEXPRESS_A15_MACHINE MACHINE_TYPE_NAME("vexpress-a15") OBJECT_DECLARE_TYPE(VexpressMachineState, VexpressMachineClass, VEXPRESS_MACHINE) -typedef void DBoardInitFn(const VexpressMachineState *machine, +typedef void DBoardInitFn(VexpressMachineState *machine, ram_addr_t ram_size, const char *cpu_type, qemu_irq *pic); @@ -263,14 +268,13 @@ static void init_cpus(MachineState *ms, const char *cpu_type, } } -static void a9_daughterboard_init(const VexpressMachineState *vms, +static void a9_daughterboard_init(VexpressMachineState *vms, ram_addr_t ram_size, const char *cpu_type, qemu_irq *pic) { MachineState *machine = MACHINE(vms); MemoryRegion *sysmem = get_system_memory(); - MemoryRegion *lowram = g_new(MemoryRegion, 1); ram_addr_t low_ram_size; if (ram_size > 0x40000000) { @@ -287,9 +291,9 @@ static void a9_daughterboard_init(const VexpressMachineState *vms, * address space should in theory be remappable to various * things including ROM or RAM; we always map the RAM there. */ - memory_region_init_alias(lowram, NULL, "vexpress.lowmem", machine->ram, - 0, low_ram_size); - memory_region_add_subregion(sysmem, 0x0, lowram); + memory_region_init_alias(&vms->lowram, NULL, "vexpress.lowmem", + machine->ram, 0, low_ram_size); + memory_region_add_subregion(sysmem, 0x0, &vms->lowram); memory_region_add_subregion(sysmem, 0x60000000, machine->ram); /* 0x1e000000 A9MPCore (SCU) private memory region */ @@ -348,14 +352,13 @@ static VEDBoardInfo a9_daughterboard = { .init = a9_daughterboard_init, }; -static void a15_daughterboard_init(const VexpressMachineState *vms, +static void a15_daughterboard_init(VexpressMachineState *vms, ram_addr_t ram_size, const char *cpu_type, qemu_irq *pic) { MachineState *machine = MACHINE(vms); MemoryRegion *sysmem = get_system_memory(); - MemoryRegion *sram = g_new(MemoryRegion, 1); { /* We have to use a separate 64 bit variable here to avoid the gcc @@ -386,9 +389,9 @@ static void a15_daughterboard_init(const VexpressMachineState *vms, /* 0x2b060000: SP805 watchdog: not modelled */ /* 0x2b0a0000: PL341 dynamic memory controller: not modelled */ /* 0x2e000000: system SRAM */ - memory_region_init_ram(sram, NULL, "vexpress.a15sram", 0x10000, + memory_region_init_ram(&vms->a15sram, NULL, "vexpress.a15sram", 0x10000, &error_fatal); - memory_region_add_subregion(sysmem, 0x2e000000, sram); + memory_region_add_subregion(sysmem, 0x2e000000, &vms->a15sram); /* 0x7ffb0000: DMA330 DMA controller: not modelled */ /* 0x7ffd0000: PL354 static memory controller: not modelled */ @@ -547,10 +550,6 @@ static void vexpress_common_init(MachineState *machine) I2CBus *i2c; ram_addr_t vram_size, sram_size; MemoryRegion *sysmem = get_system_memory(); - MemoryRegion *vram = g_new(MemoryRegion, 1); - MemoryRegion *sram = g_new(MemoryRegion, 1); - MemoryRegion *flashalias = g_new(MemoryRegion, 1); - MemoryRegion *flash0mem; const hwaddr *map = daughterboard->motherboard_map; int i; @@ -662,24 +661,25 @@ static void vexpress_common_init(MachineState *machine) if (map[VE_NORFLASHALIAS] != -1) { /* Map flash 0 as an alias into low memory */ + MemoryRegion *flash0mem; flash0mem = sysbus_mmio_get_region(SYS_BUS_DEVICE(pflash0), 0); - memory_region_init_alias(flashalias, NULL, "vexpress.flashalias", + memory_region_init_alias(&vms->flashalias, NULL, "vexpress.flashalias", flash0mem, 0, VEXPRESS_FLASH_SIZE); - memory_region_add_subregion(sysmem, map[VE_NORFLASHALIAS], flashalias); + memory_region_add_subregion(sysmem, map[VE_NORFLASHALIAS], &vms->flashalias); } dinfo = drive_get(IF_PFLASH, 0, 1); ve_pflash_cfi01_register(map[VE_NORFLASH1], "vexpress.flash1", dinfo); sram_size = 0x2000000; - memory_region_init_ram(sram, NULL, "vexpress.sram", sram_size, + memory_region_init_ram(&vms->sram, NULL, "vexpress.sram", sram_size, &error_fatal); - memory_region_add_subregion(sysmem, map[VE_SRAM], sram); + memory_region_add_subregion(sysmem, map[VE_SRAM], &vms->sram); vram_size = 0x800000; - memory_region_init_ram(vram, NULL, "vexpress.vram", vram_size, + memory_region_init_ram(&vms->vram, NULL, "vexpress.vram", vram_size, &error_fatal); - memory_region_add_subregion(sysmem, map[VE_VIDEORAM], vram); + memory_region_add_subregion(sysmem, map[VE_VIDEORAM], &vms->vram); /* 0x4e000000 LAN9118 Ethernet */ if (nd_table[0].used) { diff --git a/hw/arm/virt.c b/hw/arm/virt.c index b99ae18501..06b514b25c 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -2146,7 +2146,7 @@ static void machvirt_init(MachineState *machine) exit(1); } - if (vms->mte && (kvm_enabled() || hvf_enabled())) { + if (vms->mte && hvf_enabled()) { error_report("mach-virt: %s does not support providing " "MTE to the guest CPU", current_accel_name()); @@ -2216,39 +2216,48 @@ static void machvirt_init(MachineState *machine) } if (vms->mte) { - /* Create the memory region only once, but link to all cpus. */ - if (!tag_sysmem) { - /* - * The property exists only if MemTag is supported. - * If it is, we must allocate the ram to back that up. - */ - if (!object_property_find(cpuobj, "tag-memory")) { - error_report("MTE requested, but not supported " - "by the guest CPU"); - exit(1); + if (tcg_enabled()) { + /* Create the memory region only once, but link to all cpus. */ + if (!tag_sysmem) { + /* + * The property exists only if MemTag is supported. + * If it is, we must allocate the ram to back that up. + */ + if (!object_property_find(cpuobj, "tag-memory")) { + error_report("MTE requested, but not supported " + "by the guest CPU"); + exit(1); + } + + tag_sysmem = g_new(MemoryRegion, 1); + memory_region_init(tag_sysmem, OBJECT(machine), + "tag-memory", UINT64_MAX / 32); + + if (vms->secure) { + secure_tag_sysmem = g_new(MemoryRegion, 1); + memory_region_init(secure_tag_sysmem, OBJECT(machine), + "secure-tag-memory", + UINT64_MAX / 32); + + /* As with ram, secure-tag takes precedence over tag. */ + memory_region_add_subregion_overlap(secure_tag_sysmem, + 0, tag_sysmem, -1); + } } - tag_sysmem = g_new(MemoryRegion, 1); - memory_region_init(tag_sysmem, OBJECT(machine), - "tag-memory", UINT64_MAX / 32); - + object_property_set_link(cpuobj, "tag-memory", + OBJECT(tag_sysmem), &error_abort); if (vms->secure) { - secure_tag_sysmem = g_new(MemoryRegion, 1); - memory_region_init(secure_tag_sysmem, OBJECT(machine), - "secure-tag-memory", UINT64_MAX / 32); - - /* As with ram, secure-tag takes precedence over tag. */ - memory_region_add_subregion_overlap(secure_tag_sysmem, 0, - tag_sysmem, -1); + object_property_set_link(cpuobj, "secure-tag-memory", + OBJECT(secure_tag_sysmem), + &error_abort); } - } - - object_property_set_link(cpuobj, "tag-memory", OBJECT(tag_sysmem), - &error_abort); - if (vms->secure) { - object_property_set_link(cpuobj, "secure-tag-memory", - OBJECT(secure_tag_sysmem), - &error_abort); + } else if (kvm_enabled()) { + if (!kvm_arm_mte_supported()) { + error_report("MTE requested, but not supported by KVM"); + exit(1); + } + kvm_arm_enable_mte(cpuobj, &error_abort); } } diff --git a/hw/block/trace-events b/hw/block/trace-events index 2c45a62bd5..34be8b9135 100644 --- a/hw/block/trace-events +++ b/hw/block/trace-events @@ -44,9 +44,16 @@ pflash_write_unknown(const char *name, uint8_t cmd) "%s: unknown command 0x%02x" # virtio-blk.c virtio_blk_req_complete(void *vdev, void *req, int status) "vdev %p req %p status %d" virtio_blk_rw_complete(void *vdev, void *req, int ret) "vdev %p req %p ret %d" +virtio_blk_zone_report_complete(void *vdev, void *req, unsigned int nr_zones, int ret) "vdev %p req %p nr_zones %u ret %d" +virtio_blk_zone_mgmt_complete(void *vdev, void *req, int ret) "vdev %p req %p ret %d" +virtio_blk_zone_append_complete(void *vdev, void *req, int64_t sector, int ret) "vdev %p req %p, append sector 0x%" PRIx64 " ret %d" virtio_blk_handle_write(void *vdev, void *req, uint64_t sector, size_t nsectors) "vdev %p req %p sector %"PRIu64" nsectors %zu" virtio_blk_handle_read(void *vdev, void *req, uint64_t sector, size_t nsectors) "vdev %p req %p sector %"PRIu64" nsectors %zu" virtio_blk_submit_multireq(void *vdev, void *mrb, int start, int num_reqs, uint64_t offset, size_t size, bool is_write) "vdev %p mrb %p start %d num_reqs %d offset %"PRIu64" size %zu is_write %d" +virtio_blk_handle_zone_report(void *vdev, void *req, int64_t sector, unsigned int nr_zones) "vdev %p req %p sector 0x%" PRIx64 " nr_zones %u" +virtio_blk_handle_zone_mgmt(void *vdev, void *req, uint8_t op, int64_t sector, int64_t len) "vdev %p req %p op 0x%x sector 0x%" PRIx64 " len 0x%" PRIx64 "" +virtio_blk_handle_zone_reset_all(void *vdev, void *req, int64_t sector, int64_t len) "vdev %p req %p sector 0x%" PRIx64 " cap 0x%" PRIx64 "" +virtio_blk_handle_zone_append(void *vdev, void *req, int64_t sector) "vdev %p req %p, append sector 0x%" PRIx64 "" # hd-geometry.c hd_geometry_lchs_guess(void *blk, int cyls, int heads, int secs) "blk %p LCHS %d %d %d" diff --git a/hw/block/virtio-blk-common.c b/hw/block/virtio-blk-common.c index ac52d7c176..e2f8e2f6da 100644 --- a/hw/block/virtio-blk-common.c +++ b/hw/block/virtio-blk-common.c @@ -29,6 +29,8 @@ static const VirtIOFeature feature_sizes[] = { .end = endof(struct virtio_blk_config, discard_sector_alignment)}, {.flags = 1ULL << VIRTIO_BLK_F_WRITE_ZEROES, .end = endof(struct virtio_blk_config, write_zeroes_may_unmap)}, + {.flags = 1ULL << VIRTIO_BLK_F_ZONED, + .end = endof(struct virtio_blk_config, zoned)}, {} }; diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index cefca93b31..8f65ea4659 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -17,6 +17,7 @@ #include "qemu/module.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" +#include "block/block_int.h" #include "trace.h" #include "hw/block/block.h" #include "hw/qdev-properties.h" @@ -601,6 +602,351 @@ err: return err_status; } +typedef struct ZoneCmdData { + VirtIOBlockReq *req; + struct iovec *in_iov; + unsigned in_num; + union { + struct { + unsigned int nr_zones; + BlockZoneDescriptor *zones; + } zone_report_data; + struct { + int64_t offset; + } zone_append_data; + }; +} ZoneCmdData; + +/* + * check zoned_request: error checking before issuing requests. If all checks + * passed, return true. + * append: true if only zone append requests issued. + */ +static bool check_zoned_request(VirtIOBlock *s, int64_t offset, int64_t len, + bool append, uint8_t *status) { + BlockDriverState *bs = blk_bs(s->blk); + int index; + + if (!virtio_has_feature(s->host_features, VIRTIO_BLK_F_ZONED)) { + *status = VIRTIO_BLK_S_UNSUPP; + return false; + } + + if (offset < 0 || len < 0 || len > (bs->total_sectors << BDRV_SECTOR_BITS) + || offset > (bs->total_sectors << BDRV_SECTOR_BITS) - len) { + *status = VIRTIO_BLK_S_ZONE_INVALID_CMD; + return false; + } + + if (append) { + if (bs->bl.write_granularity) { + if ((offset % bs->bl.write_granularity) != 0) { + *status = VIRTIO_BLK_S_ZONE_UNALIGNED_WP; + return false; + } + } + + index = offset / bs->bl.zone_size; + if (BDRV_ZT_IS_CONV(bs->wps->wp[index])) { + *status = VIRTIO_BLK_S_ZONE_INVALID_CMD; + return false; + } + + if (len / 512 > bs->bl.max_append_sectors) { + if (bs->bl.max_append_sectors == 0) { + *status = VIRTIO_BLK_S_UNSUPP; + } else { + *status = VIRTIO_BLK_S_ZONE_INVALID_CMD; + } + return false; + } + } + return true; +} + +static void virtio_blk_zone_report_complete(void *opaque, int ret) +{ + ZoneCmdData *data = opaque; + VirtIOBlockReq *req = data->req; + VirtIOBlock *s = req->dev; + VirtIODevice *vdev = VIRTIO_DEVICE(req->dev); + struct iovec *in_iov = data->in_iov; + unsigned in_num = data->in_num; + int64_t zrp_size, n, j = 0; + int64_t nz = data->zone_report_data.nr_zones; + int8_t err_status = VIRTIO_BLK_S_OK; + + trace_virtio_blk_zone_report_complete(vdev, req, nz, ret); + if (ret) { + err_status = VIRTIO_BLK_S_ZONE_INVALID_CMD; + goto out; + } + + struct virtio_blk_zone_report zrp_hdr = (struct virtio_blk_zone_report) { + .nr_zones = cpu_to_le64(nz), + }; + zrp_size = sizeof(struct virtio_blk_zone_report) + + sizeof(struct virtio_blk_zone_descriptor) * nz; + n = iov_from_buf(in_iov, in_num, 0, &zrp_hdr, sizeof(zrp_hdr)); + if (n != sizeof(zrp_hdr)) { + virtio_error(vdev, "Driver provided input buffer that is too small!"); + err_status = VIRTIO_BLK_S_ZONE_INVALID_CMD; + goto out; + } + + for (size_t i = sizeof(zrp_hdr); i < zrp_size; + i += sizeof(struct virtio_blk_zone_descriptor), ++j) { + struct virtio_blk_zone_descriptor desc = + (struct virtio_blk_zone_descriptor) { + .z_start = cpu_to_le64(data->zone_report_data.zones[j].start + >> BDRV_SECTOR_BITS), + .z_cap = cpu_to_le64(data->zone_report_data.zones[j].cap + >> BDRV_SECTOR_BITS), + .z_wp = cpu_to_le64(data->zone_report_data.zones[j].wp + >> BDRV_SECTOR_BITS), + }; + + switch (data->zone_report_data.zones[j].type) { + case BLK_ZT_CONV: + desc.z_type = VIRTIO_BLK_ZT_CONV; + break; + case BLK_ZT_SWR: + desc.z_type = VIRTIO_BLK_ZT_SWR; + break; + case BLK_ZT_SWP: + desc.z_type = VIRTIO_BLK_ZT_SWP; + break; + default: + g_assert_not_reached(); + } + + switch (data->zone_report_data.zones[j].state) { + case BLK_ZS_RDONLY: + desc.z_state = VIRTIO_BLK_ZS_RDONLY; + break; + case BLK_ZS_OFFLINE: + desc.z_state = VIRTIO_BLK_ZS_OFFLINE; + break; + case BLK_ZS_EMPTY: + desc.z_state = VIRTIO_BLK_ZS_EMPTY; + break; + case BLK_ZS_CLOSED: + desc.z_state = VIRTIO_BLK_ZS_CLOSED; + break; + case BLK_ZS_FULL: + desc.z_state = VIRTIO_BLK_ZS_FULL; + break; + case BLK_ZS_EOPEN: + desc.z_state = VIRTIO_BLK_ZS_EOPEN; + break; + case BLK_ZS_IOPEN: + desc.z_state = VIRTIO_BLK_ZS_IOPEN; + break; + case BLK_ZS_NOT_WP: + desc.z_state = VIRTIO_BLK_ZS_NOT_WP; + break; + default: + g_assert_not_reached(); + } + + /* TODO: it takes O(n^2) time complexity. Optimizations required. */ + n = iov_from_buf(in_iov, in_num, i, &desc, sizeof(desc)); + if (n != sizeof(desc)) { + virtio_error(vdev, "Driver provided input buffer " + "for descriptors that is too small!"); + err_status = VIRTIO_BLK_S_ZONE_INVALID_CMD; + } + } + +out: + aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); + virtio_blk_req_complete(req, err_status); + virtio_blk_free_request(req); + aio_context_release(blk_get_aio_context(s->conf.conf.blk)); + g_free(data->zone_report_data.zones); + g_free(data); +} + +static void virtio_blk_handle_zone_report(VirtIOBlockReq *req, + struct iovec *in_iov, + unsigned in_num) +{ + VirtIOBlock *s = req->dev; + VirtIODevice *vdev = VIRTIO_DEVICE(s); + unsigned int nr_zones; + ZoneCmdData *data; + int64_t zone_size, offset; + uint8_t err_status; + + if (req->in_len < sizeof(struct virtio_blk_inhdr) + + sizeof(struct virtio_blk_zone_report) + + sizeof(struct virtio_blk_zone_descriptor)) { + virtio_error(vdev, "in buffer too small for zone report"); + return; + } + + /* start byte offset of the zone report */ + offset = virtio_ldq_p(vdev, &req->out.sector) << BDRV_SECTOR_BITS; + if (!check_zoned_request(s, offset, 0, false, &err_status)) { + goto out; + } + nr_zones = (req->in_len - sizeof(struct virtio_blk_inhdr) - + sizeof(struct virtio_blk_zone_report)) / + sizeof(struct virtio_blk_zone_descriptor); + trace_virtio_blk_handle_zone_report(vdev, req, + offset >> BDRV_SECTOR_BITS, nr_zones); + + zone_size = sizeof(BlockZoneDescriptor) * nr_zones; + data = g_malloc(sizeof(ZoneCmdData)); + data->req = req; + data->in_iov = in_iov; + data->in_num = in_num; + data->zone_report_data.nr_zones = nr_zones; + data->zone_report_data.zones = g_malloc(zone_size), + + blk_aio_zone_report(s->blk, offset, &data->zone_report_data.nr_zones, + data->zone_report_data.zones, + virtio_blk_zone_report_complete, data); + return; +out: + virtio_blk_req_complete(req, err_status); + virtio_blk_free_request(req); +} + +static void virtio_blk_zone_mgmt_complete(void *opaque, int ret) +{ + VirtIOBlockReq *req = opaque; + VirtIOBlock *s = req->dev; + VirtIODevice *vdev = VIRTIO_DEVICE(s); + int8_t err_status = VIRTIO_BLK_S_OK; + trace_virtio_blk_zone_mgmt_complete(vdev, req,ret); + + if (ret) { + err_status = VIRTIO_BLK_S_ZONE_INVALID_CMD; + } + + aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); + virtio_blk_req_complete(req, err_status); + virtio_blk_free_request(req); + aio_context_release(blk_get_aio_context(s->conf.conf.blk)); +} + +static int virtio_blk_handle_zone_mgmt(VirtIOBlockReq *req, BlockZoneOp op) +{ + VirtIOBlock *s = req->dev; + VirtIODevice *vdev = VIRTIO_DEVICE(s); + BlockDriverState *bs = blk_bs(s->blk); + int64_t offset = virtio_ldq_p(vdev, &req->out.sector) << BDRV_SECTOR_BITS; + uint64_t len; + uint64_t capacity = bs->total_sectors << BDRV_SECTOR_BITS; + uint8_t err_status = VIRTIO_BLK_S_OK; + + uint32_t type = virtio_ldl_p(vdev, &req->out.type); + if (type == VIRTIO_BLK_T_ZONE_RESET_ALL) { + /* Entire drive capacity */ + offset = 0; + len = capacity; + trace_virtio_blk_handle_zone_reset_all(vdev, req, 0, + bs->total_sectors); + } else { + if (bs->bl.zone_size > capacity - offset) { + /* The zoned device allows the last smaller zone. */ + len = capacity - bs->bl.zone_size * (bs->bl.nr_zones - 1); + } else { + len = bs->bl.zone_size; + } + trace_virtio_blk_handle_zone_mgmt(vdev, req, op, + offset >> BDRV_SECTOR_BITS, + len >> BDRV_SECTOR_BITS); + } + + if (!check_zoned_request(s, offset, len, false, &err_status)) { + goto out; + } + + blk_aio_zone_mgmt(s->blk, op, offset, len, + virtio_blk_zone_mgmt_complete, req); + + return 0; +out: + virtio_blk_req_complete(req, err_status); + virtio_blk_free_request(req); + return err_status; +} + +static void virtio_blk_zone_append_complete(void *opaque, int ret) +{ + ZoneCmdData *data = opaque; + VirtIOBlockReq *req = data->req; + VirtIOBlock *s = req->dev; + VirtIODevice *vdev = VIRTIO_DEVICE(req->dev); + int64_t append_sector, n; + uint8_t err_status = VIRTIO_BLK_S_OK; + + if (ret) { + err_status = VIRTIO_BLK_S_ZONE_INVALID_CMD; + goto out; + } + + virtio_stq_p(vdev, &append_sector, + data->zone_append_data.offset >> BDRV_SECTOR_BITS); + n = iov_from_buf(data->in_iov, data->in_num, 0, &append_sector, + sizeof(append_sector)); + if (n != sizeof(append_sector)) { + virtio_error(vdev, "Driver provided input buffer less than size of " + "append_sector"); + err_status = VIRTIO_BLK_S_ZONE_INVALID_CMD; + goto out; + } + trace_virtio_blk_zone_append_complete(vdev, req, append_sector, ret); + +out: + aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); + virtio_blk_req_complete(req, err_status); + virtio_blk_free_request(req); + aio_context_release(blk_get_aio_context(s->conf.conf.blk)); + g_free(data); +} + +static int virtio_blk_handle_zone_append(VirtIOBlockReq *req, + struct iovec *out_iov, + struct iovec *in_iov, + uint64_t out_num, + unsigned in_num) { + VirtIOBlock *s = req->dev; + VirtIODevice *vdev = VIRTIO_DEVICE(s); + uint8_t err_status = VIRTIO_BLK_S_OK; + + int64_t offset = virtio_ldq_p(vdev, &req->out.sector) << BDRV_SECTOR_BITS; + int64_t len = iov_size(out_iov, out_num); + + trace_virtio_blk_handle_zone_append(vdev, req, offset >> BDRV_SECTOR_BITS); + if (!check_zoned_request(s, offset, len, true, &err_status)) { + goto out; + } + + ZoneCmdData *data = g_malloc(sizeof(ZoneCmdData)); + data->req = req; + data->in_iov = in_iov; + data->in_num = in_num; + data->zone_append_data.offset = offset; + qemu_iovec_init_external(&req->qiov, out_iov, out_num); + + block_acct_start(blk_get_stats(s->blk), &req->acct, len, + BLOCK_ACCT_ZONE_APPEND); + + blk_aio_zone_append(s->blk, &data->zone_append_data.offset, &req->qiov, 0, + virtio_blk_zone_append_complete, data); + return 0; + +out: + aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); + virtio_blk_req_complete(req, err_status); + virtio_blk_free_request(req); + aio_context_release(blk_get_aio_context(s->conf.conf.blk)); + return err_status; +} + static int virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb) { uint32_t type; @@ -687,6 +1033,24 @@ static int virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb) case VIRTIO_BLK_T_FLUSH: virtio_blk_handle_flush(req, mrb); break; + case VIRTIO_BLK_T_ZONE_REPORT: + virtio_blk_handle_zone_report(req, in_iov, in_num); + break; + case VIRTIO_BLK_T_ZONE_OPEN: + virtio_blk_handle_zone_mgmt(req, BLK_ZO_OPEN); + break; + case VIRTIO_BLK_T_ZONE_CLOSE: + virtio_blk_handle_zone_mgmt(req, BLK_ZO_CLOSE); + break; + case VIRTIO_BLK_T_ZONE_FINISH: + virtio_blk_handle_zone_mgmt(req, BLK_ZO_FINISH); + break; + case VIRTIO_BLK_T_ZONE_RESET: + virtio_blk_handle_zone_mgmt(req, BLK_ZO_RESET); + break; + case VIRTIO_BLK_T_ZONE_RESET_ALL: + virtio_blk_handle_zone_mgmt(req, BLK_ZO_RESET); + break; case VIRTIO_BLK_T_SCSI_CMD: virtio_blk_handle_scsi(req); break; @@ -705,6 +1069,14 @@ static int virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb) virtio_blk_free_request(req); break; } + case VIRTIO_BLK_T_ZONE_APPEND & ~VIRTIO_BLK_T_OUT: + /* + * Passing out_iov/out_num and in_iov/in_num is not safe + * to access req->elem.out_sg directly because it may be + * modified by virtio_blk_handle_request(). + */ + virtio_blk_handle_zone_append(req, out_iov, in_iov, out_num, in_num); + break; /* * VIRTIO_BLK_T_DISCARD and VIRTIO_BLK_T_WRITE_ZEROES are defined with * VIRTIO_BLK_T_OUT flag set. We masked this flag in the switch statement, @@ -890,6 +1262,7 @@ static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config) { VirtIOBlock *s = VIRTIO_BLK(vdev); BlockConf *conf = &s->conf.conf; + BlockDriverState *bs = blk_bs(s->blk); struct virtio_blk_config blkcfg; uint64_t capacity; int64_t length; @@ -954,6 +1327,30 @@ static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config) blkcfg.write_zeroes_may_unmap = 1; virtio_stl_p(vdev, &blkcfg.max_write_zeroes_seg, 1); } + if (bs->bl.zoned != BLK_Z_NONE) { + switch (bs->bl.zoned) { + case BLK_Z_HM: + blkcfg.zoned.model = VIRTIO_BLK_Z_HM; + break; + case BLK_Z_HA: + blkcfg.zoned.model = VIRTIO_BLK_Z_HA; + break; + default: + g_assert_not_reached(); + } + + virtio_stl_p(vdev, &blkcfg.zoned.zone_sectors, + bs->bl.zone_size / 512); + virtio_stl_p(vdev, &blkcfg.zoned.max_active_zones, + bs->bl.max_active_zones); + virtio_stl_p(vdev, &blkcfg.zoned.max_open_zones, + bs->bl.max_open_zones); + virtio_stl_p(vdev, &blkcfg.zoned.write_granularity, blk_size); + virtio_stl_p(vdev, &blkcfg.zoned.max_append_sectors, + bs->bl.max_append_sectors); + } else { + blkcfg.zoned.model = VIRTIO_BLK_Z_NONE; + } memcpy(config, &blkcfg, s->config_size); } @@ -1163,6 +1560,14 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) return; } + BlockDriverState *bs = blk_bs(conf->conf.blk); + if (bs->bl.zoned != BLK_Z_NONE) { + virtio_add_feature(&s->host_features, VIRTIO_BLK_F_ZONED); + if (bs->bl.zoned == BLK_Z_HM) { + virtio_clear_feature(&s->host_features, VIRTIO_BLK_F_DISCARD); + } + } + if (virtio_has_feature(s->host_features, VIRTIO_BLK_F_DISCARD) && (!conf->max_discard_sectors || conf->max_discard_sectors > BDRV_REQUEST_MAX_SECTORS)) { diff --git a/hw/core/machine-qmp-cmds.c b/hw/core/machine-qmp-cmds.c index b98ff15089..3860a50c3b 100644 --- a/hw/core/machine-qmp-cmds.c +++ b/hw/core/machine-qmp-cmds.c @@ -28,18 +28,6 @@ #include "sysemu/runstate.h" #include "sysemu/sysemu.h" -static void cpustate_to_cpuinfo_s390(CpuInfoS390 *info, const CPUState *cpu) -{ -#ifdef TARGET_S390X - S390CPU *s390_cpu = S390_CPU(cpu); - CPUS390XState *env = &s390_cpu->env; - - info->cpu_state = env->cpu_state; -#else - abort(); -#endif -} - /* * fast means: we NEVER interrupt vCPU threads to retrieve * information from KVM. @@ -49,7 +37,7 @@ CpuInfoFastList *qmp_query_cpus_fast(Error **errp) MachineState *ms = MACHINE(qdev_get_machine()); MachineClass *mc = MACHINE_GET_CLASS(ms); CpuInfoFastList *head = NULL, **tail = &head; - SysEmuTarget target = qapi_enum_parse(&SysEmuTarget_lookup, TARGET_NAME, + SysEmuTarget target = qapi_enum_parse(&SysEmuTarget_lookup, target_name(), -1, &error_abort); CPUState *cpu; @@ -68,8 +56,8 @@ CpuInfoFastList *qmp_query_cpus_fast(Error **errp) } value->target = target; - if (target == SYS_EMU_TARGET_S390X) { - cpustate_to_cpuinfo_s390(&value->u.s390x, cpu); + if (cpu->cc->query_cpu_fast) { + cpu->cc->query_cpu_fast(cpu, value); } QAPI_LIST_APPEND(tail, value); @@ -129,7 +117,7 @@ TargetInfo *qmp_query_target(Error **errp) { TargetInfo *info = g_malloc0(sizeof(*info)); - info->arch = qapi_enum_parse(&SysEmuTarget_lookup, TARGET_NAME, -1, + info->arch = qapi_enum_parse(&SysEmuTarget_lookup, target_name(), -1, &error_abort); return info; diff --git a/hw/core/meson.build b/hw/core/meson.build index ae977c9396..959bc924d4 100644 --- a/hw/core/meson.build +++ b/hw/core/meson.build @@ -41,6 +41,7 @@ softmmu_ss.add(files( 'gpio.c', 'loader.c', 'machine-hmp-cmds.c', + 'machine-qmp-cmds.c', 'machine.c', 'nmi.c', 'null-machine.c', @@ -51,7 +52,3 @@ softmmu_ss.add(files( 'vm-change-state-handler.c', 'clock-vmstate.c', )) - -specific_ss.add(when: 'CONFIG_SOFTMMU', if_true: files( - 'machine-qmp-cmds.c', -)) diff --git a/hw/intc/loongarch_extioi.c b/hw/intc/loongarch_extioi.c index 4b8ec3f28a..0e7a3e32f3 100644 --- a/hw/intc/loongarch_extioi.c +++ b/hw/intc/loongarch_extioi.c @@ -254,7 +254,7 @@ static const VMStateDescription vmstate_loongarch_extioi = { .minimum_version_id = 1, .fields = (VMStateField[]) { VMSTATE_UINT32_ARRAY(bounce, LoongArchExtIOI, EXTIOI_IRQS_GROUP_COUNT), - VMSTATE_UINT32_2DARRAY(coreisr, LoongArchExtIOI, LOONGARCH_MAX_VCPUS, + VMSTATE_UINT32_2DARRAY(coreisr, LoongArchExtIOI, EXTIOI_CPUS, EXTIOI_IRQS_GROUP_COUNT), VMSTATE_UINT32_ARRAY(nodetype, LoongArchExtIOI, EXTIOI_IRQS_NODETYPE_COUNT / 2), @@ -281,7 +281,7 @@ static void loongarch_extioi_instance_init(Object *obj) qdev_init_gpio_in(DEVICE(obj), extioi_setirq, EXTIOI_IRQS); - for (cpu = 0; cpu < LOONGARCH_MAX_VCPUS; cpu++) { + for (cpu = 0; cpu < EXTIOI_CPUS; cpu++) { memory_region_init_io(&s->extioi_iocsr_mem[cpu], OBJECT(s), &extioi_ops, s, "extioi_iocsr", 0x900); sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->extioi_iocsr_mem[cpu]); diff --git a/hw/intc/loongarch_ipi.c b/hw/intc/loongarch_ipi.c index bdba0f8107..d6ab91721e 100644 --- a/hw/intc/loongarch_ipi.c +++ b/hw/intc/loongarch_ipi.c @@ -77,31 +77,42 @@ static void send_ipi_data(CPULoongArchState *env, uint64_t val, hwaddr addr) static void ipi_send(uint64_t val) { - int cpuid, data; + uint32_t cpuid; + uint8_t vector; CPULoongArchState *env; CPUState *cs; LoongArchCPU *cpu; - cpuid = (val >> 16) & 0x3ff; + cpuid = extract32(val, 16, 10); + if (cpuid >= LOONGARCH_MAX_CPUS) { + trace_loongarch_ipi_unsupported_cpuid("IOCSR_IPI_SEND", cpuid); + return; + } + /* IPI status vector */ - data = 1 << (val & 0x1f); + vector = extract8(val, 0, 5); + cs = qemu_get_cpu(cpuid); cpu = LOONGARCH_CPU(cs); env = &cpu->env; address_space_stl(&env->address_space_iocsr, 0x1008, - data, MEMTXATTRS_UNSPECIFIED, NULL); - + BIT(vector), MEMTXATTRS_UNSPECIFIED, NULL); } static void mail_send(uint64_t val) { - int cpuid; + uint32_t cpuid; hwaddr addr; CPULoongArchState *env; CPUState *cs; LoongArchCPU *cpu; - cpuid = (val >> 16) & 0x3ff; + cpuid = extract32(val, 16, 10); + if (cpuid >= LOONGARCH_MAX_CPUS) { + trace_loongarch_ipi_unsupported_cpuid("IOCSR_MAIL_SEND", cpuid); + return; + } + addr = 0x1020 + (val & 0x1c); cs = qemu_get_cpu(cpuid); cpu = LOONGARCH_CPU(cs); @@ -111,14 +122,21 @@ static void mail_send(uint64_t val) static void any_send(uint64_t val) { - int cpuid; + uint32_t cpuid; hwaddr addr; CPULoongArchState *env; + CPUState *cs; + LoongArchCPU *cpu; + + cpuid = extract32(val, 16, 10); + if (cpuid >= LOONGARCH_MAX_CPUS) { + trace_loongarch_ipi_unsupported_cpuid("IOCSR_ANY_SEND", cpuid); + return; + } - cpuid = (val >> 16) & 0x3ff; addr = val & 0xffff; - CPUState *cs = qemu_get_cpu(cpuid); - LoongArchCPU *cpu = LOONGARCH_CPU(cs); + cs = qemu_get_cpu(cpuid); + cpu = LOONGARCH_CPU(cs); env = &cpu->env; send_ipi_data(env, val, addr); } @@ -201,51 +219,43 @@ static const MemoryRegionOps loongarch_ipi64_ops = { static void loongarch_ipi_init(Object *obj) { - int cpu; - LoongArchMachineState *lams; LoongArchIPI *s = LOONGARCH_IPI(obj); SysBusDevice *sbd = SYS_BUS_DEVICE(obj); - Object *machine = qdev_get_machine(); - ObjectClass *mc = object_get_class(machine); - /* 'lams' should be initialized */ - if (!strcmp(MACHINE_CLASS(mc)->name, "none")) { - return; - } - lams = LOONGARCH_MACHINE(machine); - for (cpu = 0; cpu < MAX_IPI_CORE_NUM; cpu++) { - memory_region_init_io(&s->ipi_iocsr_mem[cpu], obj, &loongarch_ipi_ops, - &lams->ipi_core[cpu], "loongarch_ipi_iocsr", 0x48); - sysbus_init_mmio(sbd, &s->ipi_iocsr_mem[cpu]); - - memory_region_init_io(&s->ipi64_iocsr_mem[cpu], obj, &loongarch_ipi64_ops, - &lams->ipi_core[cpu], "loongarch_ipi64_iocsr", 0x118); - sysbus_init_mmio(sbd, &s->ipi64_iocsr_mem[cpu]); - qdev_init_gpio_out(DEVICE(obj), &lams->ipi_core[cpu].irq, 1); - } + + memory_region_init_io(&s->ipi_iocsr_mem, obj, &loongarch_ipi_ops, + &s->ipi_core, "loongarch_ipi_iocsr", 0x48); + + /* loongarch_ipi_iocsr performs re-entrant IO through ipi_send */ + s->ipi_iocsr_mem.disable_reentrancy_guard = true; + + sysbus_init_mmio(sbd, &s->ipi_iocsr_mem); + + memory_region_init_io(&s->ipi64_iocsr_mem, obj, &loongarch_ipi64_ops, + &s->ipi_core, "loongarch_ipi64_iocsr", 0x118); + sysbus_init_mmio(sbd, &s->ipi64_iocsr_mem); + qdev_init_gpio_out(DEVICE(obj), &s->ipi_core.irq, 1); } static const VMStateDescription vmstate_ipi_core = { .name = "ipi-single", - .version_id = 0, - .minimum_version_id = 0, + .version_id = 1, + .minimum_version_id = 1, .fields = (VMStateField[]) { VMSTATE_UINT32(status, IPICore), VMSTATE_UINT32(en, IPICore), VMSTATE_UINT32(set, IPICore), VMSTATE_UINT32(clear, IPICore), - VMSTATE_UINT32_ARRAY(buf, IPICore, MAX_IPI_MBX_NUM * 2), + VMSTATE_UINT32_ARRAY(buf, IPICore, 2), VMSTATE_END_OF_LIST() } }; static const VMStateDescription vmstate_loongarch_ipi = { .name = TYPE_LOONGARCH_IPI, - .version_id = 0, - .minimum_version_id = 0, + .version_id = 1, + .minimum_version_id = 1, .fields = (VMStateField[]) { - VMSTATE_STRUCT_ARRAY(ipi_core, LoongArchMachineState, - MAX_IPI_CORE_NUM, 0, - vmstate_ipi_core, IPICore), + VMSTATE_STRUCT(ipi_core, LoongArchIPI, 0, vmstate_ipi_core, IPICore), VMSTATE_END_OF_LIST() } }; diff --git a/hw/intc/trace-events b/hw/intc/trace-events index 50cadfb996..5c6094c457 100644 --- a/hw/intc/trace-events +++ b/hw/intc/trace-events @@ -292,6 +292,7 @@ sh_intc_set(int id, int enable) "setting interrupt group %d to %d" # loongarch_ipi.c loongarch_ipi_read(unsigned size, uint64_t addr, uint64_t val) "size: %u addr: 0x%"PRIx64 "val: 0x%"PRIx64 loongarch_ipi_write(unsigned size, uint64_t addr, uint64_t val) "size: %u addr: 0x%"PRIx64 "val: 0x%"PRIx64 +loongarch_ipi_unsupported_cpuid(const char *s, uint32_t cpuid) "%s unsupported cpuid 0x%" PRIx32 # loongarch_pch_pic.c loongarch_pch_pic_irq_handler(int irq, int level) "irq %d level %d" diff --git a/hw/loongarch/virt.c b/hw/loongarch/virt.c index f4bf14c1c8..2b7588e32a 100644 --- a/hw/loongarch/virt.c +++ b/hw/loongarch/virt.c @@ -565,9 +565,6 @@ static void loongarch_irq_init(LoongArchMachineState *lams) CPUState *cpu_state; int cpu, pin, i, start, num; - ipi = qdev_new(TYPE_LOONGARCH_IPI); - sysbus_realize_and_unref(SYS_BUS_DEVICE(ipi), &error_fatal); - extioi = qdev_new(TYPE_LOONGARCH_EXTIOI); sysbus_realize_and_unref(SYS_BUS_DEVICE(extioi), &error_fatal); @@ -598,17 +595,25 @@ static void loongarch_irq_init(LoongArchMachineState *lams) lacpu = LOONGARCH_CPU(cpu_state); env = &(lacpu->env); + ipi = qdev_new(TYPE_LOONGARCH_IPI); + sysbus_realize_and_unref(SYS_BUS_DEVICE(ipi), &error_fatal); + /* connect ipi irq to cpu irq */ - qdev_connect_gpio_out(ipi, cpu, qdev_get_gpio_in(cpudev, IRQ_IPI)); + qdev_connect_gpio_out(ipi, 0, qdev_get_gpio_in(cpudev, IRQ_IPI)); /* IPI iocsr memory region */ memory_region_add_subregion(&env->system_iocsr, SMP_IPI_MAILBOX, sysbus_mmio_get_region(SYS_BUS_DEVICE(ipi), - cpu * 2)); + 0)); memory_region_add_subregion(&env->system_iocsr, MAIL_SEND_ADDR, sysbus_mmio_get_region(SYS_BUS_DEVICE(ipi), - cpu * 2 + 1)); - /* extioi iocsr memory region */ - memory_region_add_subregion(&env->system_iocsr, APIC_BASE, + 1)); + /* + * extioi iocsr memory region + * only one extioi is added on loongarch virt machine + * external device interrupt can only be routed to cpu 0-3 + */ + if (cpu < EXTIOI_CPUS) + memory_region_add_subregion(&env->system_iocsr, APIC_BASE, sysbus_mmio_get_region(SYS_BUS_DEVICE(extioi), cpu)); } @@ -617,7 +622,7 @@ static void loongarch_irq_init(LoongArchMachineState *lams) * connect ext irq to the cpu irq * cpu_pin[9:2] <= intc_pin[7:0] */ - for (cpu = 0; cpu < ms->smp.cpus; cpu++) { + for (cpu = 0; cpu < MIN(ms->smp.cpus, EXTIOI_CPUS); cpu++) { cpudev = DEVICE(qemu_get_cpu(cpu)); for (pin = 0; pin < LS3A_INTC_IP; pin++) { qdev_connect_gpio_out(extioi, (cpu * 8 + pin), @@ -1026,7 +1031,7 @@ static void loongarch_class_init(ObjectClass *oc, void *data) mc->default_ram_size = 1 * GiB; mc->default_cpu_type = LOONGARCH_CPU_TYPE_NAME("la464"); mc->default_ram_id = "loongarch.ram"; - mc->max_cpus = LOONGARCH_MAX_VCPUS; + mc->max_cpus = LOONGARCH_MAX_CPUS; mc->is_default = 1; mc->default_kernel_irqchip_split = false; mc->block_default_type = IF_VIRTIO; diff --git a/hw/mips/malta.c b/hw/mips/malta.c index af9021316d..e3be2eea56 100644 --- a/hw/mips/malta.c +++ b/hw/mips/malta.c @@ -748,7 +748,6 @@ static void write_bootloader(uint8_t *base, uint64_t run_addr, uint64_t kernel_entry) { uint32_t *p; - void *v; /* Small bootloader */ p = (uint32_t *)base; @@ -785,9 +784,7 @@ static void write_bootloader(uint8_t *base, uint64_t run_addr, * */ - v = p; - bl_setup_gt64120_jump_kernel(&v, run_addr, kernel_entry); - p = v; + bl_setup_gt64120_jump_kernel((void **)&p, run_addr, kernel_entry); /* YAMON subroutines */ p = (uint32_t *) (base + 0x800); diff --git a/hw/net/meson.build b/hw/net/meson.build index e2be0654a1..a7860c5efe 100644 --- a/hw/net/meson.build +++ b/hw/net/meson.build @@ -43,7 +43,7 @@ softmmu_ss.add(when: 'CONFIG_NPCM7XX', if_true: files('npcm7xx_emc.c')) softmmu_ss.add(when: 'CONFIG_ETRAXFS', if_true: files('etraxfs_eth.c')) softmmu_ss.add(when: 'CONFIG_COLDFIRE', if_true: files('mcf_fec.c')) specific_ss.add(when: 'CONFIG_PSERIES', if_true: files('spapr_llan.c')) -specific_ss.add(when: 'CONFIG_XILINX_ETHLITE', if_true: files('xilinx_ethlite.c')) +softmmu_ss.add(when: 'CONFIG_XILINX_ETHLITE', if_true: files('xilinx_ethlite.c')) softmmu_ss.add(when: 'CONFIG_VIRTIO_NET', if_true: files('net_rx_pkt.c')) specific_ss.add(when: 'CONFIG_VIRTIO_NET', if_true: files('virtio-net.c')) diff --git a/hw/net/xilinx_ethlite.c b/hw/net/xilinx_ethlite.c index 99c22819ea..89f4f3b254 100644 --- a/hw/net/xilinx_ethlite.c +++ b/hw/net/xilinx_ethlite.c @@ -25,7 +25,7 @@ #include "qemu/osdep.h" #include "qemu/module.h" #include "qom/object.h" -#include "cpu.h" /* FIXME should not use tswap* */ +#include "exec/tswap.h" #include "hw/sysbus.h" #include "hw/irq.h" #include "hw/qdev-properties.h" diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c index dd5af508f9..e7b9345615 100644 --- a/hw/pci/pci_bridge.c +++ b/hw/pci/pci_bridge.c @@ -184,11 +184,11 @@ static void pci_bridge_init_vga_aliases(PCIBridge *br, PCIBus *parent, } } -static PCIBridgeWindows *pci_bridge_region_init(PCIBridge *br) +static void pci_bridge_region_init(PCIBridge *br) { PCIDevice *pd = PCI_DEVICE(br); PCIBus *parent = pci_get_bus(pd); - PCIBridgeWindows *w = g_new(PCIBridgeWindows, 1); + PCIBridgeWindows *w = &br->windows; uint16_t cmd = pci_get_word(pd->config + PCI_COMMAND); pci_bridge_init_alias(br, &w->alias_pref_mem, @@ -211,8 +211,6 @@ static PCIBridgeWindows *pci_bridge_region_init(PCIBridge *br) cmd & PCI_COMMAND_IO); pci_bridge_init_vga_aliases(br, parent, w->alias_vga); - - return w; } static void pci_bridge_region_del(PCIBridge *br, PCIBridgeWindows *w) @@ -234,19 +232,18 @@ static void pci_bridge_region_cleanup(PCIBridge *br, PCIBridgeWindows *w) object_unparent(OBJECT(&w->alias_vga[QEMU_PCI_VGA_IO_LO])); object_unparent(OBJECT(&w->alias_vga[QEMU_PCI_VGA_IO_HI])); object_unparent(OBJECT(&w->alias_vga[QEMU_PCI_VGA_MEM])); - g_free(w); } void pci_bridge_update_mappings(PCIBridge *br) { - PCIBridgeWindows *w = br->windows; + PCIBridgeWindows *w = &br->windows; /* Make updates atomic to: handle the case of one VCPU updating the bridge * while another accesses an unaffected region. */ memory_region_transaction_begin(); - pci_bridge_region_del(br, br->windows); + pci_bridge_region_del(br, w); pci_bridge_region_cleanup(br, w); - br->windows = pci_bridge_region_init(br); + pci_bridge_region_init(br); memory_region_transaction_commit(); } @@ -385,7 +382,7 @@ void pci_bridge_initfn(PCIDevice *dev, const char *typename) sec_bus->address_space_io = &br->address_space_io; memory_region_init(&br->address_space_io, OBJECT(br), "pci_bridge_io", 4 * GiB); - br->windows = pci_bridge_region_init(br); + pci_bridge_region_init(br); QLIST_INIT(&sec_bus->child); QLIST_INSERT_HEAD(&parent->child, sec_bus, sibling); } @@ -396,8 +393,8 @@ void pci_bridge_exitfn(PCIDevice *pci_dev) PCIBridge *s = PCI_BRIDGE(pci_dev); assert(QLIST_EMPTY(&s->sec_bus.child)); QLIST_REMOVE(&s->sec_bus, sibling); - pci_bridge_region_del(s, s->windows); - pci_bridge_region_cleanup(s, s->windows); + pci_bridge_region_del(s, &s->windows); + pci_bridge_region_cleanup(s, &s->windows); /* object_unparent() is called automatically during device deletion */ } diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index ddc9c7b1a1..1baea16c96 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -2166,7 +2166,7 @@ static void htab_save_first_pass(QEMUFile *f, SpaprMachineState *spapr, break; } } - } while ((index < htabslots) && !qemu_file_rate_limit(f)); + } while ((index < htabslots) && !migration_rate_exceeded(f)); if (index >= htabslots) { assert(index == htabslots); @@ -2237,7 +2237,7 @@ static int htab_save_later_pass(QEMUFile *f, SpaprMachineState *spapr, assert(index == htabslots); index = 0; } - } while ((examined < htabslots) && (!qemu_file_rate_limit(f) || final)); + } while ((examined < htabslots) && (!migration_rate_exceeded(f) || final)); if (index >= htabslots) { assert(index == htabslots); diff --git a/hw/s390x/pv.c b/hw/s390x/pv.c index 49ea38236c..b63f3784c6 100644 --- a/hw/s390x/pv.c +++ b/hw/s390x/pv.c @@ -13,6 +13,7 @@ #include <linux/kvm.h> +#include "qemu/units.h" #include "qapi/error.h" #include "qemu/error-report.h" #include "sysemu/kvm.h" @@ -115,7 +116,7 @@ static void *s390_pv_do_unprot_async_fn(void *p) return NULL; } -bool s390_pv_vm_try_disable_async(void) +bool s390_pv_vm_try_disable_async(S390CcwMachineState *ms) { /* * t is only needed to create the thread; once qemu_thread_create @@ -123,7 +124,12 @@ bool s390_pv_vm_try_disable_async(void) */ QemuThread t; - if (!kvm_check_extension(kvm_state, KVM_CAP_S390_PROTECTED_ASYNC_DISABLE)) { + /* + * If the feature is not present or if the VM is not larger than 2 GiB, + * KVM_PV_ASYNC_CLEANUP_PREPARE fill fail; no point in attempting it. + */ + if ((MACHINE(ms)->maxram_size <= 2 * GiB) || + !kvm_check_extension(kvm_state, KVM_CAP_S390_PROTECTED_ASYNC_DISABLE)) { return false; } if (s390_pv_cmd(KVM_PV_ASYNC_CLEANUP_PREPARE, NULL) != 0) { diff --git a/hw/s390x/s390-stattrib.c b/hw/s390x/s390-stattrib.c index aed919ad7d..220e845d12 100644 --- a/hw/s390x/s390-stattrib.c +++ b/hw/s390x/s390-stattrib.c @@ -209,7 +209,7 @@ static int cmma_save(QEMUFile *f, void *opaque, int final) return -ENOMEM; } - while (final ? 1 : qemu_file_rate_limit(f) == 0) { + while (final ? 1 : migration_rate_exceeded(f) == 0) { reallen = sac->get_stattr(sas, &start_gfn, buflen, buf); if (reallen < 0) { g_free(buf); diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index e6f2c62625..2516b89b32 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -330,7 +330,7 @@ static inline void s390_do_cpu_ipl(CPUState *cs, run_on_cpu_data arg) static void s390_machine_unprotect(S390CcwMachineState *ms) { - if (!s390_pv_vm_try_disable_async()) { + if (!s390_pv_vm_try_disable_async(ms)) { s390_pv_vm_disable(); } ms->pv = false; diff --git a/hw/scsi/scsi-generic.c b/hw/scsi/scsi-generic.c index ac9fa662b4..2417f0ad84 100644 --- a/hw/scsi/scsi-generic.c +++ b/hw/scsi/scsi-generic.c @@ -191,12 +191,16 @@ static int scsi_handle_inquiry_reply(SCSIGenericReq *r, SCSIDevice *s, int len) if ((s->type == TYPE_DISK || s->type == TYPE_ZBC) && (r->req.cmd.buf[1] & 0x01)) { page = r->req.cmd.buf[2]; - if (page == 0xb0) { + if (page == 0xb0 && r->buflen >= 8) { + uint8_t buf[16] = {}; + uint8_t buf_used = MIN(r->buflen, 16); uint64_t max_transfer = calculate_max_transfer(s); - stl_be_p(&r->buf[8], max_transfer); - /* Also take care of the opt xfer len. */ - stl_be_p(&r->buf[12], - MIN_NON_ZERO(max_transfer, ldl_be_p(&r->buf[12]))); + + memcpy(buf, r->buf, buf_used); + stl_be_p(&buf[8], max_transfer); + stl_be_p(&buf[12], MIN_NON_ZERO(max_transfer, ldl_be_p(&buf[12]))); + memcpy(r->buf + 8, buf + 8, buf_used - 8); + } else if (s->needs_vpd_bl_emulation && page == 0x00 && r->buflen >= 4) { /* * Now we're capable of supplying the VPD Block Limits diff --git a/hw/virtio/virtio-qmp.c b/hw/virtio/virtio-qmp.c index b70148aba9..e84316dcfd 100644 --- a/hw/virtio/virtio-qmp.c +++ b/hw/virtio/virtio-qmp.c @@ -176,6 +176,8 @@ static const qmp_virtio_feature_map_t virtio_blk_feature_map[] = { "VIRTIO_BLK_F_DISCARD: Discard command supported"), FEATURE_ENTRY(VIRTIO_BLK_F_WRITE_ZEROES, \ "VIRTIO_BLK_F_WRITE_ZEROES: Write zeroes command supported"), + FEATURE_ENTRY(VIRTIO_BLK_F_ZONED, \ + "VIRTIO_BLK_F_ZONED: Zoned block devices"), #ifndef VIRTIO_BLK_NO_LEGACY FEATURE_ENTRY(VIRTIO_BLK_F_BARRIER, \ "VIRTIO_BLK_F_BARRIER: Request barriers supported"), diff --git a/include/block/accounting.h b/include/block/accounting.h index b9caad60d5..a59e39f49d 100644 --- a/include/block/accounting.h +++ b/include/block/accounting.h @@ -37,6 +37,7 @@ enum BlockAcctType { BLOCK_ACCT_READ, BLOCK_ACCT_WRITE, BLOCK_ACCT_FLUSH, + BLOCK_ACCT_ZONE_APPEND, BLOCK_ACCT_UNMAP, BLOCK_MAX_IOTYPE, }; diff --git a/include/block/block-common.h b/include/block/block-common.h index b5122ef8ab..93196229ac 100644 --- a/include/block/block-common.h +++ b/include/block/block-common.h @@ -75,6 +75,57 @@ typedef struct BlockDriver BlockDriver; typedef struct BdrvChild BdrvChild; typedef struct BdrvChildClass BdrvChildClass; +typedef enum BlockZoneOp { + BLK_ZO_OPEN, + BLK_ZO_CLOSE, + BLK_ZO_FINISH, + BLK_ZO_RESET, +} BlockZoneOp; + +typedef enum BlockZoneModel { + BLK_Z_NONE = 0x0, /* Regular block device */ + BLK_Z_HM = 0x1, /* Host-managed zoned block device */ + BLK_Z_HA = 0x2, /* Host-aware zoned block device */ +} BlockZoneModel; + +typedef enum BlockZoneState { + BLK_ZS_NOT_WP = 0x0, + BLK_ZS_EMPTY = 0x1, + BLK_ZS_IOPEN = 0x2, + BLK_ZS_EOPEN = 0x3, + BLK_ZS_CLOSED = 0x4, + BLK_ZS_RDONLY = 0xD, + BLK_ZS_FULL = 0xE, + BLK_ZS_OFFLINE = 0xF, +} BlockZoneState; + +typedef enum BlockZoneType { + BLK_ZT_CONV = 0x1, /* Conventional random writes supported */ + BLK_ZT_SWR = 0x2, /* Sequential writes required */ + BLK_ZT_SWP = 0x3, /* Sequential writes preferred */ +} BlockZoneType; + +/* + * Zone descriptor data structure. + * Provides information on a zone with all position and size values in bytes. + */ +typedef struct BlockZoneDescriptor { + uint64_t start; + uint64_t length; + uint64_t cap; + uint64_t wp; + BlockZoneType type; + BlockZoneState state; +} BlockZoneDescriptor; + +/* + * Track write pointers of a zone in bytes. + */ +typedef struct BlockZoneWps { + CoMutex colock; + uint64_t wp[]; +} BlockZoneWps; + typedef struct BlockDriverInfo { /* in bytes, 0 if irrelevant */ int cluster_size; @@ -197,6 +248,12 @@ typedef enum { #define BDRV_SECTOR_BITS 9 #define BDRV_SECTOR_SIZE (1ULL << BDRV_SECTOR_BITS) +/* + * Get the first most significant bit of wp. If it is zero, then + * the zone type is SWR. + */ +#define BDRV_ZT_IS_CONV(wp) (wp & (1ULL << 63)) + #define BDRV_REQUEST_MAX_SECTORS MIN_CONST(SIZE_MAX >> BDRV_SECTOR_BITS, \ INT_MAX >> BDRV_SECTOR_BITS) #define BDRV_REQUEST_MAX_BYTES (BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) diff --git a/include/block/block-io.h b/include/block/block-io.h index 1f612ec5bd..a27e471a87 100644 --- a/include/block/block-io.h +++ b/include/block/block-io.h @@ -114,6 +114,19 @@ int coroutine_fn GRAPH_RDLOCK bdrv_co_flush(BlockDriverState *bs); int coroutine_fn GRAPH_RDLOCK bdrv_co_pdiscard(BdrvChild *child, int64_t offset, int64_t bytes); +/* Report zone information of zone block device. */ +int coroutine_fn GRAPH_RDLOCK bdrv_co_zone_report(BlockDriverState *bs, + int64_t offset, + unsigned int *nr_zones, + BlockZoneDescriptor *zones); +int coroutine_fn GRAPH_RDLOCK bdrv_co_zone_mgmt(BlockDriverState *bs, + BlockZoneOp op, + int64_t offset, int64_t len); +int coroutine_fn GRAPH_RDLOCK bdrv_co_zone_append(BlockDriverState *bs, + int64_t *offset, + QEMUIOVector *qiov, + BdrvRequestFlags flags); + bool bdrv_can_write_zeroes_with_unmap(BlockDriverState *bs); int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, diff --git a/include/block/block_int-common.h b/include/block/block_int-common.h index 4909876756..dbec0e3bb4 100644 --- a/include/block/block_int-common.h +++ b/include/block/block_int-common.h @@ -138,6 +138,11 @@ struct BlockDriver { bool is_format; /* + * Set to true if the BlockDriver supports zoned children. + */ + bool supports_zoned_children; + + /* * Drivers not implementing bdrv_parse_filename nor bdrv_open should have * this field set to true, except ones that are defined only by their * child's bs. @@ -713,6 +718,15 @@ struct BlockDriver { int coroutine_fn GRAPH_RDLOCK_PTR (*bdrv_co_load_vmstate)( BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos); + int coroutine_fn (*bdrv_co_zone_report)(BlockDriverState *bs, + int64_t offset, unsigned int *nr_zones, + BlockZoneDescriptor *zones); + int coroutine_fn (*bdrv_co_zone_mgmt)(BlockDriverState *bs, BlockZoneOp op, + int64_t offset, int64_t len); + int coroutine_fn (*bdrv_co_zone_append)(BlockDriverState *bs, + int64_t *offset, QEMUIOVector *qiov, + BdrvRequestFlags flags); + /* removable device specific */ bool coroutine_fn GRAPH_RDLOCK_PTR (*bdrv_co_is_inserted)( BlockDriverState *bs); @@ -862,6 +876,26 @@ typedef struct BlockLimits { * an explicit monitor command to load the disk inside the guest). */ bool has_variable_length; + + /* device zone model */ + BlockZoneModel zoned; + + /* zone size expressed in bytes */ + uint32_t zone_size; + + /* total number of zones */ + uint32_t nr_zones; + + /* maximum sectors of a zone append write operation */ + uint32_t max_append_sectors; + + /* maximum number of open zones */ + uint32_t max_open_zones; + + /* maximum number of active zones */ + uint32_t max_active_zones; + + uint32_t write_granularity; } BlockLimits; typedef struct BdrvOpBlocker BdrvOpBlocker; @@ -1223,6 +1257,9 @@ struct BlockDriverState { CoMutex bsc_modify_lock; /* Always non-NULL, but must only be dereferenced under an RCU read guard */ BdrvBlockStatusCache *block_status_cache; + + /* array of write pointers' location of each zone in the zoned device. */ + BlockZoneWps *wps; }; struct BlockBackendRootState { diff --git a/include/block/raw-aio.h b/include/block/raw-aio.h index e46a29c3f0..0fe85ade77 100644 --- a/include/block/raw-aio.h +++ b/include/block/raw-aio.h @@ -28,6 +28,9 @@ #define QEMU_AIO_WRITE_ZEROES 0x0020 #define QEMU_AIO_COPY_RANGE 0x0040 #define QEMU_AIO_TRUNCATE 0x0080 +#define QEMU_AIO_ZONE_REPORT 0x0100 +#define QEMU_AIO_ZONE_MGMT 0x0200 +#define QEMU_AIO_ZONE_APPEND 0x0400 #define QEMU_AIO_TYPE_MASK \ (QEMU_AIO_READ | \ QEMU_AIO_WRITE | \ @@ -36,7 +39,10 @@ QEMU_AIO_DISCARD | \ QEMU_AIO_WRITE_ZEROES | \ QEMU_AIO_COPY_RANGE | \ - QEMU_AIO_TRUNCATE) + QEMU_AIO_TRUNCATE | \ + QEMU_AIO_ZONE_REPORT | \ + QEMU_AIO_ZONE_MGMT | \ + QEMU_AIO_ZONE_APPEND) /* AIO flags */ #define QEMU_AIO_MISALIGNED 0x1000 diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index ad824fee52..78d258af44 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -84,11 +84,8 @@ #if defined(CONFIG_USER_ONLY) #include "exec/user/abitypes.h" +#include "exec/user/guest-base.h" -/* On some host systems the guest address space is reserved on the host. - * This allows the guest address space to be offset to a convenient location. - */ -extern uintptr_t guest_base; extern bool have_guest_base; /* diff --git a/include/exec/memop.h b/include/exec/memop.h index 07f5f88188..a86dc6743a 100644 --- a/include/exec/memop.h +++ b/include/exec/memop.h @@ -72,6 +72,43 @@ typedef enum MemOp { MO_ALIGN_64 = 6 << MO_ASHIFT, MO_ALIGN = MO_AMASK, + /* + * MO_ATOM_* describes the atomicity requirements of the operation: + * MO_ATOM_IFALIGN: the operation must be single-copy atomic if it + * is aligned; if unaligned there is no atomicity. + * MO_ATOM_IFALIGN_PAIR: the entire operation may be considered to + * be a pair of half-sized operations which are packed together + * for convenience, with single-copy atomicity on each half if + * the half is aligned. + * This is the atomicity e.g. of Arm pre-FEAT_LSE2 LDP. + * MO_ATOM_WITHIN16: the operation is single-copy atomic, even if it + * is unaligned, so long as it does not cross a 16-byte boundary; + * if it crosses a 16-byte boundary there is no atomicity. + * This is the atomicity e.g. of Arm FEAT_LSE2 LDR. + * MO_ATOM_WITHIN16_PAIR: the entire operation is single-copy atomic, + * if it happens to be within a 16-byte boundary, otherwise it + * devolves to a pair of half-sized MO_ATOM_WITHIN16 operations. + * Depending on alignment, one or both will be single-copy atomic. + * This is the atomicity e.g. of Arm FEAT_LSE2 LDP. + * MO_ATOM_SUBALIGN: the operation is single-copy atomic by parts + * by the alignment. E.g. if the address is 0 mod 4, then each + * 4-byte subobject is single-copy atomic. + * This is the atomicity e.g. of IBM Power. + * MO_ATOM_NONE: the operation has no atomicity requirements. + * + * Note the default (i.e. 0) value is single-copy atomic to the + * size of the operation, if aligned. This retains the behaviour + * from before this field was introduced. + */ + MO_ATOM_SHIFT = 8, + MO_ATOM_IFALIGN = 0 << MO_ATOM_SHIFT, + MO_ATOM_IFALIGN_PAIR = 1 << MO_ATOM_SHIFT, + MO_ATOM_WITHIN16 = 2 << MO_ATOM_SHIFT, + MO_ATOM_WITHIN16_PAIR = 3 << MO_ATOM_SHIFT, + MO_ATOM_SUBALIGN = 4 << MO_ATOM_SHIFT, + MO_ATOM_NONE = 5 << MO_ATOM_SHIFT, + MO_ATOM_MASK = 7 << MO_ATOM_SHIFT, + /* Combinations of the above, for ease of use. */ MO_UB = MO_8, MO_UW = MO_16, diff --git a/include/exec/memory.h b/include/exec/memory.h index e45ce6061f..6f24a3dd46 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -934,8 +934,11 @@ struct MemoryListener { * its @log_sync must be NULL. Vice versa. * * @listener: The #MemoryListener. + * @last_stage: The last stage to synchronize the log during migration. + * The caller should gurantee that the synchronization with true for + * @last_stage is triggered for once after all VCPUs have been stopped. */ - void (*log_sync_global)(MemoryListener *listener); + void (*log_sync_global)(MemoryListener *listener, bool last_stage); /** * @log_clear: @@ -2422,8 +2425,10 @@ MemoryRegionSection memory_region_find(MemoryRegion *mr, * memory_global_dirty_log_sync: synchronize the dirty log for all memory * * Synchronizes the dirty page log for all address spaces. + * + * @last_stage: whether this is the last stage of live migration */ -void memory_global_dirty_log_sync(void); +void memory_global_dirty_log_sync(bool last_stage); /** * memory_global_dirty_log_sync: synchronize the dirty log for all memory diff --git a/include/exec/plugin-gen.h b/include/exec/plugin-gen.h index 5f5506f1cc..3af0168e65 100644 --- a/include/exec/plugin-gen.h +++ b/include/exec/plugin-gen.h @@ -27,7 +27,7 @@ void plugin_gen_insn_start(CPUState *cpu, const struct DisasContextBase *db); void plugin_gen_insn_end(void); void plugin_gen_disable_mem_helpers(void); -void plugin_gen_empty_mem_callback(TCGv addr, uint32_t info); +void plugin_gen_empty_mem_callback(TCGv_i64 addr, uint32_t info); static inline void plugin_insn_append(abi_ptr pc, const void *from, size_t size) { @@ -69,7 +69,7 @@ static inline void plugin_gen_tb_end(CPUState *cpu) static inline void plugin_gen_disable_mem_helpers(void) { } -static inline void plugin_gen_empty_mem_callback(TCGv addr, uint32_t info) +static inline void plugin_gen_empty_mem_callback(TCGv_i64 addr, uint32_t info) { } static inline void plugin_insn_append(abi_ptr pc, const void *from, size_t size) diff --git a/include/exec/target_page.h b/include/exec/target_page.h index 96726c36a4..bbf37aea17 100644 --- a/include/exec/target_page.h +++ b/include/exec/target_page.h @@ -18,4 +18,5 @@ size_t qemu_target_page_size(void); int qemu_target_page_bits(void); int qemu_target_page_bits_min(void); +size_t qemu_target_pages_to_MiB(size_t pages); #endif diff --git a/include/exec/user/guest-base.h b/include/exec/user/guest-base.h new file mode 100644 index 0000000000..afe2ab7fbb --- /dev/null +++ b/include/exec/user/guest-base.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Declaration of guest_base. + * Copyright (c) 2003 Fabrice Bellard + */ + +#ifndef EXEC_USER_GUEST_BASE_H +#define EXEC_USER_GUEST_BASE_H + +extern uintptr_t guest_base; + +#endif diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index 397fd3ac68..39150cf8f8 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -106,6 +106,9 @@ struct SysemuCPUOps; * @has_work: Callback for checking if there is work to do. * @memory_rw_debug: Callback for GDB memory access. * @dump_state: Callback for dumping state. + * @query_cpu_fast: + * Fill in target specific information for the "query-cpus-fast" + * QAPI call. * @get_arch_id: Callback for getting architecture-dependent CPU ID. * @set_pc: Callback for setting the Program Counter register. This * should have the semantics used by the target architecture when @@ -151,6 +154,7 @@ struct CPUClass { int (*memory_rw_debug)(CPUState *cpu, vaddr addr, uint8_t *buf, int len, bool is_write); void (*dump_state)(CPUState *cpu, FILE *, int flags); + void (*query_cpu_fast)(CPUState *cpu, CpuInfoFast *value); int64_t (*get_arch_id)(CPUState *cpu); void (*set_pc)(CPUState *cpu, vaddr value); vaddr (*get_pc)(CPUState *cpu); @@ -1009,6 +1013,8 @@ void cpu_exec_unrealizefn(CPUState *cpu); */ bool target_words_bigendian(void); +const char *target_name(void); + void page_size_init(void); #ifdef NEED_CPU_H diff --git a/include/hw/intc/loongarch_extioi.h b/include/hw/intc/loongarch_extioi.h index 15b8c999f6..fbdef9a7b3 100644 --- a/include/hw/intc/loongarch_extioi.h +++ b/include/hw/intc/loongarch_extioi.h @@ -14,6 +14,8 @@ #define LS3A_INTC_IP 8 #define EXTIOI_IRQS (256) #define EXTIOI_IRQS_BITMAP_SIZE (256 / 8) +/* irq from EXTIOI is routed to no more than 4 cpus */ +#define EXTIOI_CPUS (4) /* map to ipnum per 32 irqs */ #define EXTIOI_IRQS_IPMAP_SIZE (256 / 32) #define EXTIOI_IRQS_COREMAP_SIZE 256 @@ -46,17 +48,17 @@ struct LoongArchExtIOI { uint32_t nodetype[EXTIOI_IRQS_NODETYPE_COUNT / 2]; uint32_t bounce[EXTIOI_IRQS_GROUP_COUNT]; uint32_t isr[EXTIOI_IRQS / 32]; - uint32_t coreisr[LOONGARCH_MAX_VCPUS][EXTIOI_IRQS_GROUP_COUNT]; + uint32_t coreisr[EXTIOI_CPUS][EXTIOI_IRQS_GROUP_COUNT]; uint32_t enable[EXTIOI_IRQS / 32]; uint32_t ipmap[EXTIOI_IRQS_IPMAP_SIZE / 4]; uint32_t coremap[EXTIOI_IRQS / 4]; uint32_t sw_pending[EXTIOI_IRQS / 32]; - DECLARE_BITMAP(sw_isr[LOONGARCH_MAX_VCPUS][LS3A_INTC_IP], EXTIOI_IRQS); + DECLARE_BITMAP(sw_isr[EXTIOI_CPUS][LS3A_INTC_IP], EXTIOI_IRQS); uint8_t sw_ipmap[EXTIOI_IRQS_IPMAP_SIZE]; uint8_t sw_coremap[EXTIOI_IRQS]; - qemu_irq parent_irq[LOONGARCH_MAX_VCPUS][LS3A_INTC_IP]; + qemu_irq parent_irq[EXTIOI_CPUS][LS3A_INTC_IP]; qemu_irq irq[EXTIOI_IRQS]; - MemoryRegion extioi_iocsr_mem[LOONGARCH_MAX_VCPUS]; + MemoryRegion extioi_iocsr_mem[EXTIOI_CPUS]; MemoryRegion extioi_system_mem; }; #endif /* LOONGARCH_EXTIOI_H */ diff --git a/include/hw/intc/loongarch_ipi.h b/include/hw/intc/loongarch_ipi.h index 0ee48fca55..664e050b92 100644 --- a/include/hw/intc/loongarch_ipi.h +++ b/include/hw/intc/loongarch_ipi.h @@ -28,9 +28,6 @@ #define MAIL_SEND_OFFSET 0 #define ANY_SEND_OFFSET (IOCSR_ANY_SEND - IOCSR_MAIL_SEND) -#define MAX_IPI_CORE_NUM 4 -#define MAX_IPI_MBX_NUM 4 - #define TYPE_LOONGARCH_IPI "loongarch_ipi" OBJECT_DECLARE_SIMPLE_TYPE(LoongArchIPI, LOONGARCH_IPI) @@ -40,14 +37,15 @@ typedef struct IPICore { uint32_t set; uint32_t clear; /* 64bit buf divide into 2 32bit buf */ - uint32_t buf[MAX_IPI_MBX_NUM * 2]; + uint32_t buf[2]; qemu_irq irq; } IPICore; struct LoongArchIPI { SysBusDevice parent_obj; - MemoryRegion ipi_iocsr_mem[MAX_IPI_CORE_NUM]; - MemoryRegion ipi64_iocsr_mem[MAX_IPI_CORE_NUM]; + MemoryRegion ipi_iocsr_mem; + MemoryRegion ipi64_iocsr_mem; + IPICore ipi_core; }; #endif diff --git a/include/hw/loongarch/virt.h b/include/hw/loongarch/virt.h index 7ae8a91229..f1659655c6 100644 --- a/include/hw/loongarch/virt.h +++ b/include/hw/loongarch/virt.h @@ -14,7 +14,7 @@ #include "hw/intc/loongarch_ipi.h" #include "hw/block/flash.h" -#define LOONGARCH_MAX_VCPUS 4 +#define LOONGARCH_MAX_CPUS 256 #define VIRT_ISA_IO_BASE 0x18000000UL #define VIRT_ISA_IO_SIZE 0x0004000 @@ -36,7 +36,6 @@ struct LoongArchMachineState { /*< private >*/ MachineState parent_obj; - IPICore ipi_core[MAX_IPI_CORE_NUM]; MemoryRegion lowmem; MemoryRegion highmem; MemoryRegion isa_io; diff --git a/include/hw/pci/pci_bridge.h b/include/hw/pci/pci_bridge.h index 01670e9e65..ea54a81a15 100644 --- a/include/hw/pci/pci_bridge.h +++ b/include/hw/pci/pci_bridge.h @@ -73,7 +73,7 @@ struct PCIBridge { MemoryRegion address_space_mem; MemoryRegion address_space_io; - PCIBridgeWindows *windows; + PCIBridgeWindows windows; pci_map_irq_fn map_irq; const char *bus_name; diff --git a/include/hw/s390x/pv.h b/include/hw/s390x/pv.h index 966306a9db..7b935e2246 100644 --- a/include/hw/s390x/pv.h +++ b/include/hw/s390x/pv.h @@ -14,10 +14,10 @@ #include "qapi/error.h" #include "sysemu/kvm.h" +#include "hw/s390x/s390-virtio-ccw.h" #ifdef CONFIG_KVM #include "cpu.h" -#include "hw/s390x/s390-virtio-ccw.h" static inline bool s390_is_pv(void) { @@ -41,7 +41,7 @@ static inline bool s390_is_pv(void) int s390_pv_query_info(void); int s390_pv_vm_enable(void); void s390_pv_vm_disable(void); -bool s390_pv_vm_try_disable_async(void); +bool s390_pv_vm_try_disable_async(S390CcwMachineState *ms); int s390_pv_set_sec_parms(uint64_t origin, uint64_t length); int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak); void s390_pv_prep_reset(void); @@ -61,7 +61,7 @@ static inline bool s390_is_pv(void) { return false; } static inline int s390_pv_query_info(void) { return 0; } static inline int s390_pv_vm_enable(void) { return 0; } static inline void s390_pv_vm_disable(void) {} -static inline bool s390_pv_vm_try_disable_async(void) { return false; } +static inline bool s390_pv_vm_try_disable_async(S390CcwMachineState *ms) { return false; } static inline int s390_pv_set_sec_parms(uint64_t origin, uint64_t length) { return 0; } static inline int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak) { return 0; } static inline void s390_pv_prep_reset(void) {} diff --git a/include/migration/colo.h b/include/migration/colo.h index 7ef315473e..eaac07f26d 100644 --- a/include/migration/colo.h +++ b/include/migration/colo.h @@ -28,7 +28,6 @@ bool migration_in_colo_state(void); int migration_incoming_enable_colo(void); void migration_incoming_disable_colo(void); bool migration_incoming_colo_enabled(void); -void *colo_process_incoming_thread(void *opaque); bool migration_incoming_in_colo_state(void); COLOMode get_colo_mode(void); @@ -44,5 +43,13 @@ void colo_do_failover(void); */ void colo_checkpoint_delay_set(void); +/* + * Starts COLO incoming process. Called from process_incoming_migration_co() + * after loading the state. + * + * Called with BQL locked, may temporary release BQL. + */ +int coroutine_fn colo_incoming_co(void); + void colo_shutdown(void); #endif diff --git a/include/migration/qemu-file-types.h b/include/migration/qemu-file-types.h index 1436f9ce92..9ba163f333 100644 --- a/include/migration/qemu-file-types.h +++ b/include/migration/qemu-file-types.h @@ -165,6 +165,16 @@ size_t coroutine_mixed_fn qemu_get_counted_string(QEMUFile *f, char buf[256]); void qemu_put_counted_string(QEMUFile *f, const char *name); -int qemu_file_rate_limit(QEMUFile *f); +/** + * migration_rate_exceeded: Check if we have exceeded rate for this interval + * + * Checks if we have already transferred more data that we are allowed + * in the current interval. + * + * @f: QEMUFile used for main migration channel + * + * Returns if we should stop sending data for this interval. + */ +bool migration_rate_exceeded(QEMUFile *f); #endif diff --git a/include/qemu/cpuid.h b/include/qemu/cpuid.h index 1451e8ef2f..35325f1995 100644 --- a/include/qemu/cpuid.h +++ b/include/qemu/cpuid.h @@ -71,6 +71,24 @@ #define bit_LZCNT (1 << 5) #endif +/* + * Signatures for different CPU implementations as returned from Leaf 0. + */ + +#ifndef signature_INTEL_ecx +/* "Genu" "ineI" "ntel" */ +#define signature_INTEL_ebx 0x756e6547 +#define signature_INTEL_edx 0x49656e69 +#define signature_INTEL_ecx 0x6c65746e +#endif + +#ifndef signature_AMD_ecx +/* "Auth" "enti" "cAMD" */ +#define signature_AMD_ebx 0x68747541 +#define signature_AMD_edx 0x69746e65 +#define signature_AMD_ecx 0x444d4163 +#endif + static inline unsigned xgetbv_low(unsigned c) { unsigned a, d; diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h index df4b55ac65..8e9ef252f5 100644 --- a/include/qemu/typedefs.h +++ b/include/qemu/typedefs.h @@ -41,6 +41,7 @@ typedef struct CompatProperty CompatProperty; typedef struct ConfidentialGuestSupport ConfidentialGuestSupport; typedef struct CPUAddressSpace CPUAddressSpace; typedef struct CPUArchState CPUArchState; +typedef struct CpuInfoFast CpuInfoFast; typedef struct CPUJumpCache CPUJumpCache; typedef struct CPUState CPUState; typedef struct CPUTLBEntryFull CPUTLBEntryFull; diff --git a/include/sysemu/block-backend-io.h b/include/sysemu/block-backend-io.h index 851a44de96..d62a7ee773 100644 --- a/include/sysemu/block-backend-io.h +++ b/include/sysemu/block-backend-io.h @@ -46,6 +46,16 @@ BlockAIOCB *blk_aio_pwritev(BlockBackend *blk, int64_t offset, BlockCompletionFunc *cb, void *opaque); BlockAIOCB *blk_aio_flush(BlockBackend *blk, BlockCompletionFunc *cb, void *opaque); +BlockAIOCB *blk_aio_zone_report(BlockBackend *blk, int64_t offset, + unsigned int *nr_zones, + BlockZoneDescriptor *zones, + BlockCompletionFunc *cb, void *opaque); +BlockAIOCB *blk_aio_zone_mgmt(BlockBackend *blk, BlockZoneOp op, + int64_t offset, int64_t len, + BlockCompletionFunc *cb, void *opaque); +BlockAIOCB *blk_aio_zone_append(BlockBackend *blk, int64_t *offset, + QEMUIOVector *qiov, BdrvRequestFlags flags, + BlockCompletionFunc *cb, void *opaque); BlockAIOCB *blk_aio_pdiscard(BlockBackend *blk, int64_t offset, int64_t bytes, BlockCompletionFunc *cb, void *opaque); void blk_aio_cancel_async(BlockAIOCB *acb); @@ -191,6 +201,23 @@ int co_wrapper_mixed blk_pwrite_zeroes(BlockBackend *blk, int64_t offset, int coroutine_fn blk_co_pwrite_zeroes(BlockBackend *blk, int64_t offset, int64_t bytes, BdrvRequestFlags flags); +int coroutine_fn blk_co_zone_report(BlockBackend *blk, int64_t offset, + unsigned int *nr_zones, + BlockZoneDescriptor *zones); +int co_wrapper_mixed blk_zone_report(BlockBackend *blk, int64_t offset, + unsigned int *nr_zones, + BlockZoneDescriptor *zones); +int coroutine_fn blk_co_zone_mgmt(BlockBackend *blk, BlockZoneOp op, + int64_t offset, int64_t len); +int co_wrapper_mixed blk_zone_mgmt(BlockBackend *blk, BlockZoneOp op, + int64_t offset, int64_t len); +int coroutine_fn blk_co_zone_append(BlockBackend *blk, int64_t *offset, + QEMUIOVector *qiov, + BdrvRequestFlags flags); +int co_wrapper_mixed blk_zone_append(BlockBackend *blk, int64_t *offset, + QEMUIOVector *qiov, + BdrvRequestFlags flags); + int co_wrapper_mixed blk_pdiscard(BlockBackend *blk, int64_t offset, int64_t bytes); int coroutine_fn blk_co_pdiscard(BlockBackend *blk, int64_t offset, diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h index c8281c07a7..88f5ccfbce 100644 --- a/include/sysemu/kvm.h +++ b/include/sysemu/kvm.h @@ -14,9 +14,6 @@ #ifndef QEMU_KVM_H #define QEMU_KVM_H -#include "qemu/queue.h" -#include "hw/core/cpu.h" -#include "exec/memattrs.h" #include "qemu/accel.h" #include "qom/object.h" diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h index a641c974ea..511b42bde5 100644 --- a/include/sysemu/kvm_int.h +++ b/include/sysemu/kvm_int.h @@ -115,6 +115,7 @@ struct KVMState } *as; uint64_t kvm_dirty_ring_bytes; /* Size of the per-vcpu dirty ring */ uint32_t kvm_dirty_ring_size; /* Number of dirty GFNs per ring */ + bool kvm_dirty_ring_with_bitmap; struct KVMDirtyRingReaper reaper; NotifyVmexitOption notify_vmexit; uint32_t notify_window; diff --git a/include/tcg/tcg-ldst.h b/include/tcg/tcg-ldst.h index 684e394b06..6ccfe9131d 100644 --- a/include/tcg/tcg-ldst.h +++ b/include/tcg/tcg-ldst.h @@ -25,59 +25,39 @@ #ifndef TCG_LDST_H #define TCG_LDST_H -#ifdef CONFIG_SOFTMMU - /* Value zero-extended to tcg register size. */ -tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); +tcg_target_ulong helper_ldub_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr); +tcg_target_ulong helper_lduw_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr); +tcg_target_ulong helper_ldul_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr); +uint64_t helper_ldq_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr); +Int128 helper_ld16_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr); /* Value sign-extended to tcg register size. */ -tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); +tcg_target_ulong helper_ldsb_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr); +tcg_target_ulong helper_ldsw_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr); +tcg_target_ulong helper_ldsl_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t retaddr); /* * Value extended to at least uint32_t, so that some ABIs do not require * zero-extension from uint8_t or uint16_t. */ -void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr); -void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr); -void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr); -void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - MemOpIdx oi, uintptr_t retaddr); -void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr); -void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr); -void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - MemOpIdx oi, uintptr_t retaddr); - -#else - -G_NORETURN void helper_unaligned_ld(CPUArchState *env, target_ulong addr); -G_NORETURN void helper_unaligned_st(CPUArchState *env, target_ulong addr); +void helper_stb_mmu(CPUArchState *env, uint64_t addr, uint32_t val, + MemOpIdx oi, uintptr_t retaddr); +void helper_stw_mmu(CPUArchState *env, uint64_t addr, uint32_t val, + MemOpIdx oi, uintptr_t retaddr); +void helper_stl_mmu(CPUArchState *env, uint64_t addr, uint32_t val, + MemOpIdx oi, uintptr_t retaddr); +void helper_stq_mmu(CPUArchState *env, uint64_t addr, uint64_t val, + MemOpIdx oi, uintptr_t retaddr); +void helper_st16_mmu(CPUArchState *env, uint64_t addr, Int128 val, + MemOpIdx oi, uintptr_t retaddr); -#endif /* CONFIG_SOFTMMU */ #endif /* TCG_LDST_H */ diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h index 4401fa493c..35c5700183 100644 --- a/include/tcg/tcg-op.h +++ b/include/tcg/tcg-op.h @@ -723,48 +723,27 @@ static inline void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi) #endif #if TARGET_INSN_START_WORDS == 1 -# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS static inline void tcg_gen_insn_start(target_ulong pc) { - tcg_gen_op1(INDEX_op_insn_start, pc); + TCGOp *op = tcg_emit_op(INDEX_op_insn_start, 64 / TCG_TARGET_REG_BITS); + tcg_set_insn_start_param(op, 0, pc); } -# else -static inline void tcg_gen_insn_start(target_ulong pc) -{ - tcg_gen_op2(INDEX_op_insn_start, (uint32_t)pc, (uint32_t)(pc >> 32)); -} -# endif #elif TARGET_INSN_START_WORDS == 2 -# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS -static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1) -{ - tcg_gen_op2(INDEX_op_insn_start, pc, a1); -} -# else static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1) { - tcg_gen_op4(INDEX_op_insn_start, - (uint32_t)pc, (uint32_t)(pc >> 32), - (uint32_t)a1, (uint32_t)(a1 >> 32)); + TCGOp *op = tcg_emit_op(INDEX_op_insn_start, 2 * 64 / TCG_TARGET_REG_BITS); + tcg_set_insn_start_param(op, 0, pc); + tcg_set_insn_start_param(op, 1, a1); } -# endif #elif TARGET_INSN_START_WORDS == 3 -# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS -static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1, - target_ulong a2) -{ - tcg_gen_op3(INDEX_op_insn_start, pc, a1, a2); -} -# else static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1, target_ulong a2) { - tcg_gen_op6(INDEX_op_insn_start, - (uint32_t)pc, (uint32_t)(pc >> 32), - (uint32_t)a1, (uint32_t)(a1 >> 32), - (uint32_t)a2, (uint32_t)(a2 >> 32)); + TCGOp *op = tcg_emit_op(INDEX_op_insn_start, 3 * 64 / TCG_TARGET_REG_BITS); + tcg_set_insn_start_param(op, 0, pc); + tcg_set_insn_start_param(op, 1, a1); + tcg_set_insn_start_param(op, 2, a2); } -# endif #else # error "Unhandled number of operands to insn_start" #endif @@ -824,73 +803,203 @@ static inline void tcg_gen_plugin_cb_end(void) #define tcg_temp_new() tcg_temp_new_i32() #define tcg_global_mem_new tcg_global_mem_new_i32 #define tcg_temp_free tcg_temp_free_i32 +#define tcgv_tl_temp tcgv_i32_temp #define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i32 #define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i32 #else #define tcg_temp_new() tcg_temp_new_i64() #define tcg_global_mem_new tcg_global_mem_new_i64 #define tcg_temp_free tcg_temp_free_i64 +#define tcgv_tl_temp tcgv_i64_temp #define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i64 #define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i64 #endif -void tcg_gen_qemu_ld_i32(TCGv_i32, TCGv, TCGArg, MemOp); -void tcg_gen_qemu_st_i32(TCGv_i32, TCGv, TCGArg, MemOp); -void tcg_gen_qemu_ld_i64(TCGv_i64, TCGv, TCGArg, MemOp); -void tcg_gen_qemu_st_i64(TCGv_i64, TCGv, TCGArg, MemOp); -void tcg_gen_qemu_ld_i128(TCGv_i128, TCGv, TCGArg, MemOp); -void tcg_gen_qemu_st_i128(TCGv_i128, TCGv, TCGArg, MemOp); - -void tcg_gen_atomic_cmpxchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGv_i32, - TCGArg, MemOp); -void tcg_gen_atomic_cmpxchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGv_i64, - TCGArg, MemOp); -void tcg_gen_atomic_cmpxchg_i128(TCGv_i128, TCGv, TCGv_i128, TCGv_i128, - TCGArg, MemOp); - -void tcg_gen_nonatomic_cmpxchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGv_i32, - TCGArg, MemOp); -void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGv_i64, - TCGArg, MemOp); -void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128, TCGv, TCGv_i128, TCGv_i128, - TCGArg, MemOp); - -void tcg_gen_atomic_xchg_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_xchg_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); - -void tcg_gen_atomic_fetch_add_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_fetch_add_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_fetch_and_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_fetch_and_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_fetch_or_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_fetch_or_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_fetch_xor_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_fetch_xor_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_fetch_smin_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_fetch_smin_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_fetch_umin_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_fetch_umin_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_fetch_smax_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_fetch_smax_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_fetch_umax_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_fetch_umax_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); - -void tcg_gen_atomic_add_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_add_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_and_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_and_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_or_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_or_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_xor_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_xor_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_smin_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_smin_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_umin_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_umin_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_smax_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_smax_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); -void tcg_gen_atomic_umax_fetch_i32(TCGv_i32, TCGv, TCGv_i32, TCGArg, MemOp); -void tcg_gen_atomic_umax_fetch_i64(TCGv_i64, TCGv, TCGv_i64, TCGArg, MemOp); +void tcg_gen_qemu_ld_i32_chk(TCGv_i32, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_st_i32_chk(TCGv_i32, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_ld_i64_chk(TCGv_i64, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_st_i64_chk(TCGv_i64, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_ld_i128_chk(TCGv_i128, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_st_i128_chk(TCGv_i128, TCGTemp *, TCGArg, MemOp, TCGType); + +static inline void +tcg_gen_qemu_ld_i32(TCGv_i32 v, TCGv a, TCGArg i, MemOp m) +{ + tcg_gen_qemu_ld_i32_chk(v, tcgv_tl_temp(a), i, m, TCG_TYPE_TL); +} + +static inline void +tcg_gen_qemu_st_i32(TCGv_i32 v, TCGv a, TCGArg i, MemOp m) +{ + tcg_gen_qemu_st_i32_chk(v, tcgv_tl_temp(a), i, m, TCG_TYPE_TL); +} + +static inline void +tcg_gen_qemu_ld_i64(TCGv_i64 v, TCGv a, TCGArg i, MemOp m) +{ + tcg_gen_qemu_ld_i64_chk(v, tcgv_tl_temp(a), i, m, TCG_TYPE_TL); +} + +static inline void +tcg_gen_qemu_st_i64(TCGv_i64 v, TCGv a, TCGArg i, MemOp m) +{ + tcg_gen_qemu_st_i64_chk(v, tcgv_tl_temp(a), i, m, TCG_TYPE_TL); +} + +static inline void +tcg_gen_qemu_ld_i128(TCGv_i128 v, TCGv a, TCGArg i, MemOp m) +{ + tcg_gen_qemu_ld_i128_chk(v, tcgv_tl_temp(a), i, m, TCG_TYPE_TL); +} + +static inline void +tcg_gen_qemu_st_i128(TCGv_i128 v, TCGv a, TCGArg i, MemOp m) +{ + tcg_gen_qemu_st_i128_chk(v, tcgv_tl_temp(a), i, m, TCG_TYPE_TL); +} + +void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128, + TCGv_i128, TCGArg, MemOp, TCGType); + +void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128, + TCGv_i128, TCGArg, MemOp, TCGType); + +void tcg_gen_atomic_xchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_xchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); + +void tcg_gen_atomic_fetch_add_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_add_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_and_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_and_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_or_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_or_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_xor_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_xor_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_smin_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_smin_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_umin_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_umin_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_smax_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_smax_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_umax_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_umax_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); + +void tcg_gen_atomic_add_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_add_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_and_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_and_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_or_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_or_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_xor_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_xor_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_smin_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_smin_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_umin_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_umin_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_smax_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_smax_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_umax_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_umax_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); + +#define DEF_ATOMIC2(N, S) \ + static inline void N##_##S(TCGv_##S r, TCGv a, TCGv_##S v, \ + TCGArg i, MemOp m) \ + { N##_##S##_chk(r, tcgv_tl_temp(a), v, i, m, TCG_TYPE_TL); } + +#define DEF_ATOMIC3(N, S) \ + static inline void N##_##S(TCGv_##S r, TCGv a, TCGv_##S o, \ + TCGv_##S n, TCGArg i, MemOp m) \ + { N##_##S##_chk(r, tcgv_tl_temp(a), o, n, i, m, TCG_TYPE_TL); } + +DEF_ATOMIC3(tcg_gen_atomic_cmpxchg, i32) +DEF_ATOMIC3(tcg_gen_atomic_cmpxchg, i64) +DEF_ATOMIC3(tcg_gen_atomic_cmpxchg, i128) + +DEF_ATOMIC3(tcg_gen_nonatomic_cmpxchg, i32) +DEF_ATOMIC3(tcg_gen_nonatomic_cmpxchg, i64) +DEF_ATOMIC3(tcg_gen_nonatomic_cmpxchg, i128) + +DEF_ATOMIC2(tcg_gen_atomic_xchg, i32) +DEF_ATOMIC2(tcg_gen_atomic_xchg, i64) + +DEF_ATOMIC2(tcg_gen_atomic_fetch_add, i32) +DEF_ATOMIC2(tcg_gen_atomic_fetch_add, i64) +DEF_ATOMIC2(tcg_gen_atomic_fetch_and, i32) +DEF_ATOMIC2(tcg_gen_atomic_fetch_and, i64) +DEF_ATOMIC2(tcg_gen_atomic_fetch_or, i32) +DEF_ATOMIC2(tcg_gen_atomic_fetch_or, i64) +DEF_ATOMIC2(tcg_gen_atomic_fetch_xor, i32) +DEF_ATOMIC2(tcg_gen_atomic_fetch_xor, i64) +DEF_ATOMIC2(tcg_gen_atomic_fetch_smin, i32) +DEF_ATOMIC2(tcg_gen_atomic_fetch_smin, i64) +DEF_ATOMIC2(tcg_gen_atomic_fetch_umin, i32) +DEF_ATOMIC2(tcg_gen_atomic_fetch_umin, i64) +DEF_ATOMIC2(tcg_gen_atomic_fetch_smax, i32) +DEF_ATOMIC2(tcg_gen_atomic_fetch_smax, i64) +DEF_ATOMIC2(tcg_gen_atomic_fetch_umax, i32) +DEF_ATOMIC2(tcg_gen_atomic_fetch_umax, i64) + +DEF_ATOMIC2(tcg_gen_atomic_add_fetch, i32) +DEF_ATOMIC2(tcg_gen_atomic_add_fetch, i64) +DEF_ATOMIC2(tcg_gen_atomic_and_fetch, i32) +DEF_ATOMIC2(tcg_gen_atomic_and_fetch, i64) +DEF_ATOMIC2(tcg_gen_atomic_or_fetch, i32) +DEF_ATOMIC2(tcg_gen_atomic_or_fetch, i64) +DEF_ATOMIC2(tcg_gen_atomic_xor_fetch, i32) +DEF_ATOMIC2(tcg_gen_atomic_xor_fetch, i64) +DEF_ATOMIC2(tcg_gen_atomic_smin_fetch, i32) +DEF_ATOMIC2(tcg_gen_atomic_smin_fetch, i64) +DEF_ATOMIC2(tcg_gen_atomic_umin_fetch, i32) +DEF_ATOMIC2(tcg_gen_atomic_umin_fetch, i64) +DEF_ATOMIC2(tcg_gen_atomic_smax_fetch, i32) +DEF_ATOMIC2(tcg_gen_atomic_smax_fetch, i64) +DEF_ATOMIC2(tcg_gen_atomic_umax_fetch, i32) +DEF_ATOMIC2(tcg_gen_atomic_umax_fetch, i64) + +#undef DEF_ATOMIC2 +#undef DEF_ATOMIC3 void tcg_gen_mov_vec(TCGv_vec, TCGv_vec); void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec, TCGv_i32); diff --git a/include/tcg/tcg-opc.h b/include/tcg/tcg-opc.h index dd444734d9..21594c1590 100644 --- a/include/tcg/tcg-opc.h +++ b/include/tcg/tcg-opc.h @@ -186,11 +186,10 @@ DEF(muls2_i64, 2, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_muls2_i64)) DEF(muluh_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_muluh_i64)) DEF(mulsh_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_mulsh_i64)) -#define TLADDR_ARGS (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? 1 : 2) #define DATA64_ARGS (TCG_TARGET_REG_BITS == 64 ? 1 : 2) /* QEMU specific */ -DEF(insn_start, 0, 0, TLADDR_ARGS * TARGET_INSN_START_WORDS, +DEF(insn_start, 0, 0, DATA64_ARGS * TARGET_INSN_START_WORDS, TCG_OPF_NOT_PRESENT) DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END) DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END) @@ -199,20 +198,47 @@ DEF(goto_ptr, 0, 1, 0, TCG_OPF_BB_EXIT | TCG_OPF_BB_END) DEF(plugin_cb_start, 0, 0, 3, TCG_OPF_NOT_PRESENT) DEF(plugin_cb_end, 0, 0, 0, TCG_OPF_NOT_PRESENT) -DEF(qemu_ld_i32, 1, TLADDR_ARGS, 1, +/* Replicate ld/st ops for 32 and 64-bit guest addresses. */ +DEF(qemu_ld_a32_i32, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) -DEF(qemu_st_i32, 0, TLADDR_ARGS + 1, 1, +DEF(qemu_st_a32_i32, 0, 1 + 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) -DEF(qemu_ld_i64, DATA64_ARGS, TLADDR_ARGS, 1, +DEF(qemu_ld_a32_i64, DATA64_ARGS, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT) -DEF(qemu_st_i64, 0, TLADDR_ARGS + DATA64_ARGS, 1, +DEF(qemu_st_a32_i64, 0, DATA64_ARGS + 1, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT) + +DEF(qemu_ld_a64_i32, 1, DATA64_ARGS, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) +DEF(qemu_st_a64_i32, 0, 1 + DATA64_ARGS, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) +DEF(qemu_ld_a64_i64, DATA64_ARGS, DATA64_ARGS, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT) +DEF(qemu_st_a64_i64, 0, DATA64_ARGS + DATA64_ARGS, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT) /* Only used by i386 to cope with stupid register constraints. */ -DEF(qemu_st8_i32, 0, TLADDR_ARGS + 1, 1, +DEF(qemu_st8_a32_i32, 0, 1 + 1, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | + IMPL(TCG_TARGET_HAS_qemu_st8_i32)) +DEF(qemu_st8_a64_i32, 0, 1 + DATA64_ARGS, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | IMPL(TCG_TARGET_HAS_qemu_st8_i32)) +/* Only for 64-bit hosts at the moment. */ +DEF(qemu_ld_a32_i128, 2, 1, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT | + IMPL(TCG_TARGET_HAS_qemu_ldst_i128)) +DEF(qemu_ld_a64_i128, 2, 1, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT | + IMPL(TCG_TARGET_HAS_qemu_ldst_i128)) +DEF(qemu_st_a32_i128, 0, 3, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT | + IMPL(TCG_TARGET_HAS_qemu_ldst_i128)) +DEF(qemu_st_a64_i128, 0, 3, 1, + TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT | + IMPL(TCG_TARGET_HAS_qemu_ldst_i128)) + /* Host vector support. */ #define IMPLVEC TCG_OPF_VECTOR | IMPL(TCG_TARGET_MAYBE_vec) @@ -283,7 +309,6 @@ DEF(tci_movi, 1, 0, 1, TCG_OPF_NOT_PRESENT) DEF(tci_movl, 1, 0, 1, TCG_OPF_NOT_PRESENT) #endif -#undef TLADDR_ARGS #undef DATA64_ARGS #undef IMPL #undef IMPL64 diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index b19e167e1d..cd6327b175 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -558,6 +558,13 @@ struct TCGContext { int nb_temps; int nb_indirects; int nb_ops; + TCGType addr_type; /* TCG_TYPE_I32 or TCG_TYPE_I64 */ + +#ifdef CONFIG_SOFTMMU + int page_mask; + uint8_t page_bits; + uint8_t tlb_dyn_max_bits; +#endif TCGRegSet reserved_regs; intptr_t current_frame_offset; @@ -629,7 +636,7 @@ struct TCGContext { TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS]; uint16_t gen_insn_end_off[TCG_MAX_INSNS]; - target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS]; + uint64_t gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS]; /* Exit to translator on overflow. */ sigjmp_buf jmp_trans; @@ -771,24 +778,24 @@ static inline void tcg_set_insn_param(TCGOp *op, int arg, TCGArg v) op->args[arg] = v; } -static inline target_ulong tcg_get_insn_start_param(TCGOp *op, int arg) +static inline uint64_t tcg_get_insn_start_param(TCGOp *op, int arg) { -#if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - return tcg_get_insn_param(op, arg); -#else - return tcg_get_insn_param(op, arg * 2) | - ((uint64_t)tcg_get_insn_param(op, arg * 2 + 1) << 32); -#endif + if (TCG_TARGET_REG_BITS == 64) { + return tcg_get_insn_param(op, arg); + } else { + return deposit64(tcg_get_insn_param(op, arg * 2), 32, 32, + tcg_get_insn_param(op, arg * 2 + 1)); + } } -static inline void tcg_set_insn_start_param(TCGOp *op, int arg, target_ulong v) +static inline void tcg_set_insn_start_param(TCGOp *op, int arg, uint64_t v) { -#if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - tcg_set_insn_param(op, arg, v); -#else - tcg_set_insn_param(op, arg * 2, v); - tcg_set_insn_param(op, arg * 2 + 1, v >> 32); -#endif + if (TCG_TARGET_REG_BITS == 64) { + tcg_set_insn_param(op, arg, v); + } else { + tcg_set_insn_param(op, arg * 2, v); + tcg_set_insn_param(op, arg * 2 + 1, v >> 32); + } } /* The last op that was emitted. */ @@ -852,7 +859,7 @@ void tcg_register_thread(void); void tcg_prologue_init(TCGContext *s); void tcg_func_start(TCGContext *s); -int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start); +int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start); void tb_target_set_jmp_target(const TranslationBlock *, int, uintptr_t, uintptr_t); diff --git a/linux-user/main.c b/linux-user/main.c index 5defe5a6db..5e6b2e1714 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -359,10 +359,7 @@ static void handle_arg_cpu(const char *arg) { cpu_model = strdup(arg); if (cpu_model == NULL || is_help_option(cpu_model)) { - /* XXX: implement xxx_cpu_list for targets that still miss it */ -#if defined(cpu_list) - cpu_list(); -#endif + list_cpus(); exit(EXIT_FAILURE); } } diff --git a/linux-user/mips/cpu_loop.c b/linux-user/mips/cpu_loop.c index d5c1c7941d..8735e58bad 100644 --- a/linux-user/mips/cpu_loop.c +++ b/linux-user/mips/cpu_loop.c @@ -290,7 +290,10 @@ void target_cpu_copy_regs(CPUArchState *env, struct target_pt_regs *regs) env->CP0_Status |= (1 << CP0St_FR); env->hflags |= MIPS_HFLAG_F64; } - } else if (!prog_req.fre && !prog_req.frdefault && + } else if (prog_req.fr1) { + env->CP0_Status |= (1 << CP0St_FR); + env->hflags |= MIPS_HFLAG_F64; + } else if (!prog_req.fre && !prog_req.frdefault && !prog_req.fr1 && !prog_req.single && !prog_req.soft) { fprintf(stderr, "qemu: Can't find a matching FPU mode\n"); exit(1); diff --git a/linux-user/qemu.h b/linux-user/qemu.h index e2e93fbd1d..92f9f5af41 100644 --- a/linux-user/qemu.h +++ b/linux-user/qemu.h @@ -168,6 +168,7 @@ abi_long do_brk(abi_ulong new_brk); /* user access */ +#define VERIFY_NONE 0 #define VERIFY_READ PAGE_READ #define VERIFY_WRITE (PAGE_READ | PAGE_WRITE) diff --git a/linux-user/syscall.c b/linux-user/syscall.c index 69f740ff98..89b58b386b 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -5747,7 +5747,7 @@ static abi_long do_ioctl(int fd, int cmd, abi_long arg) if (ie->target_cmd == 0) { qemu_log_mask( LOG_UNIMP, "Unsupported ioctl: cmd=0x%04lx\n", (long)cmd); - return -TARGET_ENOSYS; + return -TARGET_ENOTTY; } if (ie->target_cmd == cmd) break; @@ -5759,7 +5759,7 @@ static abi_long do_ioctl(int fd, int cmd, abi_long arg) } else if (!ie->host_cmd) { /* Some architectures define BSD ioctls in their headers that are not implemented in Linux. */ - return -TARGET_ENOSYS; + return -TARGET_ENOTTY; } switch(arg_type[0]) { @@ -5817,7 +5817,7 @@ static abi_long do_ioctl(int fd, int cmd, abi_long arg) qemu_log_mask(LOG_UNIMP, "Unsupported ioctl type: cmd=0x%04lx type=%d\n", (long)cmd, arg_type[0]); - ret = -TARGET_ENOSYS; + ret = -TARGET_ENOTTY; break; } return ret; @@ -8231,7 +8231,8 @@ void target_exception_dump(CPUArchState *env, const char *fmt, int code) } #if HOST_BIG_ENDIAN != TARGET_BIG_ENDIAN || \ - defined(TARGET_SPARC) || defined(TARGET_M68K) || defined(TARGET_HPPA) + defined(TARGET_SPARC) || defined(TARGET_M68K) || defined(TARGET_HPPA) || \ + defined(TARGET_RISCV) static int is_proc(const char *filename, const char *entry) { return strcmp(filename, entry) == 0; @@ -8309,6 +8310,35 @@ static int open_cpuinfo(CPUArchState *cpu_env, int fd) } #endif +#if defined(TARGET_RISCV) +static int open_cpuinfo(CPUArchState *cpu_env, int fd) +{ + int i; + int num_cpus = sysconf(_SC_NPROCESSORS_ONLN); + RISCVCPU *cpu = env_archcpu(cpu_env); + const RISCVCPUConfig *cfg = riscv_cpu_cfg((CPURISCVState *) cpu_env); + char *isa_string = riscv_isa_string(cpu); + const char *mmu; + + if (cfg->mmu) { + mmu = (cpu_env->xl == MXL_RV32) ? "sv32" : "sv48"; + } else { + mmu = "none"; + } + + for (i = 0; i < num_cpus; i++) { + dprintf(fd, "processor\t: %d\n", i); + dprintf(fd, "hart\t\t: %d\n", i); + dprintf(fd, "isa\t\t: %s\n", isa_string); + dprintf(fd, "mmu\t\t: %s\n", mmu); + dprintf(fd, "uarch\t\t: qemu\n\n"); + } + + g_free(isa_string); + return 0; +} +#endif + #if defined(TARGET_M68K) static int open_hardware(CPUArchState *cpu_env, int fd) { @@ -8333,7 +8363,7 @@ static int do_openat(CPUArchState *cpu_env, int dirfd, const char *pathname, int #if HOST_BIG_ENDIAN != TARGET_BIG_ENDIAN { "/proc/net/route", open_net_route, is_proc }, #endif -#if defined(TARGET_SPARC) || defined(TARGET_HPPA) +#if defined(TARGET_SPARC) || defined(TARGET_HPPA) || defined(TARGET_RISCV) { "/proc/cpuinfo", open_cpuinfo, is_proc }, #endif #if defined(TARGET_M68K) @@ -8746,6 +8776,18 @@ static int do_getdents64(abi_long dirfd, abi_long arg2, abi_long count) _syscall2(int, pivot_root, const char *, new_root, const char *, put_old) #endif +#if defined(TARGET_NR_open_tree) && defined(__NR_open_tree) +#define __NR_sys_open_tree __NR_open_tree +_syscall3(int, sys_open_tree, int, __dfd, const char *, __filename, + unsigned int, __flags) +#endif + +#if defined(TARGET_NR_move_mount) && defined(__NR_move_mount) +#define __NR_sys_move_mount __NR_move_mount +_syscall5(int, sys_move_mount, int, __from_dfd, const char *, __from_pathname, + int, __to_dfd, const char *, __to_pathname, unsigned int, flag) +#endif + /* This is an internal helper for do_syscall so that it is easier * to have a single return point, so that actions, such as logging * of syscall results, can be performed. @@ -9139,6 +9181,60 @@ static abi_long do_syscall1(CPUArchState *cpu_env, int num, abi_long arg1, unlock_user(p, arg1, 0); return ret; #endif +#if defined(TARGET_NR_move_mount) && defined(__NR_move_mount) + case TARGET_NR_move_mount: + { + void *p2, *p4; + + if (!arg2 || !arg4) { + return -TARGET_EFAULT; + } + + p2 = lock_user_string(arg2); + if (!p2) { + return -TARGET_EFAULT; + } + + p4 = lock_user_string(arg4); + if (!p4) { + unlock_user(p2, arg2, 0); + return -TARGET_EFAULT; + } + ret = get_errno(sys_move_mount(arg1, p2, arg3, p4, arg5)); + + unlock_user(p2, arg2, 0); + unlock_user(p4, arg4, 0); + + return ret; + } +#endif +#if defined(TARGET_NR_open_tree) && defined(__NR_open_tree) + case TARGET_NR_open_tree: + { + void *p2; + int host_flags; + + if (!arg2) { + return -TARGET_EFAULT; + } + + p2 = lock_user_string(arg2); + if (!p2) { + return -TARGET_EFAULT; + } + + host_flags = arg3 & ~TARGET_O_CLOEXEC; + if (arg3 & TARGET_O_CLOEXEC) { + host_flags |= O_CLOEXEC; + } + + ret = get_errno(sys_open_tree(arg1, p2, host_flags)); + + unlock_user(p2, arg2, 0); + + return ret; + } +#endif #ifdef TARGET_NR_stime /* not on alpha */ case TARGET_NR_stime: { @@ -11475,39 +11571,58 @@ static abi_long do_syscall1(CPUArchState *cpu_env, int num, abi_long arg1, { int gidsetsize = arg1; target_id *target_grouplist; - gid_t *grouplist; + g_autofree gid_t *grouplist = NULL; int i; - grouplist = alloca(gidsetsize * sizeof(gid_t)); + if (gidsetsize > NGROUPS_MAX) { + return -TARGET_EINVAL; + } + if (gidsetsize > 0) { + grouplist = g_try_new(gid_t, gidsetsize); + if (!grouplist) { + return -TARGET_ENOMEM; + } + } ret = get_errno(getgroups(gidsetsize, grouplist)); - if (gidsetsize == 0) - return ret; - if (!is_error(ret)) { - target_grouplist = lock_user(VERIFY_WRITE, arg2, gidsetsize * sizeof(target_id), 0); - if (!target_grouplist) + if (!is_error(ret) && gidsetsize > 0) { + target_grouplist = lock_user(VERIFY_WRITE, arg2, + gidsetsize * sizeof(target_id), 0); + if (!target_grouplist) { return -TARGET_EFAULT; - for(i = 0;i < ret; i++) + } + for (i = 0; i < ret; i++) { target_grouplist[i] = tswapid(high2lowgid(grouplist[i])); - unlock_user(target_grouplist, arg2, gidsetsize * sizeof(target_id)); + } + unlock_user(target_grouplist, arg2, + gidsetsize * sizeof(target_id)); } + return ret; } - return ret; case TARGET_NR_setgroups: { int gidsetsize = arg1; target_id *target_grouplist; - gid_t *grouplist = NULL; + g_autofree gid_t *grouplist = NULL; int i; - if (gidsetsize) { - grouplist = alloca(gidsetsize * sizeof(gid_t)); - target_grouplist = lock_user(VERIFY_READ, arg2, gidsetsize * sizeof(target_id), 1); + + if (gidsetsize > NGROUPS_MAX || gidsetsize < 0) { + return -TARGET_EINVAL; + } + if (gidsetsize > 0) { + grouplist = g_try_new(gid_t, gidsetsize); + if (!grouplist) { + return -TARGET_ENOMEM; + } + target_grouplist = lock_user(VERIFY_READ, arg2, + gidsetsize * sizeof(target_id), 1); if (!target_grouplist) { return -TARGET_EFAULT; } for (i = 0; i < gidsetsize; i++) { grouplist[i] = low2highgid(tswapid(target_grouplist[i])); } - unlock_user(target_grouplist, arg2, 0); + unlock_user(target_grouplist, arg2, + gidsetsize * sizeof(target_id)); } return get_errno(setgroups(gidsetsize, grouplist)); } @@ -11792,41 +11907,59 @@ static abi_long do_syscall1(CPUArchState *cpu_env, int num, abi_long arg1, { int gidsetsize = arg1; uint32_t *target_grouplist; - gid_t *grouplist; + g_autofree gid_t *grouplist = NULL; int i; - grouplist = alloca(gidsetsize * sizeof(gid_t)); + if (gidsetsize > NGROUPS_MAX) { + return -TARGET_EINVAL; + } + if (gidsetsize > 0) { + grouplist = g_try_new(gid_t, gidsetsize); + if (!grouplist) { + return -TARGET_ENOMEM; + } + } ret = get_errno(getgroups(gidsetsize, grouplist)); - if (gidsetsize == 0) - return ret; - if (!is_error(ret)) { - target_grouplist = lock_user(VERIFY_WRITE, arg2, gidsetsize * 4, 0); + if (!is_error(ret) && gidsetsize > 0) { + target_grouplist = lock_user(VERIFY_WRITE, arg2, + gidsetsize * 4, 0); if (!target_grouplist) { return -TARGET_EFAULT; } - for(i = 0;i < ret; i++) + for (i = 0; i < ret; i++) { target_grouplist[i] = tswap32(grouplist[i]); + } unlock_user(target_grouplist, arg2, gidsetsize * 4); } + return ret; } - return ret; #endif #ifdef TARGET_NR_setgroups32 case TARGET_NR_setgroups32: { int gidsetsize = arg1; uint32_t *target_grouplist; - gid_t *grouplist; + g_autofree gid_t *grouplist = NULL; int i; - grouplist = alloca(gidsetsize * sizeof(gid_t)); - target_grouplist = lock_user(VERIFY_READ, arg2, gidsetsize * 4, 1); - if (!target_grouplist) { - return -TARGET_EFAULT; + if (gidsetsize > NGROUPS_MAX || gidsetsize < 0) { + return -TARGET_EINVAL; + } + if (gidsetsize > 0) { + grouplist = g_try_new(gid_t, gidsetsize); + if (!grouplist) { + return -TARGET_ENOMEM; + } + target_grouplist = lock_user(VERIFY_READ, arg2, + gidsetsize * 4, 1); + if (!target_grouplist) { + return -TARGET_EFAULT; + } + for (i = 0; i < gidsetsize; i++) { + grouplist[i] = tswap32(target_grouplist[i]); + } + unlock_user(target_grouplist, arg2, 0); } - for(i = 0;i < gidsetsize; i++) - grouplist[i] = tswap32(target_grouplist[i]); - unlock_user(target_grouplist, arg2, 0); return get_errno(setgroups(gidsetsize, grouplist)); } #endif @@ -11897,7 +12030,7 @@ static abi_long do_syscall1(CPUArchState *cpu_env, int num, abi_long arg1, #ifdef TARGET_NR_mincore case TARGET_NR_mincore: { - void *a = lock_user(VERIFY_READ, arg1, arg2, 0); + void *a = lock_user(VERIFY_NONE, arg1, arg2, 0); if (!a) { return -TARGET_ENOMEM; } diff --git a/meson b/meson deleted file mode 160000 -Subproject 3a9b285a55b91b53b2acda987192274352ecb5b diff --git a/meson.build b/meson.build index b36124fdc4..0a5cdefd4d 100644 --- a/meson.build +++ b/meson.build @@ -1,4 +1,4 @@ -project('qemu', ['c'], meson_version: '>=0.61.3', +project('qemu', ['c'], meson_version: '>=0.63.0', default_options: ['warning_level=1', 'c_std=gnu11', 'cpp_std=gnu++11', 'b_colorout=auto', 'b_staticpic=false', 'stdsplit=false', 'optimization=2', 'b_pie=true'], version: files('VERSION')) @@ -17,11 +17,6 @@ fs = import('fs') targetos = host_machine.system() sh = find_program('sh') config_host = keyval.load(meson.current_build_dir() / 'config-host.mak') -enable_modules = 'CONFIG_MODULES' in config_host -enable_static = 'CONFIG_STATIC' in config_host - -# Allow both shared and static libraries unless --enable-static -static_kwargs = enable_static ? {'static': true} : {} cc = meson.get_compiler('c') all_languages = ['c'] @@ -88,6 +83,12 @@ have_ga = get_option('guest_agent') \ .require(targetos in ['sunos', 'linux', 'windows', 'freebsd', 'netbsd', 'openbsd'], error_message: 'unsupported OS for QEMU guest agent') \ .allowed() +enable_modules = get_option('modules') \ + .require(targetos != 'windows', + error_message: 'Modules are not available for Windows') \ + .require(not get_option('prefer_static'), + error_message: 'Modules are incompatible with static linking') \ + .allowed() have_block = have_system or have_tools python = import('python').find_installation() @@ -189,14 +190,197 @@ endif # Compiler flags # ################## -qemu_cflags = config_host['QEMU_CFLAGS'].split() -qemu_objcflags = config_host['QEMU_OBJCFLAGS'].split() -qemu_ldflags = config_host['QEMU_LDFLAGS'].split() +foreach lang : all_languages + compiler = meson.get_compiler(lang) + if compiler.get_id() == 'gcc' and compiler.version().version_compare('>=7.4') + # ok + elif compiler.get_id() == 'clang' and compiler.compiles(''' + #ifdef __apple_build_version__ + # if __clang_major__ < 12 || (__clang_major__ == 12 && __clang_minor__ < 0) + # error You need at least XCode Clang v12.0 to compile QEMU + # endif + #else + # if __clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 0) + # error You need at least Clang v10.0 to compile QEMU + # endif + #endif''') + # ok + else + error('You either need GCC v7.4 or Clang v10.0 (or XCode Clang v12.0) to compile QEMU') + endif +endforeach + +# default flags for all hosts +# We use -fwrapv to tell the compiler that we require a C dialect where +# left shift of signed integers is well defined and has the expected +# 2s-complement style results. (Both clang and gcc agree that it +# provides these semantics.) + +qemu_common_flags = [ + '-D_GNU_SOURCE', '-D_FILE_OFFSET_BITS=64', '-D_LARGEFILE_SOURCE', + '-fno-strict-aliasing', '-fno-common', '-fwrapv' ] +qemu_cflags = [] +qemu_ldflags = [] + +if targetos == 'darwin' + # Disable attempts to use ObjectiveC features in os/object.h since they + # won't work when we're compiling with gcc as a C compiler. + qemu_common_flags += '-DOS_OBJECT_USE_OBJC=0' +elif targetos == 'solaris' + # needed for CMSG_ macros in sys/socket.h + qemu_common_flags += '-D_XOPEN_SOURCE=600' + # needed for TIOCWIN* defines in termios.h + qemu_common_flags += '-D__EXTENSIONS__' +elif targetos == 'haiku' + qemu_common_flags += ['-DB_USE_POSITIVE_POSIX_ERRORS', '-D_BSD_SOURCE', '-fPIC'] +endif + +# __sync_fetch_and_and requires at least -march=i486. Many toolchains +# use i686 as default anyway, but for those that don't, an explicit +# specification is necessary +if host_arch == 'i386' and not cc.links(''' + static int sfaa(int *ptr) + { + return __sync_fetch_and_and(ptr, 0); + } + + int main(void) + { + int val = 42; + val = __sync_val_compare_and_swap(&val, 0, 1); + sfaa(&val); + return val; + }''') + qemu_common_flags = ['-march=i486'] + qemu_common_flags +endif + +if get_option('gprof') + qemu_common_flags += ['-p'] + qemu_ldflags += ['-p'] +endif -if enable_static +if get_option('prefer_static') qemu_ldflags += get_option('b_pie') ? '-static-pie' : '-static' endif +# Meson currently only handles pie as a boolean for now, so if the user +# has explicitly disabled PIE we need to extend our cflags. +if not get_option('b_pie') + qemu_common_flags += cc.get_supported_arguments('-fno-pie') + if not get_option('prefer_static') + # No PIE is implied by -static which we added above. + qemu_ldflags += cc.get_supported_link_arguments('-no-pie') + endif +endif + +if not get_option('stack_protector').disabled() + stack_protector_probe = ''' + int main(int argc, char *argv[]) + { + char arr[64], *p = arr, *c = argv[argc - 1]; + while (*c) { + *p++ = *c++; + } + return 0; + }''' + have_stack_protector = false + foreach arg : ['-fstack-protector-strong', '-fstack-protector-all'] + # We need to check both a compile and a link, since some compiler + # setups fail only on a .c->.o compile and some only at link time + if cc.compiles(stack_protector_probe, args: ['-Werror', arg]) and \ + cc.links(stack_protector_probe, args: ['-Werror', arg]) + have_stack_protector = true + qemu_cflags += arg + qemu_ldflags += arg + break + endif + endforeach + get_option('stack_protector') \ + .require(have_stack_protector, error_message: 'Stack protector not supported') +endif + +coroutine_backend = get_option('coroutine_backend') +ucontext_probe = ''' + #include <ucontext.h> + #ifdef __stub_makecontext + #error Ignoring glibc stub makecontext which will always fail + #endif + int main(void) { makecontext(0, 0, 0); return 0; }''' + +# On Windows the only valid backend is the Windows specific one. +# For POSIX prefer ucontext, but it's not always possible. The fallback +# is sigcontext. +supported_backends = [] +if targetos == 'windows' + supported_backends += ['windows'] +else + if targetos != 'darwin' and cc.links(ucontext_probe) + supported_backends += ['ucontext'] + endif + supported_backends += ['sigaltstack'] +endif + +if coroutine_backend == 'auto' + coroutine_backend = supported_backends[0] +elif coroutine_backend not in supported_backends + error('"@0@" backend requested but not available. Available backends: @1@' \ + .format(coroutine_backend, ', '.join(supported_backends))) +endif + +# Compiles if SafeStack *not* enabled +safe_stack_probe = ''' + int main(void) + { + #if defined(__has_feature) + #if __has_feature(safe_stack) + #error SafeStack Enabled + #endif + #endif + return 0; + }''' +if get_option('safe_stack') != not cc.compiles(safe_stack_probe) + safe_stack_arg = get_option('safe_stack') ? '-fsanitize=safe-stack' : '-fno-sanitize=safe-stack' + if get_option('safe_stack') != not cc.compiles(safe_stack_probe, args: safe_stack_arg) + error(get_option('safe_stack') \ + ? 'SafeStack not supported by your compiler' \ + : 'Cannot disable SafeStack') + endif + qemu_cflags += safe_stack_arg + qemu_ldflags += safe_stack_arg +endif +if get_option('safe_stack') and coroutine_backend != 'ucontext' + error('SafeStack is only supported with the ucontext coroutine backend') +endif + +if get_option('sanitizers') + if cc.has_argument('-fsanitize=address') + qemu_cflags = ['-fsanitize=address'] + qemu_cflags + qemu_ldflags = ['-fsanitize=address'] + qemu_ldflags + endif + + # Detect static linking issue with ubsan - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84285 + if cc.links('int main(int argc, char **argv) { return argc + 1; }', + args: [qemu_ldflags, '-fsanitize=undefined']) + qemu_cflags = ['-fsanitize=undefined'] + qemu_cflags + qemu_ldflags = ['-fsanitize=undefined'] + qemu_ldflags + endif +endif + +# Thread sanitizer is, for now, much noisier than the other sanitizers; +# keep it separate until that is not the case. +if get_option('tsan') + if get_option('sanitizers') + error('TSAN is not supported with other sanitizers') + endif + if not cc.has_function('__tsan_create_fiber', + args: '-fsanitize=thread', + prefix: '#include <sanitizer/tsan_interface.h>') + error('Cannot enable TSAN due to missing fiber annotation interface') + endif + qemu_cflags = ['-fsanitize=thread'] + qemu_cflags + qemu_ldflags = ['-fsanitize=thread'] + qemu_ldflags +endif + # Detect support for PT_GNU_RELRO + DT_BIND_NOW. # The combination is known as "full relro", because .got.plt is read-only too. qemu_ldflags += cc.get_supported_link_arguments('-Wl,-z,relro', '-Wl,-z,now') @@ -206,10 +390,9 @@ if targetos == 'windows' qemu_ldflags += cc.get_supported_link_arguments('-Wl,--dynamicbase', '-Wl,--high-entropy-va') endif -if get_option('gprof') - qemu_cflags += ['-p'] - qemu_objcflags += ['-p'] - qemu_ldflags += ['-p'] +# Exclude --warn-common with TSan to suppress warnings from the TSan libraries. +if targetos != 'sunos' and not get_option('tsan') + qemu_ldflags += cc.get_supported_link_arguments('-Wl,--warn-common') endif # Specify linker-script with add_project_link_arguments so that it is not placed @@ -225,8 +408,7 @@ if get_option('fuzzing') name: '-fsanitize-coverage-allowlist=/dev/null', args: ['-fsanitize-coverage-allowlist=/dev/null', '-fsanitize-coverage=trace-pc'] ) - add_global_arguments('-fsanitize-coverage-allowlist=instrumentation-filter', - native: false, language: all_languages) + qemu_common_flags += ['-fsanitize-coverage-allowlist=instrumentation-filter'] endif if get_option('fuzzing_engine') == '' @@ -234,10 +416,8 @@ if get_option('fuzzing') # compiled code. To build non-fuzzer binaries with --enable-fuzzing, link # everything with fsanitize=fuzzer-no-link. Otherwise, the linker will be # unable to bind the fuzzer-related callbacks added by instrumentation. - add_global_arguments('-fsanitize=fuzzer-no-link', - native: false, language: all_languages) - add_global_link_arguments('-fsanitize=fuzzer-no-link', - native: false, language: all_languages) + qemu_common_flags += ['-fsanitize=fuzzer-no-link'] + qemu_ldflags += ['-fsanitize=fuzzer-no-link'] # For the actual fuzzer binaries, we need to link against the libfuzzer # library. They need to be configurable, to support OSS-Fuzz fuzz_exe_ldflags = ['-fsanitize=fuzzer'] @@ -248,20 +428,50 @@ if get_option('fuzzing') endif endif +add_global_arguments(qemu_common_flags, native: false, language: all_languages) +add_global_link_arguments(qemu_ldflags, native: false, language: all_languages) + +# Collect warnings that we want to enable + +warn_flags = [ + '-Wundef', + '-Wwrite-strings', + '-Wmissing-prototypes', + '-Wstrict-prototypes', + '-Wredundant-decls', + '-Wold-style-declaration', + '-Wold-style-definition', + '-Wtype-limits', + '-Wformat-security', + '-Wformat-y2k', + '-Winit-self', + '-Wignored-qualifiers', + '-Wempty-body', + '-Wnested-externs', + '-Wendif-labels', + '-Wexpansion-to-defined', + '-Wimplicit-fallthrough=2', + '-Wmissing-format-attribute', + '-Wno-initializer-overrides', + '-Wno-missing-include-dirs', + '-Wno-shift-negative-value', + '-Wno-string-plus-int', + '-Wno-typedef-redefinition', + '-Wno-tautological-type-limit-compare', + '-Wno-psabi', + '-Wno-gnu-variable-sized-type-not-at-end', +] + +if targetos != 'darwin' + warn_flags += ['-Wthread-safety'] +endif + # Check that the C++ compiler exists and works with the C compiler. link_language = 'c' linker = cc qemu_cxxflags = [] if 'cpp' in all_languages - add_global_arguments(['-D__STDC_LIMIT_MACROS', '-D__STDC_CONSTANT_MACROS', '-D__STDC_FORMAT_MACROS'], - native: false, language: 'cpp') - foreach k: qemu_cflags - if k not in ['-Wstrict-prototypes', '-Wmissing-prototypes', '-Wnested-externs', - '-Wold-style-declaration', '-Wold-style-definition', '-Wredundant-decls'] - qemu_cxxflags += [k] - endif - endforeach - + qemu_cxxflags = ['-D__STDC_LIMIT_MACROS', '-D__STDC_CONSTANT_MACROS', '-D__STDC_FORMAT_MACROS'] + qemu_cflags if cxx.links(files('scripts/main.c'), args: qemu_cflags) link_language = 'cpp' linker = cxx @@ -271,16 +481,26 @@ if 'cpp' in all_languages endif endif -# Exclude --warn-common with TSan to suppress warnings from the TSan libraries. -if targetos != 'sunos' and not config_host.has_key('CONFIG_TSAN') - qemu_ldflags += linker.get_supported_link_arguments('-Wl,--warn-common') +# clang does not support glibc + FORTIFY_SOURCE (is it still true?) +if get_option('optimization') != '0' and targetos == 'linux' + if cc.get_id() == 'gcc' + qemu_cflags += ['-U_FORTIFY_SOURCE', '-D_FORTIFY_SOURCE=2'] + endif + if 'cpp' in all_languages and cxx.get_id() == 'gcc' + qemu_cxxflags += ['-U_FORTIFY_SOURCE', '-D_FORTIFY_SOURCE=2'] + endif endif -add_global_link_arguments(qemu_ldflags, native: false, language: all_languages) - -add_global_arguments(qemu_cflags, native: false, language: 'c') -add_global_arguments(qemu_cxxflags, native: false, language: 'cpp') -add_global_arguments(qemu_objcflags, native: false, language: 'objc') +add_project_arguments(qemu_cflags, native: false, language: 'c') +add_project_arguments(cc.get_supported_arguments(warn_flags), native: false, language: 'c') +if 'cpp' in all_languages + add_project_arguments(qemu_cxxflags, native: false, language: 'cpp') + add_project_arguments(cxx.get_supported_arguments(warn_flags), native: false, language: 'cpp') +endif +if 'objc' in all_languages + # Note sanitizer flags are not applied to Objective-C sources! + add_project_arguments(objc.get_supported_arguments(warn_flags), native: false, language: 'objc') +endif if targetos == 'linux' add_project_arguments('-isystem', meson.current_source_dir() / 'linux-headers', '-isystem', 'linux-headers', @@ -496,29 +716,85 @@ endif # Dependencies # ################ -# The path to glib.h is added to all compilation commands. This was -# grandfathered in from the QEMU Makefiles. -add_project_arguments(config_host['GLIB_CFLAGS'].split(), - native: false, language: all_languages) -glib = declare_dependency(compile_args: config_host['GLIB_CFLAGS'].split(), - link_args: config_host['GLIB_LIBS'].split(), - version: config_host['GLIB_VERSION'], - variables: { - 'bindir': config_host['GLIB_BINDIR'], - }) -# override glib dep with the configure results (for subprojects) -meson.override_dependency('glib-2.0', glib) -# pass down whether Glib has the slice allocator -if config_host.has_key('HAVE_GLIB_WITH_SLICE_ALLOCATOR') - config_host_data.set('HAVE_GLIB_WITH_SLICE_ALLOCATOR', true) +# When bumping glib minimum version, please check also whether to increase +# the _WIN32_WINNT setting in osdep.h according to the value from glib +glib_req_ver = '>=2.56.0' +glib_pc = dependency('glib-2.0', version: glib_req_ver, required: true, + method: 'pkg-config') +glib_cflags = [] +if enable_modules + gmodule = dependency('gmodule-export-2.0', version: glib_req_ver, required: true, + method: 'pkg-config') +elif config_host.has_key('CONFIG_PLUGIN') + gmodule = dependency('gmodule-no-export-2.0', version: glib_req_ver, required: true, + method: 'pkg-config') +else + gmodule = not_found +endif + +# This workaround is required due to a bug in pkg-config file for glib as it +# doesn't define GLIB_STATIC_COMPILATION for pkg-config --static +if targetos == 'windows' and get_option('prefer_static') + glib_cflags += ['-DGLIB_STATIC_COMPILATION'] endif +# Sanity check that the current size_t matches the +# size that glib thinks it should be. This catches +# problems on multi-arch where people try to build +# 32-bit QEMU while pointing at 64-bit glib headers + +if not cc.compiles(''' + #include <glib.h> + #include <unistd.h> + + #define QEMU_BUILD_BUG_ON(x) \ + typedef char qemu_build_bug_on[(x)?-1:1] __attribute__((unused)); + + int main(void) { + QEMU_BUILD_BUG_ON(sizeof(size_t) != GLIB_SIZEOF_SIZE_T); + return 0; + }''', dependencies: glib_pc, args: glib_cflags) + error('''sizeof(size_t) doesn't match GLIB_SIZEOF_SIZE_T. + You probably need to set PKG_CONFIG_LIBDIR" to point + to the right pkg-config files for your build target.''') +endif + +# Silence clang warnings triggered by glib < 2.57.2 +if not cc.compiles(''' + #include <glib.h> + typedef struct Foo { + int i; + } Foo; + static void foo_free(Foo *f) + { + g_free(f); + } + G_DEFINE_AUTOPTR_CLEANUP_FUNC(Foo, foo_free) + int main(void) { return 0; }''', dependencies: glib_pc, args: ['-Werror']) + glib_cflags += cc.get_supported_arguments('-Wno-unused-function') +endif +glib = declare_dependency(dependencies: [glib_pc, gmodule], + compile_args: glib_cflags, + version: glib_pc.version()) + +# Check whether glib has gslice, which we have to avoid for correctness. +# TODO: remove this check and the corresponding workaround (qtree) when +# the minimum supported glib is >= 2.75.3 +glib_has_gslice = glib.version().version_compare('<2.75.3') + +# override glib dep to include the above refinements +meson.override_dependency('glib-2.0', glib) + +# The path to glib.h is added to all compilation commands. +add_project_dependencies(glib.partial_dependency(compile_args: true, includes: true), + native: false, language: all_languages) + gio = not_found gdbus_codegen = not_found gdbus_codegen_error = '@0@ requires gdbus-codegen, please install libgio' if not get_option('gio').auto() or have_system gio = dependency('gio-2.0', required: get_option('gio'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if gio.found() and not cc.links(''' #include <gio/gio.h> int main(void) @@ -535,7 +811,7 @@ if not get_option('gio').auto() or have_system gdbus_codegen = find_program(gio.get_variable('gdbus_codegen'), required: get_option('gio')) gio_unix = dependency('gio-unix-2.0', required: get_option('gio'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') gio = declare_dependency(dependencies: [gio, gio_unix], version: gio.version()) endif @@ -548,20 +824,19 @@ endif lttng = not_found if 'ust' in get_option('trace_backends') lttng = dependency('lttng-ust', required: true, version: '>= 2.1', - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif pixman = not_found if have_system or have_tools pixman = dependency('pixman-1', required: have_system, version:'>=0.21.8', - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif -zlib = dependency('zlib', required: true, kwargs: static_kwargs) +zlib = dependency('zlib', required: true) libaio = not_found if not get_option('linux_aio').auto() or have_block libaio = cc.find_library('aio', has_headers: ['libaio.h'], - required: get_option('linux_aio'), - kwargs: static_kwargs) + required: get_option('linux_aio')) endif linux_io_uring_test = ''' @@ -574,7 +849,7 @@ linux_io_uring = not_found if not get_option('linux_io_uring').auto() or have_block linux_io_uring = dependency('liburing', version: '>=0.3', required: get_option('linux_io_uring'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if not cc.links(linux_io_uring_test) linux_io_uring = not_found endif @@ -584,7 +859,7 @@ libnfs = not_found if not get_option('libnfs').auto() or have_block libnfs = dependency('libnfs', version: '>=1.9.3', required: get_option('libnfs'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif libattr_test = ''' @@ -604,8 +879,7 @@ if get_option('attr').allowed() libattr = declare_dependency() else libattr = cc.find_library('attr', has_headers: ['attr/xattr.h'], - required: get_option('attr'), - kwargs: static_kwargs) + required: get_option('attr')) if libattr.found() and not \ cc.links(libattr_test, dependencies: libattr, args: '-DCONFIG_LIBATTR') libattr = not_found @@ -640,7 +914,7 @@ seccomp_has_sysrawrc = false if not get_option('seccomp').auto() or have_system or have_tools seccomp = dependency('libseccomp', version: '>=2.3.0', required: get_option('seccomp'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if seccomp.found() seccomp_has_sysrawrc = cc.has_header_symbol('seccomp.h', 'SCMP_FLTATR_API_SYSRAWRC', @@ -651,8 +925,7 @@ endif libcap_ng = not_found if not get_option('cap_ng').auto() or have_system or have_tools libcap_ng = cc.find_library('cap-ng', has_headers: ['cap-ng.h'], - required: get_option('cap_ng'), - kwargs: static_kwargs) + required: get_option('cap_ng')) endif if libcap_ng.found() and not cc.links(''' #include <cap-ng.h> @@ -673,13 +946,13 @@ if get_option('xkbcommon').auto() and not have_system and not have_tools xkbcommon = not_found else xkbcommon = dependency('xkbcommon', required: get_option('xkbcommon'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif slirp = not_found if not get_option('slirp').auto() or have_system slirp = dependency('slirp', required: get_option('slirp'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') # slirp < 4.7 is incompatible with CFI support in QEMU. This is because # it passes function pointers within libslirp as callbacks for timers. # When using a system-wide shared libslirp, the type information for the @@ -699,8 +972,7 @@ endif vde = not_found if not get_option('vde').auto() or have_system or have_tools vde = cc.find_library('vdeplug', has_headers: ['libvdeplug.h'], - required: get_option('vde'), - kwargs: static_kwargs) + required: get_option('vde')) endif if vde.found() and not cc.links(''' #include <libvdeplug.h> @@ -722,41 +994,41 @@ endif pulse = not_found if not get_option('pa').auto() or (targetos == 'linux' and have_system) pulse = dependency('libpulse', required: get_option('pa'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif alsa = not_found if not get_option('alsa').auto() or (targetos == 'linux' and have_system) alsa = dependency('alsa', required: get_option('alsa'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif jack = not_found if not get_option('jack').auto() or have_system jack = dependency('jack', required: get_option('jack'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif pipewire = not_found if not get_option('pipewire').auto() or (targetos == 'linux' and have_system) pipewire = dependency('libpipewire-0.3', version: '>=0.3.60', required: get_option('pipewire'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif sndio = not_found if not get_option('sndio').auto() or have_system sndio = dependency('sndio', required: get_option('sndio'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif spice_protocol = not_found if not get_option('spice_protocol').auto() or have_system spice_protocol = dependency('spice-protocol', version: '>=0.14.0', required: get_option('spice_protocol'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif spice = not_found if not get_option('spice').auto() or have_system spice = dependency('spice-server', version: '>=0.14.0', required: get_option('spice'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif spice_headers = spice.partial_dependency(compile_args: true, includes: true) @@ -766,13 +1038,13 @@ libiscsi = not_found if not get_option('libiscsi').auto() or have_block libiscsi = dependency('libiscsi', version: '>=1.9.0', required: get_option('libiscsi'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif zstd = not_found if not get_option('zstd').auto() or have_block zstd = dependency('libzstd', version: '>=1.4.0', required: get_option('zstd'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif virgl = not_found @@ -780,29 +1052,25 @@ have_vhost_user_gpu = have_tools and targetos == 'linux' and pixman.found() if not get_option('virglrenderer').auto() or have_system or have_vhost_user_gpu virgl = dependency('virglrenderer', method: 'pkg-config', - required: get_option('virglrenderer'), - kwargs: static_kwargs) + required: get_option('virglrenderer')) endif blkio = not_found if not get_option('blkio').auto() or have_block blkio = dependency('blkio', method: 'pkg-config', - required: get_option('blkio'), - kwargs: static_kwargs) + required: get_option('blkio')) endif curl = not_found if not get_option('curl').auto() or have_block curl = dependency('libcurl', version: '>=7.29.0', method: 'pkg-config', - required: get_option('curl'), - kwargs: static_kwargs) + required: get_option('curl')) endif libudev = not_found if targetos == 'linux' and (have_system or have_tools) libudev = dependency('libudev', method: 'pkg-config', - required: get_option('libudev'), - kwargs: static_kwargs) + required: get_option('libudev')) endif mpathlibs = [libudev] @@ -837,18 +1105,15 @@ if targetos == 'linux' and have_tools and get_option('mpath').allowed() return 0; }''' libmpathpersist = cc.find_library('mpathpersist', - required: get_option('mpath'), - kwargs: static_kwargs) + required: get_option('mpath')) if libmpathpersist.found() mpathlibs += libmpathpersist - if enable_static + if get_option('prefer_static') mpathlibs += cc.find_library('devmapper', - required: get_option('mpath'), - kwargs: static_kwargs) + required: get_option('mpath')) endif mpathlibs += cc.find_library('multipath', - required: get_option('mpath'), - kwargs: static_kwargs) + required: get_option('mpath')) foreach lib: mpathlibs if not lib.found() mpathlibs = [] @@ -898,13 +1163,13 @@ if have_system and get_option('curses').allowed() curses_dep_list = targetos == 'windows' ? ['ncurses', 'ncursesw'] : ['ncursesw'] curses = dependency(curses_dep_list, required: false, - method: 'pkg-config', - kwargs: static_kwargs) + method: 'pkg-config') msg = get_option('curses').enabled() ? 'curses library not found' : '' curses_compile_args = ['-DNCURSES_WIDECHAR=1'] if curses.found() if cc.links(curses_test, args: curses_compile_args, dependencies: [curses]) - curses = declare_dependency(compile_args: curses_compile_args, dependencies: [curses]) + curses = declare_dependency(compile_args: curses_compile_args, dependencies: [curses], + version: curses.version()) else msg = 'curses package not usable' curses = not_found @@ -921,8 +1186,7 @@ if have_system and get_option('curses').allowed() curses_libname_list = (targetos == 'windows' ? ['pdcurses'] : ['ncursesw', 'cursesw']) foreach curses_libname : curses_libname_list libcurses = cc.find_library(curses_libname, - required: false, - kwargs: static_kwargs) + required: false) if libcurses.found() if cc.links(curses_test, args: curses_compile_args, dependencies: libcurses) curses = declare_dependency(compile_args: curses_compile_args, @@ -947,7 +1211,7 @@ if have_system and get_option('curses').allowed() int main(void) { iconv_t conv = iconv_open("WCHAR_T", "UCS-2"); return conv != (iconv_t) -1; - }''', args: config_host['GLIB_CFLAGS'].split() + config_host['GLIB_LIBS'].split() + link_args) + }''', args: link_args, dependencies: glib) iconv = declare_dependency(link_args: link_args, dependencies: glib) break endif @@ -972,8 +1236,7 @@ endif brlapi = not_found if not get_option('brlapi').auto() or have_system brlapi = cc.find_library('brlapi', has_headers: ['brlapi.h'], - required: get_option('brlapi'), - kwargs: static_kwargs) + required: get_option('brlapi')) if brlapi.found() and not cc.links(''' #include <brlapi.h> #include <stddef.h> @@ -989,15 +1252,16 @@ endif sdl = not_found if not get_option('sdl').auto() or have_system - sdl = dependency('sdl2', required: get_option('sdl'), kwargs: static_kwargs) + sdl = dependency('sdl2', required: get_option('sdl')) sdl_image = not_found endif if sdl.found() # work around 2.0.8 bug sdl = declare_dependency(compile_args: '-Wno-undef', - dependencies: sdl) + dependencies: sdl, + version: sdl.version()) sdl_image = dependency('SDL2_image', required: get_option('sdl_image'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') else if get_option('sdl_image').enabled() error('sdl-image required, but SDL was @0@'.format( @@ -1008,11 +1272,9 @@ endif rbd = not_found if not get_option('rbd').auto() or have_block - librados = cc.find_library('rados', required: get_option('rbd'), - kwargs: static_kwargs) + librados = cc.find_library('rados', required: get_option('rbd')) librbd = cc.find_library('rbd', has_headers: ['rbd/librbd.h'], - required: get_option('rbd'), - kwargs: static_kwargs) + required: get_option('rbd')) if librados.found() and librbd.found() if cc.links(''' #include <stdio.h> @@ -1040,7 +1302,7 @@ glusterfs_iocb_has_stat = false if not get_option('glusterfs').auto() or have_block glusterfs = dependency('glusterfs-api', version: '>=3', required: get_option('glusterfs'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if glusterfs.found() glusterfs_ftruncate_has_stat = cc.links(''' #include <glusterfs/api/glfs.h> @@ -1075,15 +1337,13 @@ libssh = not_found if not get_option('libssh').auto() or have_block libssh = dependency('libssh', version: '>=0.8.7', method: 'pkg-config', - required: get_option('libssh'), - kwargs: static_kwargs) + required: get_option('libssh')) endif libbzip2 = not_found if not get_option('bzip2').auto() or have_block libbzip2 = cc.find_library('bz2', has_headers: ['bzlib.h'], - required: get_option('bzip2'), - kwargs: static_kwargs) + required: get_option('bzip2')) if libbzip2.found() and not cc.links(''' #include <bzlib.h> int main(void) { BZ2_bzlibVersion(); return 0; }''', dependencies: libbzip2) @@ -1099,8 +1359,7 @@ endif liblzfse = not_found if not get_option('lzfse').auto() or have_block liblzfse = cc.find_library('lzfse', has_headers: ['lzfse.h'], - required: get_option('lzfse'), - kwargs: static_kwargs) + required: get_option('lzfse')) endif if liblzfse.found() and not cc.links(''' #include <lzfse.h> @@ -1118,8 +1377,7 @@ if get_option('oss').allowed() and have_system if not cc.has_header('sys/soundcard.h') # not found elif targetos == 'netbsd' - oss = cc.find_library('ossaudio', required: get_option('oss'), - kwargs: static_kwargs) + oss = cc.find_library('ossaudio', required: get_option('oss')) else oss = declare_dependency() endif @@ -1152,7 +1410,7 @@ endif opengl = not_found if not get_option('opengl').auto() or have_system or have_vhost_user_gpu epoxy = dependency('epoxy', method: 'pkg-config', - required: get_option('opengl'), kwargs: static_kwargs) + required: get_option('opengl')) if cc.has_header('epoxy/egl.h', dependencies: epoxy) opengl = epoxy elif get_option('opengl').enabled() @@ -1161,8 +1419,7 @@ if not get_option('opengl').auto() or have_system or have_vhost_user_gpu endif gbm = not_found if (have_system or have_tools) and (virgl.found() or opengl.found()) - gbm = dependency('gbm', method: 'pkg-config', required: false, - kwargs: static_kwargs) + gbm = dependency('gbm', method: 'pkg-config', required: false) endif have_vhost_user_gpu = have_vhost_user_gpu and virgl.found() and opengl.found() and gbm.found() @@ -1184,16 +1441,14 @@ if get_option('gnutls').enabled() or (get_option('gnutls').auto() and have_syste # the platform support requirements gnutls_crypto = dependency('gnutls', version: '>=3.6.14', method: 'pkg-config', - required: false, - kwargs: static_kwargs) + required: false) if gnutls_crypto.found() gnutls = gnutls_crypto else # Our min version if all we need is TLS gnutls = dependency('gnutls', version: '>=3.5.18', method: 'pkg-config', - required: get_option('gnutls'), - kwargs: static_kwargs) + required: get_option('gnutls')) endif endif @@ -1220,34 +1475,32 @@ if not gnutls_crypto.found() if (not get_option('gcrypt').auto() or have_system) and not get_option('nettle').enabled() gcrypt = dependency('libgcrypt', version: '>=1.8', method: 'config-tool', - required: get_option('gcrypt'), - kwargs: static_kwargs) + required: get_option('gcrypt')) # Debian has removed -lgpg-error from libgcrypt-config # as it "spreads unnecessary dependencies" which in # turn breaks static builds... - if gcrypt.found() and enable_static - gcrypt = declare_dependency(dependencies: [ - gcrypt, - cc.find_library('gpg-error', required: true, kwargs: static_kwargs)]) + if gcrypt.found() and get_option('prefer_static') + gcrypt = declare_dependency(dependencies: + [gcrypt, + cc.find_library('gpg-error', required: true)], + version: gcrypt.version()) endif endif if (not get_option('nettle').auto() or have_system) and not gcrypt.found() nettle = dependency('nettle', version: '>=3.4', method: 'pkg-config', - required: get_option('nettle'), - kwargs: static_kwargs) + required: get_option('nettle')) if nettle.found() and not cc.has_header('nettle/xts.h', dependencies: nettle) xts = 'private' endif endif endif -gmp = dependency('gmp', required: false, method: 'pkg-config', kwargs: static_kwargs) +gmp = dependency('gmp', required: false, method: 'pkg-config') if nettle.found() and gmp.found() hogweed = dependency('hogweed', version: '>=3.4', method: 'pkg-config', - required: get_option('nettle'), - kwargs: static_kwargs) + required: get_option('nettle')) endif @@ -1259,20 +1512,18 @@ have_gtk_clipboard = get_option('gtk_clipboard').enabled() if not get_option('gtk').auto() or have_system gtk = dependency('gtk+-3.0', version: '>=3.22.0', method: 'pkg-config', - required: get_option('gtk'), - kwargs: static_kwargs) + required: get_option('gtk')) if gtk.found() gtkx11 = dependency('gtk+-x11-3.0', version: '>=3.22.0', method: 'pkg-config', - required: false, - kwargs: static_kwargs) - gtk = declare_dependency(dependencies: [gtk, gtkx11]) + required: false) + gtk = declare_dependency(dependencies: [gtk, gtkx11], + version: gtk.version()) if not get_option('vte').auto() or have_system vte = dependency('vte-2.91', method: 'pkg-config', - required: get_option('vte'), - kwargs: static_kwargs) + required: get_option('vte')) endif elif have_gtk_clipboard error('GTK clipboard requested, but GTK not found') @@ -1281,13 +1532,12 @@ endif x11 = not_found if gtkx11.found() - x11 = dependency('x11', method: 'pkg-config', required: gtkx11.found(), - kwargs: static_kwargs) + x11 = dependency('x11', method: 'pkg-config', required: gtkx11.found()) endif png = not_found if get_option('png').allowed() and have_system png = dependency('libpng', version: '>=1.6.34', required: get_option('png'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif vnc = not_found jpeg = not_found @@ -1295,10 +1545,9 @@ sasl = not_found if get_option('vnc').allowed() and have_system vnc = declare_dependency() # dummy dependency jpeg = dependency('libjpeg', required: get_option('vnc_jpeg'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') sasl = cc.find_library('sasl2', has_headers: ['sasl/sasl.h'], - required: get_option('vnc_sasl'), - kwargs: static_kwargs) + required: get_option('vnc_sasl')) if sasl.found() sasl = declare_dependency(dependencies: sasl, compile_args: '-DSTRUCT_IOVEC_DEFINED') @@ -1308,8 +1557,7 @@ endif pam = not_found if not get_option('auth_pam').auto() or have_system pam = cc.find_library('pam', has_headers: ['security/pam_appl.h'], - required: get_option('auth_pam'), - kwargs: static_kwargs) + required: get_option('auth_pam')) endif if pam.found() and not cc.links(''' #include <stddef.h> @@ -1333,8 +1581,7 @@ endif snappy = not_found if not get_option('snappy').auto() or have_system snappy = cc.find_library('snappy', has_headers: ['snappy-c.h'], - required: get_option('snappy'), - kwargs: static_kwargs) + required: get_option('snappy')) endif if snappy.found() and not linker.links(''' #include <snappy-c.h> @@ -1350,8 +1597,7 @@ endif lzo = not_found if not get_option('lzo').auto() or have_system lzo = cc.find_library('lzo2', has_headers: ['lzo/lzo1x.h'], - required: get_option('lzo'), - kwargs: static_kwargs) + required: get_option('lzo')) endif if lzo.found() and not cc.links(''' #include <lzo/lzo1x.h> @@ -1367,8 +1613,7 @@ endif numa = not_found if not get_option('numa').auto() or have_system or have_tools numa = cc.find_library('numa', has_headers: ['numa.h'], - required: get_option('numa'), - kwargs: static_kwargs) + required: get_option('numa')) endif if numa.found() and not cc.links(''' #include <numa.h> @@ -1386,10 +1631,8 @@ rdma = not_found if not get_option('rdma').auto() or have_system libumad = cc.find_library('ibumad', required: get_option('rdma')) rdma_libs = [cc.find_library('rdmacm', has_headers: ['rdma/rdma_cma.h'], - required: get_option('rdma'), - kwargs: static_kwargs), - cc.find_library('ibverbs', required: get_option('rdma'), - kwargs: static_kwargs), + required: get_option('rdma')), + cc.find_library('ibverbs', required: get_option('rdma')), libumad] rdma = declare_dependency(dependencies: rdma_libs) foreach lib: rdma_libs @@ -1402,30 +1645,30 @@ endif xen = not_found if get_option('xen').enabled() or (get_option('xen').auto() and have_system) xencontrol = dependency('xencontrol', required: false, - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if xencontrol.found() xen_pc = declare_dependency(version: xencontrol.version(), dependencies: [ xencontrol, # disabler: true makes xen_pc.found() return false if any is not found dependency('xenstore', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), dependency('xenforeignmemory', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), dependency('xengnttab', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), dependency('xenevtchn', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), dependency('xendevicemodel', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), # optional, no "disabler: true" dependency('xentoolcore', required: false, - method: 'pkg-config', kwargs: static_kwargs)]) + method: 'pkg-config')]) if xen_pc.found() xen = xen_pc endif @@ -1483,60 +1726,53 @@ have_xen_pci_passthrough = get_option('xen_pci_passthrough') \ cacard = not_found if not get_option('smartcard').auto() or have_system cacard = dependency('libcacard', required: get_option('smartcard'), - version: '>=2.5.1', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=2.5.1', method: 'pkg-config') endif u2f = not_found if have_system u2f = dependency('u2f-emu', required: get_option('u2f'), - method: 'pkg-config', - kwargs: static_kwargs) + method: 'pkg-config') endif canokey = not_found if have_system canokey = dependency('canokey-qemu', required: get_option('canokey'), - method: 'pkg-config', - kwargs: static_kwargs) + method: 'pkg-config') endif usbredir = not_found if not get_option('usb_redir').auto() or have_system usbredir = dependency('libusbredirparser-0.5', required: get_option('usb_redir'), - version: '>=0.6', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=0.6', method: 'pkg-config') endif libusb = not_found if not get_option('libusb').auto() or have_system libusb = dependency('libusb-1.0', required: get_option('libusb'), - version: '>=1.0.13', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=1.0.13', method: 'pkg-config') endif libpmem = not_found if not get_option('libpmem').auto() or have_system libpmem = dependency('libpmem', required: get_option('libpmem'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif libdaxctl = not_found if not get_option('libdaxctl').auto() or have_system libdaxctl = dependency('libdaxctl', required: get_option('libdaxctl'), - version: '>=57', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=57', method: 'pkg-config') endif tasn1 = not_found if gnutls.found() tasn1 = dependency('libtasn1', - method: 'pkg-config', - kwargs: static_kwargs) + method: 'pkg-config') endif keyutils = dependency('libkeyutils', required: false, - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') has_gettid = cc.has_function('gettid') # libselinux selinux = dependency('libselinux', required: get_option('selinux'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') # Malloc tests @@ -1600,8 +1836,7 @@ if get_option('fuse').disabled() and get_option('fuse_lseek').enabled() endif fuse = dependency('fuse3', required: get_option('fuse'), - version: '>=3.1', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=3.1', method: 'pkg-config') fuse_lseek = not_found if get_option('fuse_lseek').allowed() @@ -1657,10 +1892,9 @@ endif # libdw libdw = not_found if not get_option('libdw').auto() or \ - (not enable_static and (have_system or have_user)) + (not get_option('prefer_static') and (have_system or have_user)) libdw = dependency('libdw', method: 'pkg-config', - kwargs: static_kwargs, required: get_option('libdw')) endif @@ -1715,7 +1949,7 @@ if get_option('cfi') if not get_option('b_lto') error('Selected Control-Flow Integrity but LTO is disabled') endif - if config_host.has_key('CONFIG_MODULES') + if enable_modules error('Selected Control-Flow Integrity is not compatible with modules') endif # Check for cfi flags. CFI requires LTO so we can't use @@ -1766,12 +2000,17 @@ have_virtfs = get_option('virtfs') \ error_message: 'virtio-9p (virtfs) requires Linux or macOS') \ .require(targetos == 'linux' or cc.has_function('pthread_fchdir_np'), error_message: 'virtio-9p (virtfs) on macOS requires the presence of pthread_fchdir_np') \ - .require(targetos == 'darwin' or (libattr.found() and libcap_ng.found()), - error_message: 'virtio-9p (virtfs) on Linux requires libcap-ng-devel and libattr-devel') \ + .require(targetos == 'darwin' or libattr.found(), + error_message: 'virtio-9p (virtfs) on Linux requires libattr-devel') \ .disable_auto_if(not have_tools and not have_system) \ .allowed() -have_virtfs_proxy_helper = targetos != 'darwin' and have_virtfs and have_tools +have_virtfs_proxy_helper = get_option('virtfs_proxy_helper') \ + .require(targetos != 'darwin', error_message: 'the virtfs proxy helper is incompatible with macOS') \ + .require(have_virtfs, error_message: 'the virtfs proxy helper requires that virtfs is enabled') \ + .disable_auto_if(not have_tools) \ + .require(libcap_ng.found(), error_message: 'the virtfs proxy helper requires libcap-ng') \ + .allowed() if get_option('block_drv_ro_whitelist') == '' config_host_data.set('CONFIG_BDRV_RO_WHITELIST', '') @@ -1813,7 +2052,7 @@ config_host_data.set_quoted('CONFIG_QEMU_LOCALSTATEDIR', get_option('prefix') / config_host_data.set_quoted('CONFIG_QEMU_MODDIR', get_option('prefix') / qemu_moddir) config_host_data.set_quoted('CONFIG_SYSCONFDIR', get_option('prefix') / get_option('sysconfdir')) -if config_host.has_key('CONFIG_MODULES') +if enable_modules config_host_data.set('CONFIG_STAMP', run_command( meson.current_source_dir() / 'scripts/qemu-stamp.py', meson.project_version(), get_option('pkgversion'), '--', @@ -1877,6 +2116,7 @@ config_host_data.set('CONFIG_LIBSSH', libssh.found()) config_host_data.set('CONFIG_LINUX_AIO', libaio.found()) config_host_data.set('CONFIG_LINUX_IO_URING', linux_io_uring.found()) config_host_data.set('CONFIG_LIBPMEM', libpmem.found()) +config_host_data.set('CONFIG_MODULES', enable_modules) config_host_data.set('CONFIG_NUMA', numa.found()) if numa.found() config_host_data.set('HAVE_NUMA_HAS_PREFERRED_MANY', @@ -1887,6 +2127,7 @@ config_host_data.set('CONFIG_OPENGL', opengl.found()) config_host_data.set('CONFIG_PROFILER', get_option('profiler')) config_host_data.set('CONFIG_RBD', rbd.found()) config_host_data.set('CONFIG_RDMA', rdma.found()) +config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack')) config_host_data.set('CONFIG_SDL', sdl.found()) config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found()) config_host_data.set('CONFIG_SECCOMP', seccomp.found()) @@ -1895,6 +2136,7 @@ if seccomp.found() endif config_host_data.set('CONFIG_SNAPPY', snappy.found()) config_host_data.set('CONFIG_TPM', have_tpm) +config_host_data.set('CONFIG_TSAN', get_option('tsan')) config_host_data.set('CONFIG_USB_LIBUSB', libusb.found()) config_host_data.set('CONFIG_VDE', vde.found()) config_host_data.set('CONFIG_VHOST_NET', have_vhost_net) @@ -2009,6 +2251,7 @@ config_host_data.set('CONFIG_SYNC_FILE_RANGE', cc.has_function('sync_file_range' config_host_data.set('CONFIG_TIMERFD', cc.has_function('timerfd_create')) config_host_data.set('HAVE_COPY_FILE_RANGE', cc.has_function('copy_file_range')) config_host_data.set('HAVE_GETIFADDRS', cc.has_function('getifaddrs')) +config_host_data.set('HAVE_GLIB_WITH_SLICE_ALLOCATOR', glib_has_gslice) config_host_data.set('HAVE_OPENPTY', cc.has_function('openpty', dependencies: util)) config_host_data.set('HAVE_STRCHRNUL', cc.has_function('strchrnul')) config_host_data.set('HAVE_SYSTEM_FUNCTION', cc.has_function('system', prefix: '#include <stdlib.h>')) @@ -2025,7 +2268,21 @@ if rdma.found() prefix: '#include <infiniband/verbs.h>')) endif +have_asan_fiber = false +if get_option('sanitizers') and \ + not cc.has_function('__sanitizer_start_switch_fiber', + args: '-fsanitize=address', + prefix: '#include <sanitizer/asan_interface.h>') + warning('Missing ASAN due to missing fiber annotation interface') + warning('Without code annotation, the report may be inferior.') +else + have_asan_fiber = true +endif +config_host_data.set('CONFIG_ASAN_IFACE_FIBER', have_asan_fiber) + # has_header_symbol +config_host_data.set('CONFIG_BLKZONED', + cc.has_header_symbol('linux/blkzoned.h', 'BLKOPENZONE')) config_host_data.set('CONFIG_EPOLL_CREATE1', cc.has_header_symbol('sys/epoll.h', 'epoll_create1')) config_host_data.set('CONFIG_FALLOCATE_PUNCH_HOLE', @@ -2061,6 +2318,9 @@ config_host_data.set('HAVE_SIGEV_NOTIFY_THREAD_ID', config_host_data.set('HAVE_STRUCT_STAT_ST_ATIM', cc.has_member('struct stat', 'st_atim', prefix: '#include <sys/stat.h>')) +config_host_data.set('HAVE_BLK_ZONE_REP_CAPACITY', + cc.has_member('struct blk_zone', 'capacity', + prefix: '#include <linux/blkzoned.h>')) # has_type config_host_data.set('CONFIG_IOVEC', @@ -2250,23 +2510,21 @@ config_host_data.set('HAVE_BROKEN_SIZE_MAX', not cc.compiles(''' return printf("%zu", SIZE_MAX); }''', args: ['-Werror'])) -atomic_test = ''' +# See if 64-bit atomic operations are supported. +# Note that without __atomic builtins, we can only +# assume atomic loads/stores max at pointer size. +config_host_data.set('CONFIG_ATOMIC64', cc.links(''' #include <stdint.h> int main(void) { - @0@ x = 0, y = 0; + uint64_t x = 0, y = 0; y = __atomic_load_n(&x, __ATOMIC_RELAXED); __atomic_store_n(&x, y, __ATOMIC_RELAXED); __atomic_compare_exchange_n(&x, &y, x, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); __atomic_exchange_n(&x, y, __ATOMIC_RELAXED); __atomic_fetch_add(&x, y, __ATOMIC_RELAXED); return 0; - }''' - -# See if 64-bit atomic operations are supported. -# Note that without __atomic builtins, we can only -# assume atomic loads/stores max at pointer size. -config_host_data.set('CONFIG_ATOMIC64', cc.links(atomic_test.format('uint64_t'))) + }''')) has_int128 = cc.links(''' __int128_t a; @@ -2284,21 +2542,39 @@ if has_int128 # "do we have 128-bit atomics which are handled inline and specifically not # via libatomic". The reason we can't use libatomic is documented in the # comment starting "GCC is a house divided" in include/qemu/atomic128.h. - has_atomic128 = cc.links(atomic_test.format('unsigned __int128')) + # We only care about these operations on 16-byte aligned pointers, so + # force 16-byte alignment of the pointer, which may be greater than + # __alignof(unsigned __int128) for the host. + atomic_test_128 = ''' + int main(int ac, char **av) { + unsigned __int128 *p = __builtin_assume_aligned(av[ac - 1], sizeof(16)); + p[1] = __atomic_load_n(&p[0], __ATOMIC_RELAXED); + __atomic_store_n(&p[2], p[3], __ATOMIC_RELAXED); + __atomic_compare_exchange_n(&p[4], &p[5], p[6], 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + return 0; + }''' + has_atomic128 = cc.links(atomic_test_128) config_host_data.set('CONFIG_ATOMIC128', has_atomic128) if not has_atomic128 - has_cmpxchg128 = cc.links(''' - int main(void) - { - unsigned __int128 x = 0, y = 0; - __sync_val_compare_and_swap_16(&x, y, x); - return 0; - } - ''') - - config_host_data.set('CONFIG_CMPXCHG128', has_cmpxchg128) + # Even with __builtin_assume_aligned, the above test may have failed + # without optimization enabled. Try again with optimizations locally + # enabled for the function. See + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107389 + has_atomic128_opt = cc.links('__attribute__((optimize("O1")))' + atomic_test_128) + config_host_data.set('CONFIG_ATOMIC128_OPT', has_atomic128_opt) + + if not has_atomic128_opt + config_host_data.set('CONFIG_CMPXCHG128', cc.links(''' + int main(void) + { + unsigned __int128 x = 0, y = 0; + __sync_val_compare_and_swap_16(&x, y, x); + return 0; + } + ''')) + endif endif endif @@ -2736,7 +3012,7 @@ genh += custom_target('config-poison.h', capstone = not_found if not get_option('capstone').auto() or have_system or have_user capstone = dependency('capstone', version: '>=3.0.5', - kwargs: static_kwargs, method: 'pkg-config', + method: 'pkg-config', required: get_option('capstone')) # Some versions of capstone have broken pkg-config file @@ -2762,9 +3038,7 @@ if have_system and vfio_user_server_allowed libvfio_user_proj = subproject('libvfio-user') - libvfio_user_lib = libvfio_user_proj.get_variable('libvfio_user_dep') - - libvfio_user_dep = declare_dependency(dependencies: [libvfio_user_lib]) + libvfio_user_dep = libvfio_user_proj.get_variable('libvfio_user_dep') endif fdt = not_found @@ -2772,7 +3046,7 @@ if have_system fdt_opt = get_option('fdt') if fdt_opt in ['enabled', 'auto', 'system'] have_internal = fs.exists(meson.current_source_dir() / 'dtc/libfdt/Makefile.libfdt') - fdt = cc.find_library('fdt', kwargs: static_kwargs, + fdt = cc.find_library('fdt', required: fdt_opt == 'system' or fdt_opt == 'enabled' and not have_internal) if fdt.found() and cc.links(''' @@ -3078,7 +3352,6 @@ subdir('ui') subdir('hw') subdir('gdbstub') - if enable_modules libmodulecommon = static_library('module-common', files('module-common.c') + genh, pic: true, c_args: '-DBUILD_DSO') modulecommon = declare_dependency(link_whole: libmodulecommon, compile_args: '-DBUILD_DSO') @@ -3221,7 +3494,7 @@ foreach d, list : modules endif foreach m, module_ss : list - if enable_modules and targetos != 'windows' + if enable_modules module_ss = module_ss.apply(config_all, strict: false) sl = static_library(d + '-' + m, [genh, module_ss.sources()], dependencies: [modulecommon, module_ss.dependencies()], pic: true) @@ -3254,7 +3527,7 @@ endforeach foreach d, list : target_modules foreach m, module_ss : list - if enable_modules and targetos != 'windows' + if enable_modules foreach target : target_dirs if target.endswith('-softmmu') config_target = config_target_mak[target] @@ -3700,7 +3973,7 @@ if host_machine.system() == 'windows' '@OUTPUT@', get_option('prefix'), meson.current_source_dir(), - config_host['GLIB_BINDIR'], + glib_pc.get_variable('bindir'), host_machine.cpu(), '--', '-DDISPLAYVERSION=' + meson.project_version(), @@ -3774,8 +4047,8 @@ summary_info += {'system-mode emulation': have_system} summary_info += {'user-mode emulation': have_user} summary_info += {'block layer': have_block} summary_info += {'Install blobs': get_option('install_blobs')} -summary_info += {'module support': config_host.has_key('CONFIG_MODULES')} -if config_host.has_key('CONFIG_MODULES') +summary_info += {'module support': enable_modules} +if enable_modules summary_info += {'alternative module path': get_option('module_upgrades')} endif summary_info += {'fuzzing support': get_option('fuzzing')} @@ -3826,12 +4099,12 @@ link_args = get_option(link_language + '_link_args') if link_args.length() > 0 summary_info += {'LDFLAGS': ' '.join(link_args)} endif -summary_info += {'QEMU_CFLAGS': ' '.join(qemu_cflags)} +summary_info += {'QEMU_CFLAGS': ' '.join(qemu_common_flags + qemu_cflags)} if 'cpp' in all_languages - summary_info += {'QEMU_CXXFLAGS': ' '.join(qemu_cxxflags)} + summary_info += {'QEMU_CXXFLAGS': ' '.join(qemu_common_flags + qemu_cxxflags)} endif if 'objc' in all_languages - summary_info += {'QEMU_OBJCFLAGS': ' '.join(qemu_objcflags)} + summary_info += {'QEMU_OBJCFLAGS': ' '.join(qemu_common_flags)} endif summary_info += {'QEMU_LDFLAGS': ' '.join(qemu_ldflags)} summary_info += {'profiler': get_option('profiler')} @@ -3854,7 +4127,7 @@ else endif summary_info += {'gprof': gprof_info} summary_info += {'gcov': get_option('b_coverage')} -summary_info += {'thread sanitizer': config_host.has_key('CONFIG_TSAN')} +summary_info += {'thread sanitizer': get_option('tsan')} summary_info += {'CFI support': get_option('cfi')} if get_option('cfi') summary_info += {'CFI debug support': get_option('cfi_debug')} @@ -3915,13 +4188,14 @@ summary(summary_info, bool_yn: true, section: 'Targets and accelerators') # Block layer summary_info = {} -summary_info += {'coroutine backend': config_host['CONFIG_COROUTINE_BACKEND']} +summary_info += {'coroutine backend': coroutine_backend} summary_info += {'coroutine pool': have_coroutine_pool} if have_block summary_info += {'Block whitelist (rw)': get_option('block_drv_rw_whitelist')} summary_info += {'Block whitelist (ro)': get_option('block_drv_ro_whitelist')} summary_info += {'Use block whitelist in tools': get_option('block_drv_whitelist_in_tools')} - summary_info += {'VirtFS support': have_virtfs} + summary_info += {'VirtFS (9P) support': have_virtfs} + summary_info += {'VirtFS (9P) Proxy Helper support': have_virtfs_proxy_helper} summary_info += {'Live block migration': config_host_data.get('CONFIG_LIVE_BLOCK_MIGRATION')} summary_info += {'replication support': config_host_data.get('CONFIG_REPLICATION')} summary_info += {'bochs support': get_option('bochs').allowed()} diff --git a/meson_options.txt b/meson_options.txt index d8330a1f71..90237389e2 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -12,8 +12,6 @@ option('pkgversion', type : 'string', value : '', description: 'use specified string as sub-version of the package') option('smbd', type : 'string', value : '', description: 'Path to smbd for slirp networking') -option('sphinx_build', type : 'string', value : 'sphinx-build', - description: 'Use specified sphinx-build for building document') option('iasl', type : 'string', value : '', description: 'Path to ACPI disassembler') option('tls_priority', type : 'string', value : 'NORMAL', @@ -33,6 +31,9 @@ option('fuzzing_engine', type : 'string', value : '', description: 'fuzzing engine library for OSS-Fuzz') option('trace_file', type: 'string', value: 'trace', description: 'Trace file prefix for simple backend') +option('coroutine_backend', type: 'combo', + choices: ['ucontext', 'sigaltstack', 'windows', 'auto'], + value: 'auto', description: 'coroutine backend to use') # Everything else can be set via --enable/--disable-* option # on the configure script command line. After adding an option @@ -44,6 +45,8 @@ option('fuzzing', type : 'boolean', value: false, description: 'build fuzzing targets') option('gettext', type : 'feature', value : 'auto', description: 'Localization of the GTK+ user interface') +option('modules', type : 'feature', value : 'disabled', + description: 'modules support (non Windows)') option('module_upgrades', type : 'boolean', value : false, description: 'try to load modules from alternate paths for upgrades') option('install_blobs', type : 'boolean', value : true, @@ -82,6 +85,14 @@ option('tcg', type: 'feature', value: 'enabled', description: 'TCG support') option('tcg_interpreter', type: 'boolean', value: false, description: 'TCG with bytecode interpreter (slow)') +option('safe_stack', type: 'boolean', value: false, + description: 'SafeStack Stack Smash Protection (requires clang/llvm and coroutine backend ucontext)') +option('sanitizers', type: 'boolean', value: false, + description: 'enable default sanitizers') +option('tsan', type: 'boolean', value: false, + description: 'enable thread sanitizer') +option('stack_protector', type: 'feature', value: 'auto', + description: 'compiler-provided stack protection') option('cfi', type: 'boolean', value: false, description: 'Control-Flow Integrity (CFI)') option('cfi_debug', type: 'boolean', value: false, @@ -274,6 +285,8 @@ option('vhost_user_blk_server', type: 'feature', value: 'auto', description: 'build vhost-user-blk server') option('virtfs', type: 'feature', value: 'auto', description: 'virtio-9p support') +option('virtfs_proxy_helper', type: 'feature', value: 'auto', + description: 'virtio-9p proxy helper support') option('libvduse', type: 'feature', value: 'auto', description: 'build VDUSE Library') option('vduse_blk_export', type: 'feature', value: 'auto', @@ -291,6 +304,8 @@ option('live_block_migration', type: 'feature', value: 'auto', description: 'block migration in the main migration stream') option('replication', type: 'feature', value: 'auto', description: 'replication support') +option('colo_proxy', type: 'feature', value: 'auto', + description: 'colo-proxy support') option('bochs', type: 'feature', value: 'auto', description: 'bochs image format support') option('cloop', type: 'feature', value: 'auto', diff --git a/migration/block-dirty-bitmap.c b/migration/block-dirty-bitmap.c index 20f36e6bd8..032fc5f405 100644 --- a/migration/block-dirty-bitmap.c +++ b/migration/block-dirty-bitmap.c @@ -706,7 +706,7 @@ static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit) QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { while (!dbms->bulk_completed) { bulk_phase_send_chunk(f, s, dbms); - if (limit && qemu_file_rate_limit(f)) { + if (limit && migration_rate_exceeded(f)) { return; } } diff --git a/migration/block.c b/migration/block.c index a37678ce95..b9580a6c7e 100644 --- a/migration/block.c +++ b/migration/block.c @@ -23,6 +23,7 @@ #include "block/dirty-bitmap.h" #include "migration/misc.h" #include "migration.h" +#include "migration-stats.h" #include "migration/register.h" #include "qemu-file.h" #include "migration/vmstate.h" @@ -625,7 +626,7 @@ static int flush_blks(QEMUFile *f) blk_mig_lock(); while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) { - if (qemu_file_rate_limit(f)) { + if (migration_rate_exceeded(f)) { break; } if (blk->ret < 0) { @@ -747,7 +748,7 @@ static int block_save_setup(QEMUFile *f, void *opaque) static int block_save_iterate(QEMUFile *f, void *opaque) { int ret; - uint64_t last_bytes = qemu_file_total_transferred(f); + uint64_t last_bytes = qemu_file_transferred(f); trace_migration_block_save("iterate", block_mig_state.submitted, block_mig_state.transferred); @@ -762,7 +763,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque) /* control the rate of transfer */ blk_mig_lock(); while (block_mig_state.read_done * BLK_MIG_BLOCK_SIZE < - qemu_file_get_rate_limit(f) && + migration_rate_get() && block_mig_state.submitted < MAX_PARALLEL_IO && (block_mig_state.submitted + block_mig_state.read_done) < MAX_IO_BUFFERS) { @@ -799,7 +800,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque) } qemu_put_be64(f, BLK_MIG_FLAG_EOS); - uint64_t delta_bytes = qemu_file_total_transferred(f) - last_bytes; + uint64_t delta_bytes = qemu_file_transferred(f) - last_bytes; return (delta_bytes > 0); } diff --git a/migration/colo.c b/migration/colo.c index 6c7c313956..72f4f7b37e 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -145,8 +145,8 @@ static void secondary_vm_do_failover(void) qemu_sem_post(&mis->colo_incoming_sem); /* For Secondary VM, jump to incoming co */ - if (mis->migration_incoming_co) { - qemu_coroutine_enter(mis->migration_incoming_co); + if (mis->colo_incoming_co) { + qemu_coroutine_enter(mis->colo_incoming_co); } } @@ -817,7 +817,7 @@ void colo_shutdown(void) } } -void *colo_process_incoming_thread(void *opaque) +static void *colo_process_incoming_thread(void *opaque) { MigrationIncomingState *mis = opaque; QEMUFile *fb = NULL; @@ -918,3 +918,40 @@ out: rcu_unregister_thread(); return NULL; } + +int coroutine_fn colo_incoming_co(void) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + Error *local_err = NULL; + QemuThread th; + + assert(qemu_mutex_iothread_locked()); + + if (!migration_incoming_colo_enabled()) { + return 0; + } + + /* Make sure all file formats throw away their mutable metadata */ + bdrv_activate_all(&local_err); + if (local_err) { + error_report_err(local_err); + return -EINVAL; + } + + qemu_thread_create(&th, "COLO incoming", colo_process_incoming_thread, + mis, QEMU_THREAD_JOINABLE); + + mis->colo_incoming_co = qemu_coroutine_self(); + qemu_coroutine_yield(); + mis->colo_incoming_co = NULL; + + qemu_mutex_unlock_iothread(); + /* Wait checkpoint incoming thread exit before free resource */ + qemu_thread_join(&th); + qemu_mutex_lock_iothread(); + + /* We hold the global iothread lock, so it is safe here */ + colo_release_ram_cache(); + + return 0; +} diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 388337a332..84f1b0fb20 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -13,10 +13,10 @@ #include "qemu/osdep.h" #include "qemu/error-report.h" #include <zlib.h> +#include "hw/core/cpu.h" #include "qapi/error.h" -#include "cpu.h" #include "exec/ramblock.h" -#include "exec/ram_addr.h" +#include "exec/target_page.h" #include "qemu/rcu_queue.h" #include "qemu/main-loop.h" #include "qapi/qapi-commands-migration.h" @@ -29,6 +29,7 @@ #include "sysemu/kvm.h" #include "sysemu/runstate.h" #include "exec/memory.h" +#include "qemu/xxhash.h" /* * total_dirty_pages is procted by BQL and is used @@ -74,13 +75,11 @@ static inline void record_dirtypages(DirtyPageRecord *dirty_pages, static int64_t do_calculate_dirtyrate(DirtyPageRecord dirty_pages, int64_t calc_time_ms) { - uint64_t memory_size_MB; uint64_t increased_dirty_pages = dirty_pages.end_pages - dirty_pages.start_pages; + uint64_t memory_size_MiB = qemu_target_pages_to_MiB(increased_dirty_pages); - memory_size_MB = (increased_dirty_pages * TARGET_PAGE_SIZE) >> 20; - - return memory_size_MB * 1000 / calc_time_ms; + return memory_size_MiB * 1000 / calc_time_ms; } void global_dirty_log_change(unsigned int flag, bool start) @@ -102,7 +101,7 @@ void global_dirty_log_change(unsigned int flag, bool start) static void global_dirty_log_sync(unsigned int flag, bool one_shot) { qemu_mutex_lock_iothread(); - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(false); if (one_shot) { memory_global_dirty_log_stop(flag); } @@ -291,8 +290,8 @@ static void update_dirtyrate_stat(struct RamblockDirtyInfo *info) DirtyStat.page_sampling.total_dirty_samples += info->sample_dirty_count; DirtyStat.page_sampling.total_sample_count += info->sample_pages_count; /* size of total pages in MB */ - DirtyStat.page_sampling.total_block_mem_MB += (info->ramblock_pages * - TARGET_PAGE_SIZE) >> 20; + DirtyStat.page_sampling.total_block_mem_MB += + qemu_target_pages_to_MiB(info->ramblock_pages); } static void update_dirtyrate(uint64_t msec) @@ -309,19 +308,47 @@ static void update_dirtyrate(uint64_t msec) } /* + * Compute hash of a single page of size TARGET_PAGE_SIZE. + */ +static uint32_t compute_page_hash(void *ptr) +{ + size_t page_size = qemu_target_page_size(); + uint32_t i; + uint64_t v1, v2, v3, v4; + uint64_t res; + const uint64_t *p = ptr; + + v1 = QEMU_XXHASH_SEED + XXH_PRIME64_1 + XXH_PRIME64_2; + v2 = QEMU_XXHASH_SEED + XXH_PRIME64_2; + v3 = QEMU_XXHASH_SEED + 0; + v4 = QEMU_XXHASH_SEED - XXH_PRIME64_1; + for (i = 0; i < page_size / 8; i += 4) { + v1 = XXH64_round(v1, p[i + 0]); + v2 = XXH64_round(v2, p[i + 1]); + v3 = XXH64_round(v3, p[i + 2]); + v4 = XXH64_round(v4, p[i + 3]); + } + res = XXH64_mergerounds(v1, v2, v3, v4); + res += page_size; + res = XXH64_avalanche(res); + return (uint32_t)(res & UINT32_MAX); +} + + +/* * get hash result for the sampled memory with length of TARGET_PAGE_SIZE * in ramblock, which starts from ramblock base address. */ static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info, uint64_t vfn) { - uint32_t crc; + uint32_t hash; - crc = crc32(0, (info->ramblock_addr + - vfn * TARGET_PAGE_SIZE), TARGET_PAGE_SIZE); + hash = compute_page_hash(info->ramblock_addr + + vfn * qemu_target_page_size()); - trace_get_ramblock_vfn_hash(info->idstr, vfn, crc); - return crc; + trace_get_ramblock_vfn_hash(info->idstr, vfn, hash); + return hash; } static bool save_ramblock_hash(struct RamblockDirtyInfo *info) @@ -373,7 +400,7 @@ static void get_ramblock_dirty_info(RAMBlock *block, sample_pages_per_gigabytes) >> 30; /* Right shift TARGET_PAGE_BITS to calc page count */ info->ramblock_pages = qemu_ram_get_used_length(block) >> - TARGET_PAGE_BITS; + qemu_target_page_bits(); info->ramblock_addr = qemu_ram_get_host_addr(block); strcpy(info->idstr, qemu_ram_get_idstr(block)); } @@ -454,13 +481,13 @@ out: static void calc_page_dirty_rate(struct RamblockDirtyInfo *info) { - uint32_t crc; + uint32_t hash; int i; for (i = 0; i < info->sample_pages_count; i++) { - crc = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]); - if (crc != info->hash_result[i]) { - trace_calc_page_dirty_rate(info->idstr, crc, info->hash_result[i]); + hash = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]); + if (hash != info->hash_result[i]) { + trace_calc_page_dirty_rate(info->idstr, hash, info->hash_result[i]); info->sample_dirty_count++; } } @@ -484,7 +511,7 @@ find_block_matched(RAMBlock *block, int count, if (infos[i].ramblock_addr != qemu_ram_get_host_addr(block) || infos[i].ramblock_pages != - (qemu_ram_get_used_length(block) >> TARGET_PAGE_BITS)) { + (qemu_ram_get_used_length(block) >> qemu_target_page_bits())) { trace_find_page_matched(block->idstr); return NULL; } @@ -554,7 +581,7 @@ static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config) * skip it unconditionally and start dirty tracking * from 2'round of log sync */ - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(false); /* * reset page protect manually and unconditionally. diff --git a/migration/meson.build b/migration/meson.build index eb41b77db9..a8e01e70ae 100644 --- a/migration/meson.build +++ b/migration/meson.build @@ -1,5 +1,6 @@ # Files needed by unit tests migration_files = files( + 'migration-stats.c', 'page_cache.c', 'xbzrle.c', 'vmstate-types.c', @@ -13,15 +14,14 @@ softmmu_ss.add(files( 'block-dirty-bitmap.c', 'channel.c', 'channel-block.c', + 'dirtyrate.c', 'exec.c', 'fd.c', 'global_state.c', 'migration-hmp-cmds.c', - 'migration-stats.c', 'migration.c', 'multifd.c', 'multifd-zlib.c', - 'multifd-zlib.c', 'ram-compress.c', 'options.c', 'postcopy-ram.c', @@ -42,6 +42,5 @@ endif softmmu_ss.add(when: zstd, if_true: files('multifd-zstd.c')) specific_ss.add(when: 'CONFIG_SOFTMMU', - if_true: files('dirtyrate.c', - 'ram.c', + if_true: files('ram.c', 'target.c')) diff --git a/migration/migration-stats.c b/migration/migration-stats.c index 2f2cea965c..f98c8260be 100644 --- a/migration/migration-stats.c +++ b/migration/migration-stats.c @@ -12,6 +12,57 @@ #include "qemu/osdep.h" #include "qemu/stats64.h" +#include "qemu-file.h" +#include "trace.h" #include "migration-stats.h" MigrationAtomicStats mig_stats; + +bool migration_rate_exceeded(QEMUFile *f) +{ + if (qemu_file_get_error(f)) { + return true; + } + + uint64_t rate_limit_start = stat64_get(&mig_stats.rate_limit_start); + uint64_t rate_limit_current = migration_transferred_bytes(f); + uint64_t rate_limit_used = rate_limit_current - rate_limit_start; + uint64_t rate_limit_max = stat64_get(&mig_stats.rate_limit_max); + + if (rate_limit_max == RATE_LIMIT_DISABLED) { + return false; + } + if (rate_limit_max > 0 && rate_limit_used > rate_limit_max) { + return true; + } + return false; +} + +uint64_t migration_rate_get(void) +{ + return stat64_get(&mig_stats.rate_limit_max); +} + +#define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY) + +void migration_rate_set(uint64_t limit) +{ + /* + * 'limit' is per second. But we check it each BUFER_DELAY miliseconds. + */ + stat64_set(&mig_stats.rate_limit_max, limit / XFER_LIMIT_RATIO); +} + +void migration_rate_reset(QEMUFile *f) +{ + stat64_set(&mig_stats.rate_limit_start, migration_transferred_bytes(f)); +} + +uint64_t migration_transferred_bytes(QEMUFile *f) +{ + uint64_t multifd = stat64_get(&mig_stats.multifd_bytes); + uint64_t qemu_file = qemu_file_transferred(f); + + trace_migration_transferred_bytes(qemu_file, multifd); + return qemu_file + multifd; +} diff --git a/migration/migration-stats.h b/migration/migration-stats.h index cf8a4f0410..ac2260e987 100644 --- a/migration/migration-stats.h +++ b/migration/migration-stats.h @@ -16,6 +16,18 @@ #include "qemu/stats64.h" /* + * Amount of time to allocate to each "chunk" of bandwidth-throttled + * data. + */ +#define BUFFER_DELAY 100 + +/* + * If rate_limit_max is 0, there is special code to remove the rate + * limit. + */ +#define RATE_LIMIT_DISABLED 0 + +/* * These are the ram migration statistic counters. It is loosely * based on MigrationStats. We change to Stat64 any counter that * needs to be updated using atomic ops (can be accessed by more than @@ -70,6 +82,14 @@ typedef struct { */ Stat64 precopy_bytes; /* + * Amount of transferred data at the start of current cycle. + */ + Stat64 rate_limit_start; + /* + * Maximum amount of data we can send in a cycle. + */ + Stat64 rate_limit_max; + /* * Total number of bytes transferred. */ Stat64 transferred; @@ -81,4 +101,39 @@ typedef struct { extern MigrationAtomicStats mig_stats; +/** + * migration_rate_get: Get the maximum amount that can be transferred. + * + * Returns the maximum number of bytes that can be transferred in a cycle. + */ +uint64_t migration_rate_get(void); + +/** + * migration_rate_reset: Reset the rate limit counter. + * + * This is called when we know we start a new transfer cycle. + * + * @f: QEMUFile used for main migration channel + */ +void migration_rate_reset(QEMUFile *f); + +/** + * migration_rate_set: Set the maximum amount that can be transferred. + * + * Sets the maximum amount of bytes that can be transferred in one cycle. + * + * @new_rate: new maximum amount + */ +void migration_rate_set(uint64_t new_rate); + +/** + * migration_transferred_bytes: Return number of bytes transferred + * + * @f: QEMUFile used for main migration channel + * + * Returns how many bytes have we transferred since the beginning of + * the migration. It accounts for bytes sent through any migration + * channel, multifd, qemu_file, rdma, .... + */ +uint64_t migration_transferred_bytes(QEMUFile *f); #endif diff --git a/migration/migration.c b/migration/migration.c index 439e8651df..5de7f734b9 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -511,7 +511,6 @@ process_incoming_migration_co(void *opaque) MigrationIncomingState *mis = migration_incoming_get_current(); PostcopyState ps; int ret; - Error *local_err = NULL; assert(mis->from_src_file); @@ -520,12 +519,14 @@ process_incoming_migration_co(void *opaque) goto fail; } - mis->migration_incoming_co = qemu_coroutine_self(); mis->largest_page_size = qemu_ram_pagesize_largest(); postcopy_state_set(POSTCOPY_INCOMING_NONE); migrate_set_state(&mis->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_ACTIVE); + + mis->loadvm_co = qemu_coroutine_self(); ret = qemu_loadvm_state(mis->from_src_file); + mis->loadvm_co = NULL; ps = postcopy_state_get(); trace_process_incoming_migration_co_end(ret, ps); @@ -553,35 +554,14 @@ process_incoming_migration_co(void *opaque) goto fail; } - /* we get COLO info, and know if we are in COLO mode */ - if (migration_incoming_colo_enabled()) { - QemuThread colo_incoming_thread; - - /* Make sure all file formats throw away their mutable metadata */ - bdrv_activate_all(&local_err); - if (local_err) { - error_report_err(local_err); - goto fail; - } - - qemu_thread_create(&colo_incoming_thread, "COLO incoming", - colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE); - qemu_coroutine_yield(); - - qemu_mutex_unlock_iothread(); - /* Wait checkpoint incoming thread exit before free resource */ - qemu_thread_join(&colo_incoming_thread); - qemu_mutex_lock_iothread(); - /* We hold the global iothread lock, so it is safe here */ - colo_release_ram_cache(); + if (colo_incoming_co() < 0) { + goto fail; } mis->bh = qemu_bh_new(process_incoming_migration_bh, mis); qemu_bh_schedule(mis->bh); - mis->migration_incoming_co = NULL; return; fail: - local_err = NULL; migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, MIGRATION_STATUS_FAILED); qemu_fclose(mis->from_src_file); @@ -2140,12 +2120,7 @@ static int postcopy_start(MigrationState *ms) * will notice we're in POSTCOPY_ACTIVE and not actually * wrap their state up here */ - /* 0 max-postcopy-bandwidth means unlimited */ - if (!bandwidth) { - qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX); - } else { - qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO); - } + migration_rate_set(bandwidth); if (migrate_postcopy_ram()) { /* Ping just for debugging, helps line traces up */ qemu_savevm_send_ping(ms->to_dst_file, 2); @@ -2329,7 +2304,7 @@ static void migration_completion(MigrationState *s) * them if migration fails or is cancelled. */ s->block_inactive = !migrate_colo(); - qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX); + migration_rate_set(RATE_LIMIT_DISABLED); ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false, s->block_inactive); } @@ -2650,16 +2625,9 @@ static MigThrError migration_detect_error(MigrationState *s) } } -/* How many bytes have we transferred since the beginning of the migration */ -static uint64_t migration_total_bytes(MigrationState *s) -{ - return qemu_file_total_transferred(s->to_dst_file) + - stat64_get(&mig_stats.multifd_bytes); -} - static void migration_calculate_complete(MigrationState *s) { - uint64_t bytes = migration_total_bytes(s); + uint64_t bytes = migration_transferred_bytes(s->to_dst_file); int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); int64_t transfer_time; @@ -2685,7 +2653,7 @@ static void update_iteration_initial_status(MigrationState *s) * wrong speed calculation. */ s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); - s->iteration_initial_bytes = migration_total_bytes(s); + s->iteration_initial_bytes = migration_transferred_bytes(s->to_dst_file); s->iteration_initial_pages = ram_get_total_transferred_pages(); } @@ -2700,7 +2668,7 @@ static void migration_update_counters(MigrationState *s, return; } - current_bytes = migration_total_bytes(s); + current_bytes = migration_transferred_bytes(s->to_dst_file); transferred = current_bytes - s->iteration_initial_bytes; time_spent = current_time - s->iteration_start_time; bandwidth = (double)transferred / time_spent; @@ -2724,7 +2692,7 @@ static void migration_update_counters(MigrationState *s, stat64_get(&mig_stats.dirty_bytes_last_sync) / bandwidth; } - qemu_file_reset_rate_limit(s->to_dst_file); + migration_rate_reset(s->to_dst_file); update_iteration_initial_status(s); @@ -2877,7 +2845,7 @@ bool migration_rate_limit(void) bool urgent = false; migration_update_counters(s, now); - if (qemu_file_rate_limit(s->to_dst_file)) { + if (migration_rate_exceeded(s->to_dst_file)) { if (qemu_file_get_error(s->to_dst_file)) { return false; @@ -2999,7 +2967,7 @@ static void *migration_thread(void *opaque) trace_migration_thread_setup_complete(); while (migration_is_active(s)) { - if (urgent || !qemu_file_rate_limit(s->to_dst_file)) { + if (urgent || !migration_rate_exceeded(s->to_dst_file)) { MigIterateState iter_state = migration_iteration_run(s); if (iter_state == MIG_ITERATE_SKIP) { continue; @@ -3073,7 +3041,7 @@ static void *bg_migration_thread(void *opaque) rcu_register_thread(); object_ref(OBJECT(s)); - qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX); + migration_rate_set(RATE_LIMIT_DISABLED); setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST); /* @@ -3236,17 +3204,16 @@ void migrate_fd_connect(MigrationState *s, Error *error_in) if (resume) { /* This is a resumed migration */ - rate_limit = migrate_max_postcopy_bandwidth() / - XFER_LIMIT_RATIO; + rate_limit = migrate_max_postcopy_bandwidth(); } else { /* This is a fresh new migration */ - rate_limit = migrate_max_bandwidth() / XFER_LIMIT_RATIO; + rate_limit = migrate_max_bandwidth(); /* Notify before starting migration thread */ notifier_list_notify(&migration_state_notifiers, s); } - qemu_file_set_rate_limit(s->to_dst_file, rate_limit); + migration_rate_set(rate_limit); qemu_file_set_blocking(s->to_dst_file, true); /* diff --git a/migration/migration.h b/migration/migration.h index 7721c7658b..48a46123a0 100644 --- a/migration/migration.h +++ b/migration/migration.h @@ -162,8 +162,15 @@ struct MigrationIncomingState { int state; + /* + * The incoming migration coroutine, non-NULL during qemu_loadvm_state(). + * Used to wake the migration incoming coroutine from rdma code. How much is + * it safe - it's a question. + */ + Coroutine *loadvm_co; + /* The coroutine we should enter (back) after failover */ - Coroutine *migration_incoming_co; + Coroutine *colo_incoming_co; QemuSemaphore colo_incoming_sem; /* diff --git a/migration/multifd.c b/migration/multifd.c index 5c4298eadf..0bf5958a9c 100644 --- a/migration/multifd.c +++ b/migration/multifd.c @@ -175,6 +175,7 @@ void multifd_register_ops(int method, MultiFDMethods *ops) static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp) { MultiFDInit_t msg = {}; + size_t size = sizeof(msg); int ret; msg.magic = cpu_to_be32(MULTIFD_MAGIC); @@ -182,10 +183,12 @@ static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp) msg.id = p->id; memcpy(msg.uuid, &qemu_uuid.data, sizeof(msg.uuid)); - ret = qio_channel_write_all(p->c, (char *)&msg, sizeof(msg), errp); + ret = qio_channel_write_all(p->c, (char *)&msg, size, errp); if (ret != 0) { return -1; } + stat64_add(&mig_stats.multifd_bytes, size); + stat64_add(&mig_stats.transferred, size); return 0; } @@ -395,7 +398,6 @@ static int multifd_send_pages(QEMUFile *f) static int next_channel; MultiFDSendParams *p = NULL; /* make happy gcc */ MultiFDPages_t *pages = multifd_send_state->pages; - uint64_t transferred; if (qatomic_read(&multifd_send_state->exiting)) { return -1; @@ -430,11 +432,7 @@ static int multifd_send_pages(QEMUFile *f) p->packet_num = multifd_send_state->packet_num++; multifd_send_state->pages = p->pages; p->pages = pages; - transferred = ((uint64_t) pages->num) * p->page_size + p->packet_len; - qemu_file_acct_rate_limit(f, transferred); qemu_mutex_unlock(&p->mutex); - stat64_add(&mig_stats.transferred, transferred); - stat64_add(&mig_stats.multifd_bytes, transferred); qemu_sem_post(&p->sem); return 1; @@ -716,6 +714,8 @@ static void *multifd_send_thread(void *opaque) if (ret != 0) { break; } + stat64_add(&mig_stats.multifd_bytes, p->packet_len); + stat64_add(&mig_stats.transferred, p->packet_len); } else { /* Send header using the same writev call */ p->iov[0].iov_len = p->packet_len; @@ -728,6 +728,8 @@ static void *multifd_send_thread(void *opaque) break; } + stat64_add(&mig_stats.multifd_bytes, p->next_packet_size); + stat64_add(&mig_stats.transferred, p->next_packet_size); qemu_mutex_lock(&p->mutex); p->pending_job--; qemu_mutex_unlock(&p->mutex); diff --git a/migration/options.c b/migration/options.c index 7ed88b7b32..b62ab30cd5 100644 --- a/migration/options.c +++ b/migration/options.c @@ -23,6 +23,7 @@ #include "migration/colo.h" #include "migration/misc.h" #include "migration.h" +#include "migration-stats.h" #include "qemu-file.h" #include "ram.h" #include "options.h" @@ -1242,8 +1243,7 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) if (params->has_max_bandwidth) { s->parameters.max_bandwidth = params->max_bandwidth; if (s->to_dst_file && !migration_in_postcopy()) { - qemu_file_set_rate_limit(s->to_dst_file, - s->parameters.max_bandwidth / XFER_LIMIT_RATIO); + migration_rate_set(s->parameters.max_bandwidth); } } @@ -1272,8 +1272,7 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) if (params->has_max_postcopy_bandwidth) { s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth; if (s->to_dst_file && migration_in_postcopy()) { - qemu_file_set_rate_limit(s->to_dst_file, - s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO); + migration_rate_set(s->parameters.max_postcopy_bandwidth); } } if (params->has_max_cpu_throttle) { diff --git a/migration/options.h b/migration/options.h index 5cca3326d6..45991af3c2 100644 --- a/migration/options.h +++ b/migration/options.h @@ -17,13 +17,6 @@ #include "hw/qdev-properties.h" #include "hw/qdev-properties-system.h" -/* constants */ - -/* Amount of time to allocate to each "chunk" of bandwidth-throttled - * data. */ -#define BUFFER_DELAY 100 -#define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY) - /* migration properties */ extern Property migration_properties[]; diff --git a/migration/qemu-file.c b/migration/qemu-file.c index 61fb580342..acc282654a 100644 --- a/migration/qemu-file.c +++ b/migration/qemu-file.c @@ -27,8 +27,10 @@ #include "qemu/error-report.h" #include "qemu/iov.h" #include "migration.h" +#include "migration-stats.h" #include "qemu-file.h" #include "trace.h" +#include "options.h" #include "qapi/error.h" #define IO_BUF_SIZE 32768 @@ -39,17 +41,6 @@ struct QEMUFile { QIOChannel *ioc; bool is_writable; - /* - * Maximum amount of data in bytes to transfer during one - * rate limiting time window - */ - int64_t rate_limit_max; - /* - * Total amount of data in bytes queued for transfer - * during this rate limiting time window - */ - int64_t rate_limit_used; - /* The sum of bytes transferred on the wire */ uint64_t total_transferred; @@ -300,7 +291,8 @@ void qemu_fflush(QEMUFile *f) &local_error) < 0) { qemu_file_set_error_obj(f, -EIO, local_error); } else { - f->total_transferred += iov_size(f->iov, f->iovcnt); + uint64_t size = iov_size(f->iov, f->iovcnt); + f->total_transferred += size; } qemu_iovec_release_ram(f); @@ -351,9 +343,6 @@ size_t ram_control_save_page(QEMUFile *f, ram_addr_t block_offset, if (f->hooks && f->hooks->save_page) { int ret = f->hooks->save_page(f, block_offset, offset, size, bytes_sent); - if (ret != RAM_SAVE_CONTROL_NOT_SUPP) { - qemu_file_acct_rate_limit(f, size); - } if (ret != RAM_SAVE_CONTROL_DELAYED && ret != RAM_SAVE_CONTROL_NOT_SUPP) { @@ -517,7 +506,6 @@ void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size, return; } - f->rate_limit_used += size; add_to_iovec(f, buf, size, may_free); } @@ -535,7 +523,6 @@ void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size) l = size; } memcpy(f->buf + f->buf_index, buf, l); - f->rate_limit_used += l; add_buf_to_iovec(f, l); if (qemu_file_get_error(f)) { break; @@ -552,7 +539,6 @@ void qemu_put_byte(QEMUFile *f, int v) } f->buf[f->buf_index] = v; - f->rate_limit_used++; add_buf_to_iovec(f, 1); } @@ -708,7 +694,7 @@ int coroutine_mixed_fn qemu_get_byte(QEMUFile *f) return result; } -uint64_t qemu_file_total_transferred_fast(QEMUFile *f) +uint64_t qemu_file_transferred_fast(QEMUFile *f) { uint64_t ret = f->total_transferred; int i; @@ -720,43 +706,12 @@ uint64_t qemu_file_total_transferred_fast(QEMUFile *f) return ret; } -uint64_t qemu_file_total_transferred(QEMUFile *f) +uint64_t qemu_file_transferred(QEMUFile *f) { qemu_fflush(f); return f->total_transferred; } -int qemu_file_rate_limit(QEMUFile *f) -{ - if (qemu_file_get_error(f)) { - return 1; - } - if (f->rate_limit_max > 0 && f->rate_limit_used > f->rate_limit_max) { - return 1; - } - return 0; -} - -int64_t qemu_file_get_rate_limit(QEMUFile *f) -{ - return f->rate_limit_max; -} - -void qemu_file_set_rate_limit(QEMUFile *f, int64_t limit) -{ - f->rate_limit_max = limit; -} - -void qemu_file_reset_rate_limit(QEMUFile *f) -{ - f->rate_limit_used = 0; -} - -void qemu_file_acct_rate_limit(QEMUFile *f, int64_t len) -{ - f->rate_limit_used += len; -} - void qemu_put_be16(QEMUFile *f, unsigned int v) { qemu_put_byte(f, v >> 8); diff --git a/migration/qemu-file.h b/migration/qemu-file.h index 4ee58a87dd..e649718492 100644 --- a/migration/qemu-file.h +++ b/migration/qemu-file.h @@ -68,7 +68,7 @@ void qemu_file_set_hooks(QEMUFile *f, const QEMUFileHooks *hooks); int qemu_fclose(QEMUFile *f); /* - * qemu_file_total_transferred: + * qemu_file_transferred: * * Report the total number of bytes transferred with * this file. @@ -83,19 +83,19 @@ int qemu_fclose(QEMUFile *f); * * Returns: the total bytes transferred */ -uint64_t qemu_file_total_transferred(QEMUFile *f); +uint64_t qemu_file_transferred(QEMUFile *f); /* - * qemu_file_total_transferred_fast: + * qemu_file_transferred_fast: * - * As qemu_file_total_transferred except for writable + * As qemu_file_transferred except for writable * files, where no flush is performed and the reported * amount will include the size of any queued buffers, * on top of the amount actually transferred. * * Returns: the total bytes transferred and queued */ -uint64_t qemu_file_total_transferred_fast(QEMUFile *f); +uint64_t qemu_file_transferred_fast(QEMUFile *f); /* * put_buffer without copying the buffer. @@ -130,17 +130,6 @@ void qemu_file_skip(QEMUFile *f, int size); * accounting information tracks the total migration traffic. */ void qemu_file_credit_transfer(QEMUFile *f, size_t size); -void qemu_file_reset_rate_limit(QEMUFile *f); -/* - * qemu_file_acct_rate_limit: - * - * Report on a number of bytes the have been transferred - * out of band from the main file object I/O methods, and - * need to be applied to the rate limiting calcuations - */ -void qemu_file_acct_rate_limit(QEMUFile *f, int64_t len); -void qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate); -int64_t qemu_file_get_rate_limit(QEMUFile *f); int qemu_file_get_error_obj(QEMUFile *f, Error **errp); int qemu_file_get_error_obj_any(QEMUFile *f1, QEMUFile *f2, Error **errp); void qemu_file_set_error_obj(QEMUFile *f, int ret, Error *err); diff --git a/migration/ram.c b/migration/ram.c index f69d8d42b0..9fb076fa58 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -1039,7 +1039,7 @@ static void migration_trigger_throttle(RAMState *rs) } } -static void migration_bitmap_sync(RAMState *rs) +static void migration_bitmap_sync(RAMState *rs, bool last_stage) { RAMBlock *block; int64_t end_time; @@ -1051,7 +1051,7 @@ static void migration_bitmap_sync(RAMState *rs) } trace_migration_bitmap_sync_start(); - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(last_stage); qemu_mutex_lock(&rs->bitmap_mutex); WITH_RCU_READ_LOCK_GUARD() { @@ -1086,7 +1086,7 @@ static void migration_bitmap_sync(RAMState *rs) } } -static void migration_bitmap_sync_precopy(RAMState *rs) +static void migration_bitmap_sync_precopy(RAMState *rs, bool last_stage) { Error *local_err = NULL; @@ -1099,7 +1099,7 @@ static void migration_bitmap_sync_precopy(RAMState *rs) local_err = NULL; } - migration_bitmap_sync(rs); + migration_bitmap_sync(rs, last_stage); if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) { error_report_err(local_err); @@ -2699,7 +2699,7 @@ void ram_postcopy_send_discard_bitmap(MigrationState *ms) RCU_READ_LOCK_GUARD(); /* This should be our last sync, the src is now paused */ - migration_bitmap_sync(rs); + migration_bitmap_sync(rs, false); /* Easiest way to make sure we don't resume in the middle of a host-page */ rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL; @@ -2890,7 +2890,7 @@ static void ram_init_bitmaps(RAMState *rs) /* We don't use dirty log with background snapshots */ if (!migrate_background_snapshot()) { memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION); - migration_bitmap_sync_precopy(rs); + migration_bitmap_sync_precopy(rs, false); } } qemu_mutex_unlock_ramlist(); @@ -3116,7 +3116,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); i = 0; - while ((ret = qemu_file_rate_limit(f)) == 0 || + while ((ret = migration_rate_exceeded(f)) == 0 || postcopy_has_request(rs)) { int pages; @@ -3214,7 +3214,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque) WITH_RCU_READ_LOCK_GUARD() { if (!migration_in_postcopy()) { - migration_bitmap_sync_precopy(rs); + migration_bitmap_sync_precopy(rs, true); } ram_control_before_iterate(f, RAM_CONTROL_FINISH); @@ -3288,7 +3288,7 @@ static void ram_state_pending_exact(void *opaque, uint64_t *must_precopy, if (!migration_in_postcopy() && remaining_size < s->threshold_size) { qemu_mutex_lock_iothread(); WITH_RCU_READ_LOCK_GUARD() { - migration_bitmap_sync_precopy(rs); + migration_bitmap_sync_precopy(rs, false); } qemu_mutex_unlock_iothread(); remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE; @@ -3523,7 +3523,7 @@ void colo_incoming_start_dirty_log(void) qemu_mutex_lock_iothread(); qemu_mutex_lock_ramlist(); - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(false); WITH_RCU_READ_LOCK_GUARD() { RAMBLOCK_FOREACH_NOT_IGNORED(block) { ramblock_sync_dirty_bitmap(ram_state, block); @@ -3813,7 +3813,7 @@ void colo_flush_ram_cache(void) void *src_host; unsigned long offset = 0; - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(false); qemu_mutex_lock(&ram_state->bitmap_mutex); WITH_RCU_READ_LOCK_GUARD() { RAMBLOCK_FOREACH_NOT_IGNORED(block) { diff --git a/migration/rdma.c b/migration/rdma.c index 2cd8f1cc66..2e4dcff1c9 100644 --- a/migration/rdma.c +++ b/migration/rdma.c @@ -3342,9 +3342,8 @@ static void rdma_cm_poll_handler(void *opaque) } } rdma_ack_cm_event(cm_event); - - if (mis->migration_incoming_co) { - qemu_coroutine_enter(mis->migration_incoming_co); + if (mis->loadvm_co) { + qemu_coroutine_enter(mis->loadvm_co); } return; } diff --git a/migration/savevm.c b/migration/savevm.c index 032044b1d5..03795ce8dc 100644 --- a/migration/savevm.c +++ b/migration/savevm.c @@ -927,9 +927,9 @@ static int vmstate_load(QEMUFile *f, SaveStateEntry *se) static void vmstate_save_old_style(QEMUFile *f, SaveStateEntry *se, JSONWriter *vmdesc) { - uint64_t old_offset = qemu_file_total_transferred_fast(f); + uint64_t old_offset = qemu_file_transferred_fast(f); se->ops->save_state(f, se->opaque); - uint64_t size = qemu_file_total_transferred_fast(f) - old_offset; + uint64_t size = qemu_file_transferred_fast(f) - old_offset; if (vmdesc) { json_writer_int64(vmdesc, "size", size); @@ -1338,7 +1338,7 @@ int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy) !(se->ops->has_postcopy && se->ops->has_postcopy(se->opaque))) { continue; } - if (qemu_file_rate_limit(f)) { + if (migration_rate_exceeded(f)) { return 0; } trace_savevm_section_start(se->idstr, se->section_id); @@ -2956,7 +2956,7 @@ bool save_snapshot(const char *name, bool overwrite, const char *vmstate, goto the_end; } ret = qemu_savevm_state(f, errp); - vm_state_size = qemu_file_total_transferred(f); + vm_state_size = qemu_file_transferred(f); ret2 = qemu_fclose(f); if (ret < 0) { goto the_end; diff --git a/migration/trace-events b/migration/trace-events index 92161eeac5..cdaef7a1ea 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -186,6 +186,9 @@ process_incoming_migration_co_end(int ret, int ps) "ret=%d postcopy-state=%d" process_incoming_migration_co_postcopy_end_main(void) "" postcopy_preempt_enabled(bool value) "%d" +# migration-stats +migration_transferred_bytes(uint64_t qemu_file, uint64_t multifd) "qemu_file %" PRIu64 " multifd %" PRIu64 + # channel.c migration_set_incoming_channel(void *ioc, const char *ioctype) "ioc=%p ioctype=%s" migration_set_outgoing_channel(void *ioc, const char *ioctype, const char *hostname, void *err) "ioc=%p ioctype=%s hostname=%s err=%p" @@ -342,8 +345,8 @@ dirty_bitmap_load_success(void) "" # dirtyrate.c dirtyrate_set_state(const char *new_state) "new state %s" query_dirty_rate_info(const char *new_state) "current state %s" -get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t crc) "ramblock name: %s, vfn: %"PRIu64 ", crc: %" PRIu32 -calc_page_dirty_rate(const char *idstr, uint32_t new_crc, uint32_t old_crc) "ramblock name: %s, new crc: %" PRIu32 ", old crc: %" PRIu32 +get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t hash) "ramblock name: %s, vfn: %"PRIu64 ", hash: %" PRIu32 +calc_page_dirty_rate(const char *idstr, uint32_t new_hash, uint32_t old_hash) "ramblock name: %s, new hash: %" PRIu32 ", old hash: %" PRIu32 skip_sample_ramblock(const char *idstr, uint64_t ramblock_size) "ramblock name: %s, ramblock size: %" PRIu64 find_page_matched(const char *idstr) "ramblock %s addr or size changed" dirtyrate_calculate(int64_t dirtyrate) "dirty rate: %" PRIi64 " MB/s" diff --git a/migration/vmstate.c b/migration/vmstate.c index 351f56104e..af01d54b6f 100644 --- a/migration/vmstate.c +++ b/migration/vmstate.c @@ -361,7 +361,7 @@ int vmstate_save_state_v(QEMUFile *f, const VMStateDescription *vmsd, void *curr_elem = first_elem + size * i; vmsd_desc_field_start(vmsd, vmdesc_loop, field, i, n_elems); - old_offset = qemu_file_total_transferred_fast(f); + old_offset = qemu_file_transferred_fast(f); if (field->flags & VMS_ARRAY_OF_POINTER) { assert(curr_elem); curr_elem = *(void **)curr_elem; @@ -391,8 +391,7 @@ int vmstate_save_state_v(QEMUFile *f, const VMStateDescription *vmsd, return ret; } - written_bytes = qemu_file_total_transferred_fast(f) - - old_offset; + written_bytes = qemu_file_transferred_fast(f) - old_offset; vmsd_desc_field_end(vmsd, vmdesc_loop, field, written_bytes, i); /* Compressed arrays only care about the first element */ diff --git a/net/meson.build b/net/meson.build index 87afca3e93..6f4ecde57f 100644 --- a/net/meson.build +++ b/net/meson.build @@ -1,13 +1,10 @@ softmmu_ss.add(files( 'announce.c', 'checksum.c', - 'colo-compare.c', - 'colo.c', 'dump.c', 'eth.c', 'filter-buffer.c', 'filter-mirror.c', - 'filter-rewriter.c', 'filter.c', 'hub.c', 'net-hmp-cmds.c', @@ -19,6 +16,16 @@ softmmu_ss.add(files( 'util.c', )) +if get_option('replication').allowed() or \ + get_option('colo_proxy').allowed() + softmmu_ss.add(files('colo-compare.c')) + softmmu_ss.add(files('colo.c')) +endif + +if get_option('colo_proxy').allowed() + softmmu_ss.add(files('filter-rewriter.c')) +endif + softmmu_ss.add(when: 'CONFIG_TCG', if_true: files('filter-replay.c')) if have_l2tpv3 diff --git a/os-posix.c b/os-posix.c index 5adc69f560..90ea71725f 100644 --- a/os-posix.c +++ b/os-posix.c @@ -36,6 +36,8 @@ #include "qemu/log.h" #include "sysemu/runstate.h" #include "qemu/cutils.h" +#include "qemu/config-file.h" +#include "qemu/option.h" #ifdef CONFIG_LINUX #include <sys/prctl.h> @@ -152,9 +154,21 @@ int os_parse_cmd_args(int index, const char *optarg) daemonize = 1; break; #if defined(CONFIG_LINUX) + /* deprecated */ case QEMU_OPTION_asyncteardown: init_async_teardown(); break; + case QEMU_OPTION_run_with: { + QemuOpts *opts = qemu_opts_parse_noisily(qemu_find_opts("run-with"), + optarg, false); + if (!opts) { + exit(1); + } + if (qemu_opt_get_bool(opts, "async-teardown", false)) { + init_async_teardown(); + } + break; + } #endif default: return -1; diff --git a/python/Makefile b/python/Makefile index c5bd6ff83a..7c70dcc8d1 100644 --- a/python/Makefile +++ b/python/Makefile @@ -9,14 +9,14 @@ help: @echo "make check-minreqs:" @echo " Run tests in the minreqs virtual environment." @echo " These tests use the oldest dependencies." - @echo " Requires: Python 3.6" - @echo " Hint (Fedora): 'sudo dnf install python3.6'" + @echo " Requires: Python 3.7" + @echo " Hint (Fedora): 'sudo dnf install python3.7'" @echo "" @echo "make check-tox:" @echo " Run tests against multiple python versions." @echo " These tests use the newest dependencies." - @echo " Requires: Python 3.6 - 3.10, and tox." - @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.10'" + @echo " Requires: Python 3.7 - 3.11, and tox." + @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.11'" @echo " The variable QEMU_TOX_EXTRA_ARGS can be use to pass extra" @echo " arguments to tox". @echo "" @@ -54,18 +54,21 @@ pipenv check-pipenv: @echo "pipenv was dropped; try 'make check-minreqs' or 'make min-venv'" @exit 1 +PIP_INSTALL = pip install --disable-pip-version-check .PHONY: min-venv min-venv: $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate: setup.cfg tests/minreqs.txt @echo "VENV $(QEMU_MINVENV_DIR)" - @python3.6 -m venv $(QEMU_MINVENV_DIR) + @python3.7 -m venv $(QEMU_MINVENV_DIR) @( \ echo "ACTIVATE $(QEMU_MINVENV_DIR)"; \ . $(QEMU_MINVENV_DIR)/bin/activate; \ + echo "INSTALL wheel $(QEMU_MINVENV_DIR)"; \ + $(PIP_INSTALL) wheel 1>/dev/null; \ echo "INSTALL -r tests/minreqs.txt $(QEMU_MINVENV_DIR)";\ - pip install -r tests/minreqs.txt 1>/dev/null; \ + $(PIP_INSTALL) -r tests/minreqs.txt 1>/dev/null; \ echo "INSTALL -e qemu $(QEMU_MINVENV_DIR)"; \ - pip install -e . 1>/dev/null; \ + $(PIP_INSTALL) -e . 1>/dev/null; \ ) @touch $(QEMU_MINVENV_DIR) @@ -100,7 +103,7 @@ check-dev: dev-venv .PHONY: develop develop: - pip3 install --disable-pip-version-check -e .[devel] + $(PIP_INSTALL) -e .[devel] .PHONY: check check: diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py new file mode 100644 index 0000000000..8c036c019a --- /dev/null +++ b/python/scripts/mkvenv.py @@ -0,0 +1,897 @@ +""" +mkvenv - QEMU pyvenv bootstrapping utility + +usage: mkvenv [-h] command ... + +QEMU pyvenv bootstrapping utility + +options: + -h, --help show this help message and exit + +Commands: + command Description + create create a venv + post_init + post-venv initialization + ensure Ensure that the specified package is installed. + +-------------------------------------------------- + +usage: mkvenv create [-h] target + +positional arguments: + target Target directory to install virtual environment into. + +options: + -h, --help show this help message and exit + +-------------------------------------------------- + +usage: mkvenv post_init [-h] + +options: + -h, --help show this help message and exit + +-------------------------------------------------- + +usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... + +positional arguments: + dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' + +options: + -h, --help show this help message and exit + --online Install packages from PyPI, if necessary. + --dir DIR Path to vendored packages where we may install from. + +""" + +# Copyright (C) 2022-2023 Red Hat, Inc. +# +# Authors: +# John Snow <jsnow@redhat.com> +# Paolo Bonzini <pbonzini@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import argparse +from importlib.util import find_spec +import logging +import os +from pathlib import Path +import re +import shutil +import site +import subprocess +import sys +import sysconfig +from types import SimpleNamespace +from typing import ( + Any, + Iterator, + Optional, + Sequence, + Tuple, + Union, +) +import venv +import warnings + + +# Try to load distlib, with a fallback to pip's vendored version. +# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail +# outside the venv or before a potential call to ensurepip in checkpip(). +HAVE_DISTLIB = True +try: + import distlib.database + import distlib.scripts + import distlib.version +except ImportError: + try: + # Reach into pip's cookie jar. pylint and flake8 don't understand + # that these imports will be used via distlib.xxx. + from pip._vendor import distlib + import pip._vendor.distlib.database # noqa, pylint: disable=unused-import + import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import + import pip._vendor.distlib.version # noqa, pylint: disable=unused-import + except ImportError: + HAVE_DISTLIB = False + +# Do not add any mandatory dependencies from outside the stdlib: +# This script *must* be usable standalone! + +DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] +logger = logging.getLogger("mkvenv") + + +def inside_a_venv() -> bool: + """Returns True if it is executed inside of a virtual environment.""" + return sys.prefix != sys.base_prefix + + +class Ouch(RuntimeError): + """An Exception class we can't confuse with a builtin.""" + + +class QemuEnvBuilder(venv.EnvBuilder): + """ + An extension of venv.EnvBuilder for building QEMU's configure-time venv. + + The primary difference is that it emulates a "nested" virtual + environment when invoked from inside of an existing virtual + environment by including packages from the parent. Also, + "ensurepip" is replaced if possible with just recreating pip's + console_scripts inside the virtual environment. + + Parameters for base class init: + - system_site_packages: bool = False + - clear: bool = False + - symlinks: bool = False + - upgrade: bool = False + - with_pip: bool = False + - prompt: Optional[str] = None + - upgrade_deps: bool = False (Since 3.9) + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + logger.debug("QemuEnvBuilder.__init__(...)") + + # For nested venv emulation: + self.use_parent_packages = False + if inside_a_venv(): + # Include parent packages only if we're in a venv and + # system_site_packages was True. + self.use_parent_packages = kwargs.pop( + "system_site_packages", False + ) + # Include system_site_packages only when the parent, + # The venv we are currently in, also does so. + kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES + + # ensurepip is slow: venv creation can be very fast for cases where + # we allow the use of system_site_packages. Therefore, ensurepip is + # replaced with our own script generation once the virtual environment + # is setup. + self.want_pip = kwargs.get("with_pip", False) + if self.want_pip: + if ( + kwargs.get("system_site_packages", False) + and not need_ensurepip() + ): + kwargs["with_pip"] = False + else: + check_ensurepip(suggest_remedy=True) + + super().__init__(*args, **kwargs) + + # Make the context available post-creation: + self._context: Optional[SimpleNamespace] = None + + def get_parent_libpath(self) -> Optional[str]: + """Return the libpath of the parent venv, if applicable.""" + if self.use_parent_packages: + return sysconfig.get_path("purelib") + return None + + @staticmethod + def compute_venv_libpath(context: SimpleNamespace) -> str: + """ + Compatibility wrapper for context.lib_path for Python < 3.12 + """ + # Python 3.12+, not strictly necessary because it's documented + # to be the same as 3.10 code below: + if sys.version_info >= (3, 12): + return context.lib_path + + # Python 3.10+ + if "venv" in sysconfig.get_scheme_names(): + lib_path = sysconfig.get_path( + "purelib", scheme="venv", vars={"base": context.env_dir} + ) + assert lib_path is not None + return lib_path + + # For Python <= 3.9 we need to hardcode this. Fortunately the + # code below was the same in Python 3.6-3.10, so there is only + # one case. + if sys.platform == "win32": + return os.path.join(context.env_dir, "Lib", "site-packages") + return os.path.join( + context.env_dir, + "lib", + "python%d.%d" % sys.version_info[:2], + "site-packages", + ) + + def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: + logger.debug("ensure_directories(env_dir=%s)", env_dir) + self._context = super().ensure_directories(env_dir) + return self._context + + def create(self, env_dir: DirType) -> None: + logger.debug("create(env_dir=%s)", env_dir) + super().create(env_dir) + assert self._context is not None + self.post_post_setup(self._context) + + def post_post_setup(self, context: SimpleNamespace) -> None: + """ + The final, final hook. Enter the venv and run commands inside of it. + """ + if self.use_parent_packages: + # We're inside of a venv and we want to include the parent + # venv's packages. + parent_libpath = self.get_parent_libpath() + assert parent_libpath is not None + logger.debug("parent_libpath: %s", parent_libpath) + + our_libpath = self.compute_venv_libpath(context) + logger.debug("our_libpath: %s", our_libpath) + + pth_file = os.path.join(our_libpath, "nested.pth") + with open(pth_file, "w", encoding="UTF-8") as file: + file.write(parent_libpath + os.linesep) + + if self.want_pip: + args = [ + context.env_exe, + __file__, + "post_init", + ] + subprocess.run(args, check=True) + + def get_value(self, field: str) -> str: + """ + Get a string value from the context namespace after a call to build. + + For valid field names, see: + https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories + """ + ret = getattr(self._context, field) + assert isinstance(ret, str) + return ret + + +def need_ensurepip() -> bool: + """ + Tests for the presence of setuptools and pip. + + :return: `True` if we do not detect both packages. + """ + # Don't try to actually import them, it's fraught with danger: + # https://github.com/pypa/setuptools/issues/2993 + if find_spec("setuptools") and find_spec("pip"): + return False + return True + + +def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: + """ + Check that we have ensurepip. + + Raise a fatal exception with a helpful hint if it isn't available. + """ + if not find_spec("ensurepip"): + msg = ( + "Python's ensurepip module is not found.\n" + "It's normally part of the Python standard library, " + "maybe your distribution packages it separately?\n" + "(Debian puts ensurepip in its python3-venv package.)\n" + ) + if suggest_remedy: + msg += ( + "Either install ensurepip, or alleviate the need for it in the" + " first place by installing pip and setuptools for " + f"'{sys.executable}'.\n" + ) + raise Ouch(prefix + msg) + + # ensurepip uses pyexpat, which can also go missing on us: + if not find_spec("pyexpat"): + msg = ( + "Python's pyexpat module is not found.\n" + "It's normally part of the Python standard library, " + "maybe your distribution packages it separately?\n" + "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n" + ) + if suggest_remedy: + msg += ( + "Either install pyexpat, or alleviate the need for it in the " + "first place by installing pip and setuptools for " + f"'{sys.executable}'.\n" + ) + raise Ouch(prefix + msg) + + +def make_venv( # pylint: disable=too-many-arguments + env_dir: Union[str, Path], + system_site_packages: bool = False, + clear: bool = True, + symlinks: Optional[bool] = None, + with_pip: bool = True, +) -> None: + """ + Create a venv using `QemuEnvBuilder`. + + This is analogous to the `venv.create` module-level convenience + function that is part of the Python stdblib, except it uses + `QemuEnvBuilder` instead. + + :param env_dir: The directory to create/install to. + :param system_site_packages: + Allow inheriting packages from the system installation. + :param clear: When True, fully remove any prior venv and files. + :param symlinks: + Whether to use symlinks to the target interpreter or not. If + left unspecified, it will use symlinks except on Windows to + match behavior with the "venv" CLI tool. + :param with_pip: + Whether to install "pip" binaries or not. + """ + logger.debug( + "%s: make_venv(env_dir=%s, system_site_packages=%s, " + "clear=%s, symlinks=%s, with_pip=%s)", + __file__, + str(env_dir), + system_site_packages, + clear, + symlinks, + with_pip, + ) + + if symlinks is None: + # Default behavior of standard venv CLI + symlinks = os.name != "nt" + + builder = QemuEnvBuilder( + system_site_packages=system_site_packages, + clear=clear, + symlinks=symlinks, + with_pip=with_pip, + ) + + style = "non-isolated" if builder.system_site_packages else "isolated" + nested = "" + if builder.use_parent_packages: + nested = f"(with packages from '{builder.get_parent_libpath()}') " + print( + f"mkvenv: Creating {style} virtual environment" + f" {nested}at '{str(env_dir)}'", + file=sys.stderr, + ) + + try: + logger.debug("Invoking builder.create()") + try: + builder.create(str(env_dir)) + except SystemExit as exc: + # Some versions of the venv module raise SystemExit; *nasty*! + # We want the exception that prompted it. It might be a subprocess + # error that has output we *really* want to see. + logger.debug("Intercepted SystemExit from EnvBuilder.create()") + raise exc.__cause__ or exc.__context__ or exc + logger.debug("builder.create() finished") + except subprocess.CalledProcessError as exc: + logger.error("mkvenv subprocess failed:") + logger.error("cmd: %s", exc.cmd) + logger.error("returncode: %d", exc.returncode) + + def _stringify(data: Union[str, bytes]) -> str: + if isinstance(data, bytes): + return data.decode() + return data + + lines = [] + if exc.stdout: + lines.append("========== stdout ==========") + lines.append(_stringify(exc.stdout)) + lines.append("============================") + if exc.stderr: + lines.append("========== stderr ==========") + lines.append(_stringify(exc.stderr)) + lines.append("============================") + if lines: + logger.error(os.linesep.join(lines)) + + raise Ouch("VENV creation subprocess failed.") from exc + + # print the python executable to stdout for configure. + print(builder.get_value("env_exe")) + + +def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: + # pylint: disable=import-outside-toplevel + # pylint: disable=no-name-in-module + # pylint: disable=import-error + try: + # First preference: Python 3.8+ stdlib + from importlib.metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + except ImportError as exc: + logger.debug("%s", str(exc)) + # Second preference: Commonly available PyPI backport + from importlib_metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + + def _generator() -> Iterator[str]: + for package in packages: + try: + entry_points = distribution(package).entry_points + except PackageNotFoundError: + continue + + # The EntryPoints type is only available in 3.10+, + # treat this as a vanilla list and filter it ourselves. + entry_points = filter( + lambda ep: ep.group == "console_scripts", entry_points + ) + + for entry_point in entry_points: + yield f"{entry_point.name} = {entry_point.value}" + + return _generator() + + +def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: + # pylint: disable=import-outside-toplevel + # Bundled with setuptools; has a good chance of being available. + import pkg_resources + + def _generator() -> Iterator[str]: + for package in packages: + try: + eps = pkg_resources.get_entry_map(package, "console_scripts") + except pkg_resources.DistributionNotFound: + continue + + for entry_point in eps.values(): + yield str(entry_point) + + return _generator() + + +def generate_console_scripts( + packages: Sequence[str], + python_path: Optional[str] = None, + bin_path: Optional[str] = None, +) -> None: + """ + Generate script shims for console_script entry points in @packages. + """ + if python_path is None: + python_path = sys.executable + if bin_path is None: + bin_path = sysconfig.get_path("scripts") + assert bin_path is not None + + logger.debug( + "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", + packages, + python_path, + bin_path, + ) + + if not packages: + return + + def _get_entry_points() -> Iterator[str]: + """Python 3.7 compatibility shim for iterating entry points.""" + # Python 3.8+, or Python 3.7 with importlib_metadata installed. + try: + return _gen_importlib(packages) + except ImportError as exc: + logger.debug("%s", str(exc)) + + # Python 3.7 with setuptools installed. + try: + return _gen_pkg_resources(packages) + except ImportError as exc: + logger.debug("%s", str(exc)) + raise Ouch( + "Neither importlib.metadata nor pkg_resources found, " + "can't generate console script shims.\n" + "Use Python 3.8+, or install importlib-metadata or setuptools." + ) from exc + + maker = distlib.scripts.ScriptMaker(None, bin_path) + maker.variants = {""} + maker.clobber = False + + for entry_point in _get_entry_points(): + for filename in maker.make(entry_point): + logger.debug("wrote console_script '%s'", filename) + + +def checkpip() -> bool: + """ + Debian10 has a pip that's broken when used inside of a virtual environment. + + We try to detect and correct that case here. + """ + try: + # pylint: disable=import-outside-toplevel,unused-import,import-error + # pylint: disable=redefined-outer-name + import pip._internal # type: ignore # noqa: F401 + + logger.debug("pip appears to be working correctly.") + return False + except ModuleNotFoundError as exc: + if exc.name == "pip._internal": + # Uh, fair enough. They did say "internal". + # Let's just assume it's fine. + return False + logger.warning("pip appears to be malfunctioning: %s", str(exc)) + + check_ensurepip("pip appears to be non-functional, and ") + + logger.debug("Attempting to repair pip ...") + subprocess.run( + (sys.executable, "-m", "ensurepip"), + stdout=subprocess.DEVNULL, + check=True, + ) + logger.debug("Pip is now (hopefully) repaired!") + return True + + +def pkgname_from_depspec(dep_spec: str) -> str: + """ + Parse package name out of a PEP-508 depspec. + + See https://peps.python.org/pep-0508/#names + """ + match = re.match( + r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE + ) + if not match: + raise ValueError( + f"dep_spec '{dep_spec}'" + " does not appear to contain a valid package name" + ) + return match.group(0) + + +def diagnose( + dep_spec: str, + online: bool, + wheels_dir: Optional[Union[str, Path]], + prog: Optional[str], +) -> Tuple[str, bool]: + """ + Offer a summary to the user as to why a package failed to be installed. + + :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' + :param online: Did we allow PyPI access? + :param prog: + Optionally, a shell program name that can be used as a + bellwether to detect if this program is installed elsewhere on + the system. This is used to offer advice when a program is + detected for a different python version. + :param wheels_dir: + Optionally, a directory that was searched for vendored packages. + """ + # pylint: disable=too-many-branches + + # Some errors are not particularly serious + bad = False + + pkg_name = pkgname_from_depspec(dep_spec) + pkg_version = None + + has_importlib = False + try: + # Python 3.8+ stdlib + # pylint: disable=import-outside-toplevel + # pylint: disable=no-name-in-module + # pylint: disable=import-error + from importlib.metadata import ( # type: ignore + PackageNotFoundError, + version, + ) + + has_importlib = True + try: + pkg_version = version(pkg_name) + except PackageNotFoundError: + pass + except ModuleNotFoundError: + pass + + lines = [] + + if pkg_version: + lines.append( + f"Python package '{pkg_name}' version '{pkg_version}' was found," + " but isn't suitable." + ) + elif has_importlib: + lines.append( + f"Python package '{pkg_name}' was not found nor installed." + ) + else: + lines.append( + f"Python package '{pkg_name}' is either not found or" + " not a suitable version." + ) + + if wheels_dir: + lines.append( + "No suitable version found in, or failed to install from" + f" '{wheels_dir}'." + ) + bad = True + + if online: + lines.append("A suitable version could not be obtained from PyPI.") + bad = True + else: + lines.append( + "mkvenv was configured to operate offline and did not check PyPI." + ) + + if prog and not pkg_version: + which = shutil.which(prog) + if which: + if sys.base_prefix in site.PREFIXES: + pypath = Path(sys.executable).resolve() + lines.append( + f"'{prog}' was detected on your system at '{which}', " + f"but the Python package '{pkg_name}' was not found by " + f"this Python interpreter ('{pypath}'). " + f"Typically this means that '{prog}' has been installed " + "against a different Python interpreter on your system." + ) + else: + lines.append( + f"'{prog}' was detected on your system at '{which}', " + "but the build is using an isolated virtual environment." + ) + bad = True + + lines = [f" • {line}" for line in lines] + if bad: + lines.insert(0, f"Could not provide build dependency '{dep_spec}':") + else: + lines.insert(0, f"'{dep_spec}' not found:") + return os.linesep.join(lines), bad + + +def pip_install( + args: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, +) -> None: + """ + Use pip to install a package or package(s) as specified in @args. + """ + loud = bool( + os.environ.get("DEBUG") + or os.environ.get("GITLAB_CI") + or os.environ.get("V") + ) + + full_args = [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "-v" if loud else "-q", + ] + if not online: + full_args += ["--no-index"] + if wheels_dir: + full_args += ["--find-links", f"file://{str(wheels_dir)}"] + full_args += list(args) + subprocess.run( + full_args, + check=True, + ) + + +def _do_ensure( + dep_specs: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, +) -> None: + """ + Use pip to ensure we have the package specified by @dep_specs. + + If the package is already installed, do nothing. If online and + wheels_dir are both provided, prefer packages found in wheels_dir + first before connecting to PyPI. + + :param dep_specs: + PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param online: If True, fall back to PyPI. + :param wheels_dir: If specified, search this path for packages. + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=UserWarning, module="distlib" + ) + dist_path = distlib.database.DistributionPath(include_egg=True) + absent = [] + present = [] + for spec in dep_specs: + matcher = distlib.version.LegacyMatcher(spec) + dist = dist_path.get_distribution(matcher.name) + if dist is None or not matcher.match(dist.version): + absent.append(spec) + else: + logger.info("found %s", dist) + present.append(matcher.name) + + if present: + generate_console_scripts(present) + + if absent: + # Some packages are missing or aren't a suitable version, + # install a suitable (possibly vendored) package. + print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) + pip_install(args=absent, online=online, wheels_dir=wheels_dir) + + +def ensure( + dep_specs: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, + prog: Optional[str] = None, +) -> None: + """ + Use pip to ensure we have the package specified by @dep_specs. + + If the package is already installed, do nothing. If online and + wheels_dir are both provided, prefer packages found in wheels_dir + first before connecting to PyPI. + + :param dep_specs: + PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param online: If True, fall back to PyPI. + :param wheels_dir: If specified, search this path for packages. + :param prog: + If specified, use this program name for error diagnostics that will + be presented to the user. e.g., 'sphinx-build' can be used as a + bellwether for the presence of 'sphinx'. + """ + print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) + + if not HAVE_DISTLIB: + raise Ouch("a usable distlib could not be found, please install it") + + try: + _do_ensure(dep_specs, online, wheels_dir) + except subprocess.CalledProcessError as exc: + # Well, that's not good. + msg, bad = diagnose(dep_specs[0], online, wheels_dir, prog) + if bad: + raise Ouch(msg) from exc + raise SystemExit(f"\n{msg}\n\n") from exc + + +def post_venv_setup() -> None: + """ + This is intended to be run *inside the venv* after it is created. + """ + logger.debug("post_venv_setup()") + # Test for a broken pip (Debian 10 or derivative?) and fix it if needed + if not checkpip(): + # Finally, generate a 'pip' script so the venv is usable in a normal + # way from the CLI. This only happens when we inherited pip from a + # parent/system-site and haven't run ensurepip in some way. + generate_console_scripts(["pip"]) + + +def _add_create_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser("create", help="create a venv") + subparser.add_argument( + "target", + type=str, + action="store", + help="Target directory to install virtual environment into.", + ) + + +def _add_post_init_subcommand(subparsers: Any) -> None: + subparsers.add_parser("post_init", help="post-venv initialization") + + +def _add_ensure_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser( + "ensure", help="Ensure that the specified package is installed." + ) + subparser.add_argument( + "--online", + action="store_true", + help="Install packages from PyPI, if necessary.", + ) + subparser.add_argument( + "--dir", + type=str, + action="store", + help="Path to vendored packages where we may install from.", + ) + subparser.add_argument( + "--diagnose", + type=str, + action="store", + help=( + "Name of a shell utility to use for " + "diagnostics if this command fails." + ), + ) + subparser.add_argument( + "dep_specs", + type=str, + action="store", + help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", + nargs="+", + ) + + +def main() -> int: + """CLI interface to make_qemu_venv. See module docstring.""" + if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): + # You're welcome. + logging.basicConfig(level=logging.DEBUG) + else: + if os.environ.get("V"): + logging.basicConfig(level=logging.INFO) + + # These are incredibly noisy even for V=1 + logging.getLogger("distlib.metadata").addFilter(lambda record: False) + logging.getLogger("distlib.database").addFilter(lambda record: False) + + parser = argparse.ArgumentParser( + prog="mkvenv", + description="QEMU pyvenv bootstrapping utility", + ) + subparsers = parser.add_subparsers( + title="Commands", + dest="command", + required=True, + metavar="command", + help="Description", + ) + + _add_create_subcommand(subparsers) + _add_post_init_subcommand(subparsers) + _add_ensure_subcommand(subparsers) + + args = parser.parse_args() + try: + if args.command == "create": + make_venv( + args.target, + system_site_packages=True, + clear=True, + ) + if args.command == "post_init": + post_venv_setup() + if args.command == "ensure": + ensure( + dep_specs=args.dep_specs, + online=args.online, + wheels_dir=args.dir, + prog=args.diagnose, + ) + logger.debug("mkvenv.py %s: exiting", args.command) + except Ouch as exc: + print("\n*** Ouch! ***\n", file=sys.stderr) + print(str(exc), "\n\n", file=sys.stderr) + return 1 + except SystemExit: + raise + except: # pylint: disable=bare-except + logger.exception("mkvenv did not complete successfully:") + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/scripts/vendor.py b/python/scripts/vendor.py new file mode 100755 index 0000000000..34486a51f4 --- /dev/null +++ b/python/scripts/vendor.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +vendor - QEMU python vendoring utility + +usage: vendor [-h] + +QEMU python vendoring utility + +options: + -h, --help show this help message and exit +""" + +# Copyright (C) 2023 Red Hat, Inc. +# +# Authors: +# John Snow <jsnow@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import argparse +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +def main() -> int: + """Run the vendoring utility. See module-level docstring.""" + loud = False + if os.environ.get("DEBUG") or os.environ.get("V"): + loud = True + + # No options or anything for now, but I guess + # you'll figure that out when you run --help. + parser = argparse.ArgumentParser( + prog="vendor", + description="QEMU python vendoring utility", + ) + parser.parse_args() + + packages = { + "meson==0.63.3": + "d677b809c4895dcbaac9bf6c43703fcb3609a4b24c6057c78f828590049cf43a", + } + + vendor_dir = Path(__file__, "..", "..", "wheels").resolve() + + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as file: + for dep_spec, checksum in packages.items(): + file.write(f"{dep_spec} --hash=sha256:{checksum}") + file.flush() + + cli_args = [ + "pip", + "download", + "--dest", + str(vendor_dir), + "--require-hashes", + "-r", + file.name, + ] + if loud: + cli_args.append("-v") + + print(" ".join(cli_args)) + subprocess.run(cli_args, check=True) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/setup.cfg b/python/setup.cfg index 9e923d9762..5abb7d30ad 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -14,7 +14,6 @@ classifiers = Natural Language :: English Operating System :: OS Independent Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -23,7 +22,7 @@ classifiers = Typing :: Typed [options] -python_requires = >= 3.6 +python_requires = >= 3.7 packages = qemu.qmp qemu.machine @@ -36,11 +35,12 @@ packages = # Remember to update tests/minreqs.txt if changing anything below: devel = avocado-framework >= 90.0 - flake8 >= 3.6.0 + distlib >= 0.3.6 + flake8 >= 5.0.4 fusepy >= 2.0.4 isort >= 5.1.2 mypy >= 0.780 - pylint >= 2.8.0 + pylint >= 2.17.3 tox >= 3.18.0 urwid >= 2.1.2 urwid-readline >= 0.13 @@ -76,7 +76,7 @@ exclude = __pycache__, [mypy] strict = True -python_version = 3.6 +python_version = 3.7 warn_unused_configs = True namespace_packages = True warn_unused_ignores = False @@ -103,6 +103,39 @@ ignore_missing_imports = True [mypy-pygments] ignore_missing_imports = True +[mypy-importlib.metadata] +ignore_missing_imports = True + +[mypy-importlib_metadata] +ignore_missing_imports = True + +[mypy-pkg_resources] +ignore_missing_imports = True + +[mypy-distlib] +ignore_missing_imports = True + +[mypy-distlib.database] +ignore_missing_imports = True + +[mypy-distlib.scripts] +ignore_missing_imports = True + +[mypy-distlib.version] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.database] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.scripts] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.version] +ignore_missing_imports = True + [pylint.messages control] # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this @@ -132,6 +165,7 @@ good-names=i, fd, # fd = os.open(...) c, # for c in string: ... T, # for TypeVars. See pylint#3401 + SocketAddrT, # Not sure why this is invalid. [pylint.similarities] # Ignore imports when computing similarities. @@ -158,7 +192,7 @@ multi_line_output=3 # of python available on your system to run this test. [tox:tox] -envlist = py36, py37, py38, py39, py310, py311 +envlist = py37, py38, py39, py310, py311 skip_missing_interpreters = true [testenv] diff --git a/python/tests/flake8.sh b/python/tests/flake8.sh index 1cd7d40fad..e013699645 100755 --- a/python/tests/flake8.sh +++ b/python/tests/flake8.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m flake8 qemu/ +python3 -m flake8 scripts/ diff --git a/python/tests/isort.sh b/python/tests/isort.sh index 4480405bfb..66c2f7df0f 100755 --- a/python/tests/isort.sh +++ b/python/tests/isort.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m isort -c qemu/ +python3 -m isort -c scripts/ diff --git a/python/tests/minreqs.txt b/python/tests/minreqs.txt index dfb8abb155..1ce72cef6d 100644 --- a/python/tests/minreqs.txt +++ b/python/tests/minreqs.txt @@ -1,5 +1,5 @@ # This file lists the ***oldest possible dependencies*** needed to run -# "make check" successfully under ***Python 3.6***. It is used primarily +# "make check" successfully under ***Python 3.7***. It is used primarily # by GitLab CI to ensure that our stated minimum versions in setup.cfg # are truthful and regularly validated. # @@ -16,6 +16,9 @@ urwid==2.1.2 urwid-readline==0.13 Pygments==2.9.0 +# Dependencies for mkvenv +distlib==0.3.6 + # Dependencies for FUSE support for qom-fuse fusepy==2.0.4 @@ -23,23 +26,23 @@ fusepy==2.0.4 avocado-framework==90.0 # Linters -flake8==3.6.0 +flake8==5.0.4 isort==5.1.2 mypy==0.780 -pylint==2.8.0 +pylint==2.17.3 # Transitive flake8 dependencies -mccabe==0.6.0 -pycodestyle==2.4.0 -pyflakes==2.0.0 +mccabe==0.7.0 +pycodestyle==2.9.1 +pyflakes==2.5.0 # Transitive mypy dependencies mypy-extensions==0.4.3 typed-ast==1.4.0 -typing-extensions==3.7.4 +typing-extensions==4.5.0 # Transitive pylint dependencies -astroid==2.5.4 +astroid==2.15.4 lazy-object-proxy==1.4.0 toml==0.10.0 wrapt==1.12.1 diff --git a/python/tests/mypy.sh b/python/tests/mypy.sh index 5f980f563b..a33a3f58ab 100755 --- a/python/tests/mypy.sh +++ b/python/tests/mypy.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m mypy -p qemu +python3 -m mypy scripts/ diff --git a/python/tests/pylint.sh b/python/tests/pylint.sh index 03d64705a1..2b68da90df 100755 --- a/python/tests/pylint.sh +++ b/python/tests/pylint.sh @@ -1,3 +1,4 @@ #!/bin/sh -e # See commit message for environment variable explainer. SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint qemu/ +SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint scripts/ diff --git a/python/wheels/meson-0.63.3-py3-none-any.whl b/python/wheels/meson-0.63.3-py3-none-any.whl new file mode 100644 index 0000000000..8a191e3a20 --- /dev/null +++ b/python/wheels/meson-0.63.3-py3-none-any.whl Binary files differdiff --git a/qapi/block-core.json b/qapi/block-core.json index 187e35d473..98d9116dae 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -854,6 +854,10 @@ # @min_wr_latency_ns: Minimum latency of write operations in the # defined interval, in nanoseconds. # +# @min_zone_append_latency_ns: Minimum latency of zone append operations +# in the defined interval, in nanoseconds +# (since 8.1) +# # @min_flush_latency_ns: Minimum latency of flush operations in the # defined interval, in nanoseconds. # @@ -863,6 +867,10 @@ # @max_wr_latency_ns: Maximum latency of write operations in the # defined interval, in nanoseconds. # +# @max_zone_append_latency_ns: Maximum latency of zone append operations +# in the defined interval, in nanoseconds +# (since 8.1) +# # @max_flush_latency_ns: Maximum latency of flush operations in the # defined interval, in nanoseconds. # @@ -872,6 +880,10 @@ # @avg_wr_latency_ns: Average latency of write operations in the # defined interval, in nanoseconds. # +# @avg_zone_append_latency_ns: Average latency of zone append operations +# in the defined interval, in nanoseconds +# (since 8.1) +# # @avg_flush_latency_ns: Average latency of flush operations in the # defined interval, in nanoseconds. # @@ -881,15 +893,23 @@ # @avg_wr_queue_depth: Average number of pending write operations in # the defined interval. # +# @avg_zone_append_queue_depth: Average number of pending zone append +# operations in the defined interval +# (since 8.1). +# # Since: 2.5 ## { 'struct': 'BlockDeviceTimedStats', 'data': { 'interval_length': 'int', 'min_rd_latency_ns': 'int', 'max_rd_latency_ns': 'int', 'avg_rd_latency_ns': 'int', 'min_wr_latency_ns': 'int', 'max_wr_latency_ns': 'int', - 'avg_wr_latency_ns': 'int', 'min_flush_latency_ns': 'int', - 'max_flush_latency_ns': 'int', 'avg_flush_latency_ns': 'int', - 'avg_rd_queue_depth': 'number', 'avg_wr_queue_depth': 'number' } } + 'avg_wr_latency_ns': 'int', 'min_zone_append_latency_ns': 'int', + 'max_zone_append_latency_ns': 'int', + 'avg_zone_append_latency_ns': 'int', + 'min_flush_latency_ns': 'int', 'max_flush_latency_ns': 'int', + 'avg_flush_latency_ns': 'int', 'avg_rd_queue_depth': 'number', + 'avg_wr_queue_depth': 'number', + 'avg_zone_append_queue_depth': 'number' } } ## # @BlockDeviceStats: @@ -900,6 +920,9 @@ # # @wr_bytes: The number of bytes written by the device. # +# @zone_append_bytes: The number of bytes appended by the zoned devices +# (since 8.1) +# # @unmap_bytes: The number of bytes unmapped by the device (Since 4.2) # # @rd_operations: The number of read operations performed by the @@ -908,6 +931,9 @@ # @wr_operations: The number of write operations performed by the # device. # +# @zone_append_operations: The number of zone append operations performed +# by the zoned devices (since 8.1) +# # @flush_operations: The number of cache flush operations performed by # the device (since 0.15) # @@ -920,6 +946,9 @@ # @wr_total_time_ns: Total time spent on writes in nanoseconds (since # 0.15). # +# @zone_append_total_time_ns: Total time spent on zone append writes +# in nanoseconds (since 8.1) +# # @flush_total_time_ns: Total time spent on cache flushes in # nanoseconds (since 0.15). # @@ -937,6 +966,9 @@ # @wr_merged: Number of write requests that have been merged into # another request (Since 2.3). # +# @zone_append_merged: Number of zone append requests that have been merged +# into another request (since 8.1) +# # @unmap_merged: Number of unmap requests that have been merged into # another request (Since 4.2) # @@ -950,6 +982,10 @@ # @failed_wr_operations: The number of failed write operations # performed by the device (Since 2.5) # +# @failed_zone_append_operations: The number of failed zone append write +# operations performed by the zoned devices +# (since 8.1) +# # @failed_flush_operations: The number of failed flush operations # performed by the device (Since 2.5) # @@ -962,6 +998,9 @@ # @invalid_wr_operations: The number of invalid write operations # performed by the device (Since 2.5) # +# @invalid_zone_append_operations: The number of invalid zone append operations +# performed by the zoned device (since 8.1) +# # @invalid_flush_operations: The number of invalid flush operations # performed by the device (Since 2.5) # @@ -981,27 +1020,34 @@ # # @wr_latency_histogram: @BlockLatencyHistogramInfo. (Since 4.0) # +# @zone_append_latency_histogram: @BlockLatencyHistogramInfo. (since 8.1) +# # @flush_latency_histogram: @BlockLatencyHistogramInfo. (Since 4.0) # # Since: 0.14 ## { 'struct': 'BlockDeviceStats', - 'data': {'rd_bytes': 'int', 'wr_bytes': 'int', 'unmap_bytes' : 'int', - 'rd_operations': 'int', 'wr_operations': 'int', + 'data': {'rd_bytes': 'int', 'wr_bytes': 'int', 'zone_append_bytes': 'int', + 'unmap_bytes' : 'int', 'rd_operations': 'int', + 'wr_operations': 'int', 'zone_append_operations': 'int', 'flush_operations': 'int', 'unmap_operations': 'int', 'rd_total_time_ns': 'int', 'wr_total_time_ns': 'int', - 'flush_total_time_ns': 'int', 'unmap_total_time_ns': 'int', - 'wr_highest_offset': 'int', - 'rd_merged': 'int', 'wr_merged': 'int', 'unmap_merged': 'int', - '*idle_time_ns': 'int', + 'zone_append_total_time_ns': 'int', 'flush_total_time_ns': 'int', + 'unmap_total_time_ns': 'int', 'wr_highest_offset': 'int', + 'rd_merged': 'int', 'wr_merged': 'int', 'zone_append_merged': 'int', + 'unmap_merged': 'int', '*idle_time_ns': 'int', 'failed_rd_operations': 'int', 'failed_wr_operations': 'int', - 'failed_flush_operations': 'int', 'failed_unmap_operations': 'int', - 'invalid_rd_operations': 'int', 'invalid_wr_operations': 'int', + 'failed_zone_append_operations': 'int', + 'failed_flush_operations': 'int', + 'failed_unmap_operations': 'int', 'invalid_rd_operations': 'int', + 'invalid_wr_operations': 'int', + 'invalid_zone_append_operations': 'int', 'invalid_flush_operations': 'int', 'invalid_unmap_operations': 'int', 'account_invalid': 'bool', 'account_failed': 'bool', 'timed_stats': ['BlockDeviceTimedStats'], '*rd_latency_histogram': 'BlockLatencyHistogramInfo', '*wr_latency_histogram': 'BlockLatencyHistogramInfo', + '*zone_append_latency_histogram': 'BlockLatencyHistogramInfo', '*flush_latency_histogram': 'BlockLatencyHistogramInfo' } } ## diff --git a/qapi/block.json b/qapi/block.json index a1e16592fd..0f25ce3961 100644 --- a/qapi/block.json +++ b/qapi/block.json @@ -534,6 +534,9 @@ # @boundaries-write: list of interval boundary values for write # latency histogram. # +# @boundaries-zap: list of interval boundary values for zone append write +# latency histogram. +# # @boundaries-flush: list of interval boundary values for flush # latency histogram. # @@ -587,5 +590,6 @@ '*boundaries': ['uint64'], '*boundaries-read': ['uint64'], '*boundaries-write': ['uint64'], + '*boundaries-zap': ['uint64'], '*boundaries-flush': ['uint64'] }, 'allow-preconfig': true } diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c index e7a02f5b99..3f75d2f5a6 100644 --- a/qemu-io-cmds.c +++ b/qemu-io-cmds.c @@ -1730,6 +1730,224 @@ static const cmdinfo_t flush_cmd = { .oneline = "flush all in-core file state to disk", }; +static inline int64_t tosector(int64_t bytes) +{ + return bytes >> BDRV_SECTOR_BITS; +} + +static int zone_report_f(BlockBackend *blk, int argc, char **argv) +{ + int ret; + int64_t offset; + unsigned int nr_zones; + + ++optind; + offset = cvtnum(argv[optind]); + ++optind; + nr_zones = cvtnum(argv[optind]); + + g_autofree BlockZoneDescriptor *zones = NULL; + zones = g_new(BlockZoneDescriptor, nr_zones); + ret = blk_zone_report(blk, offset, &nr_zones, zones); + if (ret < 0) { + printf("zone report failed: %s\n", strerror(-ret)); + } else { + for (int i = 0; i < nr_zones; ++i) { + printf("start: 0x%" PRIx64 ", len 0x%" PRIx64 ", " + "cap"" 0x%" PRIx64 ", wptr 0x%" PRIx64 ", " + "zcond:%u, [type: %u]\n", + tosector(zones[i].start), tosector(zones[i].length), + tosector(zones[i].cap), tosector(zones[i].wp), + zones[i].state, zones[i].type); + } + } + return ret; +} + +static const cmdinfo_t zone_report_cmd = { + .name = "zone_report", + .altname = "zrp", + .cfunc = zone_report_f, + .argmin = 2, + .argmax = 2, + .args = "offset number", + .oneline = "report zone information", +}; + +static int zone_open_f(BlockBackend *blk, int argc, char **argv) +{ + int ret; + int64_t offset, len; + ++optind; + offset = cvtnum(argv[optind]); + ++optind; + len = cvtnum(argv[optind]); + ret = blk_zone_mgmt(blk, BLK_ZO_OPEN, offset, len); + if (ret < 0) { + printf("zone open failed: %s\n", strerror(-ret)); + } + return ret; +} + +static const cmdinfo_t zone_open_cmd = { + .name = "zone_open", + .altname = "zo", + .cfunc = zone_open_f, + .argmin = 2, + .argmax = 2, + .args = "offset len", + .oneline = "explicit open a range of zones in zone block device", +}; + +static int zone_close_f(BlockBackend *blk, int argc, char **argv) +{ + int ret; + int64_t offset, len; + ++optind; + offset = cvtnum(argv[optind]); + ++optind; + len = cvtnum(argv[optind]); + ret = blk_zone_mgmt(blk, BLK_ZO_CLOSE, offset, len); + if (ret < 0) { + printf("zone close failed: %s\n", strerror(-ret)); + } + return ret; +} + +static const cmdinfo_t zone_close_cmd = { + .name = "zone_close", + .altname = "zc", + .cfunc = zone_close_f, + .argmin = 2, + .argmax = 2, + .args = "offset len", + .oneline = "close a range of zones in zone block device", +}; + +static int zone_finish_f(BlockBackend *blk, int argc, char **argv) +{ + int ret; + int64_t offset, len; + ++optind; + offset = cvtnum(argv[optind]); + ++optind; + len = cvtnum(argv[optind]); + ret = blk_zone_mgmt(blk, BLK_ZO_FINISH, offset, len); + if (ret < 0) { + printf("zone finish failed: %s\n", strerror(-ret)); + } + return ret; +} + +static const cmdinfo_t zone_finish_cmd = { + .name = "zone_finish", + .altname = "zf", + .cfunc = zone_finish_f, + .argmin = 2, + .argmax = 2, + .args = "offset len", + .oneline = "finish a range of zones in zone block device", +}; + +static int zone_reset_f(BlockBackend *blk, int argc, char **argv) +{ + int ret; + int64_t offset, len; + ++optind; + offset = cvtnum(argv[optind]); + ++optind; + len = cvtnum(argv[optind]); + ret = blk_zone_mgmt(blk, BLK_ZO_RESET, offset, len); + if (ret < 0) { + printf("zone reset failed: %s\n", strerror(-ret)); + } + return ret; +} + +static const cmdinfo_t zone_reset_cmd = { + .name = "zone_reset", + .altname = "zrs", + .cfunc = zone_reset_f, + .argmin = 2, + .argmax = 2, + .args = "offset len", + .oneline = "reset a zone write pointer in zone block device", +}; + +static int do_aio_zone_append(BlockBackend *blk, QEMUIOVector *qiov, + int64_t *offset, int flags, int *total) +{ + int async_ret = NOT_DONE; + + blk_aio_zone_append(blk, offset, qiov, flags, aio_rw_done, &async_ret); + while (async_ret == NOT_DONE) { + main_loop_wait(false); + } + + *total = qiov->size; + return async_ret < 0 ? async_ret : 1; +} + +static int zone_append_f(BlockBackend *blk, int argc, char **argv) +{ + int ret; + bool pflag = false; + int flags = 0; + int total = 0; + int64_t offset; + char *buf; + int c, nr_iov; + int pattern = 0xcd; + QEMUIOVector qiov; + + if (optind > argc - 3) { + return -EINVAL; + } + + if ((c = getopt(argc, argv, "p")) != -1) { + pflag = true; + } + + offset = cvtnum(argv[optind]); + if (offset < 0) { + print_cvtnum_err(offset, argv[optind]); + return offset; + } + optind++; + nr_iov = argc - optind; + buf = create_iovec(blk, &qiov, &argv[optind], nr_iov, pattern, + flags & BDRV_REQ_REGISTERED_BUF); + if (buf == NULL) { + return -EINVAL; + } + ret = do_aio_zone_append(blk, &qiov, &offset, flags, &total); + if (ret < 0) { + printf("zone append failed: %s\n", strerror(-ret)); + goto out; + } + + if (pflag) { + printf("After zap done, the append sector is 0x%" PRIx64 "\n", + tosector(offset)); + } + +out: + qemu_io_free(blk, buf, qiov.size, + flags & BDRV_REQ_REGISTERED_BUF); + qemu_iovec_destroy(&qiov); + return ret; +} + +static const cmdinfo_t zone_append_cmd = { + .name = "zone_append", + .altname = "zap", + .cfunc = zone_append_f, + .argmin = 3, + .argmax = 4, + .args = "offset len [len..]", + .oneline = "append write a number of bytes at a specified offset", +}; + static int truncate_f(BlockBackend *blk, int argc, char **argv); static const cmdinfo_t truncate_cmd = { .name = "truncate", @@ -2523,6 +2741,12 @@ static void __attribute((constructor)) init_qemuio_commands(void) qemuio_add_command(&aio_write_cmd); qemuio_add_command(&aio_flush_cmd); qemuio_add_command(&flush_cmd); + qemuio_add_command(&zone_report_cmd); + qemuio_add_command(&zone_open_cmd); + qemuio_add_command(&zone_close_cmd); + qemuio_add_command(&zone_finish_cmd); + qemuio_add_command(&zone_reset_cmd); + qemuio_add_command(&zone_append_cmd); qemuio_add_command(&truncate_cmd); qemuio_add_command(&length_cmd); qemuio_add_command(&info_cmd); diff --git a/qemu-options.hx b/qemu-options.hx index 42b9094c10..30690d9c3f 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -4828,20 +4828,32 @@ DEF("qtest-log", HAS_ARG, QEMU_OPTION_qtest_log, "", QEMU_ARCH_ALL) DEF("async-teardown", 0, QEMU_OPTION_asyncteardown, "-async-teardown enable asynchronous teardown\n", QEMU_ARCH_ALL) -#endif SRST ``-async-teardown`` - Enable asynchronous teardown. A new process called "cleanup/<QEMU_PID>" - will be created at startup sharing the address space with the main qemu - process, using clone. It will wait for the main qemu process to - terminate completely, and then exit. - This allows qemu to terminate very quickly even if the guest was - huge, leaving the teardown of the address space to the cleanup - process. Since the cleanup process shares the same cgroups as the - main qemu process, accounting is performed correctly. This only - works if the cleanup process is not forcefully killed with SIGKILL - before the main qemu process has terminated completely. + This option is deprecated and should no longer be used. The new option + ``-run-with async-teardown=on`` is a replacement. ERST +DEF("run-with", HAS_ARG, QEMU_OPTION_run_with, + "-run-with async-teardown[=on|off]\n" + " misc QEMU process lifecycle options\n" + " async-teardown=on enables asynchronous teardown\n", + QEMU_ARCH_ALL) +SRST +``-run-with`` + Set QEMU process lifecycle options. + + ``async-teardown=on`` enables asynchronous teardown. A new process called + "cleanup/<QEMU_PID>" will be created at startup sharing the address + space with the main QEMU process, using clone. It will wait for the + main QEMU process to terminate completely, and then exit. This allows + QEMU to terminate very quickly even if the guest was huge, leaving the + teardown of the address space to the cleanup process. Since the cleanup + process shares the same cgroups as the main QEMU process, accounting is + performed correctly. This only works if the cleanup process is not + forcefully killed with SIGKILL before the main QEMU process has + terminated completely. +ERST +#endif DEF("msg", HAS_ARG, QEMU_OPTION_msg, "-msg [timestamp[=on|off]][,guest-name=[on|off]]\n" diff --git a/qga/meson.build b/qga/meson.build index ad17dc7dca..622b5f94a2 100644 --- a/qga/meson.build +++ b/qga/meson.build @@ -22,7 +22,7 @@ have_qga_vss = get_option('qga_vss') \ Then run configure with: --extra-cxxflags="-isystem /path/to/vss/inc/win2003"''') \ .require(midl.found() or widl.found(), error_message: 'VSS support requires midl or widl') \ - .require(not enable_static, + .require(not get_option('prefer_static'), error_message: 'VSS support requires dynamic linking with GLib') \ .allowed() diff --git a/scripts/ci/org.centos/stream/8/x86_64/test-avocado b/scripts/ci/org.centos/stream/8/x86_64/test-avocado index d2c0e5fb4c..7bb5b317b6 100755 --- a/scripts/ci/org.centos/stream/8/x86_64/test-avocado +++ b/scripts/ci/org.centos/stream/8/x86_64/test-avocado @@ -4,7 +4,7 @@ # KVM and x86_64, or tests that are generic enough to be valid for all # targets. Such a test list can be generated with: # -# ./tests/venv/bin/avocado list --filter-by-tags-include-empty \ +# ./pyvenv/bin/avocado list --filter-by-tags-include-empty \ # --filter-by-tags-include-empty-key -t accel:kvm,arch:x86_64 \ # tests/avocado/ # @@ -22,7 +22,7 @@ # - tests/avocado/virtio_check_params.py:VirtioMaxSegSettingsCheck.test_machine_types # make get-vm-images -./tests/venv/bin/avocado run \ +./pyvenv/bin/avocado run \ --job-results-dir=tests/results/ \ tests/avocado/boot_linux.py:BootLinuxX8664.test_pc_i440fx_kvm \ tests/avocado/boot_linux.py:BootLinuxX8664.test_pc_q35_kvm \ diff --git a/scripts/coverity-scan/COMPONENTS.md b/scripts/coverity-scan/COMPONENTS.md index 7c48e0f1d2..883da95aff 100644 --- a/scripts/coverity-scan/COMPONENTS.md +++ b/scripts/coverity-scan/COMPONENTS.md @@ -24,6 +24,9 @@ hppa i386 ~ (/qemu)?((/include)?/hw/i386/.*|/target/i386/.*|/hw/intc/[^/]*apic[^/]*\.c) +loongarch + ~ (/qemu)?((/include)?/hw/(loongarch/.*|.*/loongarch.*)|/target/loongarch/.*) + m68k ~ (/qemu)?((/include)?/hw/m68k/.*|/target/m68k/.*|(/include)?/hw(/.*)?/mcf.*|(/include)?/hw/nubus/.*) @@ -36,11 +39,14 @@ mips nios2 ~ (/qemu)?((/include)?/hw/nios2/.*|/target/nios2/.*) +openrisc + ~ (/qemu)?((/include)?/hw/openrisc/.*|/target/openrisc/.*) + ppc ~ (/qemu)?((/include)?/hw/ppc/.*|/target/ppc/.*|/hw/pci-host/(uninorth.*|dec.*|prep.*|ppc.*)|/hw/misc/macio/.*|(/include)?/hw/.*/(xics|openpic|spapr).*) riscv - ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*) + ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*|/hw/.*/(riscv_|ibex_|sifive_).*) rx ~ (/qemu)?((/include)?/hw/rx/.*|/target/rx/.*) @@ -54,12 +60,12 @@ sh4 sparc ~ (/qemu)?((/include)?/hw/sparc(64)?.*|/target/sparc/.*|/hw/.*/grlib.*|/hw/display/cg3.c) -tilegx - ~ (/qemu)?(/target/tilegx/.*) - tricore ~ (/qemu)?((/include)?/hw/tricore/.*|/target/tricore/.*) +xtensa + ~ (/qemu)?((/include)?/hw/xtensa/.*|/target/xtensa/.*) + 9pfs ~ (/qemu)?(/hw/9pfs/.*|/fsdev/.*) @@ -73,7 +79,7 @@ char ~ (/qemu)?(/qemu-char\.c|/include/sysemu/char\.h|(/include)?/hw/char/.*) crypto - ~ (/qemu)?((/include)?/crypto/.*|/hw/.*/crypto.*) + ~ (/qemu)?((/include)?/crypto/.*|/hw/.*/.*crypto.*|(/include/sysemu|/backends)/cryptodev.*) disas ~ (/qemu)?((/include)?/disas.*) @@ -100,7 +106,7 @@ net ~ (/qemu)?((/include)?(/hw)?/(net|rdma)/.*) pci - ~ (/qemu)?(/hw/pci.*|/include/hw/pci.*) + ~ (/qemu)?(/include)?/hw/(cxl/|pci).* qemu-ga ~ (/qemu)?(/qga/.*) @@ -108,9 +114,6 @@ qemu-ga scsi ~ (/qemu)?(/scsi/.*|/hw/scsi/.*|/include/hw/scsi/.*) -tcg - ~ (/qemu)?(/accel/tcg/.*|/replay/.*|/(.*/)?softmmu.*) - trace ~ (/qemu)?(/.*trace.*\.[ch]) @@ -126,9 +129,27 @@ user util ~ (/qemu)?(/util/.*|/include/qemu/.*) +vfio + ~ (/qemu)?(/include)?/hw/vfio/.* + +virtio + ~ (/qemu)?(/include)?/hw/virtio/.* + xen ~ (/qemu)?(.*/xen.*) +hvf + ~ (/qemu)?(.*/hvf.*) + +kvm + ~ (/qemu)?(.*/kvm.*) + +tcg + ~ (/qemu)?(/accel/tcg|/replay|/tcg)/.* + +sysemu + ~ (/qemu)?(/softmmu/.*|/accel/.*) + (headers) ~ (/qemu)?(/include/.*) @@ -137,9 +158,3 @@ testlibs tests ~ (/qemu)?(/tests/.*) - -loongarch - ~ (/qemu)?((/include)?/hw/(loongarch/.*|.*/loongarch.*)|/target/loongarch/.*) - -riscv - ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*|/hw/.*/(riscv_|ibex_|sifive_).*) diff --git a/scripts/device-crash-test b/scripts/device-crash-test index b74d887331..353aa575d7 100755 --- a/scripts/device-crash-test +++ b/scripts/device-crash-test @@ -43,7 +43,7 @@ except ModuleNotFoundError as exc: print(f"Module '{exc.name}' not found.") print(" Try 'make check-venv' from your build directory,") print(" and then one way to run this script is like so:") - print(f' > $builddir/tests/venv/bin/python3 "{path}"') + print(f' > $builddir/pyvenv/bin/python3 "{path}"') sys.exit(1) logger = logging.getLogger('device-crash-test') diff --git a/scripts/meson-buildoptions.py b/scripts/meson-buildoptions.py index a04dcc70a5..8d2e526132 100755 --- a/scripts/meson-buildoptions.py +++ b/scripts/meson-buildoptions.py @@ -35,6 +35,8 @@ SKIP_OPTIONS = { OPTION_NAMES = { "b_coverage": "gcov", "b_lto": "lto", + "coroutine_backend": "with-coroutine", + "debug": "debug-info", "malloc": "enable-malloc", "pkgversion": "with-pkgversion", "qemu_firmwarepath": "firmwarepath", @@ -46,6 +48,7 @@ BUILTIN_OPTIONS = { "b_coverage", "b_lto", "datadir", + "debug", "includedir", "libdir", "libexecdir", diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 2805d1c145..5714fd93d9 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -1,8 +1,8 @@ # This file is generated by meson-buildoptions.py, do not edit! meson_options_help() { - printf "%s\n" ' --audio-drv-list=CHOICES Set audio driver list [default] (choices: al' - printf "%s\n" ' sa/coreaudio/default/dsound/jack/oss/pa/' - printf "%s\n" ' pipewire/sdl/sndio)' + printf "%s\n" ' --audio-drv-list=CHOICES Set audio driver list [default] (choices: alsa/co' + printf "%s\n" ' reaudio/default/dsound/jack/oss/pa/pipewire/sdl/s' + printf "%s\n" ' ndio)' printf "%s\n" ' --block-drv-ro-whitelist=VALUE' printf "%s\n" ' set block driver read-only whitelist (by default' printf "%s\n" ' affects only QEMU, not tools like qemu-img)' @@ -11,6 +11,7 @@ meson_options_help() { printf "%s\n" ' affects only QEMU, not tools like qemu-img)' printf "%s\n" ' --datadir=VALUE Data file directory [share]' printf "%s\n" ' --disable-coroutine-pool coroutine freelist (better performance)' + printf "%s\n" ' --disable-debug-info Enable debug symbols and other information' printf "%s\n" ' --disable-hexagon-idef-parser' printf "%s\n" ' use idef-parser to automatically generate TCG' printf "%s\n" ' code for the Hexagon frontend' @@ -41,11 +42,15 @@ meson_options_help() { printf "%s\n" ' --enable-profiler profiler support' printf "%s\n" ' --enable-rng-none dummy RNG, avoid using /dev/(u)random and' printf "%s\n" ' getrandom()' + printf "%s\n" ' --enable-safe-stack SafeStack Stack Smash Protection (requires' + printf "%s\n" ' clang/llvm and coroutine backend ucontext)' + printf "%s\n" ' --enable-sanitizers enable default sanitizers' printf "%s\n" ' --enable-strip Strip targets on install' printf "%s\n" ' --enable-tcg-interpreter TCG with bytecode interpreter (slow)' printf "%s\n" ' --enable-trace-backends=CHOICES' printf "%s\n" ' Set available tracing backends [log] (choices:' printf "%s\n" ' dtrace/ftrace/log/nop/simple/syslog/ust)' + printf "%s\n" ' --enable-tsan enable thread sanitizer' printf "%s\n" ' --firmwarepath=VALUES search PATH for firmware files [share/qemu-' printf "%s\n" ' firmware]' printf "%s\n" ' --iasl=VALUE Path to ACPI disassembler' @@ -57,11 +62,11 @@ meson_options_help() { printf "%s\n" ' --localedir=VALUE Locale data directory [share/locale]' printf "%s\n" ' --localstatedir=VALUE Localstate data directory [/var/local]' printf "%s\n" ' --mandir=VALUE Manual page directory [share/man]' - printf "%s\n" ' --sphinx-build=VALUE Use specified sphinx-build for building document' - printf "%s\n" ' [sphinx-build]' printf "%s\n" ' --sysconfdir=VALUE Sysconf data directory [etc]' printf "%s\n" ' --tls-priority=VALUE Default TLS protocol/cipher priority string' printf "%s\n" ' [NORMAL]' + printf "%s\n" ' --with-coroutine=CHOICE coroutine backend to use (choices:' + printf "%s\n" ' auto/sigaltstack/ucontext/windows)' printf "%s\n" ' --with-pkgversion=VALUE use specified string as sub-version of the' printf "%s\n" ' package' printf "%s\n" ' --with-trace-file=VALUE Trace file prefix for simple backend [trace]' @@ -86,6 +91,7 @@ meson_options_help() { printf "%s\n" ' capstone Whether and how to find the capstone library' printf "%s\n" ' cloop cloop image format support' printf "%s\n" ' cocoa Cocoa user interface (macOS only)' + printf "%s\n" ' colo-proxy colo-proxy support' printf "%s\n" ' coreaudio CoreAudio sound support' printf "%s\n" ' crypto-afalg Linux AF_ALG crypto backend driver' printf "%s\n" ' curl CURL block device driver' @@ -129,6 +135,7 @@ meson_options_help() { printf "%s\n" ' lzo lzo compression support' printf "%s\n" ' malloc-trim enable libc malloc_trim() for memory optimization' printf "%s\n" ' membarrier membarrier system call (for Linux 4.14+ or Windows' + printf "%s\n" ' modules modules support (non Windows)' printf "%s\n" ' mpath Multipath persistent reservation passthrough' printf "%s\n" ' multiprocess Out of process device emulation support' printf "%s\n" ' netmap netmap network backend support' @@ -160,6 +167,7 @@ meson_options_help() { printf "%s\n" ' sparse sparse checker' printf "%s\n" ' spice Spice server support' printf "%s\n" ' spice-protocol Spice protocol support' + printf "%s\n" ' stack-protector compiler-provided stack protection' printf "%s\n" ' tcg TCG support' printf "%s\n" ' tools build support utilities that come with QEMU' printf "%s\n" ' tpm TPM support' @@ -181,6 +189,8 @@ meson_options_help() { printf "%s\n" ' vhost-vdpa vhost-vdpa kernel backend support' printf "%s\n" ' virglrenderer virgl rendering support' printf "%s\n" ' virtfs virtio-9p support' + printf "%s\n" ' virtfs-proxy-helper' + printf "%s\n" ' virtio-9p proxy helper support' printf "%s\n" ' vmdk vmdk image format support' printf "%s\n" ' vmnet vmnet.framework network backend support' printf "%s\n" ' vnc VNC server' @@ -243,8 +253,11 @@ _meson_option_parse() { --disable-cloop) printf "%s" -Dcloop=disabled ;; --enable-cocoa) printf "%s" -Dcocoa=enabled ;; --disable-cocoa) printf "%s" -Dcocoa=disabled ;; + --enable-colo-proxy) printf "%s" -Dcolo_proxy=enabled ;; + --disable-colo-proxy) printf "%s" -Dcolo_proxy=disabled ;; --enable-coreaudio) printf "%s" -Dcoreaudio=enabled ;; --disable-coreaudio) printf "%s" -Dcoreaudio=disabled ;; + --with-coroutine=*) quote_sh "-Dcoroutine_backend=$2" ;; --enable-coroutine-pool) printf "%s" -Dcoroutine_pool=true ;; --disable-coroutine-pool) printf "%s" -Dcoroutine_pool=false ;; --enable-crypto-afalg) printf "%s" -Dcrypto_afalg=enabled ;; @@ -256,6 +269,8 @@ _meson_option_parse() { --datadir=*) quote_sh "-Ddatadir=$2" ;; --enable-dbus-display) printf "%s" -Ddbus_display=enabled ;; --disable-dbus-display) printf "%s" -Ddbus_display=disabled ;; + --enable-debug-info) printf "%s" -Ddebug=true ;; + --disable-debug-info) printf "%s" -Ddebug=false ;; --enable-debug-graph-lock) printf "%s" -Ddebug_graph_lock=true ;; --disable-debug-graph-lock) printf "%s" -Ddebug_graph_lock=false ;; --enable-debug-mutex) printf "%s" -Ddebug_mutex=true ;; @@ -359,6 +374,8 @@ _meson_option_parse() { --disable-membarrier) printf "%s" -Dmembarrier=disabled ;; --enable-module-upgrades) printf "%s" -Dmodule_upgrades=true ;; --disable-module-upgrades) printf "%s" -Dmodule_upgrades=false ;; + --enable-modules) printf "%s" -Dmodules=enabled ;; + --disable-modules) printf "%s" -Dmodules=disabled ;; --enable-mpath) printf "%s" -Dmpath=enabled ;; --disable-mpath) printf "%s" -Dmpath=disabled ;; --enable-multiprocess) printf "%s" -Dmultiprocess=enabled ;; @@ -405,6 +422,10 @@ _meson_option_parse() { --disable-replication) printf "%s" -Dreplication=disabled ;; --enable-rng-none) printf "%s" -Drng_none=true ;; --disable-rng-none) printf "%s" -Drng_none=false ;; + --enable-safe-stack) printf "%s" -Dsafe_stack=true ;; + --disable-safe-stack) printf "%s" -Dsafe_stack=false ;; + --enable-sanitizers) printf "%s" -Dsanitizers=true ;; + --disable-sanitizers) printf "%s" -Dsanitizers=false ;; --enable-sdl) printf "%s" -Dsdl=enabled ;; --disable-sdl) printf "%s" -Dsdl=disabled ;; --enable-sdl-image) printf "%s" -Dsdl_image=enabled ;; @@ -425,11 +446,12 @@ _meson_option_parse() { --disable-sndio) printf "%s" -Dsndio=disabled ;; --enable-sparse) printf "%s" -Dsparse=enabled ;; --disable-sparse) printf "%s" -Dsparse=disabled ;; - --sphinx-build=*) quote_sh "-Dsphinx_build=$2" ;; --enable-spice) printf "%s" -Dspice=enabled ;; --disable-spice) printf "%s" -Dspice=disabled ;; --enable-spice-protocol) printf "%s" -Dspice_protocol=enabled ;; --disable-spice-protocol) printf "%s" -Dspice_protocol=disabled ;; + --enable-stack-protector) printf "%s" -Dstack_protector=enabled ;; + --disable-stack-protector) printf "%s" -Dstack_protector=disabled ;; --enable-strip) printf "%s" -Dstrip=true ;; --disable-strip) printf "%s" -Dstrip=false ;; --sysconfdir=*) quote_sh "-Dsysconfdir=$2" ;; @@ -444,6 +466,8 @@ _meson_option_parse() { --disable-tpm) printf "%s" -Dtpm=disabled ;; --enable-trace-backends=*) quote_sh "-Dtrace_backends=$2" ;; --with-trace-file=*) quote_sh "-Dtrace_file=$2" ;; + --enable-tsan) printf "%s" -Dtsan=true ;; + --disable-tsan) printf "%s" -Dtsan=false ;; --enable-u2f) printf "%s" -Du2f=enabled ;; --disable-u2f) printf "%s" -Du2f=disabled ;; --enable-usb-redir) printf "%s" -Dusb_redir=enabled ;; @@ -474,6 +498,8 @@ _meson_option_parse() { --disable-virglrenderer) printf "%s" -Dvirglrenderer=disabled ;; --enable-virtfs) printf "%s" -Dvirtfs=enabled ;; --disable-virtfs) printf "%s" -Dvirtfs=disabled ;; + --enable-virtfs-proxy-helper) printf "%s" -Dvirtfs_proxy_helper=enabled ;; + --disable-virtfs-proxy-helper) printf "%s" -Dvirtfs_proxy_helper=disabled ;; --enable-vmdk) printf "%s" -Dvmdk=enabled ;; --disable-vmdk) printf "%s" -Dvmdk=disabled ;; --enable-vmnet) printf "%s" -Dvmnet=enabled ;; diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini index 6625356429..3463307ddc 100644 --- a/scripts/qapi/mypy.ini +++ b/scripts/qapi/mypy.ini @@ -1,7 +1,7 @@ [mypy] strict = True disallow_untyped_calls = False -python_version = 3.6 +python_version = 3.7 [mypy-qapi.schema] disallow_untyped_defs = False diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index 4923a59d60..1ff334e6a8 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -563,11 +563,11 @@ class QAPIDoc: self._switch_section(QAPIDoc.NullSection(self._parser)) @staticmethod - def _match_at_name_colon(string: str) -> re.Match: + def _match_at_name_colon(string: str): return re.match(r'@([^:]*): *', string) @staticmethod - def _match_section_tag(string: str) -> re.Match: + def _match_section_tag(string: str): return re.match(r'(Returns|Since|Notes?|Examples?|TODO): *', string) def _append_body_line(self, line: str) -> None: diff --git a/scripts/symlink-install-tree.py b/scripts/symlink-install-tree.py index 67cb86dd52..8ed97e3c94 100644 --- a/scripts/symlink-install-tree.py +++ b/scripts/symlink-install-tree.py @@ -28,5 +28,8 @@ for source, dest in json.loads(out).items(): os.symlink(source, bundle_dest) except BaseException as e: if not isinstance(e, OSError) or e.errno != errno.EEXIST: + if os.name == 'nt': + print('Please enable Developer Mode to support soft link ' + 'without Administrator permission') print(f'error making symbolic link {dest}', file=sys.stderr) raise e diff --git a/softmmu/dirtylimit.c b/softmmu/dirtylimit.c index 71bf6dc7a4..015a9038d1 100644 --- a/softmmu/dirtylimit.c +++ b/softmmu/dirtylimit.c @@ -235,20 +235,15 @@ bool dirtylimit_vcpu_index_valid(int cpu_index) static uint64_t dirtylimit_dirty_ring_full_time(uint64_t dirtyrate) { static uint64_t max_dirtyrate; - unsigned target_page_bits = qemu_target_page_bits(); - uint64_t dirty_ring_size_MB; + uint64_t dirty_ring_size_MiB; - /* So far, the largest (non-huge) page size is 64k, i.e. 16 bits. */ - assert(target_page_bits < 20); - - /* Convert ring size (pages) to MiB (2**20). */ - dirty_ring_size_MB = kvm_dirty_ring_size() >> (20 - target_page_bits); + dirty_ring_size_MiB = qemu_target_pages_to_MiB(kvm_dirty_ring_size()); if (max_dirtyrate < dirtyrate) { max_dirtyrate = dirtyrate; } - return dirty_ring_size_MB * 1000000 / max_dirtyrate; + return dirty_ring_size_MiB * 1000000 / max_dirtyrate; } static inline bool dirtylimit_done(uint64_t quota, diff --git a/softmmu/memory.c b/softmmu/memory.c index b7b3386e9d..342c121514 100644 --- a/softmmu/memory.c +++ b/softmmu/memory.c @@ -2253,7 +2253,7 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, * If memory region `mr' is NULL, do global sync. Otherwise, sync * dirty bitmap for the specified memory region. */ -static void memory_region_sync_dirty_bitmap(MemoryRegion *mr) +static void memory_region_sync_dirty_bitmap(MemoryRegion *mr, bool last_stage) { MemoryListener *listener; AddressSpace *as; @@ -2283,7 +2283,7 @@ static void memory_region_sync_dirty_bitmap(MemoryRegion *mr) * is to do a global sync, because we are not capable to * sync in a finer granularity. */ - listener->log_sync_global(listener); + listener->log_sync_global(listener, last_stage); trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 1); } } @@ -2347,7 +2347,7 @@ DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr, { DirtyBitmapSnapshot *snapshot; assert(mr->ram_block); - memory_region_sync_dirty_bitmap(mr); + memory_region_sync_dirty_bitmap(mr, false); snapshot = cpu_physical_memory_snapshot_and_clear_dirty(mr, addr, size, client); memory_global_after_dirty_log_sync(); return snapshot; @@ -2873,9 +2873,9 @@ bool memory_region_present(MemoryRegion *container, hwaddr addr) return mr && mr != container; } -void memory_global_dirty_log_sync(void) +void memory_global_dirty_log_sync(bool last_stage) { - memory_region_sync_dirty_bitmap(NULL); + memory_region_sync_dirty_bitmap(NULL, last_stage); } void memory_global_after_dirty_log_sync(void) diff --git a/softmmu/physmem.c b/softmmu/physmem.c index 0e0182d9f2..efaed36773 100644 --- a/softmmu/physmem.c +++ b/softmmu/physmem.c @@ -3357,6 +3357,17 @@ int qemu_target_page_bits_min(void) return TARGET_PAGE_BITS_MIN; } +/* Convert target pages to MiB (2**20). */ +size_t qemu_target_pages_to_MiB(size_t pages) +{ + int page_bits = TARGET_PAGE_BITS; + + /* So far, the largest (non-huge) page size is 64k, i.e. 16 bits. */ + g_assert(page_bits < 20); + + return pages >> (20 - page_bits); +} + bool cpu_physical_memory_is_io(hwaddr phys_addr) { MemoryRegion*mr; diff --git a/stubs/colo-compare.c b/stubs/colo-compare.c new file mode 100644 index 0000000000..ec726665be --- /dev/null +++ b/stubs/colo-compare.c @@ -0,0 +1,7 @@ +#include "qemu/osdep.h" +#include "qemu/notify.h" +#include "net/colo-compare.h" + +void colo_compare_cleanup(void) +{ +} diff --git a/stubs/colo.c b/stubs/colo.c index cf9816d368..f33379d0fd 100644 --- a/stubs/colo.c +++ b/stubs/colo.c @@ -10,11 +10,9 @@ void colo_shutdown(void) { } -void *colo_process_incoming_thread(void *opaque) +int coroutine_fn colo_incoming_co(void) { - error_report("Impossible happend: trying to start COLO thread when COLO " - "module is not built in"); - abort(); + return 0; } void colo_checkpoint_delay_set(void) diff --git a/stubs/meson.build b/stubs/meson.build index 8412cad15f..a56645e2f7 100644 --- a/stubs/meson.build +++ b/stubs/meson.build @@ -46,6 +46,7 @@ stub_ss.add(files('target-monitor-defs.c')) stub_ss.add(files('trace-control.c')) stub_ss.add(files('uuid.c')) stub_ss.add(files('colo.c')) +stub_ss.add(files('colo-compare.c')) stub_ss.add(files('vmstate.c')) stub_ss.add(files('vm-stop.c')) stub_ss.add(files('win32-kbd-hook.c')) diff --git a/target/arm/Kconfig b/target/arm/Kconfig index 39f05b6420..5947366f6e 100644 --- a/target/arm/Kconfig +++ b/target/arm/Kconfig @@ -1,13 +1,8 @@ config ARM bool + select ARM_COMPATIBLE_SEMIHOSTING if TCG + select ARM_V7M if TCG config AARCH64 bool select ARM - -# This config exists just so we can make SEMIHOSTING default when TCG -# is selected without also changing it for other architectures. -config ARM_SEMIHOSTING - bool - default y if TCG && ARM - select ARM_COMPATIBLE_SEMIHOSTING diff --git a/target/arm/cortex-regs.c b/target/arm/cortex-regs.c index 17708480e7..ae817b08dd 100644 --- a/target/arm/cortex-regs.c +++ b/target/arm/cortex-regs.c @@ -15,8 +15,15 @@ static uint64_t l2ctlr_read(CPUARMState *env, const ARMCPRegInfo *ri) { ARMCPU *cpu = env_archcpu(env); - /* Number of cores is in [25:24]; otherwise we RAZ */ - return (cpu->core_count - 1) << 24; + /* + * Number of cores is in [25:24]; otherwise we RAZ. + * If the board didn't configure the CPUs into clusters, + * we default to "all CPUs in one cluster", which might be + * more than the 4 that the hardware permits and which is + * all you can report in this two-bit field. Saturate to + * 0b11 (== 4 CPUs) rather than overflowing the field. + */ + return MIN(cpu->core_count - 1, 3) << 24; } static const ARMCPRegInfo cortex_a72_a57_a53_cp_reginfo[] = { diff --git a/target/arm/cpu.c b/target/arm/cpu.c index 5182ed0c91..f6a88e52ac 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -1480,6 +1480,7 @@ void arm_cpu_post_init(Object *obj) qdev_prop_allow_set_link_before_realize, OBJ_PROP_LINK_STRONG); } + cpu->has_mte = true; } #endif } @@ -1616,7 +1617,7 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) } if (cpu->tag_memory) { error_setg(errp, - "Cannot enable %s when guest CPUs has MTE enabled", + "Cannot enable %s when guest CPUs has tag memory enabled", current_accel_name()); return; } @@ -1996,10 +1997,10 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) } #ifndef CONFIG_USER_ONLY - if (cpu->tag_memory == NULL && cpu_isar_feature(aa64_mte, cpu)) { + if (!cpu->has_mte && cpu_isar_feature(aa64_mte, cpu)) { /* - * Disable the MTE feature bits if we do not have tag-memory - * provided by the machine. + * Disable the MTE feature bits if we do not have the feature + * setup by the machine. */ cpu->isar.id_aa64pfr1 = FIELD_DP64(cpu->isar.id_aa64pfr1, ID_AA64PFR1, MTE, 0); diff --git a/target/arm/cpu.h b/target/arm/cpu.h index d469a2637b..c3463e39bc 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -935,6 +935,9 @@ struct ArchCPU { */ uint32_t psci_conduit; + /* CPU has Memory Tag Extension */ + bool has_mte; + /* For v8M, initial value of the Secure VTOR */ uint32_t init_svtor; /* For v8M, initial value of the Non-secure VTOR */ @@ -1053,6 +1056,7 @@ struct ArchCPU { bool prop_pauth; bool prop_pauth_impdef; bool prop_lpa2; + OnOffAuto prop_mte; /* DCZ blocksize, in log_2(words), ie low 4 bits of DCZID_EL0 */ uint32_t dcz_blocksize; diff --git a/target/arm/debug_helper.c b/target/arm/debug_helper.c index dfc8b2a1a5..d41cc643b1 100644 --- a/target/arm/debug_helper.c +++ b/target/arm/debug_helper.c @@ -949,8 +949,10 @@ static const ARMCPRegInfo debug_cp_reginfo[] = { .access = PL0_R, .accessfn = access_tdcc, .type = ARM_CP_CONST, .resetvalue = 0 }, /* - * OSDTRRX_EL1/OSDTRTX_EL1 are used for save and restore of DBGDTRRX_EL0. - * It is a component of the Debug Communications Channel, which is not implemented. + * These registers belong to the Debug Communications Channel, + * which is not implemented. However we implement RAZ/WI behaviour + * with trapping to prevent spurious SIGILLs if the guest OS does + * access them as the support cannot be probed for. */ { .name = "OSDTRRX_EL1", .state = ARM_CP_STATE_BOTH, .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 2, @@ -960,6 +962,11 @@ static const ARMCPRegInfo debug_cp_reginfo[] = { .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2, .access = PL1_RW, .accessfn = access_tdcc, .type = ARM_CP_CONST, .resetvalue = 0 }, + /* DBGDTRTX_EL0/DBGDTRRX_EL0 depend on direction */ + { .name = "DBGDTR_EL0", .state = ARM_CP_STATE_BOTH, .cp = 14, + .opc0 = 2, .opc1 = 3, .crn = 0, .crm = 5, .opc2 = 0, + .access = PL0_RW, .accessfn = access_tdcc, + .type = ARM_CP_CONST, .resetvalue = 0 }, /* * OSECCR_EL1 provides a mechanism for an operating system * to access the contents of EDECCR. EDECCR is not implemented though, diff --git a/target/arm/gdbstub64.c b/target/arm/gdbstub64.c index c1f7e8c934..d7b79a6589 100644 --- a/target/arm/gdbstub64.c +++ b/target/arm/gdbstub64.c @@ -233,7 +233,7 @@ int aarch64_gdb_get_pauth_reg(CPUARMState *env, GByteArray *buf, int reg) ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); ARMVAParameters param; - param = aa64_va_parameters(env, -is_high, mmu_idx, is_data); + param = aa64_va_parameters(env, -is_high, mmu_idx, is_data, false); return gdb_get_reg64(buf, pauth_ptr_mask(param)); } default: diff --git a/target/arm/helper.c b/target/arm/helper.c index 2297626bfb..0b7fd2e7e6 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -4904,7 +4904,7 @@ static TLBIRange tlbi_aa64_get_range(CPUARMState *env, ARMMMUIdx mmuidx, unsigned int page_size_granule, page_shift, num, scale, exponent; /* Extract one bit to represent the va selector in use. */ uint64_t select = sextract64(value, 36, 1); - ARMVAParameters param = aa64_va_parameters(env, select, mmuidx, true); + ARMVAParameters param = aa64_va_parameters(env, select, mmuidx, true, false); TLBIRange ret = { }; ARMGranuleSize gran; @@ -11193,7 +11193,8 @@ static ARMGranuleSize sanitize_gran_size(ARMCPU *cpu, ARMGranuleSize gran, } ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, - ARMMMUIdx mmu_idx, bool data) + ARMMMUIdx mmu_idx, bool data, + bool el1_is_aa32) { uint64_t tcr = regime_tcr(env, mmu_idx); bool epd, hpd, tsz_oob, ds, ha, hd; @@ -11289,6 +11290,16 @@ ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, } } + if (stage2 && el1_is_aa32) { + /* + * For AArch32 EL1 the min txsz (and thus max IPA size) requirements + * are loosened: a configured IPA of 40 bits is permitted even if + * the implemented PA is less than that (and so a 40 bit IPA would + * fault for an AArch64 EL1). See R_DTLMN. + */ + min_tsz = MIN(min_tsz, 24); + } + if (tsz > max_tsz) { tsz = max_tsz; tsz_oob = true; diff --git a/target/arm/helper.h b/target/arm/helper.h index 018b00ea75..3335c2b10b 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -1039,9 +1039,9 @@ DEF_HELPER_FLAGS_5(gvec_uclamp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) #ifdef TARGET_AARCH64 -#include "helper-a64.h" -#include "helper-sve.h" -#include "helper-sme.h" +#include "tcg/helper-a64.h" +#include "tcg/helper-sve.h" +#include "tcg/helper-sme.h" #endif -#include "helper-mve.h" +#include "tcg/helper-mve.h" diff --git a/target/arm/internals.h b/target/arm/internals.h index 0df8f3b8bc..c869d18c38 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -1091,8 +1091,18 @@ typedef struct ARMVAParameters { ARMGranuleSize gran : 2; } ARMVAParameters; +/** + * aa64_va_parameters: Return parameters for an AArch64 virtual address + * @env: CPU + * @va: virtual address to look up + * @mmu_idx: determines translation regime to use + * @data: true if this is a data access + * @el1_is_aa32: true if we are asking about stage 2 when EL1 is AArch32 + * (ignored if @mmu_idx is for a stage 1 regime; only affects tsz/tsz_oob) + */ ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, - ARMMMUIdx mmu_idx, bool data); + ARMMMUIdx mmu_idx, bool data, + bool el1_is_aa32); int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx); int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx); diff --git a/target/arm/kvm.c b/target/arm/kvm.c index 84da49332c..9553488ecd 100644 --- a/target/arm/kvm.c +++ b/target/arm/kvm.c @@ -31,6 +31,7 @@ #include "hw/boards.h" #include "hw/irq.h" #include "qemu/log.h" +#include "migration/blocker.h" const KVMCapabilityInfo kvm_arch_required_capabilities[] = { KVM_CAP_LAST_INFO @@ -1064,3 +1065,37 @@ bool kvm_arch_cpu_check_are_resettable(void) void kvm_arch_accel_class_init(ObjectClass *oc) { } + +void kvm_arm_enable_mte(Object *cpuobj, Error **errp) +{ + static bool tried_to_enable; + static bool succeeded_to_enable; + Error *mte_migration_blocker = NULL; + int ret; + + if (!tried_to_enable) { + /* + * MTE on KVM is enabled on a per-VM basis (and retrying doesn't make + * sense), and we only want a single migration blocker as well. + */ + tried_to_enable = true; + + ret = kvm_vm_enable_cap(kvm_state, KVM_CAP_ARM_MTE, 0); + if (ret) { + error_setg_errno(errp, -ret, "Failed to enable KVM_CAP_ARM_MTE"); + return; + } + + /* TODO: add proper migration support with MTE enabled */ + error_setg(&mte_migration_blocker, + "Live migration disabled due to MTE enabled"); + if (migrate_add_blocker(mte_migration_blocker, errp)) { + error_free(mte_migration_blocker); + return; + } + succeeded_to_enable = true; + } + if (succeeded_to_enable) { + object_property_set_bool(cpuobj, "has_mte", true, NULL); + } +} diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c index 810db33ccb..1893f38793 100644 --- a/target/arm/kvm64.c +++ b/target/arm/kvm64.c @@ -756,6 +756,11 @@ bool kvm_arm_steal_time_supported(void) return kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME); } +bool kvm_arm_mte_supported(void) +{ + return kvm_check_extension(kvm_state, KVM_CAP_ARM_MTE); +} + QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1); uint32_t kvm_arm_sve_get_vls(CPUState *cs) diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h index 330fbe5c72..2083547bf6 100644 --- a/target/arm/kvm_arm.h +++ b/target/arm/kvm_arm.h @@ -314,6 +314,13 @@ bool kvm_arm_pmu_supported(void); bool kvm_arm_sve_supported(void); /** + * kvm_arm_mte_supported: + * + * Returns: true if KVM can enable MTE, and false otherwise. + */ +bool kvm_arm_mte_supported(void); + +/** * kvm_arm_get_max_vm_ipa_size: * @ms: Machine state handle * @fixed_ipa: True when the IPA limit is fixed at 40. This is the case @@ -377,6 +384,8 @@ void kvm_arm_pvtime_init(CPUState *cs, uint64_t ipa); int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level); +void kvm_arm_enable_mte(Object *cpuobj, Error **errp); + #else /* @@ -403,6 +412,11 @@ static inline bool kvm_arm_steal_time_supported(void) return false; } +static inline bool kvm_arm_mte_supported(void) +{ + return false; +} + /* * These functions should never actually be called without KVM support. */ @@ -451,6 +465,11 @@ static inline uint32_t kvm_arm_sve_get_vls(CPUState *cs) g_assert_not_reached(); } +static inline void kvm_arm_enable_mte(Object *cpuobj, Error **errp) +{ + g_assert_not_reached(); +} + #endif static inline const char *gic_class_name(void) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index bd75da8dbc..69c05cd9da 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -103,6 +103,37 @@ ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) return stage_1_mmu_idx(arm_mmu_idx(env)); } +/* + * Return where we should do ptw loads from for a stage 2 walk. + * This depends on whether the address we are looking up is a + * Secure IPA or a NonSecure IPA, which we know from whether this is + * Stage2 or Stage2_S. + * If this is the Secure EL1&0 regime we need to check the NSW and SW bits. + */ +static ARMMMUIdx ptw_idx_for_stage_2(CPUARMState *env, ARMMMUIdx stage2idx) +{ + bool s2walk_secure; + + /* + * We're OK to check the current state of the CPU here because + * (1) we always invalidate all TLBs when the SCR_EL3.NS bit changes + * (2) there's no way to do a lookup that cares about Stage 2 for a + * different security state to the current one for AArch64, and AArch32 + * never has a secure EL2. (AArch32 ATS12NSO[UP][RW] allow EL3 to do + * an NS stage 1+2 lookup while the NS bit is 0.) + */ + if (!arm_is_secure_below_el3(env) || !arm_el_is_aa64(env, 3)) { + return ARMMMUIdx_Phys_NS; + } + if (stage2idx == ARMMMUIdx_Stage2_S) { + s2walk_secure = !(env->cp15.vstcr_el2 & VSTCR_SW); + } else { + s2walk_secure = !(env->cp15.vtcr_el2 & VTCR_NSW); + } + return s2walk_secure ? ARMMMUIdx_Phys_S : ARMMMUIdx_Phys_NS; + +} + static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx) { return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; @@ -220,7 +251,6 @@ static bool S1_ptw_translate(CPUARMState *env, S1Translate *ptw, ARMMMUIdx mmu_idx = ptw->in_mmu_idx; ARMMMUIdx s2_mmu_idx = ptw->in_ptw_idx; uint8_t pte_attrs; - bool pte_secure; ptw->out_virt = addr; @@ -232,8 +262,8 @@ static bool S1_ptw_translate(CPUARMState *env, S1Translate *ptw, if (regime_is_stage2(s2_mmu_idx)) { S1Translate s2ptw = { .in_mmu_idx = s2_mmu_idx, - .in_ptw_idx = is_secure ? ARMMMUIdx_Phys_S : ARMMMUIdx_Phys_NS, - .in_secure = is_secure, + .in_ptw_idx = ptw_idx_for_stage_2(env, s2_mmu_idx), + .in_secure = s2_mmu_idx == ARMMMUIdx_Stage2_S, .in_debug = true, }; GetPhysAddrResult s2 = { }; @@ -244,12 +274,12 @@ static bool S1_ptw_translate(CPUARMState *env, S1Translate *ptw, } ptw->out_phys = s2.f.phys_addr; pte_attrs = s2.cacheattrs.attrs; - pte_secure = s2.f.attrs.secure; + ptw->out_secure = s2.f.attrs.secure; } else { /* Regime is physical. */ ptw->out_phys = addr; pte_attrs = 0; - pte_secure = is_secure; + ptw->out_secure = s2_mmu_idx == ARMMMUIdx_Phys_S; } ptw->out_host = NULL; ptw->out_rw = false; @@ -270,7 +300,7 @@ static bool S1_ptw_translate(CPUARMState *env, S1Translate *ptw, ptw->out_phys = full->phys_addr | (addr & ~TARGET_PAGE_MASK); ptw->out_rw = full->prot & PAGE_WRITE; pte_attrs = full->pte_attrs; - pte_secure = full->attrs.secure; + ptw->out_secure = full->attrs.secure; #else g_assert_not_reached(); #endif @@ -293,11 +323,6 @@ static bool S1_ptw_translate(CPUARMState *env, S1Translate *ptw, } } - /* Check if page table walk is to secure or non-secure PA space. */ - ptw->out_secure = (is_secure - && !(pte_secure - ? env->cp15.vstcr_el2 & VSTCR_SW - : env->cp15.vtcr_el2 & VTCR_NSW)); ptw->out_be = regime_translation_big_endian(env, mmu_idx); return true; @@ -1110,17 +1135,6 @@ static int check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, uint64_t tcr, sl0 = extract32(tcr, 6, 2); if (is_aa64) { /* - * AArch64.S2InvalidTxSZ: While we checked tsz_oob near the top of - * get_phys_addr_lpae, that used aa64_va_parameters which apply - * to aarch64. If Stage1 is aarch32, the min_txsz is larger. - * See AArch64.S2MinTxSZ, where min_tsz is 24, translated to - * inputsize is 64 - 24 = 40. - */ - if (iasize < 40 && !arm_el_is_aa64(&cpu->env, 1)) { - goto fail; - } - - /* * AArch64.S2InvalidSL: Interpretation of SL depends on the page size, * so interleave AArch64.S2StartLevel. */ @@ -1259,7 +1273,8 @@ static bool get_phys_addr_lpae(CPUARMState *env, S1Translate *ptw, int ps; param = aa64_va_parameters(env, address, mmu_idx, - access_type != MMU_INST_FETCH); + access_type != MMU_INST_FETCH, + !arm_el_is_aa64(env, 1)); level = 0; /* @@ -1415,17 +1430,18 @@ static bool get_phys_addr_lpae(CPUARMState *env, S1Translate *ptw, descaddrmask &= ~indexmask_grainsize; /* - * Secure accesses start with the page table in secure memory and + * Secure stage 1 accesses start with the page table in secure memory and * can be downgraded to non-secure at any step. Non-secure accesses * remain non-secure. We implement this by just ORing in the NSTable/NS * bits at each step. + * Stage 2 never gets this kind of downgrade. */ tableattrs = is_secure ? 0 : (1 << 4); next_level: descaddr |= (address >> (stride * (4 - level))) & indexmask; descaddr &= ~7ULL; - nstable = extract32(tableattrs, 4, 1); + nstable = !regime_is_stage2(mmu_idx) && extract32(tableattrs, 4, 1); if (nstable) { /* * Stage2_S -> Stage2 or Phys_S -> Phys_NS @@ -2725,7 +2741,7 @@ static bool get_phys_addr_twostage(CPUARMState *env, S1Translate *ptw, hwaddr ipa; int s1_prot, s1_lgpgsz; bool is_secure = ptw->in_secure; - bool ret, ipa_secure, s2walk_secure; + bool ret, ipa_secure; ARMCacheAttrs cacheattrs1; bool is_el0; uint64_t hcr; @@ -2739,20 +2755,11 @@ static bool get_phys_addr_twostage(CPUARMState *env, S1Translate *ptw, ipa = result->f.phys_addr; ipa_secure = result->f.attrs.secure; - if (is_secure) { - /* Select TCR based on the NS bit from the S1 walk. */ - s2walk_secure = !(ipa_secure - ? env->cp15.vstcr_el2 & VSTCR_SW - : env->cp15.vtcr_el2 & VTCR_NSW); - } else { - assert(!ipa_secure); - s2walk_secure = false; - } is_el0 = ptw->in_mmu_idx == ARMMMUIdx_Stage1_E0; - ptw->in_mmu_idx = s2walk_secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2; - ptw->in_ptw_idx = s2walk_secure ? ARMMMUIdx_Phys_S : ARMMMUIdx_Phys_NS; - ptw->in_secure = s2walk_secure; + ptw->in_mmu_idx = ipa_secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2; + ptw->in_secure = ipa_secure; + ptw->in_ptw_idx = ptw_idx_for_stage_2(env, ptw->in_mmu_idx); /* * S1 is done, now do S2 translation. @@ -2860,6 +2867,16 @@ static bool get_phys_addr_with_struct(CPUARMState *env, S1Translate *ptw, ptw->in_ptw_idx = is_secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2; break; + case ARMMMUIdx_Stage2: + case ARMMMUIdx_Stage2_S: + /* + * Second stage lookup uses physical for ptw; whether this is S or + * NS may depend on the SW/NSW bits if this is a stage 2 lookup for + * the Secure EL2&0 regime. + */ + ptw->in_ptw_idx = ptw_idx_for_stage_2(env, mmu_idx); + break; + case ARMMMUIdx_E10_0: s1_mmu_idx = ARMMMUIdx_Stage1_E0; goto do_twostage; @@ -2883,7 +2900,7 @@ static bool get_phys_addr_with_struct(CPUARMState *env, S1Translate *ptw, /* fall through */ default: - /* Single stage and second stage uses physical for ptw. */ + /* Single stage uses physical for ptw. */ ptw->in_ptw_idx = is_secure ? ARMMMUIdx_Phys_S : ARMMMUIdx_Phys_NS; break; } diff --git a/target/arm/tcg/a64.decode b/target/arm/tcg/a64.decode new file mode 100644 index 0000000000..12a310d0a3 --- /dev/null +++ b/target/arm/tcg/a64.decode @@ -0,0 +1,152 @@ +# AArch64 A64 allowed instruction decoding +# +# Copyright (c) 2023 Linaro, Ltd +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see <http://www.gnu.org/licenses/>. + +# +# This file is processed by scripts/decodetree.py +# + +&r rn +&ri rd imm +&rri_sf rd rn imm sf +&i imm + + +### Data Processing - Immediate + +# PC-rel addressing + +%imm_pcrel 5:s19 29:2 +@pcrel . .. ..... ................... rd:5 &ri imm=%imm_pcrel + +ADR 0 .. 10000 ................... ..... @pcrel +ADRP 1 .. 10000 ................... ..... @pcrel + +# Add/subtract (immediate) + +%imm12_sh12 10:12 !function=shl_12 +@addsub_imm sf:1 .. ...... . imm:12 rn:5 rd:5 +@addsub_imm12 sf:1 .. ...... . ............ rn:5 rd:5 imm=%imm12_sh12 + +ADD_i . 00 100010 0 ............ ..... ..... @addsub_imm +ADD_i . 00 100010 1 ............ ..... ..... @addsub_imm12 +ADDS_i . 01 100010 0 ............ ..... ..... @addsub_imm +ADDS_i . 01 100010 1 ............ ..... ..... @addsub_imm12 + +SUB_i . 10 100010 0 ............ ..... ..... @addsub_imm +SUB_i . 10 100010 1 ............ ..... ..... @addsub_imm12 +SUBS_i . 11 100010 0 ............ ..... ..... @addsub_imm +SUBS_i . 11 100010 1 ............ ..... ..... @addsub_imm12 + +# Add/subtract (immediate with tags) + +&rri_tag rd rn uimm6 uimm4 +@addsub_imm_tag . .. ...... . uimm6:6 .. uimm4:4 rn:5 rd:5 &rri_tag + +ADDG_i 1 00 100011 0 ...... 00 .... ..... ..... @addsub_imm_tag +SUBG_i 1 10 100011 0 ...... 00 .... ..... ..... @addsub_imm_tag + +# Logical (immediate) + +&rri_log rd rn sf dbm +@logic_imm_64 1 .. ...... dbm:13 rn:5 rd:5 &rri_log sf=1 +@logic_imm_32 0 .. ...... 0 dbm:12 rn:5 rd:5 &rri_log sf=0 + +AND_i . 00 100100 . ...... ...... ..... ..... @logic_imm_64 +AND_i . 00 100100 . ...... ...... ..... ..... @logic_imm_32 +ORR_i . 01 100100 . ...... ...... ..... ..... @logic_imm_64 +ORR_i . 01 100100 . ...... ...... ..... ..... @logic_imm_32 +EOR_i . 10 100100 . ...... ...... ..... ..... @logic_imm_64 +EOR_i . 10 100100 . ...... ...... ..... ..... @logic_imm_32 +ANDS_i . 11 100100 . ...... ...... ..... ..... @logic_imm_64 +ANDS_i . 11 100100 . ...... ...... ..... ..... @logic_imm_32 + +# Move wide (immediate) + +&movw rd sf imm hw +@movw_64 1 .. ...... hw:2 imm:16 rd:5 &movw sf=1 +@movw_32 0 .. ...... 0 hw:1 imm:16 rd:5 &movw sf=0 + +MOVN . 00 100101 .. ................ ..... @movw_64 +MOVN . 00 100101 .. ................ ..... @movw_32 +MOVZ . 10 100101 .. ................ ..... @movw_64 +MOVZ . 10 100101 .. ................ ..... @movw_32 +MOVK . 11 100101 .. ................ ..... @movw_64 +MOVK . 11 100101 .. ................ ..... @movw_32 + +# Bitfield + +&bitfield rd rn sf immr imms +@bitfield_64 1 .. ...... 1 immr:6 imms:6 rn:5 rd:5 &bitfield sf=1 +@bitfield_32 0 .. ...... 0 0 immr:5 0 imms:5 rn:5 rd:5 &bitfield sf=0 + +SBFM . 00 100110 . ...... ...... ..... ..... @bitfield_64 +SBFM . 00 100110 . ...... ...... ..... ..... @bitfield_32 +BFM . 01 100110 . ...... ...... ..... ..... @bitfield_64 +BFM . 01 100110 . ...... ...... ..... ..... @bitfield_32 +UBFM . 10 100110 . ...... ...... ..... ..... @bitfield_64 +UBFM . 10 100110 . ...... ...... ..... ..... @bitfield_32 + +# Extract + +&extract rd rn rm imm sf + +EXTR 1 00 100111 1 0 rm:5 imm:6 rn:5 rd:5 &extract sf=1 +EXTR 0 00 100111 0 0 rm:5 0 imm:5 rn:5 rd:5 &extract sf=0 + +# Branches + +%imm26 0:s26 !function=times_4 +@branch . ..... .......................... &i imm=%imm26 + +B 0 00101 .......................... @branch +BL 1 00101 .......................... @branch + +%imm19 5:s19 !function=times_4 +&cbz rt imm sf nz + +CBZ sf:1 011010 nz:1 ................... rt:5 &cbz imm=%imm19 + +%imm14 5:s14 !function=times_4 +%imm31_19 31:1 19:5 +&tbz rt imm nz bitpos + +TBZ . 011011 nz:1 ..... .............. rt:5 &tbz imm=%imm14 bitpos=%imm31_19 + +B_cond 0101010 0 ................... 0 cond:4 imm=%imm19 + +BR 1101011 0000 11111 000000 rn:5 00000 &r +BLR 1101011 0001 11111 000000 rn:5 00000 &r +RET 1101011 0010 11111 000000 rn:5 00000 &r + +&braz rn m +BRAZ 1101011 0000 11111 00001 m:1 rn:5 11111 &braz # BRAAZ, BRABZ +BLRAZ 1101011 0001 11111 00001 m:1 rn:5 11111 &braz # BLRAAZ, BLRABZ + +&reta m +RETA 1101011 0010 11111 00001 m:1 11111 11111 &reta # RETAA, RETAB + +&bra rn rm m +BRA 1101011 1000 11111 00001 m:1 rn:5 rm:5 &bra # BRAA, BRAB +BLRA 1101011 1001 11111 00001 m:1 rn:5 rm:5 &bra # BLRAA, BLRAB + +ERET 1101011 0100 11111 000000 11111 00000 +ERETA 1101011 0100 11111 00001 m:1 11111 11111 &reta # ERETAA, ERETAB + +# We don't need to decode DRPS because it always UNDEFs except when +# the processor is in halting debug state (which we don't implement). +# The pattern is listed here as documentation. +# DRPS 1101011 0101 11111 000000 11111 00000 diff --git a/target/arm/arm_ldst.h b/target/arm/tcg/arm_ldst.h index cee0548a1c..cee0548a1c 100644 --- a/target/arm/arm_ldst.h +++ b/target/arm/tcg/arm_ldst.h diff --git a/target/arm/helper-a64.h b/target/arm/tcg/helper-a64.h index ff56807247..ff56807247 100644 --- a/target/arm/helper-a64.h +++ b/target/arm/tcg/helper-a64.h diff --git a/target/arm/helper-mve.h b/target/arm/tcg/helper-mve.h index 76bd25006d..76bd25006d 100644 --- a/target/arm/helper-mve.h +++ b/target/arm/tcg/helper-mve.h diff --git a/target/arm/helper-sme.h b/target/arm/tcg/helper-sme.h index 27eef49a11..27eef49a11 100644 --- a/target/arm/helper-sme.h +++ b/target/arm/tcg/helper-sme.h diff --git a/target/arm/helper-sve.h b/target/arm/tcg/helper-sve.h index cc4e1d8948..cc4e1d8948 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/tcg/helper-sve.h diff --git a/target/arm/tcg/meson.build b/target/arm/tcg/meson.build index 4d99f6dacb..130ed62fcd 100644 --- a/target/arm/tcg/meson.build +++ b/target/arm/tcg/meson.build @@ -13,6 +13,7 @@ gen = [ decodetree.process('a32-uncond.decode', extra_args: '--static-decode=disas_a32_uncond'), decodetree.process('t32.decode', extra_args: '--static-decode=disas_t32'), decodetree.process('t16.decode', extra_args: ['-w', '16', '--static-decode=disas_t16']), + decodetree.process('a64.decode', extra_args: ['--static-decode=disas_a64']), ] arm_ss.add(gen) diff --git a/target/arm/tcg/pauth_helper.c b/target/arm/tcg/pauth_helper.c index de067fa716..62af569341 100644 --- a/target/arm/tcg/pauth_helper.c +++ b/target/arm/tcg/pauth_helper.c @@ -293,7 +293,7 @@ static uint64_t pauth_addpac(CPUARMState *env, uint64_t ptr, uint64_t modifier, ARMPACKey *key, bool data) { ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); - ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); + ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data, false); uint64_t pac, ext_ptr, ext, test; int bot_bit, top_bit; @@ -355,7 +355,7 @@ static uint64_t pauth_auth(CPUARMState *env, uint64_t ptr, uint64_t modifier, ARMPACKey *key, bool data, int keynumber) { ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); - ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); + ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data, false); int bot_bit, top_bit; uint64_t pac, orig_ptr, test; @@ -379,7 +379,7 @@ static uint64_t pauth_auth(CPUARMState *env, uint64_t ptr, uint64_t modifier, static uint64_t pauth_strip(CPUARMState *env, uint64_t ptr, bool data) { ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); - ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); + ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data, false); return pauth_original_ptr(ptr, param); } diff --git a/target/arm/tcg/sve_helper.c b/target/arm/tcg/sve_helper.c index ccf5e5beca..0097522470 100644 --- a/target/arm/tcg/sve_helper.c +++ b/target/arm/tcg/sve_helper.c @@ -6727,6 +6727,7 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, intptr_t reg_off; SVEHostPage info; target_ulong addr, in_page; + ARMVectorReg scratch; /* Skip to the first true predicate. */ reg_off = find_next_active(vg, 0, reg_max, esz); @@ -6736,6 +6737,11 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, return; } + /* Protect against overlap between vd and vm. */ + if (unlikely(vd == vm)) { + vm = memcpy(&scratch, vm, reg_max); + } + /* * Probe the first element, allowing faults. */ diff --git a/target/arm/sve_ldst_internal.h b/target/arm/tcg/sve_ldst_internal.h index 4f159ec4ad..4f159ec4ad 100644 --- a/target/arm/sve_ldst_internal.h +++ b/target/arm/tcg/sve_ldst_internal.h diff --git a/target/arm/translate-a32.h b/target/arm/tcg/translate-a32.h index 48a15379d2..48a15379d2 100644 --- a/target/arm/translate-a32.h +++ b/target/arm/tcg/translate-a32.h diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index dff391bfe2..741a608739 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -56,6 +56,13 @@ enum a64_shift_type { A64_SHIFT_TYPE_ROR = 3 }; +/* + * Include the generated decoders. + */ + +#include "decode-sme-fa64.c.inc" +#include "decode-a64.c.inc" + /* Table based decoder typedefs - used when the relevant bits for decode * are too awkwardly scattered across the instruction (eg SIMD). */ @@ -675,83 +682,102 @@ static inline void gen_logic_CC(int sf, TCGv_i64 result) } /* dest = T0 + T1; compute C, N, V and Z flags */ -static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) +static void gen_add64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) { - if (sf) { - TCGv_i64 result, flag, tmp; - result = tcg_temp_new_i64(); - flag = tcg_temp_new_i64(); - tmp = tcg_temp_new_i64(); + TCGv_i64 result, flag, tmp; + result = tcg_temp_new_i64(); + flag = tcg_temp_new_i64(); + tmp = tcg_temp_new_i64(); - tcg_gen_movi_i64(tmp, 0); - tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp); + tcg_gen_movi_i64(tmp, 0); + tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp); - tcg_gen_extrl_i64_i32(cpu_CF, flag); + tcg_gen_extrl_i64_i32(cpu_CF, flag); - gen_set_NZ64(result); + gen_set_NZ64(result); - tcg_gen_xor_i64(flag, result, t0); - tcg_gen_xor_i64(tmp, t0, t1); - tcg_gen_andc_i64(flag, flag, tmp); - tcg_gen_extrh_i64_i32(cpu_VF, flag); + tcg_gen_xor_i64(flag, result, t0); + tcg_gen_xor_i64(tmp, t0, t1); + tcg_gen_andc_i64(flag, flag, tmp); + tcg_gen_extrh_i64_i32(cpu_VF, flag); - tcg_gen_mov_i64(dest, result); - } else { - /* 32 bit arithmetic */ - TCGv_i32 t0_32 = tcg_temp_new_i32(); - TCGv_i32 t1_32 = tcg_temp_new_i32(); - TCGv_i32 tmp = tcg_temp_new_i32(); + tcg_gen_mov_i64(dest, result); +} - tcg_gen_movi_i32(tmp, 0); - tcg_gen_extrl_i64_i32(t0_32, t0); - tcg_gen_extrl_i64_i32(t1_32, t1); - tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp); - tcg_gen_mov_i32(cpu_ZF, cpu_NF); - tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32); - tcg_gen_xor_i32(tmp, t0_32, t1_32); - tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp); - tcg_gen_extu_i32_i64(dest, cpu_NF); +static void gen_add32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) +{ + TCGv_i32 t0_32 = tcg_temp_new_i32(); + TCGv_i32 t1_32 = tcg_temp_new_i32(); + TCGv_i32 tmp = tcg_temp_new_i32(); + + tcg_gen_movi_i32(tmp, 0); + tcg_gen_extrl_i64_i32(t0_32, t0); + tcg_gen_extrl_i64_i32(t1_32, t1); + tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp); + tcg_gen_mov_i32(cpu_ZF, cpu_NF); + tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32); + tcg_gen_xor_i32(tmp, t0_32, t1_32); + tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp); + tcg_gen_extu_i32_i64(dest, cpu_NF); +} + +static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) +{ + if (sf) { + gen_add64_CC(dest, t0, t1); + } else { + gen_add32_CC(dest, t0, t1); } } /* dest = T0 - T1; compute C, N, V and Z flags */ -static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) +static void gen_sub64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) { - if (sf) { - /* 64 bit arithmetic */ - TCGv_i64 result, flag, tmp; + /* 64 bit arithmetic */ + TCGv_i64 result, flag, tmp; - result = tcg_temp_new_i64(); - flag = tcg_temp_new_i64(); - tcg_gen_sub_i64(result, t0, t1); + result = tcg_temp_new_i64(); + flag = tcg_temp_new_i64(); + tcg_gen_sub_i64(result, t0, t1); - gen_set_NZ64(result); + gen_set_NZ64(result); - tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1); - tcg_gen_extrl_i64_i32(cpu_CF, flag); + tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1); + tcg_gen_extrl_i64_i32(cpu_CF, flag); - tcg_gen_xor_i64(flag, result, t0); - tmp = tcg_temp_new_i64(); - tcg_gen_xor_i64(tmp, t0, t1); - tcg_gen_and_i64(flag, flag, tmp); - tcg_gen_extrh_i64_i32(cpu_VF, flag); - tcg_gen_mov_i64(dest, result); - } else { - /* 32 bit arithmetic */ - TCGv_i32 t0_32 = tcg_temp_new_i32(); - TCGv_i32 t1_32 = tcg_temp_new_i32(); - TCGv_i32 tmp; + tcg_gen_xor_i64(flag, result, t0); + tmp = tcg_temp_new_i64(); + tcg_gen_xor_i64(tmp, t0, t1); + tcg_gen_and_i64(flag, flag, tmp); + tcg_gen_extrh_i64_i32(cpu_VF, flag); + tcg_gen_mov_i64(dest, result); +} - tcg_gen_extrl_i64_i32(t0_32, t0); - tcg_gen_extrl_i64_i32(t1_32, t1); - tcg_gen_sub_i32(cpu_NF, t0_32, t1_32); - tcg_gen_mov_i32(cpu_ZF, cpu_NF); - tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32); - tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32); - tmp = tcg_temp_new_i32(); - tcg_gen_xor_i32(tmp, t0_32, t1_32); - tcg_gen_and_i32(cpu_VF, cpu_VF, tmp); - tcg_gen_extu_i32_i64(dest, cpu_NF); +static void gen_sub32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) +{ + /* 32 bit arithmetic */ + TCGv_i32 t0_32 = tcg_temp_new_i32(); + TCGv_i32 t1_32 = tcg_temp_new_i32(); + TCGv_i32 tmp; + + tcg_gen_extrl_i64_i32(t0_32, t0); + tcg_gen_extrl_i64_i32(t1_32, t1); + tcg_gen_sub_i32(cpu_NF, t0_32, t1_32); + tcg_gen_mov_i32(cpu_ZF, cpu_NF); + tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32); + tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32); + tmp = tcg_temp_new_i32(); + tcg_gen_xor_i32(tmp, t0_32, t1_32); + tcg_gen_and_i32(cpu_VF, cpu_VF, tmp); + tcg_gen_extu_i32_i64(dest, cpu_NF); +} + +static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) +{ + if (sf) { + gen_sub64_CC(dest, t0, t1); + } else { + gen_sub32_CC(dest, t0, t1); } } @@ -1293,116 +1319,279 @@ static inline AArch64DecodeFn *lookup_disas_fn(const AArch64DecodeTable *table, * match up with those in the manual. */ -/* Unconditional branch (immediate) - * 31 30 26 25 0 - * +----+-----------+-------------------------------------+ - * | op | 0 0 1 0 1 | imm26 | - * +----+-----------+-------------------------------------+ - */ -static void disas_uncond_b_imm(DisasContext *s, uint32_t insn) +static bool trans_B(DisasContext *s, arg_i *a) { - int64_t diff = sextract32(insn, 0, 26) * 4; - - if (insn & (1U << 31)) { - /* BL Branch with link */ - gen_pc_plus_diff(s, cpu_reg(s, 30), curr_insn_len(s)); - } + reset_btype(s); + gen_goto_tb(s, 0, a->imm); + return true; +} - /* B Branch / BL Branch with link */ +static bool trans_BL(DisasContext *s, arg_i *a) +{ + gen_pc_plus_diff(s, cpu_reg(s, 30), curr_insn_len(s)); reset_btype(s); - gen_goto_tb(s, 0, diff); + gen_goto_tb(s, 0, a->imm); + return true; } -/* Compare and branch (immediate) - * 31 30 25 24 23 5 4 0 - * +----+-------------+----+---------------------+--------+ - * | sf | 0 1 1 0 1 0 | op | imm19 | Rt | - * +----+-------------+----+---------------------+--------+ - */ -static void disas_comp_b_imm(DisasContext *s, uint32_t insn) + +static bool trans_CBZ(DisasContext *s, arg_cbz *a) { - unsigned int sf, op, rt; - int64_t diff; DisasLabel match; TCGv_i64 tcg_cmp; - sf = extract32(insn, 31, 1); - op = extract32(insn, 24, 1); /* 0: CBZ; 1: CBNZ */ - rt = extract32(insn, 0, 5); - diff = sextract32(insn, 5, 19) * 4; - - tcg_cmp = read_cpu_reg(s, rt, sf); + tcg_cmp = read_cpu_reg(s, a->rt, a->sf); reset_btype(s); match = gen_disas_label(s); - tcg_gen_brcondi_i64(op ? TCG_COND_NE : TCG_COND_EQ, + tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, tcg_cmp, 0, match.label); gen_goto_tb(s, 0, 4); set_disas_label(s, match); - gen_goto_tb(s, 1, diff); + gen_goto_tb(s, 1, a->imm); + return true; } -/* Test and branch (immediate) - * 31 30 25 24 23 19 18 5 4 0 - * +----+-------------+----+-------+-------------+------+ - * | b5 | 0 1 1 0 1 1 | op | b40 | imm14 | Rt | - * +----+-------------+----+-------+-------------+------+ - */ -static void disas_test_b_imm(DisasContext *s, uint32_t insn) +static bool trans_TBZ(DisasContext *s, arg_tbz *a) { - unsigned int bit_pos, op, rt; - int64_t diff; DisasLabel match; TCGv_i64 tcg_cmp; - bit_pos = (extract32(insn, 31, 1) << 5) | extract32(insn, 19, 5); - op = extract32(insn, 24, 1); /* 0: TBZ; 1: TBNZ */ - diff = sextract32(insn, 5, 14) * 4; - rt = extract32(insn, 0, 5); - tcg_cmp = tcg_temp_new_i64(); - tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, rt), (1ULL << bit_pos)); + tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos); reset_btype(s); match = gen_disas_label(s); - tcg_gen_brcondi_i64(op ? TCG_COND_NE : TCG_COND_EQ, + tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, tcg_cmp, 0, match.label); gen_goto_tb(s, 0, 4); set_disas_label(s, match); - gen_goto_tb(s, 1, diff); + gen_goto_tb(s, 1, a->imm); + return true; } -/* Conditional branch (immediate) - * 31 25 24 23 5 4 3 0 - * +---------------+----+---------------------+----+------+ - * | 0 1 0 1 0 1 0 | o1 | imm19 | o0 | cond | - * +---------------+----+---------------------+----+------+ - */ -static void disas_cond_b_imm(DisasContext *s, uint32_t insn) +static bool trans_B_cond(DisasContext *s, arg_B_cond *a) { - unsigned int cond; - int64_t diff; - - if ((insn & (1 << 4)) || (insn & (1 << 24))) { - unallocated_encoding(s); - return; - } - diff = sextract32(insn, 5, 19) * 4; - cond = extract32(insn, 0, 4); - reset_btype(s); - if (cond < 0x0e) { + if (a->cond < 0x0e) { /* genuinely conditional branches */ DisasLabel match = gen_disas_label(s); - arm_gen_test_cc(cond, match.label); + arm_gen_test_cc(a->cond, match.label); gen_goto_tb(s, 0, 4); set_disas_label(s, match); - gen_goto_tb(s, 1, diff); + gen_goto_tb(s, 1, a->imm); } else { /* 0xe and 0xf are both "always" conditions */ - gen_goto_tb(s, 0, diff); + gen_goto_tb(s, 0, a->imm); + } + return true; +} + +static void set_btype_for_br(DisasContext *s, int rn) +{ + if (dc_isar_feature(aa64_bti, s)) { + /* BR to {x16,x17} or !guard -> 1, else 3. */ + set_btype(s, rn == 16 || rn == 17 || !s->guarded_page ? 1 : 3); + } +} + +static void set_btype_for_blr(DisasContext *s) +{ + if (dc_isar_feature(aa64_bti, s)) { + /* BLR sets BTYPE to 2, regardless of source guarded page. */ + set_btype(s, 2); + } +} + +static bool trans_BR(DisasContext *s, arg_r *a) +{ + gen_a64_set_pc(s, cpu_reg(s, a->rn)); + set_btype_for_br(s, a->rn); + s->base.is_jmp = DISAS_JUMP; + return true; +} + +static bool trans_BLR(DisasContext *s, arg_r *a) +{ + TCGv_i64 dst = cpu_reg(s, a->rn); + TCGv_i64 lr = cpu_reg(s, 30); + if (dst == lr) { + TCGv_i64 tmp = tcg_temp_new_i64(); + tcg_gen_mov_i64(tmp, dst); + dst = tmp; + } + gen_pc_plus_diff(s, lr, curr_insn_len(s)); + gen_a64_set_pc(s, dst); + set_btype_for_blr(s); + s->base.is_jmp = DISAS_JUMP; + return true; +} + +static bool trans_RET(DisasContext *s, arg_r *a) +{ + gen_a64_set_pc(s, cpu_reg(s, a->rn)); + s->base.is_jmp = DISAS_JUMP; + return true; +} + +static TCGv_i64 auth_branch_target(DisasContext *s, TCGv_i64 dst, + TCGv_i64 modifier, bool use_key_a) +{ + TCGv_i64 truedst; + /* + * Return the branch target for a BRAA/RETA/etc, which is either + * just the destination dst, or that value with the pauth check + * done and the code removed from the high bits. + */ + if (!s->pauth_active) { + return dst; + } + + truedst = tcg_temp_new_i64(); + if (use_key_a) { + gen_helper_autia(truedst, cpu_env, dst, modifier); + } else { + gen_helper_autib(truedst, cpu_env, dst, modifier); + } + return truedst; +} + +static bool trans_BRAZ(DisasContext *s, arg_braz *a) +{ + TCGv_i64 dst; + + if (!dc_isar_feature(aa64_pauth, s)) { + return false; + } + + dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); + gen_a64_set_pc(s, dst); + set_btype_for_br(s, a->rn); + s->base.is_jmp = DISAS_JUMP; + return true; +} + +static bool trans_BLRAZ(DisasContext *s, arg_braz *a) +{ + TCGv_i64 dst, lr; + + if (!dc_isar_feature(aa64_pauth, s)) { + return false; + } + + dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); + lr = cpu_reg(s, 30); + if (dst == lr) { + TCGv_i64 tmp = tcg_temp_new_i64(); + tcg_gen_mov_i64(tmp, dst); + dst = tmp; + } + gen_pc_plus_diff(s, lr, curr_insn_len(s)); + gen_a64_set_pc(s, dst); + set_btype_for_blr(s); + s->base.is_jmp = DISAS_JUMP; + return true; +} + +static bool trans_RETA(DisasContext *s, arg_reta *a) +{ + TCGv_i64 dst; + + dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m); + gen_a64_set_pc(s, dst); + s->base.is_jmp = DISAS_JUMP; + return true; +} + +static bool trans_BRA(DisasContext *s, arg_bra *a) +{ + TCGv_i64 dst; + + if (!dc_isar_feature(aa64_pauth, s)) { + return false; + } + dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m); + gen_a64_set_pc(s, dst); + set_btype_for_br(s, a->rn); + s->base.is_jmp = DISAS_JUMP; + return true; +} + +static bool trans_BLRA(DisasContext *s, arg_bra *a) +{ + TCGv_i64 dst, lr; + + if (!dc_isar_feature(aa64_pauth, s)) { + return false; + } + dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m); + lr = cpu_reg(s, 30); + if (dst == lr) { + TCGv_i64 tmp = tcg_temp_new_i64(); + tcg_gen_mov_i64(tmp, dst); + dst = tmp; + } + gen_pc_plus_diff(s, lr, curr_insn_len(s)); + gen_a64_set_pc(s, dst); + set_btype_for_blr(s); + s->base.is_jmp = DISAS_JUMP; + return true; +} + +static bool trans_ERET(DisasContext *s, arg_ERET *a) +{ + TCGv_i64 dst; + + if (s->current_el == 0) { + return false; + } + if (s->fgt_eret) { + gen_exception_insn_el(s, 0, EXCP_UDEF, 0, 2); + return true; + } + dst = tcg_temp_new_i64(); + tcg_gen_ld_i64(dst, cpu_env, + offsetof(CPUARMState, elr_el[s->current_el])); + + if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { + gen_io_start(); } + + gen_helper_exception_return(cpu_env, dst); + /* Must exit loop to check un-masked IRQs */ + s->base.is_jmp = DISAS_EXIT; + return true; +} + +static bool trans_ERETA(DisasContext *s, arg_reta *a) +{ + TCGv_i64 dst; + + if (!dc_isar_feature(aa64_pauth, s)) { + return false; + } + if (s->current_el == 0) { + return false; + } + /* The FGT trap takes precedence over an auth trap. */ + if (s->fgt_eret) { + gen_exception_insn_el(s, 0, EXCP_UDEF, a->m ? 3 : 2, 2); + return true; + } + dst = tcg_temp_new_i64(); + tcg_gen_ld_i64(dst, cpu_env, + offsetof(CPUARMState, elr_el[s->current_el])); + + dst = auth_branch_target(s, dst, cpu_X[31], !a->m); + if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { + gen_io_start(); + } + + gen_helper_exception_return(cpu_env, dst); + /* Must exit loop to check un-masked IRQs */ + s->base.is_jmp = DISAS_EXIT; + return true; } /* HINT instruction group, including various allocated HINTs */ @@ -2173,233 +2362,10 @@ static void disas_exc(DisasContext *s, uint32_t insn) } } -/* Unconditional branch (register) - * 31 25 24 21 20 16 15 10 9 5 4 0 - * +---------------+-------+-------+-------+------+-------+ - * | 1 1 0 1 0 1 1 | opc | op2 | op3 | Rn | op4 | - * +---------------+-------+-------+-------+------+-------+ - */ -static void disas_uncond_b_reg(DisasContext *s, uint32_t insn) -{ - unsigned int opc, op2, op3, rn, op4; - unsigned btype_mod = 2; /* 0: BR, 1: BLR, 2: other */ - TCGv_i64 dst; - TCGv_i64 modifier; - - opc = extract32(insn, 21, 4); - op2 = extract32(insn, 16, 5); - op3 = extract32(insn, 10, 6); - rn = extract32(insn, 5, 5); - op4 = extract32(insn, 0, 5); - - if (op2 != 0x1f) { - goto do_unallocated; - } - - switch (opc) { - case 0: /* BR */ - case 1: /* BLR */ - case 2: /* RET */ - btype_mod = opc; - switch (op3) { - case 0: - /* BR, BLR, RET */ - if (op4 != 0) { - goto do_unallocated; - } - dst = cpu_reg(s, rn); - break; - - case 2: - case 3: - if (!dc_isar_feature(aa64_pauth, s)) { - goto do_unallocated; - } - if (opc == 2) { - /* RETAA, RETAB */ - if (rn != 0x1f || op4 != 0x1f) { - goto do_unallocated; - } - rn = 30; - modifier = cpu_X[31]; - } else { - /* BRAAZ, BRABZ, BLRAAZ, BLRABZ */ - if (op4 != 0x1f) { - goto do_unallocated; - } - modifier = tcg_constant_i64(0); - } - if (s->pauth_active) { - dst = tcg_temp_new_i64(); - if (op3 == 2) { - gen_helper_autia(dst, cpu_env, cpu_reg(s, rn), modifier); - } else { - gen_helper_autib(dst, cpu_env, cpu_reg(s, rn), modifier); - } - } else { - dst = cpu_reg(s, rn); - } - break; - - default: - goto do_unallocated; - } - /* BLR also needs to load return address */ - if (opc == 1) { - TCGv_i64 lr = cpu_reg(s, 30); - if (dst == lr) { - TCGv_i64 tmp = tcg_temp_new_i64(); - tcg_gen_mov_i64(tmp, dst); - dst = tmp; - } - gen_pc_plus_diff(s, lr, curr_insn_len(s)); - } - gen_a64_set_pc(s, dst); - break; - - case 8: /* BRAA */ - case 9: /* BLRAA */ - if (!dc_isar_feature(aa64_pauth, s)) { - goto do_unallocated; - } - if ((op3 & ~1) != 2) { - goto do_unallocated; - } - btype_mod = opc & 1; - if (s->pauth_active) { - dst = tcg_temp_new_i64(); - modifier = cpu_reg_sp(s, op4); - if (op3 == 2) { - gen_helper_autia(dst, cpu_env, cpu_reg(s, rn), modifier); - } else { - gen_helper_autib(dst, cpu_env, cpu_reg(s, rn), modifier); - } - } else { - dst = cpu_reg(s, rn); - } - /* BLRAA also needs to load return address */ - if (opc == 9) { - TCGv_i64 lr = cpu_reg(s, 30); - if (dst == lr) { - TCGv_i64 tmp = tcg_temp_new_i64(); - tcg_gen_mov_i64(tmp, dst); - dst = tmp; - } - gen_pc_plus_diff(s, lr, curr_insn_len(s)); - } - gen_a64_set_pc(s, dst); - break; - - case 4: /* ERET */ - if (s->current_el == 0) { - goto do_unallocated; - } - switch (op3) { - case 0: /* ERET */ - if (op4 != 0) { - goto do_unallocated; - } - if (s->fgt_eret) { - gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(op3), 2); - return; - } - dst = tcg_temp_new_i64(); - tcg_gen_ld_i64(dst, cpu_env, - offsetof(CPUARMState, elr_el[s->current_el])); - break; - - case 2: /* ERETAA */ - case 3: /* ERETAB */ - if (!dc_isar_feature(aa64_pauth, s)) { - goto do_unallocated; - } - if (rn != 0x1f || op4 != 0x1f) { - goto do_unallocated; - } - /* The FGT trap takes precedence over an auth trap. */ - if (s->fgt_eret) { - gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(op3), 2); - return; - } - dst = tcg_temp_new_i64(); - tcg_gen_ld_i64(dst, cpu_env, - offsetof(CPUARMState, elr_el[s->current_el])); - if (s->pauth_active) { - modifier = cpu_X[31]; - if (op3 == 2) { - gen_helper_autia(dst, cpu_env, dst, modifier); - } else { - gen_helper_autib(dst, cpu_env, dst, modifier); - } - } - break; - - default: - goto do_unallocated; - } - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } - - gen_helper_exception_return(cpu_env, dst); - /* Must exit loop to check un-masked IRQs */ - s->base.is_jmp = DISAS_EXIT; - return; - - case 5: /* DRPS */ - if (op3 != 0 || op4 != 0 || rn != 0x1f) { - goto do_unallocated; - } else { - unallocated_encoding(s); - } - return; - - default: - do_unallocated: - unallocated_encoding(s); - return; - } - - switch (btype_mod) { - case 0: /* BR */ - if (dc_isar_feature(aa64_bti, s)) { - /* BR to {x16,x17} or !guard -> 1, else 3. */ - set_btype(s, rn == 16 || rn == 17 || !s->guarded_page ? 1 : 3); - } - break; - - case 1: /* BLR */ - if (dc_isar_feature(aa64_bti, s)) { - /* BLR sets BTYPE to 2, regardless of source guarded page. */ - set_btype(s, 2); - } - break; - - default: /* RET or none of the above. */ - /* BTYPE will be set to 0 by normal end-of-insn processing. */ - break; - } - - s->base.is_jmp = DISAS_JUMP; -} - /* Branches, exception generating and system instructions */ static void disas_b_exc_sys(DisasContext *s, uint32_t insn) { switch (extract32(insn, 25, 7)) { - case 0x0a: case 0x0b: - case 0x4a: case 0x4b: /* Unconditional branch (immediate) */ - disas_uncond_b_imm(s, insn); - break; - case 0x1a: case 0x5a: /* Compare & branch (immediate) */ - disas_comp_b_imm(s, insn); - break; - case 0x1b: case 0x5b: /* Test & branch (immediate) */ - disas_test_b_imm(s, insn); - break; - case 0x2a: /* Conditional branch (immediate) */ - disas_cond_b_imm(s, insn); - break; case 0x6a: /* Exception generation / System */ if (insn & (1 << 24)) { if (extract32(insn, 22, 2) == 0) { @@ -2411,9 +2377,6 @@ static void disas_b_exc_sys(DisasContext *s, uint32_t insn) disas_exc(s, insn); } break; - case 0x6b: /* Unconditional branch (register) */ - disas_uncond_b_reg(s, insn); - break; default: unallocated_encoding(s); break; @@ -4172,132 +4135,82 @@ static void disas_ldst(DisasContext *s, uint32_t insn) } } -/* PC-rel. addressing - * 31 30 29 28 24 23 5 4 0 - * +----+-------+-----------+-------------------+------+ - * | op | immlo | 1 0 0 0 0 | immhi | Rd | - * +----+-------+-----------+-------------------+------+ - */ -static void disas_pc_rel_adr(DisasContext *s, uint32_t insn) -{ - unsigned int page, rd; - int64_t offset; +typedef void ArithTwoOp(TCGv_i64, TCGv_i64, TCGv_i64); - page = extract32(insn, 31, 1); - /* SignExtend(immhi:immlo) -> offset */ - offset = sextract64(insn, 5, 19); - offset = offset << 2 | extract32(insn, 29, 2); - rd = extract32(insn, 0, 5); +static bool gen_rri(DisasContext *s, arg_rri_sf *a, + bool rd_sp, bool rn_sp, ArithTwoOp *fn) +{ + TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn); + TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd); + TCGv_i64 tcg_imm = tcg_constant_i64(a->imm); - if (page) { - /* ADRP (page based) */ - offset <<= 12; - /* The page offset is ok for CF_PCREL. */ - offset -= s->pc_curr & 0xfff; + fn(tcg_rd, tcg_rn, tcg_imm); + if (!a->sf) { + tcg_gen_ext32u_i64(tcg_rd, tcg_rd); } - - gen_pc_plus_diff(s, cpu_reg(s, rd), offset); + return true; } /* - * Add/subtract (immediate) - * - * 31 30 29 28 23 22 21 10 9 5 4 0 - * +--+--+--+-------------+--+-------------+-----+-----+ - * |sf|op| S| 1 0 0 0 1 0 |sh| imm12 | Rn | Rd | - * +--+--+--+-------------+--+-------------+-----+-----+ - * - * sf: 0 -> 32bit, 1 -> 64bit - * op: 0 -> add , 1 -> sub - * S: 1 -> set flags - * sh: 1 -> LSL imm by 12 + * PC-rel. addressing */ -static void disas_add_sub_imm(DisasContext *s, uint32_t insn) -{ - int rd = extract32(insn, 0, 5); - int rn = extract32(insn, 5, 5); - uint64_t imm = extract32(insn, 10, 12); - bool shift = extract32(insn, 22, 1); - bool setflags = extract32(insn, 29, 1); - bool sub_op = extract32(insn, 30, 1); - bool is_64bit = extract32(insn, 31, 1); - - TCGv_i64 tcg_rn = cpu_reg_sp(s, rn); - TCGv_i64 tcg_rd = setflags ? cpu_reg(s, rd) : cpu_reg_sp(s, rd); - TCGv_i64 tcg_result; - if (shift) { - imm <<= 12; - } +static bool trans_ADR(DisasContext *s, arg_ri *a) +{ + gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm); + return true; +} - tcg_result = tcg_temp_new_i64(); - if (!setflags) { - if (sub_op) { - tcg_gen_subi_i64(tcg_result, tcg_rn, imm); - } else { - tcg_gen_addi_i64(tcg_result, tcg_rn, imm); - } - } else { - TCGv_i64 tcg_imm = tcg_constant_i64(imm); - if (sub_op) { - gen_sub_CC(is_64bit, tcg_result, tcg_rn, tcg_imm); - } else { - gen_add_CC(is_64bit, tcg_result, tcg_rn, tcg_imm); - } - } +static bool trans_ADRP(DisasContext *s, arg_ri *a) +{ + int64_t offset = (int64_t)a->imm << 12; - if (is_64bit) { - tcg_gen_mov_i64(tcg_rd, tcg_result); - } else { - tcg_gen_ext32u_i64(tcg_rd, tcg_result); - } + /* The page offset is ok for CF_PCREL. */ + offset -= s->pc_curr & 0xfff; + gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset); + return true; } /* + * Add/subtract (immediate) + */ +TRANS(ADD_i, gen_rri, a, 1, 1, tcg_gen_add_i64) +TRANS(SUB_i, gen_rri, a, 1, 1, tcg_gen_sub_i64) +TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC) +TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC) + +/* * Add/subtract (immediate, with tags) - * - * 31 30 29 28 23 22 21 16 14 10 9 5 4 0 - * +--+--+--+-------------+--+---------+--+-------+-----+-----+ - * |sf|op| S| 1 0 0 0 1 1 |o2| uimm6 |o3| uimm4 | Rn | Rd | - * +--+--+--+-------------+--+---------+--+-------+-----+-----+ - * - * op: 0 -> add, 1 -> sub */ -static void disas_add_sub_imm_with_tags(DisasContext *s, uint32_t insn) + +static bool gen_add_sub_imm_with_tags(DisasContext *s, arg_rri_tag *a, + bool sub_op) { - int rd = extract32(insn, 0, 5); - int rn = extract32(insn, 5, 5); - int uimm4 = extract32(insn, 10, 4); - int uimm6 = extract32(insn, 16, 6); - bool sub_op = extract32(insn, 30, 1); TCGv_i64 tcg_rn, tcg_rd; int imm; - /* Test all of sf=1, S=0, o2=0, o3=0. */ - if ((insn & 0xa040c000u) != 0x80000000u || - !dc_isar_feature(aa64_mte_insn_reg, s)) { - unallocated_encoding(s); - return; - } - - imm = uimm6 << LOG2_TAG_GRANULE; + imm = a->uimm6 << LOG2_TAG_GRANULE; if (sub_op) { imm = -imm; } - tcg_rn = cpu_reg_sp(s, rn); - tcg_rd = cpu_reg_sp(s, rd); + tcg_rn = cpu_reg_sp(s, a->rn); + tcg_rd = cpu_reg_sp(s, a->rd); if (s->ata) { gen_helper_addsubg(tcg_rd, cpu_env, tcg_rn, tcg_constant_i32(imm), - tcg_constant_i32(uimm4)); + tcg_constant_i32(a->uimm4)); } else { tcg_gen_addi_i64(tcg_rd, tcg_rn, imm); gen_address_with_allocation_tag0(tcg_rd, tcg_rd); } + return true; } +TRANS_FEAT(ADDG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, false) +TRANS_FEAT(SUBG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, true) + /* The input should be a value in the bottom e bits (with higher * bits zero); returns that value replicated into every element * of size e in a 64 bit integer. @@ -4312,14 +4225,12 @@ static uint64_t bitfield_replicate(uint64_t mask, unsigned int e) return mask; } -/* Return a value with the bottom len bits set (where 0 < len <= 64) */ -static inline uint64_t bitmask64(unsigned int length) -{ - assert(length > 0 && length <= 64); - return ~0ULL >> (64 - length); -} +/* + * Logical (immediate) + */ -/* Simplified variant of pseudocode DecodeBitMasks() for the case where we +/* + * Simplified variant of pseudocode DecodeBitMasks() for the case where we * only require the wmask. Returns false if the imms/immr/immn are a reserved * value (ie should cause a guest UNDEF exception), and true if they are * valid, in which case the decoded bit pattern is written to result. @@ -4374,10 +4285,10 @@ bool logic_imm_decode_wmask(uint64_t *result, unsigned int immn, /* Create the value of one element: s+1 set bits rotated * by r within the element (which is e bits wide)... */ - mask = bitmask64(s + 1); + mask = MAKE_64BIT_MASK(0, s + 1); if (r) { mask = (mask >> r) | (mask << (e - r)); - mask &= bitmask64(e); + mask &= MAKE_64BIT_MASK(0, e); } /* ...then replicate the element over the whole 64 bit value */ mask = bitfield_replicate(mask, e); @@ -4385,295 +4296,215 @@ bool logic_imm_decode_wmask(uint64_t *result, unsigned int immn, return true; } -/* Logical (immediate) - * 31 30 29 28 23 22 21 16 15 10 9 5 4 0 - * +----+-----+-------------+---+------+------+------+------+ - * | sf | opc | 1 0 0 1 0 0 | N | immr | imms | Rn | Rd | - * +----+-----+-------------+---+------+------+------+------+ - */ -static void disas_logic_imm(DisasContext *s, uint32_t insn) +static bool gen_rri_log(DisasContext *s, arg_rri_log *a, bool set_cc, + void (*fn)(TCGv_i64, TCGv_i64, int64_t)) { - unsigned int sf, opc, is_n, immr, imms, rn, rd; TCGv_i64 tcg_rd, tcg_rn; - uint64_t wmask; - bool is_and = false; - - sf = extract32(insn, 31, 1); - opc = extract32(insn, 29, 2); - is_n = extract32(insn, 22, 1); - immr = extract32(insn, 16, 6); - imms = extract32(insn, 10, 6); - rn = extract32(insn, 5, 5); - rd = extract32(insn, 0, 5); - - if (!sf && is_n) { - unallocated_encoding(s); - return; - } + uint64_t imm; - if (opc == 0x3) { /* ANDS */ - tcg_rd = cpu_reg(s, rd); - } else { - tcg_rd = cpu_reg_sp(s, rd); + /* Some immediate field values are reserved. */ + if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), + extract32(a->dbm, 0, 6), + extract32(a->dbm, 6, 6))) { + return false; } - tcg_rn = cpu_reg(s, rn); - - if (!logic_imm_decode_wmask(&wmask, is_n, imms, immr)) { - /* some immediate field values are reserved */ - unallocated_encoding(s); - return; + if (!a->sf) { + imm &= 0xffffffffull; } - if (!sf) { - wmask &= 0xffffffff; - } + tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd); + tcg_rn = cpu_reg(s, a->rn); - switch (opc) { - case 0x3: /* ANDS */ - case 0x0: /* AND */ - tcg_gen_andi_i64(tcg_rd, tcg_rn, wmask); - is_and = true; - break; - case 0x1: /* ORR */ - tcg_gen_ori_i64(tcg_rd, tcg_rn, wmask); - break; - case 0x2: /* EOR */ - tcg_gen_xori_i64(tcg_rd, tcg_rn, wmask); - break; - default: - assert(FALSE); /* must handle all above */ - break; + fn(tcg_rd, tcg_rn, imm); + if (set_cc) { + gen_logic_CC(a->sf, tcg_rd); } - - if (!sf && !is_and) { - /* zero extend final result; we know we can skip this for AND - * since the immediate had the high 32 bits clear. - */ + if (!a->sf) { tcg_gen_ext32u_i64(tcg_rd, tcg_rd); } - - if (opc == 3) { /* ANDS */ - gen_logic_CC(sf, tcg_rd); - } + return true; } +TRANS(AND_i, gen_rri_log, a, false, tcg_gen_andi_i64) +TRANS(ORR_i, gen_rri_log, a, false, tcg_gen_ori_i64) +TRANS(EOR_i, gen_rri_log, a, false, tcg_gen_xori_i64) +TRANS(ANDS_i, gen_rri_log, a, true, tcg_gen_andi_i64) + /* * Move wide (immediate) - * - * 31 30 29 28 23 22 21 20 5 4 0 - * +--+-----+-------------+-----+----------------+------+ - * |sf| opc | 1 0 0 1 0 1 | hw | imm16 | Rd | - * +--+-----+-------------+-----+----------------+------+ - * - * sf: 0 -> 32 bit, 1 -> 64 bit - * opc: 00 -> N, 10 -> Z, 11 -> K - * hw: shift/16 (0,16, and sf only 32, 48) */ -static void disas_movw_imm(DisasContext *s, uint32_t insn) + +static bool trans_MOVZ(DisasContext *s, arg_movw *a) { - int rd = extract32(insn, 0, 5); - uint64_t imm = extract32(insn, 5, 16); - int sf = extract32(insn, 31, 1); - int opc = extract32(insn, 29, 2); - int pos = extract32(insn, 21, 2) << 4; - TCGv_i64 tcg_rd = cpu_reg(s, rd); + int pos = a->hw << 4; + tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos); + return true; +} - if (!sf && (pos >= 32)) { - unallocated_encoding(s); - return; +static bool trans_MOVN(DisasContext *s, arg_movw *a) +{ + int pos = a->hw << 4; + uint64_t imm = a->imm; + + imm = ~(imm << pos); + if (!a->sf) { + imm = (uint32_t)imm; } + tcg_gen_movi_i64(cpu_reg(s, a->rd), imm); + return true; +} - switch (opc) { - case 0: /* MOVN */ - case 2: /* MOVZ */ - imm <<= pos; - if (opc == 0) { - imm = ~imm; - } - if (!sf) { - imm &= 0xffffffffu; - } - tcg_gen_movi_i64(tcg_rd, imm); - break; - case 3: /* MOVK */ - tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_constant_i64(imm), pos, 16); - if (!sf) { - tcg_gen_ext32u_i64(tcg_rd, tcg_rd); - } - break; - default: - unallocated_encoding(s); - break; +static bool trans_MOVK(DisasContext *s, arg_movw *a) +{ + int pos = a->hw << 4; + TCGv_i64 tcg_rd, tcg_im; + + tcg_rd = cpu_reg(s, a->rd); + tcg_im = tcg_constant_i64(a->imm); + tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_im, pos, 16); + if (!a->sf) { + tcg_gen_ext32u_i64(tcg_rd, tcg_rd); } + return true; } -/* Bitfield - * 31 30 29 28 23 22 21 16 15 10 9 5 4 0 - * +----+-----+-------------+---+------+------+------+------+ - * | sf | opc | 1 0 0 1 1 0 | N | immr | imms | Rn | Rd | - * +----+-----+-------------+---+------+------+------+------+ +/* + * Bitfield */ -static void disas_bitfield(DisasContext *s, uint32_t insn) + +static bool trans_SBFM(DisasContext *s, arg_SBFM *a) { - unsigned int sf, n, opc, ri, si, rn, rd, bitsize, pos, len; - TCGv_i64 tcg_rd, tcg_tmp; + TCGv_i64 tcg_rd = cpu_reg(s, a->rd); + TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); + unsigned int bitsize = a->sf ? 64 : 32; + unsigned int ri = a->immr; + unsigned int si = a->imms; + unsigned int pos, len; - sf = extract32(insn, 31, 1); - opc = extract32(insn, 29, 2); - n = extract32(insn, 22, 1); - ri = extract32(insn, 16, 6); - si = extract32(insn, 10, 6); - rn = extract32(insn, 5, 5); - rd = extract32(insn, 0, 5); - bitsize = sf ? 64 : 32; + if (si >= ri) { + /* Wd<s-r:0> = Wn<s:r> */ + len = (si - ri) + 1; + tcg_gen_sextract_i64(tcg_rd, tcg_tmp, ri, len); + if (!a->sf) { + tcg_gen_ext32u_i64(tcg_rd, tcg_rd); + } + } else { + /* Wd<32+s-r,32-r> = Wn<s:0> */ + len = si + 1; + pos = (bitsize - ri) & (bitsize - 1); - if (sf != n || ri >= bitsize || si >= bitsize || opc > 2) { - unallocated_encoding(s); - return; + if (len < ri) { + /* + * Sign extend the destination field from len to fill the + * balance of the word. Let the deposit below insert all + * of those sign bits. + */ + tcg_gen_sextract_i64(tcg_tmp, tcg_tmp, 0, len); + len = ri; + } + + /* + * We start with zero, and we haven't modified any bits outside + * bitsize, therefore no final zero-extension is unneeded for !sf. + */ + tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len); } + return true; +} - tcg_rd = cpu_reg(s, rd); +static bool trans_UBFM(DisasContext *s, arg_UBFM *a) +{ + TCGv_i64 tcg_rd = cpu_reg(s, a->rd); + TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); + unsigned int bitsize = a->sf ? 64 : 32; + unsigned int ri = a->immr; + unsigned int si = a->imms; + unsigned int pos, len; - /* Suppress the zero-extend for !sf. Since RI and SI are constrained - to be smaller than bitsize, we'll never reference data outside the - low 32-bits anyway. */ - tcg_tmp = read_cpu_reg(s, rn, 1); + tcg_rd = cpu_reg(s, a->rd); + tcg_tmp = read_cpu_reg(s, a->rn, 1); - /* Recognize simple(r) extractions. */ if (si >= ri) { /* Wd<s-r:0> = Wn<s:r> */ len = (si - ri) + 1; - if (opc == 0) { /* SBFM: ASR, SBFX, SXTB, SXTH, SXTW */ - tcg_gen_sextract_i64(tcg_rd, tcg_tmp, ri, len); - goto done; - } else if (opc == 2) { /* UBFM: UBFX, LSR, UXTB, UXTH */ - tcg_gen_extract_i64(tcg_rd, tcg_tmp, ri, len); - return; - } - /* opc == 1, BFXIL fall through to deposit */ - tcg_gen_shri_i64(tcg_tmp, tcg_tmp, ri); - pos = 0; + tcg_gen_extract_i64(tcg_rd, tcg_tmp, ri, len); } else { - /* Handle the ri > si case with a deposit - * Wd<32+s-r,32-r> = Wn<s:0> - */ + /* Wd<32+s-r,32-r> = Wn<s:0> */ len = si + 1; pos = (bitsize - ri) & (bitsize - 1); + tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len); } + return true; +} - if (opc == 0 && len < ri) { - /* SBFM: sign extend the destination field from len to fill - the balance of the word. Let the deposit below insert all - of those sign bits. */ - tcg_gen_sextract_i64(tcg_tmp, tcg_tmp, 0, len); - len = ri; - } +static bool trans_BFM(DisasContext *s, arg_BFM *a) +{ + TCGv_i64 tcg_rd = cpu_reg(s, a->rd); + TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); + unsigned int bitsize = a->sf ? 64 : 32; + unsigned int ri = a->immr; + unsigned int si = a->imms; + unsigned int pos, len; + + tcg_rd = cpu_reg(s, a->rd); + tcg_tmp = read_cpu_reg(s, a->rn, 1); - if (opc == 1) { /* BFM, BFXIL */ - tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_tmp, pos, len); + if (si >= ri) { + /* Wd<s-r:0> = Wn<s:r> */ + tcg_gen_shri_i64(tcg_tmp, tcg_tmp, ri); + len = (si - ri) + 1; + pos = 0; } else { - /* SBFM or UBFM: We start with zero, and we haven't modified - any bits outside bitsize, therefore the zero-extension - below is unneeded. */ - tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len); - return; + /* Wd<32+s-r,32-r> = Wn<s:0> */ + len = si + 1; + pos = (bitsize - ri) & (bitsize - 1); } - done: - if (!sf) { /* zero extend final result */ + tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_tmp, pos, len); + if (!a->sf) { tcg_gen_ext32u_i64(tcg_rd, tcg_rd); } + return true; } -/* Extract - * 31 30 29 28 23 22 21 20 16 15 10 9 5 4 0 - * +----+------+-------------+---+----+------+--------+------+------+ - * | sf | op21 | 1 0 0 1 1 1 | N | o0 | Rm | imms | Rn | Rd | - * +----+------+-------------+---+----+------+--------+------+------+ - */ -static void disas_extract(DisasContext *s, uint32_t insn) +static bool trans_EXTR(DisasContext *s, arg_extract *a) { - unsigned int sf, n, rm, imm, rn, rd, bitsize, op21, op0; + TCGv_i64 tcg_rd, tcg_rm, tcg_rn; - sf = extract32(insn, 31, 1); - n = extract32(insn, 22, 1); - rm = extract32(insn, 16, 5); - imm = extract32(insn, 10, 6); - rn = extract32(insn, 5, 5); - rd = extract32(insn, 0, 5); - op21 = extract32(insn, 29, 2); - op0 = extract32(insn, 21, 1); - bitsize = sf ? 64 : 32; + tcg_rd = cpu_reg(s, a->rd); - if (sf != n || op21 || op0 || imm >= bitsize) { - unallocated_encoding(s); + if (unlikely(a->imm == 0)) { + /* + * tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts, + * so an extract from bit 0 is a special case. + */ + if (a->sf) { + tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm)); + } else { + tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm)); + } } else { - TCGv_i64 tcg_rd, tcg_rm, tcg_rn; + tcg_rm = cpu_reg(s, a->rm); + tcg_rn = cpu_reg(s, a->rn); - tcg_rd = cpu_reg(s, rd); - - if (unlikely(imm == 0)) { - /* tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts, - * so an extract from bit 0 is a special case. - */ - if (sf) { - tcg_gen_mov_i64(tcg_rd, cpu_reg(s, rm)); - } else { - tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, rm)); - } + if (a->sf) { + /* Specialization to ROR happens in EXTRACT2. */ + tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm); } else { - tcg_rm = cpu_reg(s, rm); - tcg_rn = cpu_reg(s, rn); + TCGv_i32 t0 = tcg_temp_new_i32(); - if (sf) { - /* Specialization to ROR happens in EXTRACT2. */ - tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, imm); + tcg_gen_extrl_i64_i32(t0, tcg_rm); + if (a->rm == a->rn) { + tcg_gen_rotri_i32(t0, t0, a->imm); } else { - TCGv_i32 t0 = tcg_temp_new_i32(); - - tcg_gen_extrl_i64_i32(t0, tcg_rm); - if (rm == rn) { - tcg_gen_rotri_i32(t0, t0, imm); - } else { - TCGv_i32 t1 = tcg_temp_new_i32(); - tcg_gen_extrl_i64_i32(t1, tcg_rn); - tcg_gen_extract2_i32(t0, t0, t1, imm); - } - tcg_gen_extu_i32_i64(tcg_rd, t0); + TCGv_i32 t1 = tcg_temp_new_i32(); + tcg_gen_extrl_i64_i32(t1, tcg_rn); + tcg_gen_extract2_i32(t0, t0, t1, a->imm); } + tcg_gen_extu_i32_i64(tcg_rd, t0); } } -} - -/* Data processing - immediate */ -static void disas_data_proc_imm(DisasContext *s, uint32_t insn) -{ - switch (extract32(insn, 23, 6)) { - case 0x20: case 0x21: /* PC-rel. addressing */ - disas_pc_rel_adr(s, insn); - break; - case 0x22: /* Add/subtract (immediate) */ - disas_add_sub_imm(s, insn); - break; - case 0x23: /* Add/subtract (immediate, with tags) */ - disas_add_sub_imm_with_tags(s, insn); - break; - case 0x24: /* Logical (immediate) */ - disas_logic_imm(s, insn); - break; - case 0x25: /* Move wide (immediate) */ - disas_movw_imm(s, insn); - break; - case 0x26: /* Bitfield */ - disas_bitfield(s, insn); - break; - case 0x27: /* Extract */ - disas_extract(s, insn); - break; - default: - unallocated_encoding(s); - break; - } + return true; } /* Shift a TCGv src by TCGv shift_amount, put result in dst. @@ -14100,12 +13931,6 @@ static void disas_data_proc_simd_fp(DisasContext *s, uint32_t insn) } } -/* - * Include the generated SME FA64 decoder. - */ - -#include "decode-sme-fa64.c.inc" - static bool trans_OK(DisasContext *s, arg_OK *a) { return true; @@ -14200,6 +14025,33 @@ static bool btype_destination_ok(uint32_t insn, bool bt, int btype) return false; } +/* C3.1 A64 instruction index by encoding */ +static void disas_a64_legacy(DisasContext *s, uint32_t insn) +{ + switch (extract32(insn, 25, 4)) { + case 0xa: case 0xb: /* Branch, exception generation and system insns */ + disas_b_exc_sys(s, insn); + break; + case 0x4: + case 0x6: + case 0xc: + case 0xe: /* Loads and stores */ + disas_ldst(s, insn); + break; + case 0x5: + case 0xd: /* Data processing - register */ + disas_data_proc_reg(s, insn); + break; + case 0x7: + case 0xf: /* Data processing - SIMD and floating point */ + disas_data_proc_simd_fp(s, insn); + break; + default: + unallocated_encoding(s); + break; + } +} + static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cpu) { @@ -14401,43 +14253,10 @@ static void aarch64_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) disas_sme_fa64(s, insn); } - switch (extract32(insn, 25, 4)) { - case 0x0: - if (!extract32(insn, 31, 1) || !disas_sme(s, insn)) { - unallocated_encoding(s); - } - break; - case 0x1: case 0x3: /* UNALLOCATED */ - unallocated_encoding(s); - break; - case 0x2: - if (!disas_sve(s, insn)) { - unallocated_encoding(s); - } - break; - case 0x8: case 0x9: /* Data processing - immediate */ - disas_data_proc_imm(s, insn); - break; - case 0xa: case 0xb: /* Branch, exception generation and system insns */ - disas_b_exc_sys(s, insn); - break; - case 0x4: - case 0x6: - case 0xc: - case 0xe: /* Loads and stores */ - disas_ldst(s, insn); - break; - case 0x5: - case 0xd: /* Data processing - register */ - disas_data_proc_reg(s, insn); - break; - case 0x7: - case 0xf: /* Data processing - SIMD and floating point */ - disas_data_proc_simd_fp(s, insn); - break; - default: - assert(FALSE); /* all 15 cases should be handled above */ - break; + if (!disas_a64(s, insn) && + !disas_sme(s, insn) && + !disas_sve(s, insn)) { + disas_a64_legacy(s, insn); } /* diff --git a/target/arm/tcg/translate.h b/target/arm/tcg/translate.h index f02d4685b4..a9d1f4adc2 100644 --- a/target/arm/tcg/translate.h +++ b/target/arm/tcg/translate.h @@ -220,6 +220,11 @@ static inline int rsub_8(DisasContext *s, int x) return 8 - x; } +static inline int shl_12(DisasContext *s, int x) +{ + return x << 12; +} + static inline int neon_3same_fp_size(DisasContext *s, int x) { /* Convert 0==fp32, 1==fp16 into a MO_* value */ diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 4187759f10..a61cd6d99d 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -863,7 +863,7 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = { "tsx-ldtrk", NULL, NULL /* pconfig */, "arch-lbr", NULL, NULL, "amx-bf16", "avx512-fp16", "amx-tile", "amx-int8", "spec-ctrl", "stibp", - NULL, "arch-capabilities", "core-capability", "ssbd", + "flush-l1d", "arch-capabilities", "core-capability", "ssbd", }, .cpuid = { .eax = 7, @@ -1050,7 +1050,7 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = { "ssb-no", "mds-no", "pschange-mc-no", "tsx-ctrl", "taa-no", NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, + NULL, "fb-clear", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 8ade71ab55..7201a71de8 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -899,6 +899,8 @@ uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, #define CPUID_7_0_EDX_SPEC_CTRL (1U << 26) /* Single Thread Indirect Branch Predictors */ #define CPUID_7_0_EDX_STIBP (1U << 27) +/* Flush L1D cache */ +#define CPUID_7_0_EDX_FLUSH_L1D (1U << 28) /* Arch Capabilities */ #define CPUID_7_0_EDX_ARCH_CAPABILITIES (1U << 29) /* Core Capability */ @@ -1016,6 +1018,7 @@ uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, #define MSR_ARCH_CAP_PSCHANGE_MC_NO (1U << 6) #define MSR_ARCH_CAP_TSX_CTRL_MSR (1U << 7) #define MSR_ARCH_CAP_TAA_NO (1U << 8) +#define MSR_ARCH_CAP_FB_CLEAR (1U << 17) #define MSR_CORE_CAP_SPLIT_LOCK_DETECT (1U << 5) diff --git a/target/i386/ops_sse.h b/target/i386/ops_sse.h index 0bd6bfad8a..fb63af7afa 100644 --- a/target/i386/ops_sse.h +++ b/target/i386/ops_sse.h @@ -2497,6 +2497,14 @@ void helper_vpermdq_ymm(Reg *d, Reg *v, Reg *s, uint32_t order) d->Q(1) = r1; d->Q(2) = r2; d->Q(3) = r3; + if (order & 0x8) { + d->Q(0) = 0; + d->Q(1) = 0; + } + if (order & 0x80) { + d->Q(2) = 0; + d->Q(3) = 0; + } } void helper_vpermq_ymm(Reg *d, Reg *s, uint32_t order) diff --git a/target/i386/tcg/decode-new.c.inc b/target/i386/tcg/decode-new.c.inc index 4fdd87750b..46afd9960b 100644 --- a/target/i386/tcg/decode-new.c.inc +++ b/target/i386/tcg/decode-new.c.inc @@ -237,7 +237,7 @@ static void decode_group14(DisasContext *s, CPUX86State *env, X86OpEntry *entry, static void decode_0F6F(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { static const X86OpEntry opcodes_0F6F[4] = { - X86_OP_ENTRY3(MOVDQ, P,q, None,None, Q,q, vex1 mmx), /* movq */ + X86_OP_ENTRY3(MOVDQ, P,q, None,None, Q,q, vex5 mmx), /* movq */ X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex1), /* movdqa */ X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* movdqu */ {}, @@ -274,9 +274,9 @@ static void decode_0F78(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui { static const X86OpEntry opcodes_0F78[4] = { {}, - X86_OP_ENTRY3(EXTRQ_i, V,x, None,None, I,w, cpuid(SSE4A)), + X86_OP_ENTRY3(EXTRQ_i, V,x, None,None, I,w, cpuid(SSE4A)), /* AMD extension */ {}, - X86_OP_ENTRY3(INSERTQ_i, V,x, U,x, I,w, cpuid(SSE4A)), + X86_OP_ENTRY3(INSERTQ_i, V,x, U,x, I,w, cpuid(SSE4A)), /* AMD extension */ }; *entry = *decode_by_prefix(s, opcodes_0F78); } @@ -284,9 +284,9 @@ static void decode_0F78(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static void decode_0F79(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { if (s->prefix & PREFIX_REPNZ) { - entry->gen = gen_INSERTQ_r; + entry->gen = gen_INSERTQ_r; /* AMD extension */ } else if (s->prefix & PREFIX_DATA) { - entry->gen = gen_EXTRQ_r; + entry->gen = gen_EXTRQ_r; /* AMD extension */ } else { entry->gen = NULL; }; @@ -306,7 +306,7 @@ static void decode_0F7E(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static void decode_0F7F(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { static const X86OpEntry opcodes_0F7F[4] = { - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex1 mmx), /* movq */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex5 mmx), /* movq */ X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex1), /* movdqa */ X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4_unal), /* movdqu */ {}, @@ -639,15 +639,15 @@ static void decode_0F10(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static const X86OpEntry opcodes_0F10_reg[4] = { X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* MOVUPS */ X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* MOVUPD */ - X86_OP_ENTRY3(VMOVSS, V,x, H,x, W,x, vex4), - X86_OP_ENTRY3(VMOVLPx, V,x, H,x, W,x, vex4), /* MOVSD */ + X86_OP_ENTRY3(VMOVSS, V,x, H,x, W,x, vex5), + X86_OP_ENTRY3(VMOVLPx, V,x, H,x, W,x, vex5), /* MOVSD */ }; static const X86OpEntry opcodes_0F10_mem[4] = { X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* MOVUPS */ X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* MOVUPD */ - X86_OP_ENTRY3(VMOVSS_ld, V,x, H,x, M,ss, vex4), - X86_OP_ENTRY3(VMOVSD_ld, V,x, H,x, M,sd, vex4), + X86_OP_ENTRY3(VMOVSS_ld, V,x, H,x, M,ss, vex5), + X86_OP_ENTRY3(VMOVSD_ld, V,x, H,x, M,sd, vex5), }; if ((get_modrm(s, env) >> 6) == 3) { @@ -660,17 +660,17 @@ static void decode_0F10(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static void decode_0F11(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { static const X86OpEntry opcodes_0F11_reg[4] = { - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVPS */ - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVPD */ - X86_OP_ENTRY3(VMOVSS, W,x, H,x, V,x, vex4), - X86_OP_ENTRY3(VMOVLPx, W,x, H,x, V,q, vex4), /* MOVSD */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVUPS */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVUPD */ + X86_OP_ENTRY3(VMOVSS, W,x, H,x, V,x, vex5), + X86_OP_ENTRY3(VMOVLPx, W,x, H,x, V,q, vex5), /* MOVSD */ }; static const X86OpEntry opcodes_0F11_mem[4] = { - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVPS */ - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVPD */ - X86_OP_ENTRY3(VMOVSS_st, M,ss, None,None, V,x, vex4), - X86_OP_ENTRY3(VMOVLPx_st, M,sd, None,None, V,x, vex4), /* MOVSD */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVUPS */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVUPD */ + X86_OP_ENTRY3(VMOVSS_st, M,ss, None,None, V,x, vex5), + X86_OP_ENTRY3(VMOVLPx_st, M,sd, None,None, V,x, vex5), /* MOVSD */ }; if ((get_modrm(s, env) >> 6) == 3) { @@ -687,16 +687,16 @@ static void decode_0F12(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui * Use dq for operand for compatibility with gen_MOVSD and * to allow VEX128 only. */ - X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq, M,q, vex4), /* MOVLPS */ - X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq, M,q, vex4), /* MOVLPD */ + X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq, M,q, vex5), /* MOVLPS */ + X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq, M,q, vex5), /* MOVLPD */ X86_OP_ENTRY3(VMOVSLDUP, V,x, None,None, W,x, vex4 cpuid(SSE3)), - X86_OP_ENTRY3(VMOVDDUP, V,x, None,None, WM,q, vex4 cpuid(SSE3)), /* qq if VEX.256 */ + X86_OP_ENTRY3(VMOVDDUP, V,x, None,None, WM,q, vex5 cpuid(SSE3)), /* qq if VEX.256 */ }; static const X86OpEntry opcodes_0F12_reg[4] = { - X86_OP_ENTRY3(VMOVHLPS, V,dq, H,dq, U,dq, vex4), - X86_OP_ENTRY3(VMOVLPx, W,x, H,x, U,q, vex4), /* MOVLPD */ + X86_OP_ENTRY3(VMOVHLPS, V,dq, H,dq, U,dq, vex7), + X86_OP_ENTRY3(VMOVLPx, W,x, H,x, U,q, vex5), /* MOVLPD */ X86_OP_ENTRY3(VMOVSLDUP, V,x, None,None, U,x, vex4 cpuid(SSE3)), - X86_OP_ENTRY3(VMOVDDUP, V,x, None,None, U,x, vex4 cpuid(SSE3)), + X86_OP_ENTRY3(VMOVDDUP, V,x, None,None, U,x, vex5 cpuid(SSE3)), }; if ((get_modrm(s, env) >> 6) == 3) { @@ -716,15 +716,15 @@ static void decode_0F16(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui * Operand 1 technically only reads the low 64 bits, but uses dq so that * it is easier to check for op0 == op1 in an endianness-neutral manner. */ - X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq, M,q, vex4), /* MOVHPS */ - X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq, M,q, vex4), /* MOVHPD */ + X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq, M,q, vex5), /* MOVHPS */ + X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq, M,q, vex5), /* MOVHPD */ X86_OP_ENTRY3(VMOVSHDUP, V,x, None,None, W,x, vex4 cpuid(SSE3)), {}, }; static const X86OpEntry opcodes_0F16_reg[4] = { /* Same as above, operand 1 could be Hq if it wasn't for big-endian. */ - X86_OP_ENTRY3(VMOVLHPS, V,dq, H,dq, U,q, vex4), - X86_OP_ENTRY3(VMOVHPx, V,x, H,x, U,x, vex4), /* MOVHPD */ + X86_OP_ENTRY3(VMOVLHPS, V,dq, H,dq, U,q, vex7), + X86_OP_ENTRY3(VMOVHPx, V,x, H,x, U,x, vex5), /* MOVHPD */ X86_OP_ENTRY3(VMOVSHDUP, V,x, None,None, U,x, vex4 cpuid(SSE3)), {}, }; @@ -750,8 +750,9 @@ static void decode_0F2A(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static void decode_0F2B(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { static const X86OpEntry opcodes_0F2B[4] = { - X86_OP_ENTRY3(MOVDQ, M,x, None,None, V,x, vex4), /* MOVNTPS */ - X86_OP_ENTRY3(MOVDQ, M,x, None,None, V,x, vex4), /* MOVNTPD */ + X86_OP_ENTRY3(MOVDQ, M,x, None,None, V,x, vex1), /* MOVNTPS */ + X86_OP_ENTRY3(MOVDQ, M,x, None,None, V,x, vex1), /* MOVNTPD */ + /* AMD extensions */ X86_OP_ENTRY3(VMOVSS_st, M,ss, None,None, V,x, vex4 cpuid(SSE4A)), /* MOVNTSS */ X86_OP_ENTRY3(VMOVLPx_st, M,sd, None,None, V,x, vex4 cpuid(SSE4A)), /* MOVNTSD */ }; @@ -783,6 +784,17 @@ static void decode_0F2D(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui *entry = *decode_by_prefix(s, opcodes_0F2D); } +static void decode_VxCOMISx(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) +{ + /* + * VUCOMISx and VCOMISx are different and use no-prefix and 0x66 for SS and SD + * respectively. Scalar values usually are associated with 0xF2 and 0xF3, for + * which X86_VEX_REPScalar exists, but here it has to be decoded by hand. + */ + entry->s1 = entry->s2 = (s->prefix & PREFIX_DATA ? X86_SIZE_sd : X86_SIZE_ss); + entry->gen = (*b == 0x2E ? gen_VUCOMI : gen_VCOMI); +} + static void decode_sse_unary(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { if (!(s->prefix & (PREFIX_REPZ | PREFIX_REPNZ))) { @@ -813,7 +825,7 @@ static void decode_0FE6(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static const X86OpEntry opcodes_0FE6[4] = { {}, X86_OP_ENTRY2(VCVTTPD2DQ, V,x, W,x, vex2), - X86_OP_ENTRY2(VCVTDQ2PD, V,x, W,x, vex2), + X86_OP_ENTRY2(VCVTDQ2PD, V,x, W,x, vex5), X86_OP_ENTRY2(VCVTPD2DQ, V,x, W,x, vex2), }; *entry = *decode_by_prefix(s, opcodes_0FE6); @@ -831,17 +843,17 @@ static const X86OpEntry opcodes_0F[256] = { [0x10] = X86_OP_GROUP0(0F10), [0x11] = X86_OP_GROUP0(0F11), [0x12] = X86_OP_GROUP0(0F12), - [0x13] = X86_OP_ENTRY3(VMOVLPx_st, M,q, None,None, V,q, vex4 p_00_66), + [0x13] = X86_OP_ENTRY3(VMOVLPx_st, M,q, None,None, V,q, vex5 p_00_66), [0x14] = X86_OP_ENTRY3(VUNPCKLPx, V,x, H,x, W,x, vex4 p_00_66), [0x15] = X86_OP_ENTRY3(VUNPCKHPx, V,x, H,x, W,x, vex4 p_00_66), [0x16] = X86_OP_GROUP0(0F16), /* Incorrectly listed as Mq,Vq in the manual */ - [0x17] = X86_OP_ENTRY3(VMOVHPx_st, M,q, None,None, V,dq, vex4 p_00_66), + [0x17] = X86_OP_ENTRY3(VMOVHPx_st, M,q, None,None, V,dq, vex5 p_00_66), [0x50] = X86_OP_ENTRY3(MOVMSK, G,y, None,None, U,x, vex7 p_00_66), - [0x51] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), - [0x52] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex4_rep5 p_00_f3), - [0x53] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex4_rep5 p_00_f3), + [0x51] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), /* sqrtps */ + [0x52] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex4_rep5 p_00_f3), /* rsqrtps */ + [0x53] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex4_rep5 p_00_f3), /* rcpps */ [0x54] = X86_OP_ENTRY3(PAND, V,x, H,x, W,x, vex4 p_00_66), /* vand */ [0x55] = X86_OP_ENTRY3(PANDN, V,x, H,x, W,x, vex4 p_00_66), /* vandn */ [0x56] = X86_OP_ENTRY3(POR, V,x, H,x, W,x, vex4 p_00_66), /* vor */ @@ -871,15 +883,15 @@ static const X86OpEntry opcodes_0F[256] = { [0x2B] = X86_OP_GROUP0(0F2B), [0x2C] = X86_OP_GROUP0(0F2C), [0x2D] = X86_OP_GROUP0(0F2D), - [0x2E] = X86_OP_ENTRY3(VUCOMI, None,None, V,x, W,x, vex4 p_00_66), - [0x2F] = X86_OP_ENTRY3(VCOMI, None,None, V,x, W,x, vex4 p_00_66), + [0x2E] = X86_OP_GROUP3(VxCOMISx, None,None, V,x, W,x, vex3 p_00_66), /* VUCOMISS/SD */ + [0x2F] = X86_OP_GROUP3(VxCOMISx, None,None, V,x, W,x, vex3 p_00_66), /* VCOMISS/SD */ [0x38] = X86_OP_GROUP0(0F38), [0x3a] = X86_OP_GROUP0(0F3A), [0x58] = X86_OP_ENTRY3(VADD, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), [0x59] = X86_OP_ENTRY3(VMUL, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), - [0x5a] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), + [0x5a] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), /* CVTPS2PD */ [0x5b] = X86_OP_GROUP0(0F5B), [0x5c] = X86_OP_ENTRY3(VSUB, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), [0x5d] = X86_OP_ENTRY3(VMIN, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), diff --git a/target/i386/tcg/emit.c.inc b/target/i386/tcg/emit.c.inc index 95fb4f52fa..4fe8dec427 100644 --- a/target/i386/tcg/emit.c.inc +++ b/target/i386/tcg/emit.c.inc @@ -2285,7 +2285,7 @@ static void gen_VZEROALL(DisasContext *s, CPUX86State *env, X86DecodedInsn *deco { TCGv_ptr ptr = tcg_temp_new_ptr(); - tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_t0)); + tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_regs)); gen_helper_memset(ptr, ptr, tcg_constant_i32(0), tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); } diff --git a/target/openrisc/cpu.c b/target/openrisc/cpu.c index 0ce4f796fa..61d748cfdc 100644 --- a/target/openrisc/cpu.c +++ b/target/openrisc/cpu.c @@ -22,6 +22,7 @@ #include "qemu/qemu-print.h" #include "cpu.h" #include "exec/exec-all.h" +#include "fpu/softfloat-helpers.h" #include "tcg/tcg.h" static void openrisc_cpu_set_pc(CPUState *cs, vaddr value) @@ -90,6 +91,9 @@ static void openrisc_cpu_reset_hold(Object *obj) s->exception_index = -1; cpu_set_fpcsr(&cpu->env, 0); + set_float_detect_tininess(float_tininess_before_rounding, + &cpu->env.fp_status); + #ifndef CONFIG_USER_ONLY cpu->env.picmr = 0x00000000; cpu->env.picsr = 0x00000000; diff --git a/target/openrisc/fpu_helper.c b/target/openrisc/fpu_helper.c index f9e34fa2cc..8b81d2f62f 100644 --- a/target/openrisc/fpu_helper.c +++ b/target/openrisc/fpu_helper.c @@ -20,8 +20,8 @@ #include "qemu/osdep.h" #include "cpu.h" +#include "exec/exec-all.h" #include "exec/helper-proto.h" -#include "exception.h" #include "fpu/softfloat.h" static int ieee_ex_to_openrisc(int fexcp) @@ -45,6 +45,15 @@ static int ieee_ex_to_openrisc(int fexcp) return ret; } +static G_NORETURN +void do_fpe(CPUOpenRISCState *env, uintptr_t pc) +{ + CPUState *cs = env_cpu(env); + + cs->exception_index = EXCP_FPE; + cpu_loop_exit_restore(cs, pc); +} + void HELPER(update_fpcsr)(CPUOpenRISCState *env) { int tmp = get_float_exception_flags(&env->fp_status); @@ -55,7 +64,7 @@ void HELPER(update_fpcsr)(CPUOpenRISCState *env) if (tmp) { env->fpcsr |= tmp; if (env->fpcsr & FPCSR_FPEE) { - helper_exception(env, EXCP_FPE); + do_fpe(env, GETPC()); } } } diff --git a/target/openrisc/sys_helper.c b/target/openrisc/sys_helper.c index ec145960e3..ccdee3b8be 100644 --- a/target/openrisc/sys_helper.c +++ b/target/openrisc/sys_helper.c @@ -29,17 +29,37 @@ #define TO_SPR(group, number) (((group) << 11) + (number)) +static inline bool is_user(CPUOpenRISCState *env) +{ +#ifdef CONFIG_USER_ONLY + return true; +#else + return (env->sr & SR_SM) == 0; +#endif +} + void HELPER(mtspr)(CPUOpenRISCState *env, target_ulong spr, target_ulong rb) { -#ifndef CONFIG_USER_ONLY OpenRISCCPU *cpu = env_archcpu(env); +#ifndef CONFIG_USER_ONLY CPUState *cs = env_cpu(env); target_ulong mr; int idx; #endif + /* Handle user accessible SPRs first. */ switch (spr) { + case TO_SPR(0, 20): /* FPCSR */ + cpu_set_fpcsr(env, rb); + return; + } + + if (is_user(env)) { + raise_exception(cpu, EXCP_ILLEGAL); + } + #ifndef CONFIG_USER_ONLY + switch (spr) { case TO_SPR(0, 11): /* EVBAR */ env->evbar = rb; break; @@ -187,27 +207,33 @@ void HELPER(mtspr)(CPUOpenRISCState *env, target_ulong spr, target_ulong rb) cpu_openrisc_timer_update(cpu); qemu_mutex_unlock_iothread(); break; -#endif - - case TO_SPR(0, 20): /* FPCSR */ - cpu_set_fpcsr(env, rb); - break; } +#endif } target_ulong HELPER(mfspr)(CPUOpenRISCState *env, target_ulong rd, target_ulong spr) { + OpenRISCCPU *cpu = env_archcpu(env); #ifndef CONFIG_USER_ONLY uint64_t data[TARGET_INSN_START_WORDS]; MachineState *ms = MACHINE(qdev_get_machine()); - OpenRISCCPU *cpu = env_archcpu(env); CPUState *cs = env_cpu(env); int idx; #endif + /* Handle user accessible SPRs first. */ switch (spr) { + case TO_SPR(0, 20): /* FPCSR */ + return env->fpcsr; + } + + if (is_user(env)) { + raise_exception(cpu, EXCP_ILLEGAL); + } + #ifndef CONFIG_USER_ONLY + switch (spr) { case TO_SPR(0, 0): /* VR */ return env->vr; @@ -324,11 +350,8 @@ target_ulong HELPER(mfspr)(CPUOpenRISCState *env, target_ulong rd, cpu_openrisc_count_update(cpu); qemu_mutex_unlock_iothread(); return cpu_openrisc_count_get(cpu); -#endif - - case TO_SPR(0, 20): /* FPCSR */ - return env->fpcsr; } +#endif /* for rd is passed in, if rd unchanged, just keep it back. */ return rd; diff --git a/target/openrisc/translate.c b/target/openrisc/translate.c index 76e53c78d4..43ba0cc1ad 100644 --- a/target/openrisc/translate.c +++ b/target/openrisc/translate.c @@ -819,45 +819,12 @@ static bool trans_l_xori(DisasContext *dc, arg_rri *a) static bool trans_l_mfspr(DisasContext *dc, arg_l_mfspr *a) { - check_r0_write(dc, a->d); - - if (is_user(dc)) { - gen_illegal_exception(dc); - } else { - TCGv spr = tcg_temp_new(); - - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - if (dc->delayed_branch) { - tcg_gen_mov_tl(cpu_pc, jmp_pc); - tcg_gen_discard_tl(jmp_pc); - } else { - tcg_gen_movi_tl(cpu_pc, dc->base.pc_next + 4); - } - dc->base.is_jmp = DISAS_EXIT; - } + TCGv spr = tcg_temp_new(); - tcg_gen_ori_tl(spr, cpu_R(dc, a->a), a->k); - gen_helper_mfspr(cpu_R(dc, a->d), cpu_env, cpu_R(dc, a->d), spr); - } - return true; -} - -static bool trans_l_mtspr(DisasContext *dc, arg_l_mtspr *a) -{ - if (is_user(dc)) { - gen_illegal_exception(dc); - } else { - TCGv spr; + check_r0_write(dc, a->d); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } - /* For SR, we will need to exit the TB to recognize the new - * exception state. For NPC, in theory this counts as a branch - * (although the SPR only exists for use by an ICE). Save all - * of the cpu state first, allowing it to be overwritten. - */ + if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { + gen_io_start(); if (dc->delayed_branch) { tcg_gen_mov_tl(cpu_pc, jmp_pc); tcg_gen_discard_tl(jmp_pc); @@ -865,11 +832,36 @@ static bool trans_l_mtspr(DisasContext *dc, arg_l_mtspr *a) tcg_gen_movi_tl(cpu_pc, dc->base.pc_next + 4); } dc->base.is_jmp = DISAS_EXIT; + } + + tcg_gen_ori_tl(spr, cpu_R(dc, a->a), a->k); + gen_helper_mfspr(cpu_R(dc, a->d), cpu_env, cpu_R(dc, a->d), spr); + return true; +} + +static bool trans_l_mtspr(DisasContext *dc, arg_l_mtspr *a) +{ + TCGv spr = tcg_temp_new(); - spr = tcg_temp_new(); - tcg_gen_ori_tl(spr, cpu_R(dc, a->a), a->k); - gen_helper_mtspr(cpu_env, spr, cpu_R(dc, a->b)); + if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { + gen_io_start(); } + /* + * For SR, we will need to exit the TB to recognize the new + * exception state. For NPC, in theory this counts as a branch + * (although the SPR only exists for use by an ICE). Save all + * of the cpu state first, allowing it to be overwritten. + */ + if (dc->delayed_branch) { + tcg_gen_mov_tl(cpu_pc, jmp_pc); + tcg_gen_discard_tl(jmp_pc); + } else { + tcg_gen_movi_tl(cpu_pc, dc->base.pc_next + 4); + } + dc->base.is_jmp = DISAS_EXIT; + + tcg_gen_ori_tl(spr, cpu_R(dc, a->a), a->k); + gen_helper_mtspr(cpu_env, spr, cpu_R(dc, a->b)); return true; } diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c index 40fdeaa905..df167493c3 100644 --- a/target/s390x/cpu.c +++ b/target/s390x/cpu.c @@ -140,6 +140,13 @@ static bool s390_cpu_has_work(CPUState *cs) return s390_cpu_has_int(cpu); } +static void s390_query_cpu_fast(CPUState *cpu, CpuInfoFast *value) +{ + S390CPU *s390_cpu = S390_CPU(cpu); + + value->u.s390x.cpu_state = s390_cpu->env.cpu_state; +} + /* S390CPUClass::reset() */ static void s390_cpu_reset(CPUState *s, cpu_reset_type type) { @@ -332,6 +339,7 @@ static void s390_cpu_class_init(ObjectClass *oc, void *data) cc->class_by_name = s390_cpu_class_by_name, cc->has_work = s390_cpu_has_work; cc->dump_state = s390_cpu_dump_state; + cc->query_cpu_fast = s390_query_cpu_fast; cc->set_pc = s390_cpu_set_pc; cc->get_pc = s390_cpu_get_pc; cc->gdb_read_register = s390_cpu_gdb_read_register; diff --git a/target/s390x/tcg/insn-data.h.inc b/target/s390x/tcg/insn-data.h.inc index 597d968b0e..1f1ac742a9 100644 --- a/target/s390x/tcg/insn-data.h.inc +++ b/target/s390x/tcg/insn-data.h.inc @@ -606,7 +606,7 @@ F(0xed04, LDEB, RXE, Z, 0, m2_32u, new, f1, ldeb, 0, IF_BFP) F(0xed05, LXDB, RXE, Z, 0, m2_64, new_x, x1, lxdb, 0, IF_BFP) F(0xed06, LXEB, RXE, Z, 0, m2_32u, new_x, x1, lxeb, 0, IF_BFP) - F(0xb324, LDER, RXE, Z, 0, e2, new, f1, lde, 0, IF_AFP1) + F(0xb324, LDER, RRE, Z, 0, e2, new, f1, lde, 0, IF_AFP1) F(0xed24, LDE, RXE, Z, 0, m2_32u, new, f1, lde, 0, IF_AFP1) /* LOAD ROUNDED */ F(0xb344, LEDBR, RRF_e, Z, 0, f2, new, e1, ledb, 0, IF_BFP) diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index a05205beb1..d6670e6a87 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -1534,18 +1534,51 @@ static DisasJumpType op_bal(DisasContext *s, DisasOps *o) } } +/* + * Disassemble the target of a branch. The results are returned in a form + * suitable for passing into help_branch(): + * + * - bool IS_IMM reflects whether the target is fixed or computed. Non-EXECUTEd + * branches, whose DisasContext *S contains the relative immediate field RI, + * are considered fixed. All the other branches are considered computed. + * - int IMM is the value of RI. + * - TCGv_i64 CDEST is the address of the computed target. + */ +#define disas_jdest(s, ri, is_imm, imm, cdest) do { \ + if (have_field(s, ri)) { \ + if (unlikely(s->ex_value)) { \ + cdest = tcg_temp_new_i64(); \ + tcg_gen_ld_i64(cdest, cpu_env, offsetof(CPUS390XState, ex_target));\ + tcg_gen_addi_i64(cdest, cdest, (int64_t)get_field(s, ri) * 2); \ + is_imm = false; \ + } else { \ + is_imm = true; \ + } \ + } else { \ + is_imm = false; \ + } \ + imm = is_imm ? get_field(s, ri) : 0; \ +} while (false) + static DisasJumpType op_basi(DisasContext *s, DisasOps *o) { + DisasCompare c; + bool is_imm; + int imm; + pc_to_link_info(o->out, s, s->pc_tmp); - return help_goto_direct(s, s->base.pc_next + (int64_t)get_field(s, i2) * 2); + + disas_jdest(s, i2, is_imm, imm, o->in2); + disas_jcc(s, &c, 0xf); + return help_branch(s, &c, is_imm, imm, o->in2); } static DisasJumpType op_bc(DisasContext *s, DisasOps *o) { int m1 = get_field(s, m1); - bool is_imm = have_field(s, i2); - int imm = is_imm ? get_field(s, i2) : 0; DisasCompare c; + bool is_imm; + int imm; /* BCR with R2 = 0 causes no branching */ if (have_field(s, r2) && get_field(s, r2) == 0) { @@ -1562,6 +1595,7 @@ static DisasJumpType op_bc(DisasContext *s, DisasOps *o) return DISAS_NEXT; } + disas_jdest(s, i2, is_imm, imm, o->in2); disas_jcc(s, &c, m1); return help_branch(s, &c, is_imm, imm, o->in2); } @@ -1569,10 +1603,10 @@ static DisasJumpType op_bc(DisasContext *s, DisasOps *o) static DisasJumpType op_bct32(DisasContext *s, DisasOps *o) { int r1 = get_field(s, r1); - bool is_imm = have_field(s, i2); - int imm = is_imm ? get_field(s, i2) : 0; DisasCompare c; + bool is_imm; TCGv_i64 t; + int imm; c.cond = TCG_COND_NE; c.is_64 = false; @@ -1584,6 +1618,7 @@ static DisasJumpType op_bct32(DisasContext *s, DisasOps *o) c.u.s32.b = tcg_constant_i32(0); tcg_gen_extrl_i64_i32(c.u.s32.a, t); + disas_jdest(s, i2, is_imm, imm, o->in2); return help_branch(s, &c, is_imm, imm, o->in2); } @@ -1611,9 +1646,9 @@ static DisasJumpType op_bcth(DisasContext *s, DisasOps *o) static DisasJumpType op_bct64(DisasContext *s, DisasOps *o) { int r1 = get_field(s, r1); - bool is_imm = have_field(s, i2); - int imm = is_imm ? get_field(s, i2) : 0; DisasCompare c; + bool is_imm; + int imm; c.cond = TCG_COND_NE; c.is_64 = true; @@ -1622,6 +1657,7 @@ static DisasJumpType op_bct64(DisasContext *s, DisasOps *o) c.u.s64.a = regs[r1]; c.u.s64.b = tcg_constant_i64(0); + disas_jdest(s, i2, is_imm, imm, o->in2); return help_branch(s, &c, is_imm, imm, o->in2); } @@ -1629,10 +1665,10 @@ static DisasJumpType op_bx32(DisasContext *s, DisasOps *o) { int r1 = get_field(s, r1); int r3 = get_field(s, r3); - bool is_imm = have_field(s, i2); - int imm = is_imm ? get_field(s, i2) : 0; DisasCompare c; + bool is_imm; TCGv_i64 t; + int imm; c.cond = (s->insn->data ? TCG_COND_LE : TCG_COND_GT); c.is_64 = false; @@ -1645,6 +1681,7 @@ static DisasJumpType op_bx32(DisasContext *s, DisasOps *o) tcg_gen_extrl_i64_i32(c.u.s32.b, regs[r3 | 1]); store_reg32_i64(r1, t); + disas_jdest(s, i2, is_imm, imm, o->in2); return help_branch(s, &c, is_imm, imm, o->in2); } @@ -1652,9 +1689,9 @@ static DisasJumpType op_bx64(DisasContext *s, DisasOps *o) { int r1 = get_field(s, r1); int r3 = get_field(s, r3); - bool is_imm = have_field(s, i2); - int imm = is_imm ? get_field(s, i2) : 0; DisasCompare c; + bool is_imm; + int imm; c.cond = (s->insn->data ? TCG_COND_LE : TCG_COND_GT); c.is_64 = true; @@ -1668,6 +1705,7 @@ static DisasJumpType op_bx64(DisasContext *s, DisasOps *o) tcg_gen_add_i64(regs[r1], regs[r1], regs[r3]); c.u.s64.a = regs[r1]; + disas_jdest(s, i2, is_imm, imm, o->in2); return help_branch(s, &c, is_imm, imm, o->in2); } @@ -1685,10 +1723,9 @@ static DisasJumpType op_cj(DisasContext *s, DisasOps *o) c.u.s64.a = o->in1; c.u.s64.b = o->in2; - is_imm = have_field(s, i4); - if (is_imm) { - imm = get_field(s, i4); - } else { + o->out = NULL; + disas_jdest(s, i4, is_imm, imm, o->out); + if (!is_imm && !o->out) { imm = 0; o->out = get_address(s, 0, get_field(s, b4), get_field(s, d4)); @@ -5764,15 +5801,13 @@ static void in2_a2(DisasContext *s, DisasOps *o) static TCGv gen_ri2(DisasContext *s) { - int64_t delta = (int64_t)get_field(s, i2) * 2; - TCGv ri2; + TCGv ri2 = NULL; + bool is_imm; + int imm; - if (unlikely(s->ex_value)) { - ri2 = tcg_temp_new_i64(); - tcg_gen_ld_i64(ri2, cpu_env, offsetof(CPUS390XState, ex_target)); - tcg_gen_addi_i64(ri2, ri2, delta); - } else { - ri2 = tcg_constant_i64(s->base.pc_next + delta); + disas_jdest(s, i2, is_imm, imm, ri2); + if (is_imm) { + ri2 = tcg_constant_i64(s->base.pc_next + imm * 2); } return ri2; diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index 62dd22d73c..bc6b99a1bd 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -13,6 +13,12 @@ #include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" #include "qemu/bitops.h" +#ifdef __linux__ +#include <asm/hwcap.h> +#endif +#ifdef CONFIG_DARWIN +#include <sys/sysctl.h> +#endif /* We're going to re-use TCGType in setting of the SF bit, which controls the size of the operation performed. If we know the values match, it @@ -71,15 +77,13 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) return TCG_REG_X0 + slot; } +bool have_lse; +bool have_lse2; + #define TCG_REG_TMP TCG_REG_X30 #define TCG_VEC_TMP TCG_REG_V31 #ifndef CONFIG_SOFTMMU -/* Note that XZR cannot be encoded in the address base register slot, - as that actaully encodes SP. So if we need to zero-extend the guest - address, via the address index register slot, we need to load even - a zero guest base into a register. */ -#define USE_GUEST_BASE (guest_base != 0 || TARGET_LONG_BITS == 32) #define TCG_REG_GUEST_BASE TCG_REG_X28 #endif @@ -1584,41 +1588,13 @@ typedef struct { TCGReg base; TCGReg index; TCGType index_ext; + TCGAtomAlign aa; } HostAddress; -#ifdef CONFIG_SOFTMMU -/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, - * MemOpIdx oi, uintptr_t ra) - */ -static void * const qemu_ld_helpers[MO_SIZE + 1] = { - [MO_8] = helper_ret_ldub_mmu, -#if HOST_BIG_ENDIAN - [MO_16] = helper_be_lduw_mmu, - [MO_32] = helper_be_ldul_mmu, - [MO_64] = helper_be_ldq_mmu, -#else - [MO_16] = helper_le_lduw_mmu, - [MO_32] = helper_le_ldul_mmu, - [MO_64] = helper_le_ldq_mmu, -#endif -}; - -/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, - * uintxx_t val, MemOpIdx oi, - * uintptr_t ra) - */ -static void * const qemu_st_helpers[MO_SIZE + 1] = { - [MO_8] = helper_ret_stb_mmu, -#if HOST_BIG_ENDIAN - [MO_16] = helper_be_stw_mmu, - [MO_32] = helper_be_stl_mmu, - [MO_64] = helper_be_stq_mmu, -#else - [MO_16] = helper_le_stw_mmu, - [MO_32] = helper_le_stl_mmu, - [MO_64] = helper_le_stq_mmu, -#endif -}; +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return false; +} static const TCGLdstHelperParam ldst_helper_param = { .ntmp = 1, .tmp = { TCG_REG_TMP } @@ -1652,40 +1628,6 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) tcg_out_goto(s, lb->raddr); return true; } -#else -static void tcg_out_adr(TCGContext *s, TCGReg rd, const void *target) -{ - ptrdiff_t offset = tcg_pcrel_diff(s, target); - tcg_debug_assert(offset == sextract64(offset, 0, 21)); - tcg_out_insn(s, 3406, ADR, rd, offset); -} - -static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) -{ - if (!reloc_pc19(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { - return false; - } - - tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_X1, l->addrlo_reg); - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0); - - /* "Tail call" to the helper, with the return address back inline. */ - tcg_out_adr(s, TCG_REG_LR, l->raddr); - tcg_out_goto_long(s, (const void *)(l->is_ld ? helper_unaligned_ld - : helper_unaligned_st)); - return true; -} - -static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} - -static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} -#endif /* CONFIG_SOFTMMU */ /* * For softmmu, perform the TLB load and compare. @@ -1697,11 +1639,16 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, TCGReg addr_reg, MemOpIdx oi, bool is_ld) { - TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + TCGType addr_type = s->addr_type; TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); - unsigned a_bits = get_alignment_bits(opc); - unsigned a_mask = (1u << a_bits) - 1; + unsigned a_mask; + + h->aa = atom_and_align_for_opc(s, opc, + have_lse2 ? MO_ATOM_WITHIN16 + : MO_ATOM_IFALIGN, + false); + a_mask = (1 << h->aa.align) - 1; #ifdef CONFIG_SOFTMMU unsigned s_bits = opc & MO_SIZE; @@ -1716,7 +1663,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->oi = oi; ldst->addrlo_reg = addr_reg; - mask_type = (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32 + mask_type = (s->page_bits + s->tlb_dyn_max_bits > 32 ? TCG_TYPE_I64 : TCG_TYPE_I32); /* Load env_tlb(env)->f[mmu_idx].{mask,table} into {x0,x1}. */ @@ -1730,13 +1677,13 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, /* Extract the TLB index from the address into X0. */ tcg_out_insn(s, 3502S, AND_LSR, mask_type == TCG_TYPE_I64, TCG_REG_X0, TCG_REG_X0, addr_reg, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + s->page_bits - CPU_TLB_ENTRY_BITS); /* Add the tlb_table pointer, creating the CPUTLBEntry address into X1. */ tcg_out_insn(s, 3502, ADD, 1, TCG_REG_X1, TCG_REG_X1, TCG_REG_X0); /* Load the tlb comparator into X0, and the fast path addend into X1. */ - tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_X0, TCG_REG_X1, + tcg_out_ld(s, addr_type, TCG_REG_X0, TCG_REG_X1, is_ld ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write)); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_X1, TCG_REG_X1, @@ -1747,31 +1694,28 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, * bits within the address. For unaligned access, we check that we don't * cross pages using the address of the last byte of the access. */ - if (a_bits >= s_bits) { + if (a_mask >= s_mask) { x3 = addr_reg; } else { - tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64, + tcg_out_insn(s, 3401, ADDI, addr_type, TCG_REG_X3, addr_reg, s_mask - a_mask); x3 = TCG_REG_X3; } - compare_mask = (uint64_t)TARGET_PAGE_MASK | a_mask; + compare_mask = (uint64_t)s->page_mask | a_mask; /* Store the page mask part of the address into X3. */ - tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64, - TCG_REG_X3, x3, compare_mask); + tcg_out_logicali(s, I3404_ANDI, addr_type, TCG_REG_X3, x3, compare_mask); /* Perform the address comparison. */ - tcg_out_cmp(s, TARGET_LONG_BITS == 64, TCG_REG_X0, TCG_REG_X3, 0); + tcg_out_cmp(s, addr_type, TCG_REG_X0, TCG_REG_X3, 0); /* If not equal, we jump to the slow path. */ ldst->label_ptr[0] = s->code_ptr; tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0); - *h = (HostAddress){ - .base = TCG_REG_X1, - .index = addr_reg, - .index_ext = addr_type - }; + h->base = TCG_REG_X1, + h->index = addr_reg; + h->index_ext = addr_type; #else if (a_mask) { ldst = new_ldst_label(s); @@ -1788,18 +1732,14 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0); } - if (USE_GUEST_BASE) { - *h = (HostAddress){ - .base = TCG_REG_GUEST_BASE, - .index = addr_reg, - .index_ext = addr_type - }; + if (guest_base || addr_type == TCG_TYPE_I32) { + h->base = TCG_REG_GUEST_BASE; + h->index = addr_reg; + h->index_ext = addr_type; } else { - *h = (HostAddress){ - .base = addr_reg, - .index = TCG_REG_XZR, - .index_ext = TCG_TYPE_I64 - }; + h->base = addr_reg; + h->index = TCG_REG_XZR; + h->index_ext = TCG_TYPE_I64; } #endif @@ -2218,12 +2158,16 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_insn(s, 3506, CSEL, ext, a0, REG0(3), REG0(4), args[5]); break; - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: tcg_out_qemu_ld(s, a0, a1, a2, ext); break; - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: tcg_out_qemu_st(s, REG0(0), a1, a2, ext); break; @@ -2860,11 +2804,15 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_movcond_i64: return C_O1_I4(r, r, rA, rZ, rZ); - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: return C_O1_I1(r, l); - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: return C_O0_I2(lZ, l); case INDEX_op_deposit_i32: @@ -2930,8 +2878,39 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) } } +#ifdef CONFIG_DARWIN +static bool sysctl_for_bool(const char *name) +{ + int val = 0; + size_t len = sizeof(val); + + if (sysctlbyname(name, &val, &len, NULL, 0) == 0) { + return val != 0; + } + + /* + * We might in the future ask for properties not present in older kernels, + * but we're only asking about static properties, all of which should be + * 'int'. So we shouln't see ENOMEM (val too small), or any of the other + * more exotic errors. + */ + assert(errno == ENOENT); + return false; +} +#endif + static void tcg_target_init(TCGContext *s) { +#ifdef __linux__ + unsigned long hwcap = qemu_getauxval(AT_HWCAP); + have_lse = hwcap & HWCAP_ATOMICS; + have_lse2 = hwcap & HWCAP_USCAT; +#endif +#ifdef CONFIG_DARWIN + have_lse = sysctl_for_bool("hw.optional.arm.FEAT_LSE"); + have_lse2 = sysctl_for_bool("hw.optional.arm.FEAT_LSE2"); +#endif + tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffffu; tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffffu; tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull; @@ -3008,10 +2987,14 @@ static void tcg_target_qemu_prologue(TCGContext *s) CPU_TEMP_BUF_NLONGS * sizeof(long)); #if !defined(CONFIG_SOFTMMU) - if (USE_GUEST_BASE) { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base); - tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE); - } + /* + * Note that XZR cannot be encoded in the address base register slot, + * as that actaully encodes SP. Depending on the guest, we may need + * to zero-extend the guest address via the address index register slot, + * therefore we need to load even a zero guest base into a register. + */ + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base); + tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE); #endif tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index c0b0f614ba..74ee2ed255 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -57,6 +57,9 @@ typedef enum { #define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_EVEN #define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL +extern bool have_lse; +extern bool have_lse2; + /* optional instructions */ #define TCG_TARGET_HAS_div_i32 1 #define TCG_TARGET_HAS_rem_i32 1 @@ -126,6 +129,8 @@ typedef enum { #define TCG_TARGET_HAS_muluh_i64 1 #define TCG_TARGET_HAS_mulsh_i64 1 +#define TCG_TARGET_HAS_qemu_ldst_i128 0 + #define TCG_TARGET_HAS_v64 1 #define TCG_TARGET_HAS_v128 1 #define TCG_TARGET_HAS_v256 0 @@ -151,7 +156,6 @@ typedef enum { #define TCG_TARGET_HAS_cmpsel_vec 0 #define TCG_TARGET_DEFAULT_MO (0) -#define TCG_TARGET_HAS_MEMORY_BSWAP 0 #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/arm/tcg-target-con-set.h b/tcg/arm/tcg-target-con-set.h index b8849b2478..229ae258ac 100644 --- a/tcg/arm/tcg-target-con-set.h +++ b/tcg/arm/tcg-target-con-set.h @@ -12,19 +12,19 @@ C_O0_I1(r) C_O0_I2(r, r) C_O0_I2(r, rIN) -C_O0_I2(s, s) +C_O0_I2(q, q) C_O0_I2(w, r) -C_O0_I3(s, s, s) -C_O0_I3(S, p, s) +C_O0_I3(q, q, q) +C_O0_I3(Q, p, q) C_O0_I4(r, r, rI, rI) -C_O0_I4(S, p, s, s) -C_O1_I1(r, l) +C_O0_I4(Q, p, q, q) +C_O1_I1(r, q) C_O1_I1(r, r) C_O1_I1(w, r) C_O1_I1(w, w) C_O1_I1(w, wr) C_O1_I2(r, 0, rZ) -C_O1_I2(r, l, l) +C_O1_I2(r, q, q) C_O1_I2(r, r, r) C_O1_I2(r, r, rI) C_O1_I2(r, r, rIK) @@ -39,8 +39,8 @@ C_O1_I2(w, w, wZ) C_O1_I3(w, w, w, w) C_O1_I4(r, r, r, rI, rI) C_O1_I4(r, r, rIN, rIK, 0) -C_O2_I1(e, p, l) -C_O2_I2(e, p, l, l) +C_O2_I1(e, p, q) +C_O2_I2(e, p, q, q) C_O2_I2(r, r, r, r) C_O2_I4(r, r, r, r, rIN, rIK) C_O2_I4(r, r, rI, rI, rIN, rIK) diff --git a/tcg/arm/tcg-target-con-str.h b/tcg/arm/tcg-target-con-str.h index 24b4b59feb..f83f1d3919 100644 --- a/tcg/arm/tcg-target-con-str.h +++ b/tcg/arm/tcg-target-con-str.h @@ -10,9 +10,8 @@ */ REGS('e', ALL_GENERAL_REGS & 0x5555) /* even regs */ REGS('r', ALL_GENERAL_REGS) -REGS('l', ALL_QLOAD_REGS) -REGS('s', ALL_QSTORE_REGS) -REGS('S', ALL_QSTORE_REGS & 0x5555) /* even qstore */ +REGS('q', ALL_QLDST_REGS) +REGS('Q', ALL_QLDST_REGS & 0x5555) /* even qldst */ REGS('w', ALL_VECTOR_REGS) /* diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index df514e56fc..20cc1cc477 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -353,23 +353,16 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, #define ALL_VECTOR_REGS 0xffff0000u /* - * r0-r2 will be overwritten when reading the tlb entry (softmmu only) - * and r0-r1 doing the byte swapping, so don't use these. - * r3 is removed for softmmu to avoid clashes with helper arguments. + * r0-r3 will be overwritten when reading the tlb entry (softmmu only); + * r14 will be overwritten by the BLNE branching to the slow path. */ #ifdef CONFIG_SOFTMMU -#define ALL_QLOAD_REGS \ +#define ALL_QLDST_REGS \ (ALL_GENERAL_REGS & ~((1 << TCG_REG_R0) | (1 << TCG_REG_R1) | \ (1 << TCG_REG_R2) | (1 << TCG_REG_R3) | \ (1 << TCG_REG_R14))) -#define ALL_QSTORE_REGS \ - (ALL_GENERAL_REGS & ~((1 << TCG_REG_R0) | (1 << TCG_REG_R1) | \ - (1 << TCG_REG_R2) | (1 << TCG_REG_R14) | \ - ((TARGET_LONG_BITS == 64) << TCG_REG_R3))) #else -#define ALL_QLOAD_REGS ALL_GENERAL_REGS -#define ALL_QSTORE_REGS \ - (ALL_GENERAL_REGS & ~((1 << TCG_REG_R0) | (1 << TCG_REG_R1))) +#define ALL_QLDST_REGS (ALL_GENERAL_REGS & ~(1 << TCG_REG_R14)) #endif /* @@ -1330,45 +1323,13 @@ typedef struct { TCGReg base; int index; bool index_scratch; + TCGAtomAlign aa; } HostAddress; -#ifdef CONFIG_SOFTMMU -/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, - * int mmu_idx, uintptr_t ra) - */ -static void * const qemu_ld_helpers[MO_SSIZE + 1] = { - [MO_UB] = helper_ret_ldub_mmu, - [MO_SB] = helper_ret_ldsb_mmu, -#if HOST_BIG_ENDIAN - [MO_UW] = helper_be_lduw_mmu, - [MO_UL] = helper_be_ldul_mmu, - [MO_UQ] = helper_be_ldq_mmu, - [MO_SW] = helper_be_ldsw_mmu, - [MO_SL] = helper_be_ldul_mmu, -#else - [MO_UW] = helper_le_lduw_mmu, - [MO_UL] = helper_le_ldul_mmu, - [MO_UQ] = helper_le_ldq_mmu, - [MO_SW] = helper_le_ldsw_mmu, - [MO_SL] = helper_le_ldul_mmu, -#endif -}; - -/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, - * uintxx_t val, int mmu_idx, uintptr_t ra) - */ -static void * const qemu_st_helpers[MO_SIZE + 1] = { - [MO_8] = helper_ret_stb_mmu, -#if HOST_BIG_ENDIAN - [MO_16] = helper_be_stw_mmu, - [MO_32] = helper_be_stl_mmu, - [MO_64] = helper_be_stq_mmu, -#else - [MO_16] = helper_le_stw_mmu, - [MO_32] = helper_le_stl_mmu, - [MO_64] = helper_le_stq_mmu, -#endif -}; +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return false; +} static TCGReg ldst_ra_gen(TCGContext *s, const TCGLabelQemuLdst *l, int arg) { @@ -1412,50 +1373,6 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) tcg_out_goto(s, COND_AL, qemu_st_helpers[opc & MO_SIZE]); return true; } -#else -static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) -{ - if (!reloc_pc24(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { - return false; - } - - if (TARGET_LONG_BITS == 64) { - /* 64-bit target address is aligned into R2:R3. */ - TCGMovExtend ext[2] = { - { .dst = TCG_REG_R2, .dst_type = TCG_TYPE_I32, - .src = l->addrlo_reg, - .src_type = TCG_TYPE_I32, .src_ext = MO_UL }, - { .dst = TCG_REG_R3, .dst_type = TCG_TYPE_I32, - .src = l->addrhi_reg, - .src_type = TCG_TYPE_I32, .src_ext = MO_UL }, - }; - tcg_out_movext2(s, &ext[0], &ext[1], TCG_REG_TMP); - } else { - tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R1, l->addrlo_reg); - } - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_AREG0); - - /* - * Tail call to the helper, with the return address back inline, - * just for the clarity of the debugging traceback -- the helper - * cannot return. We have used BLNE to arrive here, so LR is - * already set. - */ - tcg_out_goto(s, COND_AL, (const void *) - (l->is_ld ? helper_unaligned_ld : helper_unaligned_st)); - return true; -} - -static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} - -static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} -#endif /* SOFTMMU */ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, TCGReg addrlo, TCGReg addrhi, @@ -1463,8 +1380,26 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, { TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); - MemOp a_bits = get_alignment_bits(opc); - unsigned a_mask = (1 << a_bits) - 1; + unsigned a_mask; + +#ifdef CONFIG_SOFTMMU + *h = (HostAddress){ + .cond = COND_AL, + .base = addrlo, + .index = TCG_REG_R1, + .index_scratch = true, + }; +#else + *h = (HostAddress){ + .cond = COND_AL, + .base = addrlo, + .index = guest_base ? TCG_REG_GUEST_BASE : -1, + .index_scratch = false, + }; +#endif + + h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false); + a_mask = (1 << h->aa.align) - 1; #ifdef CONFIG_SOFTMMU int mem_index = get_mmuidx(oi); @@ -1489,25 +1424,25 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, /* Extract the tlb index from the address into R0. */ tcg_out_dat_reg(s, COND_AL, ARITH_AND, TCG_REG_R0, TCG_REG_R0, addrlo, - SHIFT_IMM_LSR(TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS)); + SHIFT_IMM_LSR(s->page_bits - CPU_TLB_ENTRY_BITS)); /* * Add the tlb_table pointer, creating the CPUTLBEntry address in R1. * Load the tlb comparator into R2/R3 and the fast path addend into R1. */ if (cmp_off == 0) { - if (TARGET_LONG_BITS == 64) { - tcg_out_ldrd_rwb(s, COND_AL, TCG_REG_R2, TCG_REG_R1, TCG_REG_R0); - } else { + if (s->addr_type == TCG_TYPE_I32) { tcg_out_ld32_rwb(s, COND_AL, TCG_REG_R2, TCG_REG_R1, TCG_REG_R0); + } else { + tcg_out_ldrd_rwb(s, COND_AL, TCG_REG_R2, TCG_REG_R1, TCG_REG_R0); } } else { tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_R1, TCG_REG_R1, TCG_REG_R0, 0); - if (TARGET_LONG_BITS == 64) { - tcg_out_ldrd_8(s, COND_AL, TCG_REG_R2, TCG_REG_R1, cmp_off); - } else { + if (s->addr_type == TCG_TYPE_I32) { tcg_out_ld32_12(s, COND_AL, TCG_REG_R2, TCG_REG_R1, cmp_off); + } else { + tcg_out_ldrd_8(s, COND_AL, TCG_REG_R2, TCG_REG_R1, cmp_off); } } @@ -1533,8 +1468,8 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_dat_imm(s, COND_AL, ARITH_ADD, t_addr, addrlo, s_mask - a_mask); } - if (use_armv7_instructions && TARGET_PAGE_BITS <= 16) { - tcg_out_movi32(s, COND_AL, TCG_REG_TMP, ~(TARGET_PAGE_MASK | a_mask)); + if (use_armv7_instructions && s->page_bits <= 16) { + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, ~(s->page_mask | a_mask)); tcg_out_dat_reg(s, COND_AL, ARITH_BIC, TCG_REG_TMP, t_addr, TCG_REG_TMP, 0); tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0, TCG_REG_R2, TCG_REG_TMP, 0); @@ -1544,22 +1479,15 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, addrlo, a_mask); } tcg_out_dat_reg(s, COND_AL, ARITH_MOV, TCG_REG_TMP, 0, t_addr, - SHIFT_IMM_LSR(TARGET_PAGE_BITS)); + SHIFT_IMM_LSR(s->page_bits)); tcg_out_dat_reg(s, (a_mask ? COND_EQ : COND_AL), ARITH_CMP, 0, TCG_REG_R2, TCG_REG_TMP, - SHIFT_IMM_LSL(TARGET_PAGE_BITS)); + SHIFT_IMM_LSL(s->page_bits)); } - if (TARGET_LONG_BITS == 64) { + if (s->addr_type != TCG_TYPE_I32) { tcg_out_dat_reg(s, COND_EQ, ARITH_CMP, 0, TCG_REG_R3, addrhi, 0); } - - *h = (HostAddress){ - .cond = COND_AL, - .base = addrlo, - .index = TCG_REG_R1, - .index_scratch = true, - }; #else if (a_mask) { ldst = new_ldst_label(s); @@ -1568,18 +1496,11 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->addrlo_reg = addrlo; ldst->addrhi_reg = addrhi; - /* We are expecting a_bits to max out at 7 */ + /* We are expecting alignment to max out at 7 */ tcg_debug_assert(a_mask <= 0xff); /* tst addr, #mask */ tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, addrlo, a_mask); } - - *h = (HostAddress){ - .cond = COND_AL, - .base = addrlo, - .index = guest_base ? TCG_REG_GUEST_BASE : -1, - .index_scratch = false, - }; #endif return ldst; @@ -2064,41 +1985,36 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, ARITH_MOV, args[0], 0, 0); break; - case INDEX_op_qemu_ld_i32: - if (TARGET_LONG_BITS == 32) { - tcg_out_qemu_ld(s, args[0], -1, args[1], -1, - args[2], TCG_TYPE_I32); - } else { - tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], - args[3], TCG_TYPE_I32); - } + case INDEX_op_qemu_ld_a32_i32: + tcg_out_qemu_ld(s, args[0], -1, args[1], -1, args[2], TCG_TYPE_I32); break; - case INDEX_op_qemu_ld_i64: - if (TARGET_LONG_BITS == 32) { - tcg_out_qemu_ld(s, args[0], args[1], args[2], -1, - args[3], TCG_TYPE_I64); - } else { - tcg_out_qemu_ld(s, args[0], args[1], args[2], args[3], - args[4], TCG_TYPE_I64); - } + case INDEX_op_qemu_ld_a64_i32: + tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); break; - case INDEX_op_qemu_st_i32: - if (TARGET_LONG_BITS == 32) { - tcg_out_qemu_st(s, args[0], -1, args[1], -1, - args[2], TCG_TYPE_I32); - } else { - tcg_out_qemu_st(s, args[0], -1, args[1], args[2], - args[3], TCG_TYPE_I32); - } + case INDEX_op_qemu_ld_a32_i64: + tcg_out_qemu_ld(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); break; - case INDEX_op_qemu_st_i64: - if (TARGET_LONG_BITS == 32) { - tcg_out_qemu_st(s, args[0], args[1], args[2], -1, - args[3], TCG_TYPE_I64); - } else { - tcg_out_qemu_st(s, args[0], args[1], args[2], args[3], - args[4], TCG_TYPE_I64); - } + case INDEX_op_qemu_ld_a64_i64: + tcg_out_qemu_ld(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); + break; + + case INDEX_op_qemu_st_a32_i32: + tcg_out_qemu_st(s, args[0], -1, args[1], -1, args[2], TCG_TYPE_I32); + break; + case INDEX_op_qemu_st_a64_i32: + tcg_out_qemu_st(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); + break; + case INDEX_op_qemu_st_a32_i64: + tcg_out_qemu_st(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); + break; + case INDEX_op_qemu_st_a64_i64: + tcg_out_qemu_st(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); break; case INDEX_op_bswap16_i32: @@ -2239,14 +2155,22 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_setcond2_i32: return C_O1_I4(r, r, r, rI, rI); - case INDEX_op_qemu_ld_i32: - return TARGET_LONG_BITS == 32 ? C_O1_I1(r, l) : C_O1_I2(r, l, l); - case INDEX_op_qemu_ld_i64: - return TARGET_LONG_BITS == 32 ? C_O2_I1(e, p, l) : C_O2_I2(e, p, l, l); - case INDEX_op_qemu_st_i32: - return TARGET_LONG_BITS == 32 ? C_O0_I2(s, s) : C_O0_I3(s, s, s); - case INDEX_op_qemu_st_i64: - return TARGET_LONG_BITS == 32 ? C_O0_I3(S, p, s) : C_O0_I4(S, p, s, s); + case INDEX_op_qemu_ld_a32_i32: + return C_O1_I1(r, q); + case INDEX_op_qemu_ld_a64_i32: + return C_O1_I2(r, q, q); + case INDEX_op_qemu_ld_a32_i64: + return C_O2_I1(e, p, q); + case INDEX_op_qemu_ld_a64_i64: + return C_O2_I2(e, p, q, q); + case INDEX_op_qemu_st_a32_i32: + return C_O0_I2(q, q); + case INDEX_op_qemu_st_a64_i32: + return C_O0_I3(q, q, q); + case INDEX_op_qemu_st_a32_i64: + return C_O0_I3(Q, p, q); + case INDEX_op_qemu_st_a64_i64: + return C_O0_I4(Q, p, q, q); case INDEX_op_st_vec: return C_O0_I2(w, r); diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index def2a189e6..65efc538f4 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -125,6 +125,8 @@ extern bool use_neon_instructions; #define TCG_TARGET_HAS_rem_i32 0 #define TCG_TARGET_HAS_qemu_st8_i32 0 +#define TCG_TARGET_HAS_qemu_ldst_i128 0 + #define TCG_TARGET_HAS_v64 use_neon_instructions #define TCG_TARGET_HAS_v128 use_neon_instructions #define TCG_TARGET_HAS_v256 0 @@ -150,7 +152,6 @@ extern bool use_neon_instructions; #define TCG_TARGET_HAS_cmpsel_vec 0 #define TCG_TARGET_DEFAULT_MO (0) -#define TCG_TARGET_HAS_MEMORY_BSWAP 0 #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index a01bfad773..8b9a5f00e5 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -185,6 +185,7 @@ bool have_avx512dq; bool have_avx512vbmi2; bool have_avx512vl; bool have_movbe; +bool have_atomic16; #ifdef CONFIG_CPUID_H static bool have_bmi2; @@ -1091,7 +1092,7 @@ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, { /* This function is only used for passing structs by reference. */ tcg_debug_assert(imm == (int32_t)imm); - tcg_out_modrm_offset(s, OPC_LEA, rd, rs, imm); + tcg_out_modrm_offset(s, OPC_LEA | P_REXW, rd, rs, imm); } static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val) @@ -1314,7 +1315,9 @@ static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src) static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src) { - tcg_out_ext32u(s, dest, src); + if (dest != src) { + tcg_out_ext32u(s, dest, src); + } } static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg dest, TCGReg src) @@ -1773,34 +1776,13 @@ typedef struct { int index; int ofs; int seg; + TCGAtomAlign aa; } HostAddress; -#if defined(CONFIG_SOFTMMU) -/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, - * int mmu_idx, uintptr_t ra) - */ -static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = { - [MO_UB] = helper_ret_ldub_mmu, - [MO_LEUW] = helper_le_lduw_mmu, - [MO_LEUL] = helper_le_ldul_mmu, - [MO_LEUQ] = helper_le_ldq_mmu, - [MO_BEUW] = helper_be_lduw_mmu, - [MO_BEUL] = helper_be_ldul_mmu, - [MO_BEUQ] = helper_be_ldq_mmu, -}; - -/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, - * uintxx_t val, int mmu_idx, uintptr_t ra) - */ -static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { - [MO_UB] = helper_ret_stb_mmu, - [MO_LEUW] = helper_le_stw_mmu, - [MO_LEUL] = helper_le_stl_mmu, - [MO_LEUQ] = helper_le_stq_mmu, - [MO_BEUW] = helper_be_stw_mmu, - [MO_BEUL] = helper_be_stl_mmu, - [MO_BEUQ] = helper_be_stq_mmu, -}; +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return have_movbe; +} /* * Because i686 has no register parameters and because x86_64 has xchg @@ -1837,12 +1819,12 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) /* resolve label address */ tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4); - if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { + if (label_ptr[1]) { tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4); } tcg_out_ld_helper_args(s, l, &ldst_helper_param); - tcg_out_branch(s, 1, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_branch(s, 1, qemu_ld_helpers[opc & MO_SIZE]); tcg_out_ld_helper_ret(s, l, false, &ldst_helper_param); tcg_out_jmp(s, l->raddr); @@ -1859,61 +1841,18 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) /* resolve label address */ tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4); - if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { + if (label_ptr[1]) { tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4); } tcg_out_st_helper_args(s, l, &ldst_helper_param); - tcg_out_branch(s, 1, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_branch(s, 1, qemu_st_helpers[opc & MO_SIZE]); tcg_out_jmp(s, l->raddr); return true; } -#else -static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) -{ - /* resolve label address */ - tcg_patch32(l->label_ptr[0], s->code_ptr - l->label_ptr[0] - 4); - - if (TCG_TARGET_REG_BITS == 32) { - int ofs = 0; - - tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs); - ofs += 4; - - tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs); - ofs += 4; - if (TARGET_LONG_BITS == 64) { - tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs); - ofs += 4; - } - - tcg_out_pushi(s, (uintptr_t)l->raddr); - } else { - tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1], - l->addrlo_reg); - tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0); - - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RAX, (uintptr_t)l->raddr); - tcg_out_push(s, TCG_REG_RAX); - } - - /* "Tail call" to the helper, with the return address back inline. */ - tcg_out_jmp(s, (const void *)(l->is_ld ? helper_unaligned_ld - : helper_unaligned_st)); - return true; -} - -static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} - -static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} +#ifndef CONFIG_SOFTMMU static HostAddress x86_guest_base = { .index = -1 }; @@ -1945,7 +1884,7 @@ static inline int setup_guest_base_seg(void) return 0; } #endif /* setup_guest_base_seg */ -#endif /* SOFTMMU */ +#endif /* !SOFTMMU */ /* * For softmmu, perform the TLB load and compare. @@ -1959,8 +1898,18 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, { TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); - unsigned a_bits = get_alignment_bits(opc); - unsigned a_mask = (1 << a_bits) - 1; + unsigned a_mask; + +#ifdef CONFIG_SOFTMMU + h->index = TCG_REG_L0; + h->ofs = 0; + h->seg = 0; +#else + *h = x86_guest_base; +#endif + h->base = addrlo; + h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false); + a_mask = (1 << h->aa.align) - 1; #ifdef CONFIG_SOFTMMU int cmp_ofs = is_ld ? offsetof(CPUTLBEntry, addr_read) @@ -1971,7 +1920,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, unsigned mem_index = get_mmuidx(oi); unsigned s_bits = opc & MO_SIZE; unsigned s_mask = (1 << s_bits) - 1; - target_ulong tlb_mask; + int tlb_mask; ldst = new_ldst_label(s); ldst->is_ld = is_ld; @@ -1980,13 +1929,11 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->addrhi_reg = addrhi; if (TCG_TARGET_REG_BITS == 64) { - if (TARGET_LONG_BITS == 64) { - ttype = TCG_TYPE_I64; - trexw = P_REXW; - } + ttype = s->addr_type; + trexw = (ttype == TCG_TYPE_I32 ? 0 : P_REXW); if (TCG_TYPE_PTR == TCG_TYPE_I64) { hrexw = P_REXW; - if (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32) { + if (s->page_bits + s->tlb_dyn_max_bits > 32) { tlbtype = TCG_TYPE_I64; tlbrexw = P_REXW; } @@ -1995,7 +1942,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_mov(s, tlbtype, TCG_REG_L0, addrlo); tcg_out_shifti(s, SHIFT_SHR + tlbrexw, TCG_REG_L0, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + s->page_bits - CPU_TLB_ENTRY_BITS); tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, TCG_REG_L0, TCG_AREG0, TLB_MASK_TABLE_OFS(mem_index) + @@ -2010,13 +1957,13 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, * copy the address and mask. For lesser alignments, check that we don't * cross pages for the complete access. */ - if (a_bits >= s_bits) { + if (a_mask >= s_mask) { tcg_out_mov(s, ttype, TCG_REG_L1, addrlo); } else { tcg_out_modrm_offset(s, OPC_LEA + trexw, TCG_REG_L1, addrlo, s_mask - a_mask); } - tlb_mask = (target_ulong)TARGET_PAGE_MASK | a_mask; + tlb_mask = s->page_mask | a_mask; tgen_arithi(s, ARITH_AND + trexw, TCG_REG_L1, tlb_mask, 0); /* cmp 0(TCG_REG_L0), TCG_REG_L1 */ @@ -2028,7 +1975,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->label_ptr[0] = s->code_ptr; s->code_ptr += 4; - if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { + if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I64) { /* cmp 4(TCG_REG_L0), addrhi */ tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, TCG_REG_L0, cmp_ofs + 4); @@ -2041,13 +1988,8 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, /* TLB Hit. */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_L0, TCG_REG_L0, offsetof(CPUTLBEntry, addend)); - - *h = (HostAddress) { - .base = addrlo, - .index = TCG_REG_L0, - }; #else - if (a_bits) { + if (a_mask) { ldst = new_ldst_label(s); ldst->is_ld = is_ld; @@ -2061,9 +2003,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->label_ptr[0] = s->code_ptr; s->code_ptr += 4; } - - *h = x86_guest_base; - h->base = addrlo; #endif return ldst; @@ -2536,35 +2475,51 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0); break; - case INDEX_op_qemu_ld_i32: - if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { - tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); - } else { + case INDEX_op_qemu_ld_a64_i32: + if (TCG_TARGET_REG_BITS == 32) { tcg_out_qemu_ld(s, a0, -1, a1, a2, args[3], TCG_TYPE_I32); + break; } + /* fall through */ + case INDEX_op_qemu_ld_a32_i32: + tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i64: if (TCG_TARGET_REG_BITS == 64) { tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); - } else if (TARGET_LONG_BITS == 32) { + } else { tcg_out_qemu_ld(s, a0, a1, a2, -1, args[3], TCG_TYPE_I64); + } + break; + case INDEX_op_qemu_ld_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); } else { tcg_out_qemu_ld(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); } break; - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st8_i32: - if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { - tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); - } else { + + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_st8_a64_i32: + if (TCG_TARGET_REG_BITS == 32) { tcg_out_qemu_st(s, a0, -1, a1, a2, args[3], TCG_TYPE_I32); + break; } + /* fall through */ + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st8_a32_i32: + tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i64: if (TCG_TARGET_REG_BITS == 64) { tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); - } else if (TARGET_LONG_BITS == 32) { + } else { tcg_out_qemu_st(s, a0, a1, a2, -1, args[3], TCG_TYPE_I64); + } + break; + case INDEX_op_qemu_st_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); } else { tcg_out_qemu_st(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); } @@ -3242,26 +3197,29 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_clz_i64: return have_lzcnt ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); - case INDEX_op_qemu_ld_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); - - case INDEX_op_qemu_st_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O0_I2(L, L) : C_O0_I3(L, L, L)); - case INDEX_op_qemu_st8_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O0_I2(s, L) : C_O0_I3(s, L, L)); - - case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L) - : C_O2_I2(r, r, L, L)); - - case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(L, L, L) - : C_O0_I4(L, L, L, L)); + case INDEX_op_qemu_ld_a32_i32: + return C_O1_I1(r, L); + case INDEX_op_qemu_ld_a64_i32: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) : C_O1_I2(r, L, L); + + case INDEX_op_qemu_st_a32_i32: + return C_O0_I2(L, L); + case INDEX_op_qemu_st_a64_i32: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) : C_O0_I3(L, L, L); + case INDEX_op_qemu_st8_a32_i32: + return C_O0_I2(s, L); + case INDEX_op_qemu_st8_a64_i32: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(s, L) : C_O0_I3(s, L, L); + + case INDEX_op_qemu_ld_a32_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) : C_O2_I1(r, r, L); + case INDEX_op_qemu_ld_a64_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) : C_O2_I2(r, r, L, L); + + case INDEX_op_qemu_st_a32_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) : C_O0_I3(L, L, L); + case INDEX_op_qemu_st_a64_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) : C_O0_I4(L, L, L, L); case INDEX_op_brcond2_i32: return C_O0_I4(r, r, ri, ri); @@ -4052,6 +4010,32 @@ static void tcg_target_init(TCGContext *s) have_avx512dq = (b7 & bit_AVX512DQ) != 0; have_avx512vbmi2 = (c7 & bit_AVX512VBMI2) != 0; } + + /* + * The Intel SDM has added: + * Processors that enumerate support for Intel® AVX + * (by setting the feature flag CPUID.01H:ECX.AVX[bit 28]) + * guarantee that the 16-byte memory operations performed + * by the following instructions will always be carried + * out atomically: + * - MOVAPD, MOVAPS, and MOVDQA. + * - VMOVAPD, VMOVAPS, and VMOVDQA when encoded with VEX.128. + * - VMOVAPD, VMOVAPS, VMOVDQA32, and VMOVDQA64 when encoded + * with EVEX.128 and k0 (masking disabled). + * Note that these instructions require the linear addresses + * of their memory operands to be 16-byte aligned. + * + * AMD has provided an even stronger guarantee that processors + * with AVX provide 16-byte atomicity for all cachable, + * naturally aligned single loads and stores, e.g. MOVDQU. + * + * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 + */ + if (have_avx1) { + __cpuid(0, a, b, c, d); + have_atomic16 = (c == signature_INTEL_ecx || + c == signature_AMD_ecx); + } } } } diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index d4f2a6f8c2..0b5a2c68c5 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -120,6 +120,7 @@ extern bool have_avx512dq; extern bool have_avx512vbmi2; extern bool have_avx512vl; extern bool have_movbe; +extern bool have_atomic16; /* optional instructions */ #define TCG_TARGET_HAS_div2_i32 1 @@ -153,9 +154,9 @@ extern bool have_movbe; #define TCG_TARGET_HAS_mulsh_i32 0 #if TCG_TARGET_REG_BITS == 64 -/* Keep target addresses zero-extended in a register. */ -#define TCG_TARGET_HAS_extrl_i64_i32 (TARGET_LONG_BITS == 32) -#define TCG_TARGET_HAS_extrh_i64_i32 (TARGET_LONG_BITS == 32) +/* Keep 32-bit values zero-extended in a register. */ +#define TCG_TARGET_HAS_extrl_i64_i32 1 +#define TCG_TARGET_HAS_extrh_i64_i32 1 #define TCG_TARGET_HAS_div2_i64 1 #define TCG_TARGET_HAS_rot_i64 1 #define TCG_TARGET_HAS_ext8s_i64 1 @@ -193,6 +194,8 @@ extern bool have_movbe; #define TCG_TARGET_HAS_qemu_st8_i32 1 #endif +#define TCG_TARGET_HAS_qemu_ldst_i128 0 + /* We do not support older SSE systems, only beginning with AVX1. */ #define TCG_TARGET_HAS_v64 have_avx1 #define TCG_TARGET_HAS_v128 have_avx1 @@ -239,9 +242,6 @@ extern bool have_movbe; #include "tcg/tcg-mo.h" #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) - -#define TCG_TARGET_HAS_MEMORY_BSWAP have_movbe - #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index 83fa45c802..0bae922982 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -30,6 +30,7 @@ */ #include "../tcg-ldst.c.inc" +#include <asm/hwcap.h> #ifdef CONFIG_DEBUG_TCG static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { @@ -783,30 +784,6 @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, * Load/store helpers for SoftMMU, and qemu_ld/st implementations */ -#if defined(CONFIG_SOFTMMU) -/* - * helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, - * MemOpIdx oi, uintptr_t ra) - */ -static void * const qemu_ld_helpers[4] = { - [MO_8] = helper_ret_ldub_mmu, - [MO_16] = helper_le_lduw_mmu, - [MO_32] = helper_le_ldul_mmu, - [MO_64] = helper_le_ldq_mmu, -}; - -/* - * helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, - * uintxx_t val, MemOpIdx oi, - * uintptr_t ra) - */ -static void * const qemu_st_helpers[4] = { - [MO_8] = helper_ret_stb_mmu, - [MO_16] = helper_le_stw_mmu, - [MO_32] = helper_le_stl_mmu, - [MO_64] = helper_le_stq_mmu, -}; - static bool tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) { tcg_out_opc_b(s, 0); @@ -845,41 +822,18 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE], false); return tcg_out_goto(s, l->raddr); } -#else -static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) -{ - /* resolve label address */ - if (!reloc_br_sk16(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { - return false; - } - - tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_A1, l->addrlo_reg); - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0); - - /* tail call, with the return address back inline. */ - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (uintptr_t)l->raddr); - tcg_out_call_int(s, (const void *)(l->is_ld ? helper_unaligned_ld - : helper_unaligned_st), true); - return true; -} - -static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} - -static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} - -#endif /* CONFIG_SOFTMMU */ typedef struct { TCGReg base; TCGReg index; + TCGAtomAlign aa; } HostAddress; +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return false; +} + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -890,9 +844,13 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, TCGReg addr_reg, MemOpIdx oi, bool is_ld) { + TCGType addr_type = s->addr_type; TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); - unsigned a_bits = get_alignment_bits(opc); + MemOp a_bits; + + h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false); + a_bits = h->aa.align; #ifdef CONFIG_SOFTMMU unsigned s_bits = opc & MO_SIZE; @@ -900,7 +858,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, int fast_ofs = TLB_MASK_TABLE_OFS(mem_index); int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); - tcg_target_long compare_mask; ldst = new_ldst_label(s); ldst->is_ld = is_ld; @@ -913,25 +870,31 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_AREG0, table_ofs); tcg_out_opc_srli_d(s, TCG_REG_TMP2, addr_reg, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + s->page_bits - CPU_TLB_ENTRY_BITS); tcg_out_opc_and(s, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); tcg_out_opc_add_d(s, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); /* Load the tlb comparator and the addend. */ - tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP0, TCG_REG_TMP2, + tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP2, is_ld ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write)); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2, offsetof(CPUTLBEntry, addend)); - /* We don't support unaligned accesses. */ + /* + * For aligned accesses, we check the first byte and include the alignment + * bits within the address. For unaligned access, we check that we don't + * cross pages using the address of the last byte of the access. + */ if (a_bits < s_bits) { - a_bits = s_bits; + unsigned a_mask = (1u << a_bits) - 1; + unsigned s_mask = (1u << s_bits) - 1; + tcg_out_addi(s, addr_type, TCG_REG_TMP1, addr_reg, s_mask - a_mask); + } else { + tcg_out_mov(s, addr_type, TCG_REG_TMP1, addr_reg); } - /* Clear the non-page, non-alignment bits from the address. */ - compare_mask = (tcg_target_long)TARGET_PAGE_MASK | ((1 << a_bits) - 1); - tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); - tcg_out_opc_and(s, TCG_REG_TMP1, TCG_REG_TMP1, addr_reg); + tcg_out_opc_bstrins_d(s, TCG_REG_TMP1, TCG_REG_ZERO, + a_bits, s->page_bits - 1); /* Compare masked address with the TLB entry. */ ldst->label_ptr[0] = s->code_ptr; @@ -961,7 +924,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, h->index = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; #endif - if (TARGET_LONG_BITS == 32) { + if (addr_type == TCG_TYPE_I32) { h->base = TCG_REG_TMP0; tcg_out_ext32u(s, h->base, addr_reg); } else { @@ -1481,16 +1444,20 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_ldst(s, OPC_ST_D, a0, a1, a2); break; - case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; - case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; @@ -1530,8 +1497,10 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_st32_i64: case INDEX_op_st_i32: case INDEX_op_st_i64: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: return C_O0_I2(rZ, r); case INDEX_op_brcond_i32: @@ -1573,8 +1542,10 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_ld32u_i64: case INDEX_op_ld_i32: case INDEX_op_ld_i64: - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: return C_O1_I1(r, r); case INDEX_op_andc_i32: @@ -1727,6 +1698,14 @@ static void tcg_target_qemu_prologue(TCGContext *s) static void tcg_target_init(TCGContext *s) { + unsigned long hwcap = qemu_getauxval(AT_HWCAP); + + /* Server and desktop class cpus have UAL; embedded cpus do not. */ + if (!(hwcap & HWCAP_LOONGARCH_UAL)) { + error_report("TCG: unaligned access support required; exiting"); + exit(EXIT_FAILURE); + } + tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS; diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index 17b8193aa5..482901ac15 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -168,11 +168,10 @@ typedef enum { #define TCG_TARGET_HAS_muls2_i64 0 #define TCG_TARGET_HAS_muluh_i64 1 #define TCG_TARGET_HAS_mulsh_i64 1 +#define TCG_TARGET_HAS_qemu_ldst_i128 0 #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_NEED_LDST_LABELS -#define TCG_TARGET_HAS_MEMORY_BSWAP 0 - #endif /* LOONGARCH_TCG_TARGET_H */ diff --git a/tcg/meson.build b/tcg/meson.build index c4c63b19d4..bdc185a485 100644 --- a/tcg/meson.build +++ b/tcg/meson.build @@ -6,13 +6,14 @@ tcg_ss.add(files( 'tcg.c', 'tcg-common.c', 'tcg-op.c', + 'tcg-op-ldst.c', 'tcg-op-gvec.c', 'tcg-op-vec.c', )) if get_option('tcg_interpreter') libffi = dependency('libffi', version: '>=3.0', required: true, - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') specific_ss.add(libffi) specific_ss.add(files('tci.c')) endif diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index 5ad9867882..ef146b193c 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -354,10 +354,6 @@ typedef enum { /* Aliases for convenience. */ ALIAS_PADD = sizeof(void *) == 4 ? OPC_ADDU : OPC_DADDU, ALIAS_PADDI = sizeof(void *) == 4 ? OPC_ADDIU : OPC_DADDIU, - ALIAS_TSRL = TARGET_LONG_BITS == 32 || TCG_TARGET_REG_BITS == 32 - ? OPC_SRL : OPC_DSRL, - ALIAS_TADDI = TARGET_LONG_BITS == 32 || TCG_TARGET_REG_BITS == 32 - ? OPC_ADDIU : OPC_DADDIU, } MIPSInsn; /* @@ -1075,38 +1071,6 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg, tcg_out_nop(s); } -#if defined(CONFIG_SOFTMMU) -static void * const qemu_ld_helpers[MO_SSIZE + 1] = { - [MO_UB] = helper_ret_ldub_mmu, - [MO_SB] = helper_ret_ldsb_mmu, -#if HOST_BIG_ENDIAN - [MO_UW] = helper_be_lduw_mmu, - [MO_SW] = helper_be_ldsw_mmu, - [MO_UL] = helper_be_ldul_mmu, - [MO_SL] = helper_be_ldsl_mmu, - [MO_UQ] = helper_be_ldq_mmu, -#else - [MO_UW] = helper_le_lduw_mmu, - [MO_SW] = helper_le_ldsw_mmu, - [MO_UL] = helper_le_ldul_mmu, - [MO_UQ] = helper_le_ldq_mmu, - [MO_SL] = helper_le_ldsl_mmu, -#endif -}; - -static void * const qemu_st_helpers[MO_SIZE + 1] = { - [MO_UB] = helper_ret_stb_mmu, -#if HOST_BIG_ENDIAN - [MO_UW] = helper_be_stw_mmu, - [MO_UL] = helper_be_stl_mmu, - [MO_UQ] = helper_be_stq_mmu, -#else - [MO_UW] = helper_le_stw_mmu, - [MO_UL] = helper_le_stl_mmu, - [MO_UQ] = helper_le_stq_mmu, -#endif -}; - /* We have four temps, we might as well expose three of them. */ static const TCGLdstHelperParam ldst_helper_param = { .ntmp = 3, .tmp = { TCG_TMP0, TCG_TMP1, TCG_TMP2 } @@ -1119,8 +1083,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) /* resolve label address */ if (!reloc_pc16(l->label_ptr[0], tgt_rx) - || (TCG_TARGET_REG_BITS < TARGET_LONG_BITS - && !reloc_pc16(l->label_ptr[1], tgt_rx))) { + || (l->label_ptr[1] && !reloc_pc16(l->label_ptr[1], tgt_rx))) { return false; } @@ -1149,8 +1112,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) /* resolve label address */ if (!reloc_pc16(l->label_ptr[0], tgt_rx) - || (TCG_TARGET_REG_BITS < TARGET_LONG_BITS - && !reloc_pc16(l->label_ptr[1], tgt_rx))) { + || (l->label_ptr[1] && !reloc_pc16(l->label_ptr[1], tgt_rx))) { return false; } @@ -1170,61 +1132,16 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) return true; } -#else -static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) -{ - void *target; - - if (!reloc_pc16(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { - return false; - } - - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - /* A0 is env, A1 is skipped, A2:A3 is the uint64_t address. */ - TCGReg a2 = MIPS_BE ? l->addrhi_reg : l->addrlo_reg; - TCGReg a3 = MIPS_BE ? l->addrlo_reg : l->addrhi_reg; - - if (a3 != TCG_REG_A2) { - tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A2, a2); - tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A3, a3); - } else if (a2 != TCG_REG_A3) { - tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A3, a3); - tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A2, a2); - } else { - tcg_out_mov(s, TCG_TYPE_I32, TCG_TMP0, TCG_REG_A2); - tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A2, TCG_REG_A3); - tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_A3, TCG_TMP0); - } - } else { - tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_A1, l->addrlo_reg); - } - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0); - - /* - * Tail call to the helper, with the return address back inline. - * We have arrived here via BNEL, so $31 is already set. - */ - target = (l->is_ld ? helper_unaligned_ld : helper_unaligned_st); - tcg_out_call_int(s, target, true); - return true; -} - -static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} - -static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} -#endif /* SOFTMMU */ - typedef struct { TCGReg base; - MemOp align; + TCGAtomAlign aa; } HostAddress; +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return false; +} + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -1235,13 +1152,18 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, TCGReg addrlo, TCGReg addrhi, MemOpIdx oi, bool is_ld) { + TCGType addr_type = s->addr_type; TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); - unsigned a_bits = get_alignment_bits(opc); + MemOp a_bits; unsigned s_bits = opc & MO_SIZE; - unsigned a_mask = (1 << a_bits) - 1; + unsigned a_mask; TCGReg base; + h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false); + a_bits = h->aa.align; + a_mask = (1 << a_bits) - 1; + #ifdef CONFIG_SOFTMMU unsigned s_mask = (1 << s_bits) - 1; int mem_index = get_mmuidx(oi); @@ -1265,23 +1187,26 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP1, TCG_AREG0, table_off); /* Extract the TLB index from the address into TMP3. */ - tcg_out_opc_sa(s, ALIAS_TSRL, TCG_TMP3, addrlo, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + if (TCG_TARGET_REG_BITS == 32 || addr_type == TCG_TYPE_I32) { + tcg_out_opc_sa(s, OPC_SRL, TCG_TMP3, addrlo, + s->page_bits - CPU_TLB_ENTRY_BITS); + } else { + tcg_out_dsrl(s, TCG_TMP3, addrlo, + s->page_bits - CPU_TLB_ENTRY_BITS); + } tcg_out_opc_reg(s, OPC_AND, TCG_TMP3, TCG_TMP3, TCG_TMP0); /* Add the tlb_table pointer, creating the CPUTLBEntry address in TMP3. */ tcg_out_opc_reg(s, ALIAS_PADD, TCG_TMP3, TCG_TMP3, TCG_TMP1); - /* Load the (low-half) tlb comparator. */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - tcg_out_ldst(s, OPC_LW, TCG_TMP0, TCG_TMP3, cmp_off + LO_OFF); - } else { - tcg_out_ld(s, TCG_TYPE_TL, TCG_TMP0, TCG_TMP3, cmp_off); - } - - if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS == 64 || addr_type == TCG_TYPE_I32) { + /* Load the tlb comparator. */ + tcg_out_ld(s, addr_type, TCG_TMP0, TCG_TMP3, cmp_off); /* Load the tlb addend for the fast path. */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP3, TCG_TMP3, add_off); + } else { + /* Load the low half of the tlb comparator. */ + tcg_out_ldst(s, OPC_LW, TCG_TMP0, TCG_TMP3, cmp_off + LO_OFF); } /* @@ -1289,16 +1214,20 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, * For unaligned accesses, compare against the end of the access to * verify that it does not cross a page boundary. */ - tcg_out_movi(s, TCG_TYPE_TL, TCG_TMP1, TARGET_PAGE_MASK | a_mask); + tcg_out_movi(s, addr_type, TCG_TMP1, s->page_mask | a_mask); if (a_mask < s_mask) { - tcg_out_opc_imm(s, ALIAS_TADDI, TCG_TMP2, addrlo, s_mask - a_mask); + if (TCG_TARGET_REG_BITS == 32 || addr_type == TCG_TYPE_I32) { + tcg_out_opc_imm(s, OPC_ADDIU, TCG_TMP2, addrlo, s_mask - a_mask); + } else { + tcg_out_opc_imm(s, OPC_DADDIU, TCG_TMP2, addrlo, s_mask - a_mask); + } tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, TCG_TMP2); } else { tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, addrlo); } /* Zero extend a 32-bit guest address for a 64-bit host. */ - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS == 64 && addr_type == TCG_TYPE_I32) { tcg_out_ext32u(s, TCG_TMP2, addrlo); addrlo = TCG_TMP2; } @@ -1307,7 +1236,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_opc_br(s, OPC_BNE, TCG_TMP1, TCG_TMP0); /* Load and test the high half tlb comparator. */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS == 32 && addr_type != TCG_TYPE_I32) { /* delay slot */ tcg_out_ldst(s, OPC_LW, TCG_TMP0, TCG_TMP3, cmp_off + HI_OFF); @@ -1344,7 +1273,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, } base = addrlo; - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS == 64 && addr_type == TCG_TYPE_I32) { tcg_out_ext32u(s, TCG_REG_A0, base); base = TCG_REG_A0; } @@ -1360,7 +1289,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, #endif h->base = base; - h->align = a_bits; return ldst; } @@ -1473,7 +1401,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, ldst = prepare_host_addr(s, &h, addrlo, addrhi, oi, true); - if (use_mips32r6_instructions || h.align >= (opc & MO_SIZE)) { + if (use_mips32r6_instructions || h.aa.align >= (opc & MO_SIZE)) { tcg_out_qemu_ld_direct(s, datalo, datahi, h.base, opc, data_type); } else { tcg_out_qemu_ld_unalign(s, datalo, datahi, h.base, opc, data_type); @@ -1560,7 +1488,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, ldst = prepare_host_addr(s, &h, addrlo, addrhi, oi, false); - if (use_mips32r6_instructions || h.align >= (opc & MO_SIZE)) { + if (use_mips32r6_instructions || h.aa.align >= (opc & MO_SIZE)) { tcg_out_qemu_st_direct(s, datalo, datahi, h.base, opc); } else { tcg_out_qemu_st_unalign(s, datalo, datahi, h.base, opc); @@ -2030,34 +1958,49 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]); break; - case INDEX_op_qemu_ld_i32: - if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { - tcg_out_qemu_ld(s, a0, 0, a1, 0, a2, TCG_TYPE_I32); - } else { + case INDEX_op_qemu_ld_a64_i32: + if (TCG_TARGET_REG_BITS == 32) { tcg_out_qemu_ld(s, a0, 0, a1, a2, args[3], TCG_TYPE_I32); + break; } + /* fall through */ + case INDEX_op_qemu_ld_a32_i32: + tcg_out_qemu_ld(s, a0, 0, a1, 0, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i64: if (TCG_TARGET_REG_BITS == 64) { tcg_out_qemu_ld(s, a0, 0, a1, 0, a2, TCG_TYPE_I64); - } else if (TARGET_LONG_BITS == 32) { + } else { tcg_out_qemu_ld(s, a0, a1, a2, 0, args[3], TCG_TYPE_I64); + } + break; + case INDEX_op_qemu_ld_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_ld(s, a0, 0, a1, 0, a2, TCG_TYPE_I64); } else { tcg_out_qemu_ld(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); } break; - case INDEX_op_qemu_st_i32: - if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { - tcg_out_qemu_st(s, a0, 0, a1, 0, a2, TCG_TYPE_I32); - } else { + + case INDEX_op_qemu_st_a64_i32: + if (TCG_TARGET_REG_BITS == 32) { tcg_out_qemu_st(s, a0, 0, a1, a2, args[3], TCG_TYPE_I32); + break; } + /* fall through */ + case INDEX_op_qemu_st_a32_i32: + tcg_out_qemu_st(s, a0, 0, a1, 0, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i64: if (TCG_TARGET_REG_BITS == 64) { tcg_out_qemu_st(s, a0, 0, a1, 0, a2, TCG_TYPE_I64); - } else if (TARGET_LONG_BITS == 32) { + } else { tcg_out_qemu_st(s, a0, a1, a2, 0, args[3], TCG_TYPE_I64); + } + break; + case INDEX_op_qemu_st_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_st(s, a0, 0, a1, 0, a2, TCG_TYPE_I64); } else { tcg_out_qemu_st(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); } @@ -2216,19 +2159,22 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_brcond2_i32: return C_O0_I4(rZ, rZ, rZ, rZ); - case INDEX_op_qemu_ld_i32: - return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 - ? C_O1_I1(r, r) : C_O1_I2(r, r, r)); - case INDEX_op_qemu_st_i32: - return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 - ? C_O0_I2(rZ, r) : C_O0_I3(rZ, r, r)); - case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) - : TARGET_LONG_BITS == 32 ? C_O2_I1(r, r, r) - : C_O2_I2(r, r, r, r)); - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_ld_a32_i32: + return C_O1_I1(r, r); + case INDEX_op_qemu_ld_a64_i32: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O1_I2(r, r, r); + case INDEX_op_qemu_st_a32_i32: + return C_O0_I2(rZ, r); + case INDEX_op_qemu_st_a64_i32: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(rZ, r) : C_O0_I3(rZ, r, r); + case INDEX_op_qemu_ld_a32_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O2_I1(r, r, r); + case INDEX_op_qemu_ld_a64_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O2_I2(r, r, r, r); + case INDEX_op_qemu_st_a32_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(rZ, r) : C_O0_I3(rZ, rZ, r); + case INDEX_op_qemu_st_a64_i64: return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(rZ, r) - : TARGET_LONG_BITS == 32 ? C_O0_I3(rZ, rZ, r) : C_O0_I4(rZ, rZ, r, r)); default: diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index 42bd7fff01..7277a117ef 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -204,9 +204,9 @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_HAS_ext16u_i64 0 /* andi rt, rs, 0xffff */ #endif -#define TCG_TARGET_DEFAULT_MO 0 -#define TCG_TARGET_HAS_MEMORY_BSWAP 0 +#define TCG_TARGET_HAS_qemu_ldst_i128 0 +#define TCG_TARGET_DEFAULT_MO 0 #define TCG_TARGET_NEED_LDST_LABELS #endif diff --git a/tcg/optimize.c b/tcg/optimize.c index 9614fa3638..bf975a3a6c 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -2184,13 +2184,22 @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(orc): done = fold_orc(&ctx, op); break; - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: + case INDEX_op_qemu_ld_a32_i128: + case INDEX_op_qemu_ld_a64_i128: done = fold_qemu_ld(&ctx, op); break; - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st8_i32: - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st8_a32_i32: + case INDEX_op_qemu_st8_a64_i32: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: + case INDEX_op_qemu_st_a32_i128: + case INDEX_op_qemu_st_a64_i128: done = fold_qemu_st(&ctx, op); break; CASE_OP_32_64(rem): diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 29bfbfcc61..d4269dffcf 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -1962,33 +1962,6 @@ static const uint32_t qemu_stx_opc[(MO_SIZE + MO_BSWAP) + 1] = { [MO_BSWAP | MO_UQ] = STDBRX, }; -#if defined (CONFIG_SOFTMMU) -/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr, - * int mmu_idx, uintptr_t ra) - */ -static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = { - [MO_UB] = helper_ret_ldub_mmu, - [MO_LEUW] = helper_le_lduw_mmu, - [MO_LEUL] = helper_le_ldul_mmu, - [MO_LEUQ] = helper_le_ldq_mmu, - [MO_BEUW] = helper_be_lduw_mmu, - [MO_BEUL] = helper_be_ldul_mmu, - [MO_BEUQ] = helper_be_ldq_mmu, -}; - -/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr, - * uintxx_t val, int mmu_idx, uintptr_t ra) - */ -static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { - [MO_UB] = helper_ret_stb_mmu, - [MO_LEUW] = helper_le_stw_mmu, - [MO_LEUL] = helper_le_stl_mmu, - [MO_LEUQ] = helper_le_stq_mmu, - [MO_BEUW] = helper_be_stw_mmu, - [MO_BEUL] = helper_be_stl_mmu, - [MO_BEUQ] = helper_be_stq_mmu, -}; - static TCGReg ldst_ra_gen(TCGContext *s, const TCGLabelQemuLdst *l, int arg) { if (arg < 0) { @@ -2017,7 +1990,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) } tcg_out_ld_helper_args(s, lb, &ldst_helper_param); - tcg_out_call_int(s, LK, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_call_int(s, LK, qemu_ld_helpers[opc & MO_SIZE]); tcg_out_ld_helper_ret(s, lb, false, &ldst_helper_param); tcg_out_b(s, 0, lb->raddr); @@ -2033,60 +2006,23 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) } tcg_out_st_helper_args(s, lb, &ldst_helper_param); - tcg_out_call_int(s, LK, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_call_int(s, LK, qemu_st_helpers[opc & MO_SIZE]); tcg_out_b(s, 0, lb->raddr); return true; } -#else -static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) -{ - if (!reloc_pc14(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { - return false; - } - - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - TCGReg arg = TCG_REG_R4; - - arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN); - if (l->addrlo_reg != arg) { - tcg_out_mov(s, TCG_TYPE_I32, arg, l->addrhi_reg); - tcg_out_mov(s, TCG_TYPE_I32, arg + 1, l->addrlo_reg); - } else if (l->addrhi_reg != arg + 1) { - tcg_out_mov(s, TCG_TYPE_I32, arg + 1, l->addrlo_reg); - tcg_out_mov(s, TCG_TYPE_I32, arg, l->addrhi_reg); - } else { - tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R0, arg); - tcg_out_mov(s, TCG_TYPE_I32, arg, arg + 1); - tcg_out_mov(s, TCG_TYPE_I32, arg + 1, TCG_REG_R0); - } - } else { - tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R4, l->addrlo_reg); - } - tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R3, TCG_AREG0); - - /* "Tail call" to the helper, with the return address back inline. */ - tcg_out_call_int(s, 0, (const void *)(l->is_ld ? helper_unaligned_ld - : helper_unaligned_st)); - return true; -} - -static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} - -static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} -#endif /* SOFTMMU */ typedef struct { TCGReg base; TCGReg index; + TCGAtomAlign aa; } HostAddress; +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return true; +} + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -2099,7 +2035,23 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, { TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); - unsigned a_bits = get_alignment_bits(opc); + MemOp a_bits; + + /* + * Book II, Section 1.4, Single-Copy Atomicity, specifies: + * + * Before 3.0, "An access that is not atomic is performed as a set of + * smaller disjoint atomic accesses. In general, the number and alignment + * of these accesses are implementation-dependent." Thus MO_ATOM_IFALIGN. + * + * As of 3.0, "the non-atomic access is performed as described in + * the corresponding list", which matches MO_ATOM_SUBALIGN. + */ + h->aa = atom_and_align_for_opc(s, opc, + have_isa_3_00 ? MO_ATOM_SUBALIGN + : MO_ATOM_IFALIGN, + false); + a_bits = h->aa.align; #ifdef CONFIG_SOFTMMU int mem_index = get_mmuidx(oi); @@ -2125,10 +2077,10 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, /* Extract the page index, shifted into place for tlb index. */ if (TCG_TARGET_REG_BITS == 32) { tcg_out_shri32(s, TCG_REG_R0, addrlo, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + s->page_bits - CPU_TLB_ENTRY_BITS); } else { tcg_out_shri64(s, TCG_REG_R0, addrlo, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + s->page_bits - CPU_TLB_ENTRY_BITS); } tcg_out32(s, AND | SAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_R0)); @@ -2167,7 +2119,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, a_bits = s_bits; } tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0, - (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS); + (32 - a_bits) & 31, 31 - s->page_bits); } else { TCGReg t = addrlo; @@ -2188,13 +2140,13 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, /* Mask the address for the requested alignment. */ if (TARGET_LONG_BITS == 32) { tcg_out_rlw(s, RLWINM, TCG_REG_R0, t, 0, - (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS); + (32 - a_bits) & 31, 31 - s->page_bits); } else if (a_bits == 0) { - tcg_out_rld(s, RLDICR, TCG_REG_R0, t, 0, 63 - TARGET_PAGE_BITS); + tcg_out_rld(s, RLDICR, TCG_REG_R0, t, 0, 63 - s->page_bits); } else { tcg_out_rld(s, RLDICL, TCG_REG_R0, t, - 64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - a_bits); - tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0); + 64 - s->page_bits, s->page_bits - a_bits); + tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, s->page_bits, 0); } } @@ -2880,43 +2832,58 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out32(s, MODUD | TAB(args[0], args[1], args[2])); break; - case INDEX_op_qemu_ld_i32: - if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { - tcg_out_qemu_ld(s, args[0], -1, args[1], -1, - args[2], TCG_TYPE_I32); - } else { + case INDEX_op_qemu_ld_a64_i32: + if (TCG_TARGET_REG_BITS == 32) { tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], args[3], TCG_TYPE_I32); + break; } + /* fall through */ + case INDEX_op_qemu_ld_a32_i32: + tcg_out_qemu_ld(s, args[0], -1, args[1], -1, args[2], TCG_TYPE_I32); break; - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i64: if (TCG_TARGET_REG_BITS == 64) { tcg_out_qemu_ld(s, args[0], -1, args[1], -1, args[2], TCG_TYPE_I64); - } else if (TARGET_LONG_BITS == 32) { + } else { tcg_out_qemu_ld(s, args[0], args[1], args[2], -1, args[3], TCG_TYPE_I64); + } + break; + case INDEX_op_qemu_ld_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_ld(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I64); } else { tcg_out_qemu_ld(s, args[0], args[1], args[2], args[3], args[4], TCG_TYPE_I64); } break; - case INDEX_op_qemu_st_i32: - if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { - tcg_out_qemu_st(s, args[0], -1, args[1], -1, - args[2], TCG_TYPE_I32); - } else { + + case INDEX_op_qemu_st_a64_i32: + if (TCG_TARGET_REG_BITS == 32) { tcg_out_qemu_st(s, args[0], -1, args[1], args[2], args[3], TCG_TYPE_I32); + break; } + /* fall through */ + case INDEX_op_qemu_st_a32_i32: + tcg_out_qemu_st(s, args[0], -1, args[1], -1, args[2], TCG_TYPE_I32); break; - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i64: if (TCG_TARGET_REG_BITS == 64) { tcg_out_qemu_st(s, args[0], -1, args[1], -1, args[2], TCG_TYPE_I64); - } else if (TARGET_LONG_BITS == 32) { + } else { tcg_out_qemu_st(s, args[0], args[1], args[2], -1, args[3], TCG_TYPE_I64); + } + break; + case INDEX_op_qemu_st_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_st(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I64); } else { tcg_out_qemu_st(s, args[0], args[1], args[2], args[3], args[4], TCG_TYPE_I64); @@ -3737,25 +3704,23 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_sub2_i32: return C_O2_I4(r, r, rI, rZM, r, r); - case INDEX_op_qemu_ld_i32: - return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 - ? C_O1_I1(r, r) - : C_O1_I2(r, r, r)); - - case INDEX_op_qemu_st_i32: - return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 - ? C_O0_I2(r, r) - : C_O0_I3(r, r, r)); - - case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) - : TARGET_LONG_BITS == 32 ? C_O2_I1(r, r, r) - : C_O2_I2(r, r, r, r)); - - case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) - : TARGET_LONG_BITS == 32 ? C_O0_I3(r, r, r) - : C_O0_I4(r, r, r, r)); + case INDEX_op_qemu_ld_a32_i32: + return C_O1_I1(r, r); + case INDEX_op_qemu_ld_a64_i32: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O1_I2(r, r, r); + case INDEX_op_qemu_ld_a32_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O2_I1(r, r, r); + case INDEX_op_qemu_ld_a64_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O2_I2(r, r, r, r); + + case INDEX_op_qemu_st_a32_i32: + return C_O0_I2(r, r); + case INDEX_op_qemu_st_a64_i32: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) : C_O0_I3(r, r, r); + case INDEX_op_qemu_st_a32_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) : C_O0_I3(r, r, r); + case INDEX_op_qemu_st_a64_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) : C_O0_I4(r, r, r, r); case INDEX_op_add_vec: case INDEX_op_sub_vec: diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index af81c5a57f..0914380bd7 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -149,6 +149,8 @@ extern bool have_vsx; #define TCG_TARGET_HAS_mulsh_i64 1 #endif +#define TCG_TARGET_HAS_qemu_ldst_i128 0 + /* * While technically Altivec could support V64, it has no 64-bit store * instruction and substituting two 32-bit stores makes the generated @@ -179,7 +181,6 @@ extern bool have_vsx; #define TCG_TARGET_HAS_cmpsel_vec 0 #define TCG_TARGET_DEFAULT_MO (0) -#define TCG_TARGET_HAS_MEMORY_BSWAP 1 #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index d12b824d8c..ff6334980f 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -846,49 +846,6 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) * Load/store and TLB */ -#if defined(CONFIG_SOFTMMU) -/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, - * MemOpIdx oi, uintptr_t ra) - */ -static void * const qemu_ld_helpers[MO_SSIZE + 1] = { - [MO_UB] = helper_ret_ldub_mmu, - [MO_SB] = helper_ret_ldsb_mmu, -#if HOST_BIG_ENDIAN - [MO_UW] = helper_be_lduw_mmu, - [MO_SW] = helper_be_ldsw_mmu, - [MO_UL] = helper_be_ldul_mmu, -#if TCG_TARGET_REG_BITS == 64 - [MO_SL] = helper_be_ldsl_mmu, -#endif - [MO_UQ] = helper_be_ldq_mmu, -#else - [MO_UW] = helper_le_lduw_mmu, - [MO_SW] = helper_le_ldsw_mmu, - [MO_UL] = helper_le_ldul_mmu, -#if TCG_TARGET_REG_BITS == 64 - [MO_SL] = helper_le_ldsl_mmu, -#endif - [MO_UQ] = helper_le_ldq_mmu, -#endif -}; - -/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, - * uintxx_t val, MemOpIdx oi, - * uintptr_t ra) - */ -static void * const qemu_st_helpers[MO_SIZE + 1] = { - [MO_8] = helper_ret_stb_mmu, -#if HOST_BIG_ENDIAN - [MO_16] = helper_be_stw_mmu, - [MO_32] = helper_be_stl_mmu, - [MO_64] = helper_be_stq_mmu, -#else - [MO_16] = helper_le_stw_mmu, - [MO_32] = helper_le_stl_mmu, - [MO_64] = helper_le_stq_mmu, -#endif -}; - static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) { tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); @@ -896,6 +853,11 @@ static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) tcg_debug_assert(ok); } +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return false; +} + /* We have three temps, we might as well expose them. */ static const TCGLdstHelperParam ldst_helper_param = { .ntmp = 3, .tmp = { TCG_REG_TMP0, TCG_REG_TMP1, TCG_REG_TMP2 } @@ -935,34 +897,6 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_goto(s, l->raddr); return true; } -#else -static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) -{ - /* resolve label address */ - if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { - return false; - } - - tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_A1, l->addrlo_reg); - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0); - - /* tail call, with the return address back inline. */ - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (uintptr_t)l->raddr); - tcg_out_call_int(s, (const void *)(l->is_ld ? helper_unaligned_ld - : helper_unaligned_st), true); - return true; -} - -static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} - -static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} -#endif /* CONFIG_SOFTMMU */ /* * For softmmu, perform the TLB load and compare. @@ -976,17 +910,21 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, { TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); - unsigned a_bits = get_alignment_bits(opc); - unsigned a_mask = (1u << a_bits) - 1; + TCGAtomAlign aa; + unsigned a_mask; + + aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false); + a_mask = (1u << aa.align) - 1; #ifdef CONFIG_SOFTMMU unsigned s_bits = opc & MO_SIZE; + unsigned s_mask = (1u << s_bits) - 1; int mem_index = get_mmuidx(oi); int fast_ofs = TLB_MASK_TABLE_OFS(mem_index); int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); - TCGReg mask_base = TCG_AREG0, table_base = TCG_AREG0; - tcg_target_long compare_mask; + int compare_mask; + TCGReg addr_adj; ldst = new_ldst_label(s); ldst->is_ld = is_ld; @@ -995,14 +933,33 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, mask_base, mask_ofs); - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, table_base, table_ofs); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_AREG0, mask_ofs); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_AREG0, table_ofs); tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addr_reg, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + s->page_bits - CPU_TLB_ENTRY_BITS); tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); + /* + * For aligned accesses, we check the first byte and include the alignment + * bits within the address. For unaligned access, we check that we don't + * cross pages using the address of the last byte of the access. + */ + addr_adj = addr_reg; + if (a_mask < s_mask) { + addr_adj = TCG_REG_TMP0; + tcg_out_opc_imm(s, TARGET_LONG_BITS == 32 ? OPC_ADDIW : OPC_ADDI, + addr_adj, addr_reg, s_mask - a_mask); + } + compare_mask = s->page_mask | a_mask; + if (compare_mask == sextreg(compare_mask, 0, 12)) { + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_adj, compare_mask); + } else { + tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); + tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addr_adj); + } + /* Load the tlb comparator and the addend. */ tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP0, TCG_REG_TMP2, is_ld ? offsetof(CPUTLBEntry, addr_read) @@ -1010,29 +967,17 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2, offsetof(CPUTLBEntry, addend)); - /* We don't support unaligned accesses. */ - if (a_bits < s_bits) { - a_bits = s_bits; - } - /* Clear the non-page, non-alignment bits from the address. */ - compare_mask = (tcg_target_long)TARGET_PAGE_MASK | a_mask; - if (compare_mask == sextreg(compare_mask, 0, 12)) { - tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, compare_mask); - } else { - tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); - tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addr_reg); - } - /* Compare masked address with the TLB entry. */ ldst->label_ptr[0] = s->code_ptr; tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); /* TLB Hit - translate address using addend. */ + addr_adj = addr_reg; if (TARGET_LONG_BITS == 32) { - tcg_out_ext32u(s, TCG_REG_TMP0, addr_reg); - addr_reg = TCG_REG_TMP0; + addr_adj = TCG_REG_TMP0; + tcg_out_ext32u(s, addr_adj, addr_reg); } - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addr_reg); + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addr_adj); *pbase = TCG_REG_TMP0; #else if (a_mask) { @@ -1041,8 +986,8 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, ldst->oi = oi; ldst->addrlo_reg = addr_reg; - /* We are expecting a_bits max 7, so we can always use andi. */ - tcg_debug_assert(a_bits < 12); + /* We are expecting alignment max 7, so we can always use andi. */ + tcg_debug_assert(a_mask == sextreg(a_mask, 0, 12)); tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, a_mask); ldst->label_ptr[0] = s->code_ptr; @@ -1437,16 +1382,20 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_setcond(s, args[3], a0, a1, a2); break; - case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; - case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; @@ -1588,11 +1537,15 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_sub2_i64: return C_O2_I4(r, r, rZ, rZ, rM, rM); - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: return C_O1_I1(r, r); - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: return C_O0_I2(rZ, r); default: diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index dddf2486c1..494c986b49 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -163,11 +163,11 @@ typedef enum { #define TCG_TARGET_HAS_muluh_i64 1 #define TCG_TARGET_HAS_mulsh_i64 1 +#define TCG_TARGET_HAS_qemu_ldst_i128 0 + #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS -#define TCG_TARGET_HAS_MEMORY_BSWAP 0 - #endif diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index aacbaf21d5..dfaa34c264 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -438,33 +438,6 @@ static const uint8_t tcg_cond_to_ltr_cond[] = { [TCG_COND_GEU] = S390_CC_ALWAYS, }; -#ifdef CONFIG_SOFTMMU -static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = { - [MO_UB] = helper_ret_ldub_mmu, - [MO_SB] = helper_ret_ldsb_mmu, - [MO_LEUW] = helper_le_lduw_mmu, - [MO_LESW] = helper_le_ldsw_mmu, - [MO_LEUL] = helper_le_ldul_mmu, - [MO_LESL] = helper_le_ldsl_mmu, - [MO_LEUQ] = helper_le_ldq_mmu, - [MO_BEUW] = helper_be_lduw_mmu, - [MO_BESW] = helper_be_ldsw_mmu, - [MO_BEUL] = helper_be_ldul_mmu, - [MO_BESL] = helper_be_ldsl_mmu, - [MO_BEUQ] = helper_be_ldq_mmu, -}; - -static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { - [MO_UB] = helper_ret_stb_mmu, - [MO_LEUW] = helper_le_stw_mmu, - [MO_LEUL] = helper_le_stl_mmu, - [MO_LEUQ] = helper_le_stq_mmu, - [MO_BEUW] = helper_be_stw_mmu, - [MO_BEUL] = helper_be_stl_mmu, - [MO_BEUQ] = helper_be_stq_mmu, -}; -#endif - static const tcg_insn_unit *tb_ret_addr; uint64_t s390_facilities[3]; @@ -1599,8 +1572,14 @@ typedef struct { TCGReg base; TCGReg index; int disp; + TCGAtomAlign aa; } HostAddress; +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return true; +} + static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data, HostAddress h) { @@ -1706,7 +1685,6 @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data, } } -#if defined(CONFIG_SOFTMMU) static const TCGLdstHelperParam ldst_helper_param = { .ntmp = 1, .tmp = { TCG_TMP0 } }; @@ -1721,7 +1699,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) } tcg_out_ld_helper_args(s, lb, &ldst_helper_param); - tcg_out_call_int(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SIZE]); tcg_out_ld_helper_ret(s, lb, false, &ldst_helper_param); tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr); @@ -1738,39 +1716,11 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) } tcg_out_st_helper_args(s, lb, &ldst_helper_param); - tcg_out_call_int(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE]); tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr); return true; } -#else -static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) -{ - if (!patch_reloc(l->label_ptr[0], R_390_PC16DBL, - (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) { - return false; - } - - tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_R3, l->addrlo_reg); - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0); - - /* "Tail call" to the helper, with the return address back inline. */ - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R14, (uintptr_t)l->raddr); - tgen_gotoi(s, S390_CC_ALWAYS, (const void *)(l->is_ld ? helper_unaligned_ld - : helper_unaligned_st)); - return true; -} - -static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} - -static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) -{ - return tcg_out_fail_alignment(s, l); -} -#endif /* CONFIG_SOFTMMU */ /* * For softmmu, perform the TLB load and compare. @@ -1784,8 +1734,10 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, { TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); - unsigned a_bits = get_alignment_bits(opc); - unsigned a_mask = (1u << a_bits) - 1; + unsigned a_mask; + + h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false); + a_mask = (1 << h->aa.align) - 1; #ifdef CONFIG_SOFTMMU unsigned s_bits = opc & MO_SIZE; @@ -1803,7 +1755,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->addrlo_reg = addr_reg; tcg_out_sh64(s, RSY_SRLG, TCG_TMP0, addr_reg, TCG_REG_NONE, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + s->page_bits - CPU_TLB_ENTRY_BITS); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 19)); @@ -1815,8 +1767,8 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, * bits within the address. For unaligned access, we check that we don't * cross pages using the address of the last byte of the access. */ - a_off = (a_bits >= s_bits ? 0 : s_mask - a_mask); - tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask; + a_off = (a_mask >= s_mask ? 0 : s_mask - a_mask); + tlb_mask = (uint64_t)s->page_mask | a_mask; if (a_off == 0) { tgen_andi_risbg(s, TCG_REG_R0, addr_reg, tlb_mask); } else { @@ -1857,7 +1809,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->addrlo_reg = addr_reg; /* We are expecting a_bits to max out at 7, much lower than TMLL. */ - tcg_debug_assert(a_bits < 16); + tcg_debug_assert(a_mask <= 0xffff); tcg_out_insn(s, RI, TMLL, addr_reg, a_mask); tcg_out16(s, RI_BRC | (7 << 4)); /* CC in {1,2,3} */ @@ -2258,16 +2210,20 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, args[2], const_args[2], args[3], const_args[3], args[4]); break; - case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: tcg_out_qemu_ld(s, args[0], args[1], args[2], TCG_TYPE_I32); break; - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: tcg_out_qemu_ld(s, args[0], args[1], args[2], TCG_TYPE_I64); break; - case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: tcg_out_qemu_st(s, args[0], args[1], args[2], TCG_TYPE_I32); break; - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: tcg_out_qemu_st(s, args[0], args[1], args[2], TCG_TYPE_I64); break; @@ -3141,11 +3097,15 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_ctpop_i64: return C_O1_I1(r, r); - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: return C_O1_I1(r, r); - case INDEX_op_qemu_st_i64: - case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: return C_O0_I2(r, r); case INDEX_op_deposit_i32: diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index a05b473117..170007bea5 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -140,6 +140,8 @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_muluh_i64 0 #define TCG_TARGET_HAS_mulsh_i64 0 +#define TCG_TARGET_HAS_qemu_ldst_i128 0 + #define TCG_TARGET_HAS_v64 HAVE_FACILITY(VECTOR) #define TCG_TARGET_HAS_v128 HAVE_FACILITY(VECTOR) #define TCG_TARGET_HAS_v256 0 @@ -172,8 +174,6 @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_BY_REF #define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_BY_REF -#define TCG_TARGET_HAS_MEMORY_BSWAP 1 - #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/sparc64/tcg-target-con-set.h b/tcg/sparc64/tcg-target-con-set.h index 31e6fea1fc..434bf25072 100644 --- a/tcg/sparc64/tcg-target-con-set.h +++ b/tcg/sparc64/tcg-target-con-set.h @@ -12,8 +12,6 @@ C_O0_I1(r) C_O0_I2(rZ, r) C_O0_I2(rZ, rJ) -C_O0_I2(sZ, s) -C_O1_I1(r, s) C_O1_I1(r, r) C_O1_I2(r, r, r) C_O1_I2(r, rZ, rJ) diff --git a/tcg/sparc64/tcg-target-con-str.h b/tcg/sparc64/tcg-target-con-str.h index 8f5c7aef97..0577ec4942 100644 --- a/tcg/sparc64/tcg-target-con-str.h +++ b/tcg/sparc64/tcg-target-con-str.h @@ -9,7 +9,6 @@ * REGS(letter, register_mask) */ REGS('r', ALL_GENERAL_REGS) -REGS('s', ALL_QLDST_REGS) /* * Define constraint letters for constants: diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index 7e6466d3b6..d2d0f604c2 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -27,6 +27,7 @@ #error "unsupported code generation mode" #endif +#include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" #ifdef CONFIG_DEBUG_TCG @@ -70,22 +71,12 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { #define TCG_CT_CONST_S13 0x200 #define TCG_CT_CONST_ZERO 0x400 -/* - * For softmmu, we need to avoid conflicts with the first 3 - * argument registers to perform the tlb lookup, and to call - * the helper function. - */ -#ifdef CONFIG_SOFTMMU -#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_O0, 3) -#else -#define SOFTMMU_RESERVE_REGS 0 -#endif -#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) -#define ALL_QLDST_REGS (ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS) +#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) -/* Define some temporary registers. T2 is used for constant generation. */ +/* Define some temporary registers. T3 is used for constant generation. */ #define TCG_REG_T1 TCG_REG_G1 -#define TCG_REG_T2 TCG_REG_O7 +#define TCG_REG_T2 TCG_REG_G2 +#define TCG_REG_T3 TCG_REG_O7 #ifndef CONFIG_SOFTMMU # define TCG_GUEST_BASE_REG TCG_REG_I5 @@ -110,7 +101,6 @@ static const int tcg_target_reg_alloc_order[] = { TCG_REG_I4, TCG_REG_I5, - TCG_REG_G2, TCG_REG_G3, TCG_REG_G4, TCG_REG_G5, @@ -399,22 +389,25 @@ static void tcg_out_sethi(TCGContext *s, TCGReg ret, uint32_t arg) tcg_out32(s, SETHI | INSN_RD(ret) | ((arg & 0xfffffc00) >> 10)); } -static void tcg_out_movi_imm13(TCGContext *s, TCGReg ret, int32_t arg) +/* A 13-bit constant sign-extended to 64 bits. */ +static void tcg_out_movi_s13(TCGContext *s, TCGReg ret, int32_t arg) { tcg_out_arithi(s, ret, TCG_REG_G0, arg, ARITH_OR); } -static void tcg_out_movi_imm32(TCGContext *s, TCGReg ret, int32_t arg) +/* A 32-bit constant sign-extended to 64 bits. */ +static void tcg_out_movi_s32(TCGContext *s, TCGReg ret, int32_t arg) { - if (check_fit_i32(arg, 13)) { - /* A 13-bit constant sign-extended to 64-bits. */ - tcg_out_movi_imm13(s, ret, arg); - } else { - /* A 32-bit constant zero-extended to 64 bits. */ - tcg_out_sethi(s, ret, arg); - if (arg & 0x3ff) { - tcg_out_arithi(s, ret, ret, arg & 0x3ff, ARITH_OR); - } + tcg_out_sethi(s, ret, ~arg); + tcg_out_arithi(s, ret, ret, (arg & 0x3ff) | -0x400, ARITH_XOR); +} + +/* A 32-bit constant zero-extended to 64 bits. */ +static void tcg_out_movi_u32(TCGContext *s, TCGReg ret, uint32_t arg) +{ + tcg_out_sethi(s, ret, arg); + if (arg & 0x3ff) { + tcg_out_arithi(s, ret, ret, arg & 0x3ff, ARITH_OR); } } @@ -425,15 +418,15 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long hi, lo = (int32_t)arg; tcg_target_long test, lsb; - /* A 32-bit constant, or 32-bit zero-extended to 64-bits. */ - if (type == TCG_TYPE_I32 || arg == (uint32_t)arg) { - tcg_out_movi_imm32(s, ret, arg); + /* A 13-bit constant sign-extended to 64-bits. */ + if (check_fit_tl(arg, 13)) { + tcg_out_movi_s13(s, ret, arg); return; } - /* A 13-bit constant sign-extended to 64-bits. */ - if (check_fit_tl(arg, 13)) { - tcg_out_movi_imm13(s, ret, arg); + /* A 32-bit constant, or 32-bit zero-extended to 64-bits. */ + if (type == TCG_TYPE_I32 || arg == (uint32_t)arg) { + tcg_out_movi_u32(s, ret, arg); return; } @@ -448,8 +441,7 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, /* A 32-bit constant sign-extended to 64-bits. */ if (arg == lo) { - tcg_out_sethi(s, ret, ~arg); - tcg_out_arithi(s, ret, ret, (arg & 0x3ff) | -0x400, ARITH_XOR); + tcg_out_movi_s32(s, ret, arg); return; } @@ -477,13 +469,13 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, /* A 64-bit constant decomposed into 2 32-bit pieces. */ if (check_fit_i32(lo, 13)) { hi = (arg - lo) >> 32; - tcg_out_movi_imm32(s, ret, hi); + tcg_out_movi_u32(s, ret, hi); tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX); tcg_out_arithi(s, ret, ret, lo, ARITH_ADD); } else { hi = arg >> 32; - tcg_out_movi_imm32(s, ret, hi); - tcg_out_movi_imm32(s, scratch, lo); + tcg_out_movi_u32(s, ret, hi); + tcg_out_movi_u32(s, scratch, lo); tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX); tcg_out_arith(s, ret, ret, scratch, ARITH_OR); } @@ -492,8 +484,8 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret, tcg_target_long arg) { - tcg_debug_assert(ret != TCG_REG_T2); - tcg_out_movi_int(s, type, ret, arg, false, TCG_REG_T2); + tcg_debug_assert(ret != TCG_REG_T3); + tcg_out_movi_int(s, type, ret, arg, false, TCG_REG_T3); } static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) @@ -767,7 +759,7 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, default: tcg_out_cmp(s, c1, c2, c2const); - tcg_out_movi_imm13(s, ret, 0); + tcg_out_movi_s13(s, ret, 0); tcg_out_movcc(s, cond, MOVCC_ICC, ret, 1, 1); return; } @@ -803,11 +795,11 @@ static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, /* For 64-bit signed comparisons vs zero, we can avoid the compare if the input does not overlap the output. */ if (c2 == 0 && !is_unsigned_cond(cond) && c1 != ret) { - tcg_out_movi_imm13(s, ret, 0); + tcg_out_movi_s13(s, ret, 0); tcg_out_movr(s, cond, ret, c1, 1, 1); } else { tcg_out_cmp(s, c1, c2, c2const); - tcg_out_movi_imm13(s, ret, 0); + tcg_out_movi_s13(s, ret, 0); tcg_out_movcc(s, cond, MOVCC_XCC, ret, 1, 1); } } @@ -844,7 +836,7 @@ static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh, if (use_vis3_instructions && !is_sub) { /* Note that ADDXC doesn't accept immediates. */ if (bhconst && bh != 0) { - tcg_out_movi_imm13(s, TCG_REG_T2, bh); + tcg_out_movi_s13(s, TCG_REG_T2, bh); bh = TCG_REG_T2; } tcg_out_arith(s, rh, ah, bh, ARITH_ADDXC); @@ -866,7 +858,7 @@ static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh, * so the adjustment fits 12 bits. */ if (bhconst) { - tcg_out_movi_imm13(s, TCG_REG_T2, bh + (is_sub ? -1 : 1)); + tcg_out_movi_s13(s, TCG_REG_T2, bh + (is_sub ? -1 : 1)); } else { tcg_out_arithi(s, TCG_REG_T2, bh, 1, is_sub ? ARITH_SUB : ARITH_ADD); @@ -885,10 +877,8 @@ static void tcg_out_jmpl_const(TCGContext *s, const tcg_insn_unit *dest, { uintptr_t desti = (uintptr_t)dest; - /* Be careful not to clobber %o7 for a tail call. */ tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_REG_T1, - desti & ~0xfff, in_prologue, - tail_call ? TCG_REG_G2 : TCG_REG_O7); + desti & ~0xfff, in_prologue, TCG_REG_T2); tcg_out_arithi(s, tail_call ? TCG_REG_G0 : TCG_REG_O7, TCG_REG_T1, desti & 0xfff, JMPL); } @@ -918,104 +908,6 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) tcg_out32(s, MEMBAR | (a0 & TCG_MO_ALL)); } -#ifdef CONFIG_SOFTMMU -static const tcg_insn_unit *qemu_ld_trampoline[(MO_SSIZE | MO_BSWAP) + 1]; -static const tcg_insn_unit *qemu_st_trampoline[(MO_SIZE | MO_BSWAP) + 1]; - -static void build_trampolines(TCGContext *s) -{ - static void * const qemu_ld_helpers[] = { - [MO_UB] = helper_ret_ldub_mmu, - [MO_SB] = helper_ret_ldsb_mmu, - [MO_LEUW] = helper_le_lduw_mmu, - [MO_LESW] = helper_le_ldsw_mmu, - [MO_LEUL] = helper_le_ldul_mmu, - [MO_LEUQ] = helper_le_ldq_mmu, - [MO_BEUW] = helper_be_lduw_mmu, - [MO_BESW] = helper_be_ldsw_mmu, - [MO_BEUL] = helper_be_ldul_mmu, - [MO_BEUQ] = helper_be_ldq_mmu, - }; - static void * const qemu_st_helpers[] = { - [MO_UB] = helper_ret_stb_mmu, - [MO_LEUW] = helper_le_stw_mmu, - [MO_LEUL] = helper_le_stl_mmu, - [MO_LEUQ] = helper_le_stq_mmu, - [MO_BEUW] = helper_be_stw_mmu, - [MO_BEUL] = helper_be_stl_mmu, - [MO_BEUQ] = helper_be_stq_mmu, - }; - - int i; - - for (i = 0; i < ARRAY_SIZE(qemu_ld_helpers); ++i) { - if (qemu_ld_helpers[i] == NULL) { - continue; - } - - /* May as well align the trampoline. */ - while ((uintptr_t)s->code_ptr & 15) { - tcg_out_nop(s); - } - qemu_ld_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr); - - /* Set the retaddr operand. */ - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O3, TCG_REG_O7); - /* Tail call. */ - tcg_out_jmpl_const(s, qemu_ld_helpers[i], true, true); - /* delay slot -- set the env argument */ - tcg_out_mov_delay(s, TCG_REG_O0, TCG_AREG0); - } - - for (i = 0; i < ARRAY_SIZE(qemu_st_helpers); ++i) { - if (qemu_st_helpers[i] == NULL) { - continue; - } - - /* May as well align the trampoline. */ - while ((uintptr_t)s->code_ptr & 15) { - tcg_out_nop(s); - } - qemu_st_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr); - - /* Set the retaddr operand. */ - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O4, TCG_REG_O7); - - /* Tail call. */ - tcg_out_jmpl_const(s, qemu_st_helpers[i], true, true); - /* delay slot -- set the env argument */ - tcg_out_mov_delay(s, TCG_REG_O0, TCG_AREG0); - } -} -#else -static const tcg_insn_unit *qemu_unalign_ld_trampoline; -static const tcg_insn_unit *qemu_unalign_st_trampoline; - -static void build_trampolines(TCGContext *s) -{ - for (int ld = 0; ld < 2; ++ld) { - void *helper; - - while ((uintptr_t)s->code_ptr & 15) { - tcg_out_nop(s); - } - - if (ld) { - helper = helper_unaligned_ld; - qemu_unalign_ld_trampoline = tcg_splitwx_to_rx(s->code_ptr); - } else { - helper = helper_unaligned_st; - qemu_unalign_st_trampoline = tcg_splitwx_to_rx(s->code_ptr); - } - - /* Tail call. */ - tcg_out_jmpl_const(s, helper, true, true); - /* delay slot -- set the env argument */ - tcg_out_mov_delay(s, TCG_REG_O0, TCG_AREG0); - } -} -#endif - /* Generate global QEMU prologue and epilogue code */ static void tcg_target_qemu_prologue(TCGContext *s) { @@ -1060,9 +952,7 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); /* delay slot */ - tcg_out_movi_imm13(s, TCG_REG_O0, 0); - - build_trampolines(s); + tcg_out_movi_s13(s, TCG_REG_O0, 0); } static void tcg_out_nop_fill(tcg_insn_unit *p, int count) @@ -1073,388 +963,239 @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count) } } -#if defined(CONFIG_SOFTMMU) - -/* We expect to use a 13-bit negative offset from ENV. */ -QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); -QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 12)); +static const TCGLdstHelperParam ldst_helper_param = { + .ntmp = 1, .tmp = { TCG_REG_T1 } +}; -/* Perform the TLB load and compare. +static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) +{ + MemOp opc = get_memop(lb->oi); + MemOp sgn; - Inputs: - ADDRLO and ADDRHI contain the possible two parts of the address. + if (!patch_reloc(lb->label_ptr[0], R_SPARC_WDISP19, + (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 0)) { + return false; + } - MEM_INDEX and S_BITS are the memory context and log2 size of the load. + /* Use inline tcg_out_ext32s; otherwise let the helper sign-extend. */ + sgn = (opc & MO_SIZE) < MO_32 ? MO_SIGN : 0; - WHICH is the offset into the CPUTLBEntry structure of the slot to read. - This should be offsetof addr_read or addr_write. + tcg_out_ld_helper_args(s, lb, &ldst_helper_param); + tcg_out_call(s, qemu_ld_helpers[opc & (MO_SIZE | sgn)], NULL); + tcg_out_ld_helper_ret(s, lb, sgn, &ldst_helper_param); - The result of the TLB comparison is in %[ix]cc. The sanitized address - is in the returned register, maybe %o0. The TLB addend is in %o1. */ + tcg_out_bpcc0(s, COND_A, BPCC_A | BPCC_PT, 0); + return patch_reloc(s->code_ptr - 1, R_SPARC_WDISP19, + (intptr_t)lb->raddr, 0); +} -static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index, - MemOp opc, int which) +static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { - int fast_off = TLB_MASK_TABLE_OFS(mem_index); - int mask_off = fast_off + offsetof(CPUTLBDescFast, mask); - int table_off = fast_off + offsetof(CPUTLBDescFast, table); - const TCGReg r0 = TCG_REG_O0; - const TCGReg r1 = TCG_REG_O1; - const TCGReg r2 = TCG_REG_O2; - unsigned s_bits = opc & MO_SIZE; - unsigned a_bits = get_alignment_bits(opc); - tcg_target_long compare_mask; - - /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */ - tcg_out_ld(s, TCG_TYPE_PTR, r0, TCG_AREG0, mask_off); - tcg_out_ld(s, TCG_TYPE_PTR, r1, TCG_AREG0, table_off); - - /* Extract the page index, shifted into place for tlb index. */ - tcg_out_arithi(s, r2, addr, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS, - SHIFT_SRL); - tcg_out_arith(s, r2, r2, r0, ARITH_AND); - - /* Add the tlb_table pointer, creating the CPUTLBEntry address into R2. */ - tcg_out_arith(s, r2, r2, r1, ARITH_ADD); - - /* Load the tlb comparator and the addend. */ - tcg_out_ld(s, TCG_TYPE_TL, r0, r2, which); - tcg_out_ld(s, TCG_TYPE_PTR, r1, r2, offsetof(CPUTLBEntry, addend)); + MemOp opc = get_memop(lb->oi); - /* Mask out the page offset, except for the required alignment. - We don't support unaligned accesses. */ - if (a_bits < s_bits) { - a_bits = s_bits; + if (!patch_reloc(lb->label_ptr[0], R_SPARC_WDISP19, + (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 0)) { + return false; } - compare_mask = (tcg_target_ulong)TARGET_PAGE_MASK | ((1 << a_bits) - 1); - if (check_fit_tl(compare_mask, 13)) { - tcg_out_arithi(s, r2, addr, compare_mask, ARITH_AND); - } else { - tcg_out_movi(s, TCG_TYPE_TL, r2, compare_mask); - tcg_out_arith(s, r2, addr, r2, ARITH_AND); - } - tcg_out_cmp(s, r0, r2, 0); - /* If the guest address must be zero-extended, do so now. */ - if (TARGET_LONG_BITS == 32) { - tcg_out_ext32u(s, r0, addr); - return r0; - } - return addr; -} -#endif /* CONFIG_SOFTMMU */ - -static const int qemu_ld_opc[(MO_SSIZE | MO_BSWAP) + 1] = { - [MO_UB] = LDUB, - [MO_SB] = LDSB, - [MO_UB | MO_LE] = LDUB, - [MO_SB | MO_LE] = LDSB, - - [MO_BEUW] = LDUH, - [MO_BESW] = LDSH, - [MO_BEUL] = LDUW, - [MO_BESL] = LDSW, - [MO_BEUQ] = LDX, - [MO_BESQ] = LDX, - - [MO_LEUW] = LDUH_LE, - [MO_LESW] = LDSH_LE, - [MO_LEUL] = LDUW_LE, - [MO_LESL] = LDSW_LE, - [MO_LEUQ] = LDX_LE, - [MO_LESQ] = LDX_LE, -}; + tcg_out_st_helper_args(s, lb, &ldst_helper_param); + tcg_out_call(s, qemu_st_helpers[opc & MO_SIZE], NULL); -static const int qemu_st_opc[(MO_SIZE | MO_BSWAP) + 1] = { - [MO_UB] = STB, + tcg_out_bpcc0(s, COND_A, BPCC_A | BPCC_PT, 0); + return patch_reloc(s->code_ptr - 1, R_SPARC_WDISP19, + (intptr_t)lb->raddr, 0); +} - [MO_BEUW] = STH, - [MO_BEUL] = STW, - [MO_BEUQ] = STX, +typedef struct { + TCGReg base; + TCGReg index; + TCGAtomAlign aa; +} HostAddress; - [MO_LEUW] = STH_LE, - [MO_LEUL] = STW_LE, - [MO_LEUQ] = STX_LE, -}; +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return true; +} -static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, - MemOpIdx oi, TCGType data_type) +/* + * For softmmu, perform the TLB load and compare. + * For useronly, perform any required alignment tests. + * In both cases, return a TCGLabelQemuLdst structure if the slow path + * is required and fill in @h with the host address for the fast path. + */ +static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, + TCGReg addr_reg, MemOpIdx oi, + bool is_ld) { - MemOp memop = get_memop(oi); - tcg_insn_unit *label_ptr; + TCGLabelQemuLdst *ldst = NULL; + MemOp opc = get_memop(oi); + MemOp s_bits = opc & MO_SIZE; + unsigned a_mask; -#ifdef CONFIG_SOFTMMU - unsigned memi = get_mmuidx(oi); - TCGReg addrz; - const tcg_insn_unit *func; + /* We don't support unaligned accesses. */ + h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false); + h->aa.align = MAX(h->aa.align, s_bits); + a_mask = (1u << h->aa.align) - 1; - addrz = tcg_out_tlb_load(s, addr, memi, memop, - offsetof(CPUTLBEntry, addr_read)); +#ifdef CONFIG_SOFTMMU + int mem_index = get_mmuidx(oi); + int fast_off = TLB_MASK_TABLE_OFS(mem_index); + int mask_off = fast_off + offsetof(CPUTLBDescFast, mask); + int table_off = fast_off + offsetof(CPUTLBDescFast, table); + int cmp_off = is_ld ? offsetof(CPUTLBEntry, addr_read) + : offsetof(CPUTLBEntry, addr_write); + int add_off = offsetof(CPUTLBEntry, addend); + int compare_mask; + int cc; - /* The fast path is exactly one insn. Thus we can perform the - entire TLB Hit in the (annulled) delay slot of the branch - over the TLB Miss case. */ + /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */ + QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); + QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 12)); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T2, TCG_AREG0, mask_off); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T3, TCG_AREG0, table_off); - /* beq,a,pt %[xi]cc, label0 */ - label_ptr = s->code_ptr; - tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT - | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0); - /* delay slot */ - tcg_out_ldst_rr(s, data, addrz, TCG_REG_O1, - qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]); + /* Extract the page index, shifted into place for tlb index. */ + tcg_out_arithi(s, TCG_REG_T1, addr_reg, + s->page_bits - CPU_TLB_ENTRY_BITS, SHIFT_SRL); + tcg_out_arith(s, TCG_REG_T1, TCG_REG_T1, TCG_REG_T2, ARITH_AND); - /* TLB Miss. */ + /* Add the tlb_table pointer, creating the CPUTLBEntry address into R2. */ + tcg_out_arith(s, TCG_REG_T1, TCG_REG_T1, TCG_REG_T3, ARITH_ADD); - tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_O1, addrz); + /* Load the tlb comparator and the addend. */ + tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_T2, TCG_REG_T1, cmp_off); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T1, TCG_REG_T1, add_off); + h->base = TCG_REG_T1; - /* We use the helpers to extend SB and SW data, leaving the case - of SL needing explicit extending below. */ - if ((memop & MO_SSIZE) == MO_SL) { - func = qemu_ld_trampoline[memop & (MO_BSWAP | MO_SIZE)]; + /* Mask out the page offset, except for the required alignment. */ + compare_mask = s->page_mask | a_mask; + if (check_fit_tl(compare_mask, 13)) { + tcg_out_arithi(s, TCG_REG_T3, addr_reg, compare_mask, ARITH_AND); } else { - func = qemu_ld_trampoline[memop & (MO_BSWAP | MO_SSIZE)]; + tcg_out_movi_s32(s, TCG_REG_T3, compare_mask); + tcg_out_arith(s, TCG_REG_T3, addr_reg, TCG_REG_T3, ARITH_AND); } - tcg_debug_assert(func != NULL); - tcg_out_call_nodelay(s, func, false); - /* delay slot */ - tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_O2, oi); + tcg_out_cmp(s, TCG_REG_T2, TCG_REG_T3, 0); - /* We let the helper sign-extend SB and SW, but leave SL for here. */ - if ((memop & MO_SSIZE) == MO_SL) { - tcg_out_ext32s(s, data, TCG_REG_O0); - } else { - tcg_out_mov(s, TCG_TYPE_REG, data, TCG_REG_O0); - } + ldst = new_ldst_label(s); + ldst->is_ld = is_ld; + ldst->oi = oi; + ldst->addrlo_reg = addr_reg; + ldst->label_ptr[0] = s->code_ptr; - *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); + /* bne,pn %[xi]cc, label0 */ + cc = TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC; + tcg_out_bpcc0(s, COND_NE, BPCC_PN | cc, 0); #else - TCGReg index = (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0); - unsigned a_bits = get_alignment_bits(memop); - unsigned s_bits = memop & MO_SIZE; - unsigned t_bits; - - if (TARGET_LONG_BITS == 32) { - tcg_out_ext32u(s, TCG_REG_T1, addr); - addr = TCG_REG_T1; - } - /* - * Normal case: alignment equal to access size. + * If the size equals the required alignment, we can skip the test + * and allow host SIGBUS to deliver SIGBUS to the guest. + * Otherwise, test for at least natural alignment and defer + * everything else to the helper functions. */ - if (a_bits == s_bits) { - tcg_out_ldst_rr(s, data, addr, index, - qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]); - return; + if (s_bits != get_alignment_bits(opc)) { + tcg_debug_assert(check_fit_tl(a_mask, 13)); + tcg_out_arithi(s, TCG_REG_G0, addr_reg, a_mask, ARITH_ANDCC); + + ldst = new_ldst_label(s); + ldst->is_ld = is_ld; + ldst->oi = oi; + ldst->addrlo_reg = addr_reg; + ldst->label_ptr[0] = s->code_ptr; + + /* bne,pn %icc, label0 */ + tcg_out_bpcc0(s, COND_NE, BPCC_PN | BPCC_ICC, 0); } + h->base = guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0; +#endif - /* - * Test for at least natural alignment, and assume most accesses - * will be aligned -- perform a straight load in the delay slot. - * This is required to preserve atomicity for aligned accesses. - */ - t_bits = MAX(a_bits, s_bits); - tcg_debug_assert(t_bits < 13); - tcg_out_arithi(s, TCG_REG_G0, addr, (1u << t_bits) - 1, ARITH_ANDCC); - - /* beq,a,pt %icc, label */ - label_ptr = s->code_ptr; - tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT | BPCC_ICC, 0); - /* delay slot */ - tcg_out_ldst_rr(s, data, addr, index, - qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]); - - if (a_bits >= s_bits) { - /* - * Overalignment: A successful alignment test will perform the memory - * operation in the delay slot, and failure need only invoke the - * handler for SIGBUS. - */ - tcg_out_call_nodelay(s, qemu_unalign_ld_trampoline, false); - /* delay slot -- move to low part of argument reg */ - tcg_out_mov_delay(s, TCG_REG_O1, addr); + /* If the guest address must be zero-extended, do in the delay slot. */ + if (TARGET_LONG_BITS == 32) { + tcg_out_ext32u(s, TCG_REG_T2, addr_reg); + h->index = TCG_REG_T2; } else { - /* Underalignment: load by pieces of minimum alignment. */ - int ld_opc, a_size, s_size, i; - - /* - * Force full address into T1 early; avoids problems with - * overlap between @addr and @data. - */ - tcg_out_arith(s, TCG_REG_T1, addr, index, ARITH_ADD); - - a_size = 1 << a_bits; - s_size = 1 << s_bits; - if ((memop & MO_BSWAP) == MO_BE) { - ld_opc = qemu_ld_opc[a_bits | MO_BE | (memop & MO_SIGN)]; - tcg_out_ldst(s, data, TCG_REG_T1, 0, ld_opc); - ld_opc = qemu_ld_opc[a_bits | MO_BE]; - for (i = a_size; i < s_size; i += a_size) { - tcg_out_ldst(s, TCG_REG_T2, TCG_REG_T1, i, ld_opc); - tcg_out_arithi(s, data, data, a_size, SHIFT_SLLX); - tcg_out_arith(s, data, data, TCG_REG_T2, ARITH_OR); - } - } else if (a_bits == 0) { - ld_opc = LDUB; - tcg_out_ldst(s, data, TCG_REG_T1, 0, ld_opc); - for (i = a_size; i < s_size; i += a_size) { - if ((memop & MO_SIGN) && i == s_size - a_size) { - ld_opc = LDSB; - } - tcg_out_ldst(s, TCG_REG_T2, TCG_REG_T1, i, ld_opc); - tcg_out_arithi(s, TCG_REG_T2, TCG_REG_T2, i * 8, SHIFT_SLLX); - tcg_out_arith(s, data, data, TCG_REG_T2, ARITH_OR); - } - } else { - ld_opc = qemu_ld_opc[a_bits | MO_LE]; - tcg_out_ldst_rr(s, data, TCG_REG_T1, TCG_REG_G0, ld_opc); - for (i = a_size; i < s_size; i += a_size) { - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, a_size, ARITH_ADD); - if ((memop & MO_SIGN) && i == s_size - a_size) { - ld_opc = qemu_ld_opc[a_bits | MO_LE | MO_SIGN]; - } - tcg_out_ldst_rr(s, TCG_REG_T2, TCG_REG_T1, TCG_REG_G0, ld_opc); - tcg_out_arithi(s, TCG_REG_T2, TCG_REG_T2, i * 8, SHIFT_SLLX); - tcg_out_arith(s, data, data, TCG_REG_T2, ARITH_OR); - } + if (ldst) { + tcg_out_nop(s); } + h->index = addr_reg; } - - *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); -#endif /* CONFIG_SOFTMMU */ + return ldst; } -static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, +static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, MemOpIdx oi, TCGType data_type) { - MemOp memop = get_memop(oi); - tcg_insn_unit *label_ptr; + static const int ld_opc[(MO_SSIZE | MO_BSWAP) + 1] = { + [MO_UB] = LDUB, + [MO_SB] = LDSB, + [MO_UB | MO_LE] = LDUB, + [MO_SB | MO_LE] = LDSB, + + [MO_BEUW] = LDUH, + [MO_BESW] = LDSH, + [MO_BEUL] = LDUW, + [MO_BESL] = LDSW, + [MO_BEUQ] = LDX, + [MO_BESQ] = LDX, + + [MO_LEUW] = LDUH_LE, + [MO_LESW] = LDSH_LE, + [MO_LEUL] = LDUW_LE, + [MO_LESL] = LDSW_LE, + [MO_LEUQ] = LDX_LE, + [MO_LESQ] = LDX_LE, + }; -#ifdef CONFIG_SOFTMMU - unsigned memi = get_mmuidx(oi); - TCGReg addrz; - const tcg_insn_unit *func; - - addrz = tcg_out_tlb_load(s, addr, memi, memop, - offsetof(CPUTLBEntry, addr_write)); - - /* The fast path is exactly one insn. Thus we can perform the entire - TLB Hit in the (annulled) delay slot of the branch over TLB Miss. */ - /* beq,a,pt %[xi]cc, label0 */ - label_ptr = s->code_ptr; - tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT - | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0); - /* delay slot */ - tcg_out_ldst_rr(s, data, addrz, TCG_REG_O1, - qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]); + TCGLabelQemuLdst *ldst; + HostAddress h; - /* TLB Miss. */ + ldst = prepare_host_addr(s, &h, addr, oi, true); - tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_O1, addrz); - tcg_out_movext(s, (memop & MO_SIZE) == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32, - TCG_REG_O2, data_type, memop & MO_SIZE, data); + tcg_out_ldst_rr(s, data, h.base, h.index, + ld_opc[get_memop(oi) & (MO_BSWAP | MO_SSIZE)]); - func = qemu_st_trampoline[memop & (MO_BSWAP | MO_SIZE)]; - tcg_debug_assert(func != NULL); - tcg_out_call_nodelay(s, func, false); - /* delay slot */ - tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_O3, oi); + if (ldst) { + ldst->type = data_type; + ldst->datalo_reg = data; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); + } +} - *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); -#else - TCGReg index = (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0); - unsigned a_bits = get_alignment_bits(memop); - unsigned s_bits = memop & MO_SIZE; - unsigned t_bits; +static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, + MemOpIdx oi, TCGType data_type) +{ + static const int st_opc[(MO_SIZE | MO_BSWAP) + 1] = { + [MO_UB] = STB, - if (TARGET_LONG_BITS == 32) { - tcg_out_ext32u(s, TCG_REG_T1, addr); - addr = TCG_REG_T1; - } + [MO_BEUW] = STH, + [MO_BEUL] = STW, + [MO_BEUQ] = STX, - /* - * Normal case: alignment equal to access size. - */ - if (a_bits == s_bits) { - tcg_out_ldst_rr(s, data, addr, index, - qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]); - return; - } + [MO_LEUW] = STH_LE, + [MO_LEUL] = STW_LE, + [MO_LEUQ] = STX_LE, + }; - /* - * Test for at least natural alignment, and assume most accesses - * will be aligned -- perform a straight store in the delay slot. - * This is required to preserve atomicity for aligned accesses. - */ - t_bits = MAX(a_bits, s_bits); - tcg_debug_assert(t_bits < 13); - tcg_out_arithi(s, TCG_REG_G0, addr, (1u << t_bits) - 1, ARITH_ANDCC); + TCGLabelQemuLdst *ldst; + HostAddress h; - /* beq,a,pt %icc, label */ - label_ptr = s->code_ptr; - tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT | BPCC_ICC, 0); - /* delay slot */ - tcg_out_ldst_rr(s, data, addr, index, - qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]); + ldst = prepare_host_addr(s, &h, addr, oi, false); - if (a_bits >= s_bits) { - /* - * Overalignment: A successful alignment test will perform the memory - * operation in the delay slot, and failure need only invoke the - * handler for SIGBUS. - */ - tcg_out_call_nodelay(s, qemu_unalign_st_trampoline, false); - /* delay slot -- move to low part of argument reg */ - tcg_out_mov_delay(s, TCG_REG_O1, addr); - } else { - /* Underalignment: store by pieces of minimum alignment. */ - int st_opc, a_size, s_size, i; + tcg_out_ldst_rr(s, data, h.base, h.index, + st_opc[get_memop(oi) & (MO_BSWAP | MO_SIZE)]); - /* - * Force full address into T1 early; avoids problems with - * overlap between @addr and @data. - */ - tcg_out_arith(s, TCG_REG_T1, addr, index, ARITH_ADD); - - a_size = 1 << a_bits; - s_size = 1 << s_bits; - if ((memop & MO_BSWAP) == MO_BE) { - st_opc = qemu_st_opc[a_bits | MO_BE]; - for (i = 0; i < s_size; i += a_size) { - TCGReg d = data; - int shift = (s_size - a_size - i) * 8; - if (shift) { - d = TCG_REG_T2; - tcg_out_arithi(s, d, data, shift, SHIFT_SRLX); - } - tcg_out_ldst(s, d, TCG_REG_T1, i, st_opc); - } - } else if (a_bits == 0) { - tcg_out_ldst(s, data, TCG_REG_T1, 0, STB); - for (i = 1; i < s_size; i++) { - tcg_out_arithi(s, TCG_REG_T2, data, i * 8, SHIFT_SRLX); - tcg_out_ldst(s, TCG_REG_T2, TCG_REG_T1, i, STB); - } - } else { - /* Note that ST*A with immediate asi must use indexed address. */ - st_opc = qemu_st_opc[a_bits + MO_LE]; - tcg_out_ldst_rr(s, data, TCG_REG_T1, TCG_REG_G0, st_opc); - for (i = a_size; i < s_size; i += a_size) { - tcg_out_arithi(s, TCG_REG_T2, data, i * 8, SHIFT_SRLX); - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, a_size, ARITH_ADD); - tcg_out_ldst_rr(s, TCG_REG_T2, TCG_REG_T1, TCG_REG_G0, st_opc); - } - } + if (ldst) { + ldst->type = data_type; + ldst->datalo_reg = data; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } - - *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); -#endif /* CONFIG_SOFTMMU */ } static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) { if (check_fit_ptr(a0, 13)) { tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); - tcg_out_movi_imm13(s, TCG_REG_O0, a0); + tcg_out_movi_s13(s, TCG_REG_O0, a0); return; } else { intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0); @@ -1635,16 +1376,20 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX); break; - case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; - case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); break; - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; @@ -1766,6 +1511,10 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_extu_i32_i64: case INDEX_op_extrl_i64_i32: case INDEX_op_extrh_i64_i32: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: return C_O1_I1(r, r); case INDEX_op_st8_i32: @@ -1775,6 +1524,10 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_st_i32: case INDEX_op_st32_i64: case INDEX_op_st_i64: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: return C_O0_I2(rZ, r); case INDEX_op_add_i32: @@ -1824,13 +1577,6 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_muluh_i64: return C_O1_I2(r, r, r); - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: - return C_O1_I1(r, s); - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: - return C_O0_I2(sZ, s); - default: g_assert_not_reached(); } @@ -1878,6 +1624,7 @@ static void tcg_target_init(TCGContext *s) tcg_regset_set_reg(s->reserved_regs, TCG_REG_O6); /* stack pointer */ tcg_regset_set_reg(s->reserved_regs, TCG_REG_T1); /* for internal use */ tcg_regset_set_reg(s->reserved_regs, TCG_REG_T2); /* for internal use */ + tcg_regset_set_reg(s->reserved_regs, TCG_REG_T3); /* for internal use */ } #define ELF_HOST_MACHINE EM_SPARCV9 diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h index ffe22b1d21..31c5537379 100644 --- a/tcg/sparc64/tcg-target.h +++ b/tcg/sparc64/tcg-target.h @@ -151,10 +151,12 @@ extern bool use_vis3_instructions; #define TCG_TARGET_HAS_muluh_i64 use_vis3_instructions #define TCG_TARGET_HAS_mulsh_i64 0 +#define TCG_TARGET_HAS_qemu_ldst_i128 0 + #define TCG_AREG0 TCG_REG_I0 #define TCG_TARGET_DEFAULT_MO (0) -#define TCG_TARGET_HAS_MEMORY_BSWAP 1 +#define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS #endif diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index 0f1ba01a9a..67b698bd5c 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -126,4 +126,6 @@ static inline TCGv_i64 TCGV128_HIGH(TCGv_i128 t) return temp_tcgv_i64(tcgv_i128_temp(t) + o); } +bool tcg_target_has_memory_bswap(MemOp memop); + #endif /* TCG_INTERNAL_H */ diff --git a/tcg/tcg-op-ldst.c b/tcg/tcg-op-ldst.c new file mode 100644 index 0000000000..f4e508cb68 --- /dev/null +++ b/tcg/tcg-op-ldst.c @@ -0,0 +1,1232 @@ +/* + * Tiny Code Generator for QEMU + * + * Copyright (c) 2008 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "qemu/osdep.h" +#include "exec/exec-all.h" +#include "tcg/tcg.h" +#include "tcg/tcg-temp-internal.h" +#include "tcg/tcg-op.h" +#include "tcg/tcg-mo.h" +#include "exec/plugin-gen.h" +#include "tcg-internal.h" + + +static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) +{ + /* Trigger the asserts within as early as possible. */ + unsigned a_bits = get_alignment_bits(op); + + /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */ + if (a_bits == (op & MO_SIZE)) { + op = (op & ~MO_AMASK) | MO_ALIGN; + } + + switch (op & MO_SIZE) { + case MO_8: + op &= ~MO_BSWAP; + break; + case MO_16: + break; + case MO_32: + if (!is64) { + op &= ~MO_SIGN; + } + break; + case MO_64: + if (is64) { + op &= ~MO_SIGN; + break; + } + /* fall through */ + default: + g_assert_not_reached(); + } + if (st) { + op &= ~MO_SIGN; + } + return op; +} + +static void gen_ldst(TCGOpcode opc, TCGTemp *vl, TCGTemp *vh, + TCGTemp *addr, MemOpIdx oi) +{ + if (TCG_TARGET_REG_BITS == 64 || tcg_ctx->addr_type == TCG_TYPE_I32) { + if (vh) { + tcg_gen_op4(opc, temp_arg(vl), temp_arg(vh), temp_arg(addr), oi); + } else { + tcg_gen_op3(opc, temp_arg(vl), temp_arg(addr), oi); + } + } else { + /* See TCGV_LOW/HIGH. */ + TCGTemp *al = addr + HOST_BIG_ENDIAN; + TCGTemp *ah = addr + !HOST_BIG_ENDIAN; + + if (vh) { + tcg_gen_op5(opc, temp_arg(vl), temp_arg(vh), + temp_arg(al), temp_arg(ah), oi); + } else { + tcg_gen_op4(opc, temp_arg(vl), temp_arg(al), temp_arg(ah), oi); + } + } +} + +static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) +{ + if (TCG_TARGET_REG_BITS == 32) { + TCGTemp *vl = tcgv_i32_temp(TCGV_LOW(v)); + TCGTemp *vh = tcgv_i32_temp(TCGV_HIGH(v)); + gen_ldst(opc, vl, vh, addr, oi); + } else { + gen_ldst(opc, tcgv_i64_temp(v), NULL, addr, oi); + } +} + +static void tcg_gen_req_mo(TCGBar type) +{ +#ifdef TCG_GUEST_DEFAULT_MO + type &= TCG_GUEST_DEFAULT_MO; +#endif + type &= ~TCG_TARGET_DEFAULT_MO; + if (type) { + tcg_gen_mb(type | TCG_BAR_SC); + } +} + +/* Only required for loads, where value might overlap addr. */ +static TCGv_i64 plugin_maybe_preserve_addr(TCGTemp *addr) +{ +#ifdef CONFIG_PLUGIN + if (tcg_ctx->plugin_insn != NULL) { + /* Save a copy of the vaddr for use after a load. */ + TCGv_i64 temp = tcg_temp_ebb_new_i64(); + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + tcg_gen_extu_i32_i64(temp, temp_tcgv_i32(addr)); + } else { + tcg_gen_mov_i64(temp, temp_tcgv_i64(addr)); + } + return temp; + } +#endif + return NULL; +} + +static void +plugin_gen_mem_callbacks(TCGv_i64 copy_addr, TCGTemp *orig_addr, MemOpIdx oi, + enum qemu_plugin_mem_rw rw) +{ +#ifdef CONFIG_PLUGIN + if (tcg_ctx->plugin_insn != NULL) { + qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw); + + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + if (!copy_addr) { + copy_addr = tcg_temp_ebb_new_i64(); + tcg_gen_extu_i32_i64(copy_addr, temp_tcgv_i32(orig_addr)); + } + plugin_gen_empty_mem_callback(copy_addr, info); + tcg_temp_free_i64(copy_addr); + } else { + if (copy_addr) { + plugin_gen_empty_mem_callback(copy_addr, info); + tcg_temp_free_i64(copy_addr); + } else { + plugin_gen_empty_mem_callback(temp_tcgv_i64(orig_addr), info); + } + } + } +#endif +} + +static void tcg_gen_qemu_ld_i32_int(TCGv_i32 val, TCGTemp *addr, + TCGArg idx, MemOp memop) +{ + MemOp orig_memop; + MemOpIdx orig_oi, oi; + TCGv_i64 copy_addr; + TCGOpcode opc; + + tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + orig_memop = memop = tcg_canonicalize_memop(memop, 0, 0); + orig_oi = oi = make_memop_idx(memop, idx); + + if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { + memop &= ~MO_BSWAP; + /* The bswap primitive benefits from zero-extended input. */ + if ((memop & MO_SSIZE) == MO_SW) { + memop &= ~MO_SIGN; + } + oi = make_memop_idx(memop, idx); + } + + copy_addr = plugin_maybe_preserve_addr(addr); + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + opc = INDEX_op_qemu_ld_a32_i32; + } else { + opc = INDEX_op_qemu_ld_a64_i32; + } + gen_ldst(opc, tcgv_i32_temp(val), NULL, addr, oi); + plugin_gen_mem_callbacks(copy_addr, addr, orig_oi, QEMU_PLUGIN_MEM_R); + + if ((orig_memop ^ memop) & MO_BSWAP) { + switch (orig_memop & MO_SIZE) { + case MO_16: + tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN + ? TCG_BSWAP_IZ | TCG_BSWAP_OS + : TCG_BSWAP_IZ | TCG_BSWAP_OZ)); + break; + case MO_32: + tcg_gen_bswap32_i32(val, val); + break; + default: + g_assert_not_reached(); + } + } +} + +void tcg_gen_qemu_ld_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx, + MemOp memop, TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) <= MO_32); + tcg_gen_qemu_ld_i32_int(val, addr, idx, memop); +} + +static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr, + TCGArg idx, MemOp memop) +{ + TCGv_i32 swap = NULL; + MemOpIdx orig_oi, oi; + TCGOpcode opc; + + tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + memop = tcg_canonicalize_memop(memop, 0, 1); + orig_oi = oi = make_memop_idx(memop, idx); + + if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { + swap = tcg_temp_ebb_new_i32(); + switch (memop & MO_SIZE) { + case MO_16: + tcg_gen_bswap16_i32(swap, val, 0); + break; + case MO_32: + tcg_gen_bswap32_i32(swap, val); + break; + default: + g_assert_not_reached(); + } + val = swap; + memop &= ~MO_BSWAP; + oi = make_memop_idx(memop, idx); + } + + if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) { + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + opc = INDEX_op_qemu_st8_a32_i32; + } else { + opc = INDEX_op_qemu_st8_a64_i32; + } + } else { + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + opc = INDEX_op_qemu_st_a32_i32; + } else { + opc = INDEX_op_qemu_st_a64_i32; + } + } + gen_ldst(opc, tcgv_i32_temp(val), NULL, addr, oi); + plugin_gen_mem_callbacks(NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); + + if (swap) { + tcg_temp_free_i32(swap); + } +} + +void tcg_gen_qemu_st_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx, + MemOp memop, TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) <= MO_32); + tcg_gen_qemu_st_i32_int(val, addr, idx, memop); +} + +static void tcg_gen_qemu_ld_i64_int(TCGv_i64 val, TCGTemp *addr, + TCGArg idx, MemOp memop) +{ + MemOp orig_memop; + MemOpIdx orig_oi, oi; + TCGv_i64 copy_addr; + TCGOpcode opc; + + if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { + tcg_gen_qemu_ld_i32_int(TCGV_LOW(val), addr, idx, memop); + if (memop & MO_SIGN) { + tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); + } else { + tcg_gen_movi_i32(TCGV_HIGH(val), 0); + } + return; + } + + tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + orig_memop = memop = tcg_canonicalize_memop(memop, 1, 0); + orig_oi = oi = make_memop_idx(memop, idx); + + if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { + memop &= ~MO_BSWAP; + /* The bswap primitive benefits from zero-extended input. */ + if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { + memop &= ~MO_SIGN; + } + oi = make_memop_idx(memop, idx); + } + + copy_addr = plugin_maybe_preserve_addr(addr); + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + opc = INDEX_op_qemu_ld_a32_i64; + } else { + opc = INDEX_op_qemu_ld_a64_i64; + } + gen_ldst_i64(opc, val, addr, oi); + plugin_gen_mem_callbacks(copy_addr, addr, orig_oi, QEMU_PLUGIN_MEM_R); + + if ((orig_memop ^ memop) & MO_BSWAP) { + int flags = (orig_memop & MO_SIGN + ? TCG_BSWAP_IZ | TCG_BSWAP_OS + : TCG_BSWAP_IZ | TCG_BSWAP_OZ); + switch (orig_memop & MO_SIZE) { + case MO_16: + tcg_gen_bswap16_i64(val, val, flags); + break; + case MO_32: + tcg_gen_bswap32_i64(val, val, flags); + break; + case MO_64: + tcg_gen_bswap64_i64(val, val); + break; + default: + g_assert_not_reached(); + } + } +} + +void tcg_gen_qemu_ld_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx, + MemOp memop, TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) <= MO_64); + tcg_gen_qemu_ld_i64_int(val, addr, idx, memop); +} + +static void tcg_gen_qemu_st_i64_int(TCGv_i64 val, TCGTemp *addr, + TCGArg idx, MemOp memop) +{ + TCGv_i64 swap = NULL; + MemOpIdx orig_oi, oi; + TCGOpcode opc; + + if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { + tcg_gen_qemu_st_i32_int(TCGV_LOW(val), addr, idx, memop); + return; + } + + tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + memop = tcg_canonicalize_memop(memop, 1, 1); + orig_oi = oi = make_memop_idx(memop, idx); + + if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { + swap = tcg_temp_ebb_new_i64(); + switch (memop & MO_SIZE) { + case MO_16: + tcg_gen_bswap16_i64(swap, val, 0); + break; + case MO_32: + tcg_gen_bswap32_i64(swap, val, 0); + break; + case MO_64: + tcg_gen_bswap64_i64(swap, val); + break; + default: + g_assert_not_reached(); + } + val = swap; + memop &= ~MO_BSWAP; + oi = make_memop_idx(memop, idx); + } + + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + opc = INDEX_op_qemu_st_a32_i64; + } else { + opc = INDEX_op_qemu_st_a64_i64; + } + gen_ldst_i64(opc, val, addr, oi); + plugin_gen_mem_callbacks(NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); + + if (swap) { + tcg_temp_free_i64(swap); + } +} + +void tcg_gen_qemu_st_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx, + MemOp memop, TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) <= MO_64); + tcg_gen_qemu_st_i64_int(val, addr, idx, memop); +} + +/* + * Return true if @mop, without knowledge of the pointer alignment, + * does not require 16-byte atomicity, and it would be adventagous + * to avoid a call to a helper function. + */ +static bool use_two_i64_for_i128(MemOp mop) +{ +#ifdef CONFIG_SOFTMMU + /* Two softmmu tlb lookups is larger than one function call. */ + return false; +#else + /* + * For user-only, two 64-bit operations may well be smaller than a call. + * Determine if that would be legal for the requested atomicity. + */ + switch (mop & MO_ATOM_MASK) { + case MO_ATOM_NONE: + case MO_ATOM_IFALIGN_PAIR: + return true; + case MO_ATOM_IFALIGN: + case MO_ATOM_SUBALIGN: + case MO_ATOM_WITHIN16: + case MO_ATOM_WITHIN16_PAIR: + /* In a serialized context, no atomicity is required. */ + return !(tcg_ctx->gen_tb->cflags & CF_PARALLEL); + default: + g_assert_not_reached(); + } +#endif +} + +static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig) +{ + MemOp mop_1 = orig, mop_2; + + /* Reduce the size to 64-bit. */ + mop_1 = (mop_1 & ~MO_SIZE) | MO_64; + + /* Retain the alignment constraints of the original. */ + switch (orig & MO_AMASK) { + case MO_UNALN: + case MO_ALIGN_2: + case MO_ALIGN_4: + mop_2 = mop_1; + break; + case MO_ALIGN_8: + /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */ + mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN; + mop_2 = mop_1; + break; + case MO_ALIGN: + /* Second has 8-byte alignment; first has 16-byte alignment. */ + mop_2 = mop_1; + mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16; + break; + case MO_ALIGN_16: + case MO_ALIGN_32: + case MO_ALIGN_64: + /* Second has 8-byte alignment; first retains original. */ + mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN; + break; + default: + g_assert_not_reached(); + } + + /* Use a memory ordering implemented by the host. */ + if ((orig & MO_BSWAP) && !tcg_target_has_memory_bswap(mop_1)) { + mop_1 &= ~MO_BSWAP; + mop_2 &= ~MO_BSWAP; + } + + ret[0] = mop_1; + ret[1] = mop_2; +} + +static TCGv_i64 maybe_extend_addr64(TCGTemp *addr) +{ + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + TCGv_i64 a64 = tcg_temp_ebb_new_i64(); + tcg_gen_extu_i32_i64(a64, temp_tcgv_i32(addr)); + return a64; + } + return temp_tcgv_i64(addr); +} + +static void maybe_free_addr64(TCGv_i64 a64) +{ + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + tcg_temp_free_i64(a64); + } +} + +static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, + TCGArg idx, MemOp memop) +{ + const MemOpIdx orig_oi = make_memop_idx(memop, idx); + TCGv_i64 ext_addr = NULL; + TCGOpcode opc; + + tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + + /* TODO: For now, force 32-bit hosts to use the helper. */ + if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) { + TCGv_i64 lo, hi; + bool need_bswap = false; + MemOpIdx oi = orig_oi; + + if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { + lo = TCGV128_HIGH(val); + hi = TCGV128_LOW(val); + oi = make_memop_idx(memop & ~MO_BSWAP, idx); + need_bswap = true; + } else { + lo = TCGV128_LOW(val); + hi = TCGV128_HIGH(val); + } + + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + opc = INDEX_op_qemu_ld_a32_i128; + } else { + opc = INDEX_op_qemu_ld_a64_i128; + } + gen_ldst(opc, tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi); + + if (need_bswap) { + tcg_gen_bswap64_i64(lo, lo); + tcg_gen_bswap64_i64(hi, hi); + } + } else if (use_two_i64_for_i128(memop)) { + MemOp mop[2]; + TCGTemp *addr_p8; + TCGv_i64 x, y; + bool need_bswap; + + canonicalize_memop_i128_as_i64(mop, memop); + need_bswap = (mop[0] ^ memop) & MO_BSWAP; + + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + opc = INDEX_op_qemu_ld_a32_i64; + } else { + opc = INDEX_op_qemu_ld_a64_i64; + } + + /* + * Since there are no global TCGv_i128, there is no visible state + * changed if the second load faults. Load directly into the two + * subwords. + */ + if ((memop & MO_BSWAP) == MO_LE) { + x = TCGV128_LOW(val); + y = TCGV128_HIGH(val); + } else { + x = TCGV128_HIGH(val); + y = TCGV128_LOW(val); + } + + gen_ldst_i64(opc, x, addr, make_memop_idx(mop[0], idx)); + + if (need_bswap) { + tcg_gen_bswap64_i64(x, x); + } + + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + TCGv_i32 t = tcg_temp_ebb_new_i32(); + tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8); + addr_p8 = tcgv_i32_temp(t); + } else { + TCGv_i64 t = tcg_temp_ebb_new_i64(); + tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8); + addr_p8 = tcgv_i64_temp(t); + } + + gen_ldst_i64(opc, y, addr_p8, make_memop_idx(mop[1], idx)); + tcg_temp_free_internal(addr_p8); + + if (need_bswap) { + tcg_gen_bswap64_i64(y, y); + } + } else { + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + ext_addr = tcg_temp_ebb_new_i64(); + tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr)); + addr = tcgv_i64_temp(ext_addr); + } + gen_helper_ld_i128(val, cpu_env, temp_tcgv_i64(addr), + tcg_constant_i32(orig_oi)); + } + + plugin_gen_mem_callbacks(ext_addr, addr, orig_oi, QEMU_PLUGIN_MEM_R); +} + +void tcg_gen_qemu_ld_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx, + MemOp memop, TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) == MO_128); + tcg_debug_assert((memop & MO_SIGN) == 0); + tcg_gen_qemu_ld_i128_int(val, addr, idx, memop); +} + +static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, + TCGArg idx, MemOp memop) +{ + const MemOpIdx orig_oi = make_memop_idx(memop, idx); + TCGv_i64 ext_addr = NULL; + TCGOpcode opc; + + tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST); + + /* TODO: For now, force 32-bit hosts to use the helper. */ + + if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) { + TCGv_i64 lo, hi; + MemOpIdx oi = orig_oi; + bool need_bswap = false; + + if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { + lo = tcg_temp_ebb_new_i64(); + hi = tcg_temp_ebb_new_i64(); + tcg_gen_bswap64_i64(lo, TCGV128_HIGH(val)); + tcg_gen_bswap64_i64(hi, TCGV128_LOW(val)); + oi = make_memop_idx(memop & ~MO_BSWAP, idx); + need_bswap = true; + } else { + lo = TCGV128_LOW(val); + hi = TCGV128_HIGH(val); + } + + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + opc = INDEX_op_qemu_st_a32_i128; + } else { + opc = INDEX_op_qemu_st_a64_i128; + } + gen_ldst(opc, tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi); + + if (need_bswap) { + tcg_temp_free_i64(lo); + tcg_temp_free_i64(hi); + } + } else if (use_two_i64_for_i128(memop)) { + MemOp mop[2]; + TCGTemp *addr_p8; + TCGv_i64 x, y, b = NULL; + + canonicalize_memop_i128_as_i64(mop, memop); + + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + opc = INDEX_op_qemu_st_a32_i64; + } else { + opc = INDEX_op_qemu_st_a64_i64; + } + + if ((memop & MO_BSWAP) == MO_LE) { + x = TCGV128_LOW(val); + y = TCGV128_HIGH(val); + } else { + x = TCGV128_HIGH(val); + y = TCGV128_LOW(val); + } + + if ((mop[0] ^ memop) & MO_BSWAP) { + b = tcg_temp_ebb_new_i64(); + tcg_gen_bswap64_i64(b, x); + x = b; + } + + gen_ldst_i64(opc, x, addr, make_memop_idx(mop[0], idx)); + + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + TCGv_i32 t = tcg_temp_ebb_new_i32(); + tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8); + addr_p8 = tcgv_i32_temp(t); + } else { + TCGv_i64 t = tcg_temp_ebb_new_i64(); + tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8); + addr_p8 = tcgv_i64_temp(t); + } + + if (b) { + tcg_gen_bswap64_i64(b, y); + gen_ldst_i64(opc, b, addr_p8, make_memop_idx(mop[1], idx)); + tcg_temp_free_i64(b); + } else { + gen_ldst_i64(opc, y, addr_p8, make_memop_idx(mop[1], idx)); + } + tcg_temp_free_internal(addr_p8); + } else { + if (tcg_ctx->addr_type == TCG_TYPE_I32) { + ext_addr = tcg_temp_ebb_new_i64(); + tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr)); + addr = tcgv_i64_temp(ext_addr); + } + gen_helper_st_i128(cpu_env, temp_tcgv_i64(addr), val, + tcg_constant_i32(orig_oi)); + } + + plugin_gen_mem_callbacks(ext_addr, addr, orig_oi, QEMU_PLUGIN_MEM_W); +} + +void tcg_gen_qemu_st_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx, + MemOp memop, TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) == MO_128); + tcg_debug_assert((memop & MO_SIGN) == 0); + tcg_gen_qemu_st_i128_int(val, addr, idx, memop); +} + +static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc) +{ + switch (opc & MO_SSIZE) { + case MO_SB: + tcg_gen_ext8s_i32(ret, val); + break; + case MO_UB: + tcg_gen_ext8u_i32(ret, val); + break; + case MO_SW: + tcg_gen_ext16s_i32(ret, val); + break; + case MO_UW: + tcg_gen_ext16u_i32(ret, val); + break; + default: + tcg_gen_mov_i32(ret, val); + break; + } +} + +static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc) +{ + switch (opc & MO_SSIZE) { + case MO_SB: + tcg_gen_ext8s_i64(ret, val); + break; + case MO_UB: + tcg_gen_ext8u_i64(ret, val); + break; + case MO_SW: + tcg_gen_ext16s_i64(ret, val); + break; + case MO_UW: + tcg_gen_ext16u_i64(ret, val); + break; + case MO_SL: + tcg_gen_ext32s_i64(ret, val); + break; + case MO_UL: + tcg_gen_ext32u_i64(ret, val); + break; + default: + tcg_gen_mov_i64(ret, val); + break; + } +} + +typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv_i64, + TCGv_i32, TCGv_i32, TCGv_i32); +typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv_i64, + TCGv_i64, TCGv_i64, TCGv_i32); +typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv_i64, + TCGv_i128, TCGv_i128, TCGv_i32); +typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv_i64, + TCGv_i32, TCGv_i32); +typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv_i64, + TCGv_i64, TCGv_i32); + +#ifdef CONFIG_ATOMIC64 +# define WITH_ATOMIC64(X) X, +#else +# define WITH_ATOMIC64(X) +#endif +#ifdef CONFIG_CMPXCHG128 +# define WITH_ATOMIC128(X) X, +#else +# define WITH_ATOMIC128(X) +#endif + +static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = { + [MO_8] = gen_helper_atomic_cmpxchgb, + [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, + [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, + [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, + [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, + WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) + WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) + WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le) + WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be) +}; + +static void tcg_gen_nonatomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr, + TCGv_i32 cmpv, TCGv_i32 newv, + TCGArg idx, MemOp memop) +{ + TCGv_i32 t1 = tcg_temp_ebb_new_i32(); + TCGv_i32 t2 = tcg_temp_ebb_new_i32(); + + tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); + + tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop & ~MO_SIGN); + tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); + tcg_gen_qemu_st_i32_int(t2, addr, idx, memop); + tcg_temp_free_i32(t2); + + if (memop & MO_SIGN) { + tcg_gen_ext_i32(retv, t1, memop); + } else { + tcg_gen_mov_i32(retv, t1); + } + tcg_temp_free_i32(t1); +} + +void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr, + TCGv_i32 cmpv, TCGv_i32 newv, + TCGArg idx, MemOp memop, + TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) <= MO_32); + tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); +} + +static void tcg_gen_atomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr, + TCGv_i32 cmpv, TCGv_i32 newv, + TCGArg idx, MemOp memop) +{ + gen_atomic_cx_i32 gen; + TCGv_i64 a64; + MemOpIdx oi; + + if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { + tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); + return; + } + + memop = tcg_canonicalize_memop(memop, 0, 0); + gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; + tcg_debug_assert(gen != NULL); + + oi = make_memop_idx(memop & ~MO_SIGN, idx); + a64 = maybe_extend_addr64(addr); + gen(retv, cpu_env, a64, cmpv, newv, tcg_constant_i32(oi)); + maybe_free_addr64(a64); + + if (memop & MO_SIGN) { + tcg_gen_ext_i32(retv, retv, memop); + } +} + +void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr, + TCGv_i32 cmpv, TCGv_i32 newv, + TCGArg idx, MemOp memop, + TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) <= MO_32); + tcg_gen_atomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); +} + +static void tcg_gen_nonatomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr, + TCGv_i64 cmpv, TCGv_i64 newv, + TCGArg idx, MemOp memop) +{ + TCGv_i64 t1, t2; + + if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { + tcg_gen_nonatomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), + TCGV_LOW(newv), idx, memop); + if (memop & MO_SIGN) { + tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); + } else { + tcg_gen_movi_i32(TCGV_HIGH(retv), 0); + } + return; + } + + t1 = tcg_temp_ebb_new_i64(); + t2 = tcg_temp_ebb_new_i64(); + + tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); + + tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop & ~MO_SIGN); + tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); + tcg_gen_qemu_st_i64_int(t2, addr, idx, memop); + tcg_temp_free_i64(t2); + + if (memop & MO_SIGN) { + tcg_gen_ext_i64(retv, t1, memop); + } else { + tcg_gen_mov_i64(retv, t1); + } + tcg_temp_free_i64(t1); +} + +void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr, + TCGv_i64 cmpv, TCGv_i64 newv, + TCGArg idx, MemOp memop, + TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) <= MO_64); + tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); +} + +static void tcg_gen_atomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr, + TCGv_i64 cmpv, TCGv_i64 newv, + TCGArg idx, MemOp memop) +{ + if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { + tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); + return; + } + + if ((memop & MO_SIZE) == MO_64) { + gen_atomic_cx_i64 gen; + + memop = tcg_canonicalize_memop(memop, 1, 0); + gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; + if (gen) { + MemOpIdx oi = make_memop_idx(memop, idx); + TCGv_i64 a64 = maybe_extend_addr64(addr); + gen(retv, cpu_env, a64, cmpv, newv, tcg_constant_i32(oi)); + maybe_free_addr64(a64); + return; + } + + gen_helper_exit_atomic(cpu_env); + + /* + * Produce a result for a well-formed opcode stream. This satisfies + * liveness for set before used, which happens before this dead code + * is removed. + */ + tcg_gen_movi_i64(retv, 0); + return; + } + + if (TCG_TARGET_REG_BITS == 32) { + tcg_gen_atomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), + TCGV_LOW(newv), idx, memop); + if (memop & MO_SIGN) { + tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); + } else { + tcg_gen_movi_i32(TCGV_HIGH(retv), 0); + } + } else { + TCGv_i32 c32 = tcg_temp_ebb_new_i32(); + TCGv_i32 n32 = tcg_temp_ebb_new_i32(); + TCGv_i32 r32 = tcg_temp_ebb_new_i32(); + + tcg_gen_extrl_i64_i32(c32, cmpv); + tcg_gen_extrl_i64_i32(n32, newv); + tcg_gen_atomic_cmpxchg_i32_int(r32, addr, c32, n32, + idx, memop & ~MO_SIGN); + tcg_temp_free_i32(c32); + tcg_temp_free_i32(n32); + + tcg_gen_extu_i32_i64(retv, r32); + tcg_temp_free_i32(r32); + + if (memop & MO_SIGN) { + tcg_gen_ext_i64(retv, retv, memop); + } + } +} + +void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr, + TCGv_i64 cmpv, TCGv_i64 newv, + TCGArg idx, MemOp memop, TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & MO_SIZE) <= MO_64); + tcg_gen_atomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); +} + +static void tcg_gen_nonatomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr, + TCGv_i128 cmpv, TCGv_i128 newv, + TCGArg idx, MemOp memop) +{ + if (TCG_TARGET_REG_BITS == 32) { + /* Inline expansion below is simply too large for 32-bit hosts. */ + gen_atomic_cx_i128 gen = ((memop & MO_BSWAP) == MO_LE + ? gen_helper_nonatomic_cmpxchgo_le + : gen_helper_nonatomic_cmpxchgo_be); + MemOpIdx oi = make_memop_idx(memop, idx); + TCGv_i64 a64 = maybe_extend_addr64(addr); + + gen(retv, cpu_env, a64, cmpv, newv, tcg_constant_i32(oi)); + maybe_free_addr64(a64); + } else { + TCGv_i128 oldv = tcg_temp_ebb_new_i128(); + TCGv_i128 tmpv = tcg_temp_ebb_new_i128(); + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + TCGv_i64 t1 = tcg_temp_ebb_new_i64(); + TCGv_i64 z = tcg_constant_i64(0); + + tcg_gen_qemu_ld_i128_int(oldv, addr, idx, memop); + + /* Compare i128 */ + tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv)); + tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv)); + tcg_gen_or_i64(t0, t0, t1); + + /* tmpv = equal ? newv : oldv */ + tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z, + TCGV128_LOW(newv), TCGV128_LOW(oldv)); + tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z, + TCGV128_HIGH(newv), TCGV128_HIGH(oldv)); + + /* Unconditional writeback. */ + tcg_gen_qemu_st_i128_int(tmpv, addr, idx, memop); + tcg_gen_mov_i128(retv, oldv); + + tcg_temp_free_i64(t0); + tcg_temp_free_i64(t1); + tcg_temp_free_i128(tmpv); + tcg_temp_free_i128(oldv); + } +} + +void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr, + TCGv_i128 cmpv, TCGv_i128 newv, + TCGArg idx, MemOp memop, + TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128); + tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); +} + +static void tcg_gen_atomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr, + TCGv_i128 cmpv, TCGv_i128 newv, + TCGArg idx, MemOp memop) +{ + gen_atomic_cx_i128 gen; + + if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { + tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); + return; + } + + gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; + if (gen) { + MemOpIdx oi = make_memop_idx(memop, idx); + TCGv_i64 a64 = maybe_extend_addr64(addr); + gen(retv, cpu_env, a64, cmpv, newv, tcg_constant_i32(oi)); + maybe_free_addr64(a64); + return; + } + + gen_helper_exit_atomic(cpu_env); + + /* + * Produce a result for a well-formed opcode stream. This satisfies + * liveness for set before used, which happens before this dead code + * is removed. + */ + tcg_gen_movi_i64(TCGV128_LOW(retv), 0); + tcg_gen_movi_i64(TCGV128_HIGH(retv), 0); +} + +void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr, + TCGv_i128 cmpv, TCGv_i128 newv, + TCGArg idx, MemOp memop, + TCGType addr_type) +{ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); + tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128); + tcg_gen_atomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); +} + +static void do_nonatomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val, + TCGArg idx, MemOp memop, bool new_val, + void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) +{ + TCGv_i32 t1 = tcg_temp_ebb_new_i32(); + TCGv_i32 t2 = tcg_temp_ebb_new_i32(); + + memop = tcg_canonicalize_memop(memop, 0, 0); + + tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop); + tcg_gen_ext_i32(t2, val, memop); + gen(t2, t1, t2); + tcg_gen_qemu_st_i32_int(t2, addr, idx, memop); + + tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); + tcg_temp_free_i32(t1); + tcg_temp_free_i32(t2); +} + +static void do_atomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val, + TCGArg idx, MemOp memop, void * const table[]) +{ + gen_atomic_op_i32 gen; + TCGv_i64 a64; + MemOpIdx oi; + + memop = tcg_canonicalize_memop(memop, 0, 0); + + gen = table[memop & (MO_SIZE | MO_BSWAP)]; + tcg_debug_assert(gen != NULL); + + oi = make_memop_idx(memop & ~MO_SIGN, idx); + a64 = maybe_extend_addr64(addr); + gen(ret, cpu_env, a64, val, tcg_constant_i32(oi)); + maybe_free_addr64(a64); + + if (memop & MO_SIGN) { + tcg_gen_ext_i32(ret, ret, memop); + } +} + +static void do_nonatomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val, + TCGArg idx, MemOp memop, bool new_val, + void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) +{ + TCGv_i64 t1 = tcg_temp_ebb_new_i64(); + TCGv_i64 t2 = tcg_temp_ebb_new_i64(); + + memop = tcg_canonicalize_memop(memop, 1, 0); + + tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop); + tcg_gen_ext_i64(t2, val, memop); + gen(t2, t1, t2); + tcg_gen_qemu_st_i64_int(t2, addr, idx, memop); + + tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); + tcg_temp_free_i64(t1); + tcg_temp_free_i64(t2); +} + +static void do_atomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val, + TCGArg idx, MemOp memop, void * const table[]) +{ + memop = tcg_canonicalize_memop(memop, 1, 0); + + if ((memop & MO_SIZE) == MO_64) { + gen_atomic_op_i64 gen = table[memop & (MO_SIZE | MO_BSWAP)]; + + if (gen) { + MemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx); + TCGv_i64 a64 = maybe_extend_addr64(addr); + gen(ret, cpu_env, a64, val, tcg_constant_i32(oi)); + maybe_free_addr64(a64); + return; + } + + gen_helper_exit_atomic(cpu_env); + /* Produce a result, so that we have a well-formed opcode stream + with respect to uses of the result in the (dead) code following. */ + tcg_gen_movi_i64(ret, 0); + } else { + TCGv_i32 v32 = tcg_temp_ebb_new_i32(); + TCGv_i32 r32 = tcg_temp_ebb_new_i32(); + + tcg_gen_extrl_i64_i32(v32, val); + do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); + tcg_temp_free_i32(v32); + + tcg_gen_extu_i32_i64(ret, r32); + tcg_temp_free_i32(r32); + + if (memop & MO_SIGN) { + tcg_gen_ext_i64(ret, ret, memop); + } + } +} + +#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ +static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ + [MO_8] = gen_helper_atomic_##NAME##b, \ + [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ + [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ + [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ + [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ + WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ + WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ +}; \ +void tcg_gen_atomic_##NAME##_i32_chk(TCGv_i32 ret, TCGTemp *addr, \ + TCGv_i32 val, TCGArg idx, \ + MemOp memop, TCGType addr_type) \ +{ \ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); \ + tcg_debug_assert((memop & MO_SIZE) <= MO_32); \ + if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ + do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ + } else { \ + do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ + tcg_gen_##OP##_i32); \ + } \ +} \ +void tcg_gen_atomic_##NAME##_i64_chk(TCGv_i64 ret, TCGTemp *addr, \ + TCGv_i64 val, TCGArg idx, \ + MemOp memop, TCGType addr_type) \ +{ \ + tcg_debug_assert(addr_type == tcg_ctx->addr_type); \ + tcg_debug_assert((memop & MO_SIZE) <= MO_64); \ + if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ + do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ + } else { \ + do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ + tcg_gen_##OP##_i64); \ + } \ +} + +GEN_ATOMIC_HELPER(fetch_add, add, 0) +GEN_ATOMIC_HELPER(fetch_and, and, 0) +GEN_ATOMIC_HELPER(fetch_or, or, 0) +GEN_ATOMIC_HELPER(fetch_xor, xor, 0) +GEN_ATOMIC_HELPER(fetch_smin, smin, 0) +GEN_ATOMIC_HELPER(fetch_umin, umin, 0) +GEN_ATOMIC_HELPER(fetch_smax, smax, 0) +GEN_ATOMIC_HELPER(fetch_umax, umax, 0) + +GEN_ATOMIC_HELPER(add_fetch, add, 1) +GEN_ATOMIC_HELPER(and_fetch, and, 1) +GEN_ATOMIC_HELPER(or_fetch, or, 1) +GEN_ATOMIC_HELPER(xor_fetch, xor, 1) +GEN_ATOMIC_HELPER(smin_fetch, smin, 1) +GEN_ATOMIC_HELPER(umin_fetch, umin, 1) +GEN_ATOMIC_HELPER(smax_fetch, smax, 1) +GEN_ATOMIC_HELPER(umax_fetch, umax, 1) + +static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) +{ + tcg_gen_mov_i32(r, b); +} + +static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) +{ + tcg_gen_mov_i64(r, b); +} + +GEN_ATOMIC_HELPER(xchg, mov2, 0) + +#undef GEN_ATOMIC_HELPER diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index 3136cef81a..edbd1c61d7 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -27,7 +27,6 @@ #include "tcg/tcg.h" #include "tcg/tcg-temp-internal.h" #include "tcg/tcg-op.h" -#include "tcg/tcg-mo.h" #include "exec/plugin-gen.h" #include "tcg-internal.h" @@ -2841,866 +2840,3 @@ void tcg_gen_lookup_and_goto_ptr(void) tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); tcg_temp_free_ptr(ptr); } - -static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) -{ - /* Trigger the asserts within as early as possible. */ - unsigned a_bits = get_alignment_bits(op); - - /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */ - if (a_bits == (op & MO_SIZE)) { - op = (op & ~MO_AMASK) | MO_ALIGN; - } - - switch (op & MO_SIZE) { - case MO_8: - op &= ~MO_BSWAP; - break; - case MO_16: - break; - case MO_32: - if (!is64) { - op &= ~MO_SIGN; - } - break; - case MO_64: - if (is64) { - op &= ~MO_SIGN; - break; - } - /* fall through */ - default: - g_assert_not_reached(); - } - if (st) { - op &= ~MO_SIGN; - } - return op; -} - -static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, - MemOp memop, TCGArg idx) -{ - MemOpIdx oi = make_memop_idx(memop, idx); -#if TARGET_LONG_BITS == 32 - tcg_gen_op3i_i32(opc, val, addr, oi); -#else - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); - } else { - tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); - } -#endif -} - -static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, - MemOp memop, TCGArg idx) -{ - MemOpIdx oi = make_memop_idx(memop, idx); -#if TARGET_LONG_BITS == 32 - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); - } else { - tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); - } -#else - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), - TCGV_LOW(addr), TCGV_HIGH(addr), oi); - } else { - tcg_gen_op3i_i64(opc, val, addr, oi); - } -#endif -} - -static void tcg_gen_req_mo(TCGBar type) -{ -#ifdef TCG_GUEST_DEFAULT_MO - type &= TCG_GUEST_DEFAULT_MO; -#endif - type &= ~TCG_TARGET_DEFAULT_MO; - if (type) { - tcg_gen_mb(type | TCG_BAR_SC); - } -} - -static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr) -{ -#ifdef CONFIG_PLUGIN - if (tcg_ctx->plugin_insn != NULL) { - /* Save a copy of the vaddr for use after a load. */ - TCGv temp = tcg_temp_new(); - tcg_gen_mov_tl(temp, vaddr); - return temp; - } -#endif - return vaddr; -} - -static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi, - enum qemu_plugin_mem_rw rw) -{ -#ifdef CONFIG_PLUGIN - if (tcg_ctx->plugin_insn != NULL) { - qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw); - plugin_gen_empty_mem_callback(vaddr, info); - tcg_temp_free(vaddr); - } -#endif -} - -void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) -{ - MemOp orig_memop; - MemOpIdx oi; - - tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); - memop = tcg_canonicalize_memop(memop, 0, 0); - oi = make_memop_idx(memop, idx); - - orig_memop = memop; - if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { - memop &= ~MO_BSWAP; - /* The bswap primitive benefits from zero-extended input. */ - if ((memop & MO_SSIZE) == MO_SW) { - memop &= ~MO_SIGN; - } - } - - addr = plugin_prep_mem_callbacks(addr); - gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); - plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); - - if ((orig_memop ^ memop) & MO_BSWAP) { - switch (orig_memop & MO_SIZE) { - case MO_16: - tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN - ? TCG_BSWAP_IZ | TCG_BSWAP_OS - : TCG_BSWAP_IZ | TCG_BSWAP_OZ)); - break; - case MO_32: - tcg_gen_bswap32_i32(val, val); - break; - default: - g_assert_not_reached(); - } - } -} - -void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) -{ - TCGv_i32 swap = NULL; - MemOpIdx oi; - - tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); - memop = tcg_canonicalize_memop(memop, 0, 1); - oi = make_memop_idx(memop, idx); - - if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { - swap = tcg_temp_ebb_new_i32(); - switch (memop & MO_SIZE) { - case MO_16: - tcg_gen_bswap16_i32(swap, val, 0); - break; - case MO_32: - tcg_gen_bswap32_i32(swap, val); - break; - default: - g_assert_not_reached(); - } - val = swap; - memop &= ~MO_BSWAP; - } - - addr = plugin_prep_mem_callbacks(addr); - if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) { - gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx); - } else { - gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); - } - plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); - - if (swap) { - tcg_temp_free_i32(swap); - } -} - -void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) -{ - MemOp orig_memop; - MemOpIdx oi; - - if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { - tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); - if (memop & MO_SIGN) { - tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); - } else { - tcg_gen_movi_i32(TCGV_HIGH(val), 0); - } - return; - } - - tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); - memop = tcg_canonicalize_memop(memop, 1, 0); - oi = make_memop_idx(memop, idx); - - orig_memop = memop; - if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { - memop &= ~MO_BSWAP; - /* The bswap primitive benefits from zero-extended input. */ - if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { - memop &= ~MO_SIGN; - } - } - - addr = plugin_prep_mem_callbacks(addr); - gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); - plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); - - if ((orig_memop ^ memop) & MO_BSWAP) { - int flags = (orig_memop & MO_SIGN - ? TCG_BSWAP_IZ | TCG_BSWAP_OS - : TCG_BSWAP_IZ | TCG_BSWAP_OZ); - switch (orig_memop & MO_SIZE) { - case MO_16: - tcg_gen_bswap16_i64(val, val, flags); - break; - case MO_32: - tcg_gen_bswap32_i64(val, val, flags); - break; - case MO_64: - tcg_gen_bswap64_i64(val, val); - break; - default: - g_assert_not_reached(); - } - } -} - -void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) -{ - TCGv_i64 swap = NULL; - MemOpIdx oi; - - if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { - tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); - return; - } - - tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); - memop = tcg_canonicalize_memop(memop, 1, 1); - oi = make_memop_idx(memop, idx); - - if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { - swap = tcg_temp_ebb_new_i64(); - switch (memop & MO_SIZE) { - case MO_16: - tcg_gen_bswap16_i64(swap, val, 0); - break; - case MO_32: - tcg_gen_bswap32_i64(swap, val, 0); - break; - case MO_64: - tcg_gen_bswap64_i64(swap, val); - break; - default: - g_assert_not_reached(); - } - val = swap; - memop &= ~MO_BSWAP; - } - - addr = plugin_prep_mem_callbacks(addr); - gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); - plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); - - if (swap) { - tcg_temp_free_i64(swap); - } -} - -static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig) -{ - MemOp mop_1 = orig, mop_2; - - tcg_debug_assert((orig & MO_SIZE) == MO_128); - tcg_debug_assert((orig & MO_SIGN) == 0); - - /* Use a memory ordering implemented by the host. */ - if (!TCG_TARGET_HAS_MEMORY_BSWAP && (orig & MO_BSWAP)) { - mop_1 &= ~MO_BSWAP; - } - - /* Reduce the size to 64-bit. */ - mop_1 = (mop_1 & ~MO_SIZE) | MO_64; - - /* Retain the alignment constraints of the original. */ - switch (orig & MO_AMASK) { - case MO_UNALN: - case MO_ALIGN_2: - case MO_ALIGN_4: - mop_2 = mop_1; - break; - case MO_ALIGN_8: - /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */ - mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN; - mop_2 = mop_1; - break; - case MO_ALIGN: - /* Second has 8-byte alignment; first has 16-byte alignment. */ - mop_2 = mop_1; - mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16; - break; - case MO_ALIGN_16: - case MO_ALIGN_32: - case MO_ALIGN_64: - /* Second has 8-byte alignment; first retains original. */ - mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN; - break; - default: - g_assert_not_reached(); - } - ret[0] = mop_1; - ret[1] = mop_2; -} - -void tcg_gen_qemu_ld_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop) -{ - MemOp mop[2]; - TCGv addr_p8; - TCGv_i64 x, y; - - canonicalize_memop_i128_as_i64(mop, memop); - - tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); - addr = plugin_prep_mem_callbacks(addr); - - /* TODO: respect atomicity of the operation. */ - /* TODO: allow the tcg backend to see the whole operation. */ - - /* - * Since there are no global TCGv_i128, there is no visible state - * changed if the second load faults. Load directly into the two - * subwords. - */ - if ((memop & MO_BSWAP) == MO_LE) { - x = TCGV128_LOW(val); - y = TCGV128_HIGH(val); - } else { - x = TCGV128_HIGH(val); - y = TCGV128_LOW(val); - } - - gen_ldst_i64(INDEX_op_qemu_ld_i64, x, addr, mop[0], idx); - - if ((mop[0] ^ memop) & MO_BSWAP) { - tcg_gen_bswap64_i64(x, x); - } - - addr_p8 = tcg_temp_new(); - tcg_gen_addi_tl(addr_p8, addr, 8); - gen_ldst_i64(INDEX_op_qemu_ld_i64, y, addr_p8, mop[1], idx); - tcg_temp_free(addr_p8); - - if ((mop[0] ^ memop) & MO_BSWAP) { - tcg_gen_bswap64_i64(y, y); - } - - plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx), - QEMU_PLUGIN_MEM_R); -} - -void tcg_gen_qemu_st_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop) -{ - MemOp mop[2]; - TCGv addr_p8; - TCGv_i64 x, y; - - canonicalize_memop_i128_as_i64(mop, memop); - - tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST); - addr = plugin_prep_mem_callbacks(addr); - - /* TODO: respect atomicity of the operation. */ - /* TODO: allow the tcg backend to see the whole operation. */ - - if ((memop & MO_BSWAP) == MO_LE) { - x = TCGV128_LOW(val); - y = TCGV128_HIGH(val); - } else { - x = TCGV128_HIGH(val); - y = TCGV128_LOW(val); - } - - addr_p8 = tcg_temp_new(); - if ((mop[0] ^ memop) & MO_BSWAP) { - TCGv_i64 t = tcg_temp_ebb_new_i64(); - - tcg_gen_bswap64_i64(t, x); - gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr, mop[0], idx); - tcg_gen_bswap64_i64(t, y); - tcg_gen_addi_tl(addr_p8, addr, 8); - gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr_p8, mop[1], idx); - tcg_temp_free_i64(t); - } else { - gen_ldst_i64(INDEX_op_qemu_st_i64, x, addr, mop[0], idx); - tcg_gen_addi_tl(addr_p8, addr, 8); - gen_ldst_i64(INDEX_op_qemu_st_i64, y, addr_p8, mop[1], idx); - } - tcg_temp_free(addr_p8); - - plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx), - QEMU_PLUGIN_MEM_W); -} - -static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc) -{ - switch (opc & MO_SSIZE) { - case MO_SB: - tcg_gen_ext8s_i32(ret, val); - break; - case MO_UB: - tcg_gen_ext8u_i32(ret, val); - break; - case MO_SW: - tcg_gen_ext16s_i32(ret, val); - break; - case MO_UW: - tcg_gen_ext16u_i32(ret, val); - break; - default: - tcg_gen_mov_i32(ret, val); - break; - } -} - -static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc) -{ - switch (opc & MO_SSIZE) { - case MO_SB: - tcg_gen_ext8s_i64(ret, val); - break; - case MO_UB: - tcg_gen_ext8u_i64(ret, val); - break; - case MO_SW: - tcg_gen_ext16s_i64(ret, val); - break; - case MO_UW: - tcg_gen_ext16u_i64(ret, val); - break; - case MO_SL: - tcg_gen_ext32s_i64(ret, val); - break; - case MO_UL: - tcg_gen_ext32u_i64(ret, val); - break; - default: - tcg_gen_mov_i64(ret, val); - break; - } -} - -typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, - TCGv_i32, TCGv_i32, TCGv_i32); -typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, - TCGv_i64, TCGv_i64, TCGv_i32); -typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv, - TCGv_i128, TCGv_i128, TCGv_i32); -typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, - TCGv_i32, TCGv_i32); -typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, - TCGv_i64, TCGv_i32); - -#ifdef CONFIG_ATOMIC64 -# define WITH_ATOMIC64(X) X, -#else -# define WITH_ATOMIC64(X) -#endif -#ifdef CONFIG_CMPXCHG128 -# define WITH_ATOMIC128(X) X, -#else -# define WITH_ATOMIC128(X) -#endif - -static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = { - [MO_8] = gen_helper_atomic_cmpxchgb, - [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, - [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, - [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, - [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, - WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) - WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) - WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le) - WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be) -}; - -void tcg_gen_nonatomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, - TCGv_i32 newv, TCGArg idx, MemOp memop) -{ - TCGv_i32 t1 = tcg_temp_ebb_new_i32(); - TCGv_i32 t2 = tcg_temp_ebb_new_i32(); - - tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); - - tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); - tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); - tcg_gen_qemu_st_i32(t2, addr, idx, memop); - tcg_temp_free_i32(t2); - - if (memop & MO_SIGN) { - tcg_gen_ext_i32(retv, t1, memop); - } else { - tcg_gen_mov_i32(retv, t1); - } - tcg_temp_free_i32(t1); -} - -void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, - TCGv_i32 newv, TCGArg idx, MemOp memop) -{ - gen_atomic_cx_i32 gen; - MemOpIdx oi; - - if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { - tcg_gen_nonatomic_cmpxchg_i32(retv, addr, cmpv, newv, idx, memop); - return; - } - - memop = tcg_canonicalize_memop(memop, 0, 0); - gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; - tcg_debug_assert(gen != NULL); - - oi = make_memop_idx(memop & ~MO_SIGN, idx); - gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); - - if (memop & MO_SIGN) { - tcg_gen_ext_i32(retv, retv, memop); - } -} - -void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, - TCGv_i64 newv, TCGArg idx, MemOp memop) -{ - TCGv_i64 t1, t2; - - if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { - tcg_gen_nonatomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), - TCGV_LOW(newv), idx, memop); - if (memop & MO_SIGN) { - tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); - } else { - tcg_gen_movi_i32(TCGV_HIGH(retv), 0); - } - return; - } - - t1 = tcg_temp_ebb_new_i64(); - t2 = tcg_temp_ebb_new_i64(); - - tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); - - tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); - tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); - tcg_gen_qemu_st_i64(t2, addr, idx, memop); - tcg_temp_free_i64(t2); - - if (memop & MO_SIGN) { - tcg_gen_ext_i64(retv, t1, memop); - } else { - tcg_gen_mov_i64(retv, t1); - } - tcg_temp_free_i64(t1); -} - -void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, - TCGv_i64 newv, TCGArg idx, MemOp memop) -{ - if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { - tcg_gen_nonatomic_cmpxchg_i64(retv, addr, cmpv, newv, idx, memop); - return; - } - - if ((memop & MO_SIZE) == MO_64) { - gen_atomic_cx_i64 gen; - - memop = tcg_canonicalize_memop(memop, 1, 0); - gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; - if (gen) { - MemOpIdx oi = make_memop_idx(memop, idx); - gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); - return; - } - - gen_helper_exit_atomic(cpu_env); - - /* - * Produce a result for a well-formed opcode stream. This satisfies - * liveness for set before used, which happens before this dead code - * is removed. - */ - tcg_gen_movi_i64(retv, 0); - return; - } - - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_atomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), - TCGV_LOW(newv), idx, memop); - if (memop & MO_SIGN) { - tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); - } else { - tcg_gen_movi_i32(TCGV_HIGH(retv), 0); - } - } else { - TCGv_i32 c32 = tcg_temp_ebb_new_i32(); - TCGv_i32 n32 = tcg_temp_ebb_new_i32(); - TCGv_i32 r32 = tcg_temp_ebb_new_i32(); - - tcg_gen_extrl_i64_i32(c32, cmpv); - tcg_gen_extrl_i64_i32(n32, newv); - tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); - tcg_temp_free_i32(c32); - tcg_temp_free_i32(n32); - - tcg_gen_extu_i32_i64(retv, r32); - tcg_temp_free_i32(r32); - - if (memop & MO_SIGN) { - tcg_gen_ext_i64(retv, retv, memop); - } - } -} - -void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv, - TCGv_i128 newv, TCGArg idx, MemOp memop) -{ - if (TCG_TARGET_REG_BITS == 32) { - /* Inline expansion below is simply too large for 32-bit hosts. */ - gen_atomic_cx_i128 gen = ((memop & MO_BSWAP) == MO_LE - ? gen_helper_nonatomic_cmpxchgo_le - : gen_helper_nonatomic_cmpxchgo_be); - MemOpIdx oi = make_memop_idx(memop, idx); - - tcg_debug_assert((memop & MO_SIZE) == MO_128); - tcg_debug_assert((memop & MO_SIGN) == 0); - - gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); - } else { - TCGv_i128 oldv = tcg_temp_ebb_new_i128(); - TCGv_i128 tmpv = tcg_temp_ebb_new_i128(); - TCGv_i64 t0 = tcg_temp_ebb_new_i64(); - TCGv_i64 t1 = tcg_temp_ebb_new_i64(); - TCGv_i64 z = tcg_constant_i64(0); - - tcg_gen_qemu_ld_i128(oldv, addr, idx, memop); - - /* Compare i128 */ - tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv)); - tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv)); - tcg_gen_or_i64(t0, t0, t1); - - /* tmpv = equal ? newv : oldv */ - tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z, - TCGV128_LOW(newv), TCGV128_LOW(oldv)); - tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z, - TCGV128_HIGH(newv), TCGV128_HIGH(oldv)); - - /* Unconditional writeback. */ - tcg_gen_qemu_st_i128(tmpv, addr, idx, memop); - tcg_gen_mov_i128(retv, oldv); - - tcg_temp_free_i64(t0); - tcg_temp_free_i64(t1); - tcg_temp_free_i128(tmpv); - tcg_temp_free_i128(oldv); - } -} - -void tcg_gen_atomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv, - TCGv_i128 newv, TCGArg idx, MemOp memop) -{ - gen_atomic_cx_i128 gen; - - if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { - tcg_gen_nonatomic_cmpxchg_i128(retv, addr, cmpv, newv, idx, memop); - return; - } - - tcg_debug_assert((memop & MO_SIZE) == MO_128); - tcg_debug_assert((memop & MO_SIGN) == 0); - gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; - - if (gen) { - MemOpIdx oi = make_memop_idx(memop, idx); - gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); - return; - } - - gen_helper_exit_atomic(cpu_env); - - /* - * Produce a result for a well-formed opcode stream. This satisfies - * liveness for set before used, which happens before this dead code - * is removed. - */ - tcg_gen_movi_i64(TCGV128_LOW(retv), 0); - tcg_gen_movi_i64(TCGV128_HIGH(retv), 0); -} - -static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, - TCGArg idx, MemOp memop, bool new_val, - void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) -{ - TCGv_i32 t1 = tcg_temp_ebb_new_i32(); - TCGv_i32 t2 = tcg_temp_ebb_new_i32(); - - memop = tcg_canonicalize_memop(memop, 0, 0); - - tcg_gen_qemu_ld_i32(t1, addr, idx, memop); - tcg_gen_ext_i32(t2, val, memop); - gen(t2, t1, t2); - tcg_gen_qemu_st_i32(t2, addr, idx, memop); - - tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); - tcg_temp_free_i32(t1); - tcg_temp_free_i32(t2); -} - -static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, - TCGArg idx, MemOp memop, void * const table[]) -{ - gen_atomic_op_i32 gen; - MemOpIdx oi; - - memop = tcg_canonicalize_memop(memop, 0, 0); - - gen = table[memop & (MO_SIZE | MO_BSWAP)]; - tcg_debug_assert(gen != NULL); - - oi = make_memop_idx(memop & ~MO_SIGN, idx); - gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); - - if (memop & MO_SIGN) { - tcg_gen_ext_i32(ret, ret, memop); - } -} - -static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, - TCGArg idx, MemOp memop, bool new_val, - void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) -{ - TCGv_i64 t1 = tcg_temp_ebb_new_i64(); - TCGv_i64 t2 = tcg_temp_ebb_new_i64(); - - memop = tcg_canonicalize_memop(memop, 1, 0); - - tcg_gen_qemu_ld_i64(t1, addr, idx, memop); - tcg_gen_ext_i64(t2, val, memop); - gen(t2, t1, t2); - tcg_gen_qemu_st_i64(t2, addr, idx, memop); - - tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); - tcg_temp_free_i64(t1); - tcg_temp_free_i64(t2); -} - -static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, - TCGArg idx, MemOp memop, void * const table[]) -{ - memop = tcg_canonicalize_memop(memop, 1, 0); - - if ((memop & MO_SIZE) == MO_64) { -#ifdef CONFIG_ATOMIC64 - gen_atomic_op_i64 gen; - MemOpIdx oi; - - gen = table[memop & (MO_SIZE | MO_BSWAP)]; - tcg_debug_assert(gen != NULL); - - oi = make_memop_idx(memop & ~MO_SIGN, idx); - gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); -#else - gen_helper_exit_atomic(cpu_env); - /* Produce a result, so that we have a well-formed opcode stream - with respect to uses of the result in the (dead) code following. */ - tcg_gen_movi_i64(ret, 0); -#endif /* CONFIG_ATOMIC64 */ - } else { - TCGv_i32 v32 = tcg_temp_ebb_new_i32(); - TCGv_i32 r32 = tcg_temp_ebb_new_i32(); - - tcg_gen_extrl_i64_i32(v32, val); - do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); - tcg_temp_free_i32(v32); - - tcg_gen_extu_i32_i64(ret, r32); - tcg_temp_free_i32(r32); - - if (memop & MO_SIGN) { - tcg_gen_ext_i64(ret, ret, memop); - } - } -} - -#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ -static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ - [MO_8] = gen_helper_atomic_##NAME##b, \ - [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ - [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ - [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ - [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ - WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ - WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ -}; \ -void tcg_gen_atomic_##NAME##_i32 \ - (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \ -{ \ - if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ - do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ - } else { \ - do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ - tcg_gen_##OP##_i32); \ - } \ -} \ -void tcg_gen_atomic_##NAME##_i64 \ - (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \ -{ \ - if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ - do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ - } else { \ - do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ - tcg_gen_##OP##_i64); \ - } \ -} - -GEN_ATOMIC_HELPER(fetch_add, add, 0) -GEN_ATOMIC_HELPER(fetch_and, and, 0) -GEN_ATOMIC_HELPER(fetch_or, or, 0) -GEN_ATOMIC_HELPER(fetch_xor, xor, 0) -GEN_ATOMIC_HELPER(fetch_smin, smin, 0) -GEN_ATOMIC_HELPER(fetch_umin, umin, 0) -GEN_ATOMIC_HELPER(fetch_smax, smax, 0) -GEN_ATOMIC_HELPER(fetch_umax, umax, 0) - -GEN_ATOMIC_HELPER(add_fetch, add, 1) -GEN_ATOMIC_HELPER(and_fetch, and, 1) -GEN_ATOMIC_HELPER(or_fetch, or, 1) -GEN_ATOMIC_HELPER(xor_fetch, xor, 1) -GEN_ATOMIC_HELPER(smin_fetch, smin, 1) -GEN_ATOMIC_HELPER(umin_fetch, umin, 1) -GEN_ATOMIC_HELPER(smax_fetch, smax, 1) -GEN_ATOMIC_HELPER(umax_fetch, umax, 1) - -static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) -{ - tcg_gen_mov_i32(r, b); -} - -static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) -{ - tcg_gen_mov_i64(r, b); -} - -GEN_ATOMIC_HELPER(xchg, mov2, 0) - -#undef GEN_ATOMIC_HELPER diff --git a/tcg/tcg.c b/tcg/tcg.c index 1231c8ab4c..0b0fe9c7ad 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -63,6 +63,9 @@ #include "tcg/tcg-temp-internal.h" #include "tcg-internal.h" #include "accel/tcg/perf.h" +#ifdef CONFIG_USER_ONLY +#include "exec/user/guest-base.h" +#endif /* Forward declarations for functions declared in tcg-target.c.inc and used here. */ @@ -197,6 +200,38 @@ static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l, const TCGLdstHelperParam *p) __attribute__((unused)); +static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = { + [MO_UB] = helper_ldub_mmu, + [MO_SB] = helper_ldsb_mmu, + [MO_UW] = helper_lduw_mmu, + [MO_SW] = helper_ldsw_mmu, + [MO_UL] = helper_ldul_mmu, + [MO_UQ] = helper_ldq_mmu, +#if TCG_TARGET_REG_BITS == 64 + [MO_SL] = helper_ldsl_mmu, + [MO_128] = helper_ld16_mmu, +#endif +}; + +static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = { + [MO_8] = helper_stb_mmu, + [MO_16] = helper_stw_mmu, + [MO_32] = helper_stl_mmu, + [MO_64] = helper_stq_mmu, +#if TCG_TARGET_REG_BITS == 64 + [MO_128] = helper_st16_mmu, +#endif +}; + +typedef struct { + MemOp atom; /* lg2 bits of atomicity required */ + MemOp align; /* lg2 bits of alignment to use */ +} TCGAtomAlign; + +static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc, + MemOp host_atom, bool allow_two_ops) + __attribute__((unused)); + TCGContext tcg_init_ctx; __thread TCGContext *tcg_ctx; @@ -513,6 +548,82 @@ static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1, tcg_out_movext1_new_src(s, i1, src1); } +/** + * tcg_out_movext3 -- move and extend three pair + * @s: tcg context + * @i1: first move description + * @i2: second move description + * @i3: third move description + * @scratch: temporary register, or -1 for none + * + * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap + * between the sources and destinations. + */ + +static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1, + const TCGMovExtend *i2, const TCGMovExtend *i3, + int scratch) +{ + TCGReg src1 = i1->src; + TCGReg src2 = i2->src; + TCGReg src3 = i3->src; + + if (i1->dst != src2 && i1->dst != src3) { + tcg_out_movext1(s, i1); + tcg_out_movext2(s, i2, i3, scratch); + return; + } + if (i2->dst != src1 && i2->dst != src3) { + tcg_out_movext1(s, i2); + tcg_out_movext2(s, i1, i3, scratch); + return; + } + if (i3->dst != src1 && i3->dst != src2) { + tcg_out_movext1(s, i3); + tcg_out_movext2(s, i1, i2, scratch); + return; + } + + /* + * There is a cycle. Since there are only 3 nodes, the cycle is + * either "clockwise" or "anti-clockwise", and can be solved with + * a single scratch or two xchg. + */ + if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) { + /* "Clockwise" */ + if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) { + tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3); + /* The data is now in the correct registers, now extend. */ + tcg_out_movext1_new_src(s, i1, i1->dst); + tcg_out_movext1_new_src(s, i2, i2->dst); + tcg_out_movext1_new_src(s, i3, i3->dst); + } else { + tcg_debug_assert(scratch >= 0); + tcg_out_mov(s, i1->src_type, scratch, src1); + tcg_out_movext1(s, i3); + tcg_out_movext1(s, i2); + tcg_out_movext1_new_src(s, i1, scratch); + } + } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) { + /* "Anti-clockwise" */ + if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) { + tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2); + /* The data is now in the correct registers, now extend. */ + tcg_out_movext1_new_src(s, i1, i1->dst); + tcg_out_movext1_new_src(s, i2, i2->dst); + tcg_out_movext1_new_src(s, i3, i3->dst); + } else { + tcg_debug_assert(scratch >= 0); + tcg_out_mov(s, i1->src_type, scratch, src1); + tcg_out_movext1(s, i2); + tcg_out_movext1(s, i3); + tcg_out_movext1_new_src(s, i1, scratch); + } + } else { + g_assert_not_reached(); + } +} + #define C_PFX1(P, A) P##A #define C_PFX2(P, A, B) P##A##_##B #define C_PFX3(P, A, B, C) P##A##_##B##_##C @@ -757,7 +868,7 @@ static TCGHelperInfo info_helper_ld32_mmu = { .flags = TCG_CALL_NO_WG, .typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */ | dh_typemask(env, 1) - | dh_typemask(tl, 2) /* target_ulong addr */ + | dh_typemask(i64, 2) /* uint64_t addr */ | dh_typemask(i32, 3) /* unsigned oi */ | dh_typemask(ptr, 4) /* uintptr_t ra */ }; @@ -766,7 +877,16 @@ static TCGHelperInfo info_helper_ld64_mmu = { .flags = TCG_CALL_NO_WG, .typemask = dh_typemask(i64, 0) /* return uint64_t */ | dh_typemask(env, 1) - | dh_typemask(tl, 2) /* target_ulong addr */ + | dh_typemask(i64, 2) /* uint64_t addr */ + | dh_typemask(i32, 3) /* unsigned oi */ + | dh_typemask(ptr, 4) /* uintptr_t ra */ +}; + +static TCGHelperInfo info_helper_ld128_mmu = { + .flags = TCG_CALL_NO_WG, + .typemask = dh_typemask(i128, 0) /* return Int128 */ + | dh_typemask(env, 1) + | dh_typemask(i64, 2) /* uint64_t addr */ | dh_typemask(i32, 3) /* unsigned oi */ | dh_typemask(ptr, 4) /* uintptr_t ra */ }; @@ -775,7 +895,7 @@ static TCGHelperInfo info_helper_st32_mmu = { .flags = TCG_CALL_NO_WG, .typemask = dh_typemask(void, 0) | dh_typemask(env, 1) - | dh_typemask(tl, 2) /* target_ulong addr */ + | dh_typemask(i64, 2) /* uint64_t addr */ | dh_typemask(i32, 3) /* uint32_t data */ | dh_typemask(i32, 4) /* unsigned oi */ | dh_typemask(ptr, 5) /* uintptr_t ra */ @@ -785,12 +905,22 @@ static TCGHelperInfo info_helper_st64_mmu = { .flags = TCG_CALL_NO_WG, .typemask = dh_typemask(void, 0) | dh_typemask(env, 1) - | dh_typemask(tl, 2) /* target_ulong addr */ + | dh_typemask(i64, 2) /* uint64_t addr */ | dh_typemask(i64, 3) /* uint64_t data */ | dh_typemask(i32, 4) /* unsigned oi */ | dh_typemask(ptr, 5) /* uintptr_t ra */ }; +static TCGHelperInfo info_helper_st128_mmu = { + .flags = TCG_CALL_NO_WG, + .typemask = dh_typemask(void, 0) + | dh_typemask(env, 1) + | dh_typemask(i64, 2) /* uint64_t addr */ + | dh_typemask(i128, 3) /* Int128 data */ + | dh_typemask(i32, 4) /* unsigned oi */ + | dh_typemask(ptr, 5) /* uintptr_t ra */ +}; + #ifdef CONFIG_TCG_INTERPRETER static ffi_type *typecode_to_ffi(int argmask) { @@ -1204,8 +1334,10 @@ static void tcg_context_init(unsigned max_cpus) init_call_layout(&info_helper_ld32_mmu); init_call_layout(&info_helper_ld64_mmu); + init_call_layout(&info_helper_ld128_mmu); init_call_layout(&info_helper_st32_mmu); init_call_layout(&info_helper_st64_mmu); + init_call_layout(&info_helper_st128_mmu); #ifdef CONFIG_TCG_INTERPRETER init_ffi_layouts(); @@ -1391,6 +1523,9 @@ void tcg_func_start(TCGContext *s) QTAILQ_INIT(&s->ops); QTAILQ_INIT(&s->free_ops); QSIMPLEQ_INIT(&s->labels); + + tcg_debug_assert(s->addr_type == TCG_TYPE_I32 || + s->addr_type == TCG_TYPE_I64); } static TCGTemp *tcg_temp_alloc(TCGContext *s) @@ -1707,15 +1842,26 @@ bool tcg_op_supported(TCGOpcode op) case INDEX_op_exit_tb: case INDEX_op_goto_tb: case INDEX_op_goto_ptr: - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: return true; - case INDEX_op_qemu_st8_i32: + case INDEX_op_qemu_st8_a32_i32: + case INDEX_op_qemu_st8_a64_i32: return TCG_TARGET_HAS_qemu_st8_i32; + case INDEX_op_qemu_ld_a32_i128: + case INDEX_op_qemu_ld_a64_i128: + case INDEX_op_qemu_st_a32_i128: + case INDEX_op_qemu_st_a64_i128: + return TCG_TARGET_HAS_qemu_ldst_i128; + case INDEX_op_mov_i32: case INDEX_op_setcond_i32: case INDEX_op_brcond_i32: @@ -2168,7 +2314,7 @@ static const char * const cond_name[] = [TCG_COND_GTU] = "gtu" }; -static const char * const ldst_name[] = +static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] = { [MO_UB] = "ub", [MO_SB] = "sb", @@ -2182,6 +2328,8 @@ static const char * const ldst_name[] = [MO_BEUL] = "beul", [MO_BESL] = "besl", [MO_BEUQ] = "beq", + [MO_128 + MO_BE] = "beo", + [MO_128 + MO_LE] = "leo", }; static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { @@ -2195,6 +2343,15 @@ static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", }; +static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = { + [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "", + [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+", + [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+", + [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+", + [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+", + [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+", +}; + static const char bswap_flag_name[][6] = { [TCG_BSWAP_IZ] = "iz", [TCG_BSWAP_OZ] = "oz", @@ -2240,13 +2397,8 @@ static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) col += ne_fprintf(f, "\n ----"); for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { - target_ulong a; -#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS - a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); -#else - a = op->args[i]; -#endif - col += ne_fprintf(f, " " TARGET_FMT_lx, a); + col += ne_fprintf(f, " %016" PRIx64, + tcg_get_insn_start_param(op, i)); } } else if (c == INDEX_op_call) { const TCGHelperInfo *info = tcg_call_info(op); @@ -2324,23 +2476,38 @@ static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) } i = 1; break; - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st8_i32: - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_st8_a32_i32: + case INDEX_op_qemu_st8_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: + case INDEX_op_qemu_ld_a32_i128: + case INDEX_op_qemu_ld_a64_i128: + case INDEX_op_qemu_st_a32_i128: + case INDEX_op_qemu_st_a64_i128: { + const char *s_al, *s_op, *s_at; MemOpIdx oi = op->args[k++]; MemOp op = get_memop(oi); unsigned ix = get_mmuidx(oi); - if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { - col += ne_fprintf(f, ",$0x%x,%u", op, ix); + s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; + s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; + s_at = atom_name[(op & MO_ATOM_MASK) >> MO_ATOM_SHIFT]; + op &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK); + + /* If all fields are accounted for, print symbolically. */ + if (!op && s_al && s_op && s_at) { + col += ne_fprintf(f, ",%s%s%s,%u", + s_at, s_al, s_op, ix); } else { - const char *s_al, *s_op; - s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; - s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; - col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix); + op = get_memop(oi); + col += ne_fprintf(f, ",$0x%x,%u", op, ix); } i = 1; } @@ -5087,6 +5254,92 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) } } +/** + * atom_and_align_for_opc: + * @s: tcg context + * @opc: memory operation code + * @host_atom: MO_ATOM_{IFALIGN,WITHIN16,SUBALIGN} for host operations + * @allow_two_ops: true if we are prepared to issue two operations + * + * Return the alignment and atomicity to use for the inline fast path + * for the given memory operation. The alignment may be larger than + * that specified in @opc, and the correct alignment will be diagnosed + * by the slow path helper. + * + * If @allow_two_ops, the host is prepared to test for 2x alignment, + * and issue two loads or stores for subalignment. + */ +static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc, + MemOp host_atom, bool allow_two_ops) +{ + MemOp align = get_alignment_bits(opc); + MemOp size = opc & MO_SIZE; + MemOp half = size ? size - 1 : 0; + MemOp atmax; + MemOp atom; + + /* When serialized, no further atomicity required. */ + if (s->gen_tb->cflags & CF_PARALLEL) { + atom = opc & MO_ATOM_MASK; + } else { + atom = MO_ATOM_NONE; + } + + switch (atom) { + case MO_ATOM_NONE: + /* The operation requires no specific atomicity. */ + atmax = MO_8; + break; + + case MO_ATOM_IFALIGN: + atmax = size; + break; + + case MO_ATOM_IFALIGN_PAIR: + atmax = half; + break; + + case MO_ATOM_WITHIN16: + atmax = size; + if (size == MO_128) { + /* Misalignment implies !within16, and therefore no atomicity. */ + } else if (host_atom != MO_ATOM_WITHIN16) { + /* The host does not implement within16, so require alignment. */ + align = MAX(align, size); + } + break; + + case MO_ATOM_WITHIN16_PAIR: + atmax = size; + /* + * Misalignment implies !within16, and therefore half atomicity. + * Any host prepared for two operations can implement this with + * half alignment. + */ + if (host_atom != MO_ATOM_WITHIN16 && allow_two_ops) { + align = MAX(align, half); + } + break; + + case MO_ATOM_SUBALIGN: + atmax = size; + if (host_atom != MO_ATOM_SUBALIGN) { + /* If unaligned but not odd, there are subobjects up to half. */ + if (allow_two_ops) { + align = MAX(align, half); + } else { + align = MAX(align, size); + } + } + break; + + default: + g_assert_not_reached(); + } + + return (TCGAtomAlign){ .atom = atmax, .align = align }; +} + /* * Similarly for qemu_ld/st slow path helpers. * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously, @@ -5109,57 +5362,12 @@ static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot) return ofs; } -static void tcg_out_helper_load_regs(TCGContext *s, - unsigned nmov, TCGMovExtend *mov, - unsigned ntmp, const int *tmp) -{ - switch (nmov) { - default: - /* The backend must have provided enough temps for the worst case. */ - tcg_debug_assert(ntmp + 1 >= nmov); - - for (unsigned i = nmov - 1; i >= 2; --i) { - TCGReg dst = mov[i].dst; - - for (unsigned j = 0; j < i; ++j) { - if (dst == mov[j].src) { - /* - * Conflict. - * Copy the source to a temporary, recurse for the - * remaining moves, perform the extension from our - * scratch on the way out. - */ - TCGReg scratch = tmp[--ntmp]; - tcg_out_mov(s, mov[i].src_type, scratch, mov[i].src); - mov[i].src = scratch; - - tcg_out_helper_load_regs(s, i, mov, ntmp, tmp); - tcg_out_movext1(s, &mov[i]); - return; - } - } - - /* No conflicts: perform this move and continue. */ - tcg_out_movext1(s, &mov[i]); - } - /* fall through for the final two moves */ - - case 2: - tcg_out_movext2(s, mov, mov + 1, ntmp ? tmp[0] : -1); - return; - case 1: - tcg_out_movext1(s, mov); - return; - case 0: - g_assert_not_reached(); - } -} - static void tcg_out_helper_load_slots(TCGContext *s, unsigned nmov, TCGMovExtend *mov, const TCGLdstHelperParam *parm) { unsigned i; + TCGReg dst3; /* * Start from the end, storing to the stack first. @@ -5197,7 +5405,47 @@ static void tcg_out_helper_load_slots(TCGContext *s, for (i = 0; i < nmov; ++i) { mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst]; } - tcg_out_helper_load_regs(s, nmov, mov, parm->ntmp, parm->tmp); + + switch (nmov) { + case 4: + /* The backend must have provided enough temps for the worst case. */ + tcg_debug_assert(parm->ntmp >= 2); + + dst3 = mov[3].dst; + for (unsigned j = 0; j < 3; ++j) { + if (dst3 == mov[j].src) { + /* + * Conflict. Copy the source to a temporary, perform the + * remaining moves, then the extension from our scratch + * on the way out. + */ + TCGReg scratch = parm->tmp[1]; + + tcg_out_mov(s, mov[3].src_type, scratch, mov[3].src); + tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]); + tcg_out_movext1_new_src(s, &mov[3], scratch); + break; + } + } + + /* No conflicts: perform this move and continue. */ + tcg_out_movext1(s, &mov[3]); + /* fall through */ + + case 3: + tcg_out_movext3(s, mov, mov + 1, mov + 2, + parm->ntmp ? parm->tmp[0] : -1); + break; + case 2: + tcg_out_movext2(s, mov, mov + 1, + parm->ntmp ? parm->tmp[0] : -1); + break; + case 1: + tcg_out_movext1(s, mov); + break; + default: + g_assert_not_reached(); + } } static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot, @@ -5288,6 +5536,8 @@ static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov, TCGType dst_type, TCGType src_type, TCGReg lo, TCGReg hi) { + MemOp reg_mo; + if (dst_type <= TCG_TYPE_REG) { MemOp src_ext; @@ -5315,19 +5565,25 @@ static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov, return 1; } - assert(TCG_TARGET_REG_BITS == 32); + if (TCG_TARGET_REG_BITS == 32) { + assert(dst_type == TCG_TYPE_I64); + reg_mo = MO_32; + } else { + assert(dst_type == TCG_TYPE_I128); + reg_mo = MO_64; + } mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot; mov[0].src = lo; - mov[0].dst_type = TCG_TYPE_I32; - mov[0].src_type = TCG_TYPE_I32; - mov[0].src_ext = MO_32; + mov[0].dst_type = TCG_TYPE_REG; + mov[0].src_type = TCG_TYPE_REG; + mov[0].src_ext = reg_mo; mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot; mov[1].src = hi; - mov[1].dst_type = TCG_TYPE_I32; - mov[1].src_type = TCG_TYPE_I32; - mov[1].src_ext = MO_32; + mov[1].dst_type = TCG_TYPE_REG; + mov[1].src_type = TCG_TYPE_REG; + mov[1].src_ext = reg_mo; return 2; } @@ -5350,6 +5606,9 @@ static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, case MO_64: info = &info_helper_ld64_mmu; break; + case MO_128: + info = &info_helper_ld128_mmu; + break; default: g_assert_not_reached(); } @@ -5358,14 +5617,54 @@ static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, next_arg = 1; loc = &info->in[next_arg]; - nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_TL, TCG_TYPE_TL, - ldst->addrlo_reg, ldst->addrhi_reg); - next_arg += nmov; + if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) { + /* + * 32-bit host with 32-bit guest: zero-extend the guest address + * to 64-bits for the helper by storing the low part, then + * load a zero for the high part. + */ + tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN, + TCG_TYPE_I32, TCG_TYPE_I32, + ldst->addrlo_reg, -1); + tcg_out_helper_load_slots(s, 1, mov, parm); + + tcg_out_helper_load_imm(s, loc[!HOST_BIG_ENDIAN].arg_slot, + TCG_TYPE_I32, 0, parm); + next_arg += 2; + } else { + nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, + ldst->addrlo_reg, ldst->addrhi_reg); + tcg_out_helper_load_slots(s, nmov, mov, parm); + next_arg += nmov; + } - tcg_out_helper_load_slots(s, nmov, mov, parm); + switch (info->out_kind) { + case TCG_CALL_RET_NORMAL: + case TCG_CALL_RET_BY_VEC: + break; + case TCG_CALL_RET_BY_REF: + /* + * The return reference is in the first argument slot. + * We need memory in which to return: re-use the top of stack. + */ + { + int ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET; - /* No special attention for 32 and 64-bit return values. */ - tcg_debug_assert(info->out_kind == TCG_CALL_RET_NORMAL); + if (arg_slot_reg_p(0)) { + tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[0], + TCG_REG_CALL_STACK, ofs_slot0); + } else { + tcg_debug_assert(parm->ntmp != 0); + tcg_out_addi_ptr(s, parm->tmp[0], + TCG_REG_CALL_STACK, ofs_slot0); + tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], + TCG_REG_CALL_STACK, ofs_slot0); + } + } + break; + default: + g_assert_not_reached(); + } tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); } @@ -5374,11 +5673,18 @@ static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst, bool load_sign, const TCGLdstHelperParam *parm) { + MemOp mop = get_memop(ldst->oi); TCGMovExtend mov[2]; + int ofs_slot0; - if (ldst->type <= TCG_TYPE_REG) { - MemOp mop = get_memop(ldst->oi); + switch (ldst->type) { + case TCG_TYPE_I64: + if (TCG_TARGET_REG_BITS == 32) { + break; + } + /* fall through */ + case TCG_TYPE_I32: mov[0].dst = ldst->datalo_reg; mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0); mov[0].dst_type = ldst->type; @@ -5404,25 +5710,49 @@ static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst, mov[0].src_ext = mop & MO_SSIZE; } tcg_out_movext1(s, mov); - } else { - assert(TCG_TARGET_REG_BITS == 32); - - mov[0].dst = ldst->datalo_reg; - mov[0].src = - tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN); - mov[0].dst_type = TCG_TYPE_I32; - mov[0].src_type = TCG_TYPE_I32; - mov[0].src_ext = MO_32; + return; - mov[1].dst = ldst->datahi_reg; - mov[1].src = - tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN); - mov[1].dst_type = TCG_TYPE_REG; - mov[1].src_type = TCG_TYPE_REG; - mov[1].src_ext = MO_32; + case TCG_TYPE_I128: + tcg_debug_assert(TCG_TARGET_REG_BITS == 64); + ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET; + switch (TCG_TARGET_CALL_RET_I128) { + case TCG_CALL_RET_NORMAL: + break; + case TCG_CALL_RET_BY_VEC: + tcg_out_st(s, TCG_TYPE_V128, + tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), + TCG_REG_CALL_STACK, ofs_slot0); + /* fall through */ + case TCG_CALL_RET_BY_REF: + tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg, + TCG_REG_CALL_STACK, ofs_slot0 + 8 * HOST_BIG_ENDIAN); + tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg, + TCG_REG_CALL_STACK, ofs_slot0 + 8 * !HOST_BIG_ENDIAN); + return; + default: + g_assert_not_reached(); + } + break; - tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1); + default: + g_assert_not_reached(); } + + mov[0].dst = ldst->datalo_reg; + mov[0].src = + tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN); + mov[0].dst_type = TCG_TYPE_I32; + mov[0].src_type = TCG_TYPE_I32; + mov[0].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64; + + mov[1].dst = ldst->datahi_reg; + mov[1].src = + tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN); + mov[1].dst_type = TCG_TYPE_REG; + mov[1].src_type = TCG_TYPE_REG; + mov[1].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64; + + tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1); } static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, @@ -5446,6 +5776,10 @@ static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, info = &info_helper_st64_mmu; data_type = TCG_TYPE_I64; break; + case MO_128: + info = &info_helper_st128_mmu; + data_type = TCG_TYPE_I128; + break; default: g_assert_not_reached(); } @@ -5456,20 +5790,74 @@ static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, /* Handle addr argument. */ loc = &info->in[next_arg]; - n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_TL, TCG_TYPE_TL, - ldst->addrlo_reg, ldst->addrhi_reg); - next_arg += n; - nmov += n; + if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) { + /* + * 32-bit host with 32-bit guest: zero-extend the guest address + * to 64-bits for the helper by storing the low part. Later, + * after we have processed the register inputs, we will load a + * zero for the high part. + */ + tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN, + TCG_TYPE_I32, TCG_TYPE_I32, + ldst->addrlo_reg, -1); + next_arg += 2; + nmov += 1; + } else { + n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, + ldst->addrlo_reg, ldst->addrhi_reg); + next_arg += n; + nmov += n; + } /* Handle data argument. */ loc = &info->in[next_arg]; - n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type, - ldst->datalo_reg, ldst->datahi_reg); - next_arg += n; - nmov += n; - tcg_debug_assert(nmov <= ARRAY_SIZE(mov)); + switch (loc->kind) { + case TCG_CALL_ARG_NORMAL: + case TCG_CALL_ARG_EXTEND_U: + case TCG_CALL_ARG_EXTEND_S: + n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type, + ldst->datalo_reg, ldst->datahi_reg); + next_arg += n; + nmov += n; + tcg_out_helper_load_slots(s, nmov, mov, parm); + break; + + case TCG_CALL_ARG_BY_REF: + tcg_debug_assert(TCG_TARGET_REG_BITS == 64); + tcg_debug_assert(data_type == TCG_TYPE_I128); + tcg_out_st(s, TCG_TYPE_I64, + HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg, + TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[0].ref_slot)); + tcg_out_st(s, TCG_TYPE_I64, + HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg, + TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[1].ref_slot)); + + tcg_out_helper_load_slots(s, nmov, mov, parm); + + if (arg_slot_reg_p(loc->arg_slot)) { + tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot], + TCG_REG_CALL_STACK, + arg_slot_stk_ofs(loc->ref_slot)); + } else { + tcg_debug_assert(parm->ntmp != 0); + tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK, + arg_slot_stk_ofs(loc->ref_slot)); + tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], + TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot)); + } + next_arg += 2; + break; + + default: + g_assert_not_reached(); + } + + if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) { + /* Zero extend the address by loading a zero for the high part. */ + loc = &info->in[1 + !HOST_BIG_ENDIAN]; + tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm); + } - tcg_out_helper_load_slots(s, nmov, mov, parm); tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); } @@ -5582,7 +5970,7 @@ int64_t tcg_cpu_exec_time(void) #endif -int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) +int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) { #ifdef CONFIG_PROFILER TCGProfile *prof = &s->prof; @@ -5743,13 +6131,8 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) } num_insns++; for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { - target_ulong a; -#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS - a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); -#else - a = op->args[i]; -#endif - s->gen_insn_data[num_insns][i] = a; + s->gen_insn_data[num_insns][i] = + tcg_get_insn_start_param(op, i); } break; case INDEX_op_discard: diff --git a/tcg/tci.c b/tcg/tci.c index fc67e7e767..bab4397bc5 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -286,162 +286,54 @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) return result; } -static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, +static uint64_t tci_qemu_ld(CPUArchState *env, uint64_t taddr, MemOpIdx oi, const void *tb_ptr) { MemOp mop = get_memop(oi); uintptr_t ra = (uintptr_t)tb_ptr; -#ifdef CONFIG_SOFTMMU - switch (mop & (MO_BSWAP | MO_SSIZE)) { + switch (mop & MO_SSIZE) { case MO_UB: - return helper_ret_ldub_mmu(env, taddr, oi, ra); + return helper_ldub_mmu(env, taddr, oi, ra); case MO_SB: - return helper_ret_ldsb_mmu(env, taddr, oi, ra); - case MO_LEUW: - return helper_le_lduw_mmu(env, taddr, oi, ra); - case MO_LESW: - return helper_le_ldsw_mmu(env, taddr, oi, ra); - case MO_LEUL: - return helper_le_ldul_mmu(env, taddr, oi, ra); - case MO_LESL: - return helper_le_ldsl_mmu(env, taddr, oi, ra); - case MO_LEUQ: - return helper_le_ldq_mmu(env, taddr, oi, ra); - case MO_BEUW: - return helper_be_lduw_mmu(env, taddr, oi, ra); - case MO_BESW: - return helper_be_ldsw_mmu(env, taddr, oi, ra); - case MO_BEUL: - return helper_be_ldul_mmu(env, taddr, oi, ra); - case MO_BESL: - return helper_be_ldsl_mmu(env, taddr, oi, ra); - case MO_BEUQ: - return helper_be_ldq_mmu(env, taddr, oi, ra); + return helper_ldsb_mmu(env, taddr, oi, ra); + case MO_UW: + return helper_lduw_mmu(env, taddr, oi, ra); + case MO_SW: + return helper_ldsw_mmu(env, taddr, oi, ra); + case MO_UL: + return helper_ldul_mmu(env, taddr, oi, ra); + case MO_SL: + return helper_ldsl_mmu(env, taddr, oi, ra); + case MO_UQ: + return helper_ldq_mmu(env, taddr, oi, ra); default: g_assert_not_reached(); } -#else - void *haddr = g2h(env_cpu(env), taddr); - unsigned a_mask = (1u << get_alignment_bits(mop)) - 1; - uint64_t ret; - - set_helper_retaddr(ra); - if (taddr & a_mask) { - helper_unaligned_ld(env, taddr); - } - switch (mop & (MO_BSWAP | MO_SSIZE)) { - case MO_UB: - ret = ldub_p(haddr); - break; - case MO_SB: - ret = ldsb_p(haddr); - break; - case MO_LEUW: - ret = lduw_le_p(haddr); - break; - case MO_LESW: - ret = ldsw_le_p(haddr); - break; - case MO_LEUL: - ret = (uint32_t)ldl_le_p(haddr); - break; - case MO_LESL: - ret = (int32_t)ldl_le_p(haddr); - break; - case MO_LEUQ: - ret = ldq_le_p(haddr); - break; - case MO_BEUW: - ret = lduw_be_p(haddr); - break; - case MO_BESW: - ret = ldsw_be_p(haddr); - break; - case MO_BEUL: - ret = (uint32_t)ldl_be_p(haddr); - break; - case MO_BESL: - ret = (int32_t)ldl_be_p(haddr); - break; - case MO_BEUQ: - ret = ldq_be_p(haddr); - break; - default: - g_assert_not_reached(); - } - clear_helper_retaddr(); - return ret; -#endif } -static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, +static void tci_qemu_st(CPUArchState *env, uint64_t taddr, uint64_t val, MemOpIdx oi, const void *tb_ptr) { MemOp mop = get_memop(oi); uintptr_t ra = (uintptr_t)tb_ptr; -#ifdef CONFIG_SOFTMMU - switch (mop & (MO_BSWAP | MO_SIZE)) { + switch (mop & MO_SIZE) { case MO_UB: - helper_ret_stb_mmu(env, taddr, val, oi, ra); + helper_stb_mmu(env, taddr, val, oi, ra); break; - case MO_LEUW: - helper_le_stw_mmu(env, taddr, val, oi, ra); + case MO_UW: + helper_stw_mmu(env, taddr, val, oi, ra); break; - case MO_LEUL: - helper_le_stl_mmu(env, taddr, val, oi, ra); + case MO_UL: + helper_stl_mmu(env, taddr, val, oi, ra); break; - case MO_LEUQ: - helper_le_stq_mmu(env, taddr, val, oi, ra); - break; - case MO_BEUW: - helper_be_stw_mmu(env, taddr, val, oi, ra); - break; - case MO_BEUL: - helper_be_stl_mmu(env, taddr, val, oi, ra); - break; - case MO_BEUQ: - helper_be_stq_mmu(env, taddr, val, oi, ra); + case MO_UQ: + helper_stq_mmu(env, taddr, val, oi, ra); break; default: g_assert_not_reached(); } -#else - void *haddr = g2h(env_cpu(env), taddr); - unsigned a_mask = (1u << get_alignment_bits(mop)) - 1; - - set_helper_retaddr(ra); - if (taddr & a_mask) { - helper_unaligned_st(env, taddr); - } - switch (mop & (MO_BSWAP | MO_SIZE)) { - case MO_UB: - stb_p(haddr, val); - break; - case MO_LEUW: - stw_le_p(haddr, val); - break; - case MO_LEUL: - stl_le_p(haddr, val); - break; - case MO_LEUQ: - stq_le_p(haddr, val); - break; - case MO_BEUW: - stw_be_p(haddr, val); - break; - case MO_BEUL: - stl_be_p(haddr, val); - break; - case MO_BEUQ: - stq_be_p(haddr, val); - break; - default: - g_assert_not_reached(); - } - clear_helper_retaddr(); -#endif } #if TCG_TARGET_REG_BITS == 64 @@ -480,10 +372,9 @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, TCGReg r0, r1, r2, r3, r4, r5; tcg_target_ulong t1; TCGCond condition; - target_ulong taddr; uint8_t pos, len; uint32_t tmp32; - uint64_t tmp64; + uint64_t tmp64, taddr; uint64_t T1, T2; MemOpIdx oi; int32_t ofs; @@ -1030,30 +921,41 @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tb_ptr = ptr; break; - case INDEX_op_qemu_ld_i32: - if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { + case INDEX_op_qemu_ld_a32_i32: + tci_args_rrm(insn, &r0, &r1, &oi); + taddr = (uint32_t)regs[r1]; + goto do_ld_i32; + case INDEX_op_qemu_ld_a64_i32: + if (TCG_TARGET_REG_BITS == 64) { tci_args_rrm(insn, &r0, &r1, &oi); taddr = regs[r1]; } else { tci_args_rrrm(insn, &r0, &r1, &r2, &oi); taddr = tci_uint64(regs[r2], regs[r1]); } - tmp32 = tci_qemu_ld(env, taddr, oi, tb_ptr); - regs[r0] = tmp32; + do_ld_i32: + regs[r0] = tci_qemu_ld(env, taddr, oi, tb_ptr); break; - case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_ld_a32_i64: if (TCG_TARGET_REG_BITS == 64) { tci_args_rrm(insn, &r0, &r1, &oi); - taddr = regs[r1]; - } else if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { + taddr = (uint32_t)regs[r1]; + } else { tci_args_rrrm(insn, &r0, &r1, &r2, &oi); - taddr = regs[r2]; + taddr = (uint32_t)regs[r2]; + } + goto do_ld_i64; + case INDEX_op_qemu_ld_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + tci_args_rrm(insn, &r0, &r1, &oi); + taddr = regs[r1]; } else { tci_args_rrrrr(insn, &r0, &r1, &r2, &r3, &r4); taddr = tci_uint64(regs[r3], regs[r2]); oi = regs[r4]; } + do_ld_i64: tmp64 = tci_qemu_ld(env, taddr, oi, tb_ptr); if (TCG_TARGET_REG_BITS == 32) { tci_write_reg64(regs, r1, r0, tmp64); @@ -1062,34 +964,45 @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, } break; - case INDEX_op_qemu_st_i32: - if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { + case INDEX_op_qemu_st_a32_i32: + tci_args_rrm(insn, &r0, &r1, &oi); + taddr = (uint32_t)regs[r1]; + goto do_st_i32; + case INDEX_op_qemu_st_a64_i32: + if (TCG_TARGET_REG_BITS == 64) { tci_args_rrm(insn, &r0, &r1, &oi); taddr = regs[r1]; } else { tci_args_rrrm(insn, &r0, &r1, &r2, &oi); taddr = tci_uint64(regs[r2], regs[r1]); } - tmp32 = regs[r0]; - tci_qemu_st(env, taddr, tmp32, oi, tb_ptr); + do_st_i32: + tci_qemu_st(env, taddr, regs[r0], oi, tb_ptr); break; - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_st_a32_i64: if (TCG_TARGET_REG_BITS == 64) { tci_args_rrm(insn, &r0, &r1, &oi); - taddr = regs[r1]; tmp64 = regs[r0]; + taddr = (uint32_t)regs[r1]; } else { - if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { - tci_args_rrrm(insn, &r0, &r1, &r2, &oi); - taddr = regs[r2]; - } else { - tci_args_rrrrr(insn, &r0, &r1, &r2, &r3, &r4); - taddr = tci_uint64(regs[r3], regs[r2]); - oi = regs[r4]; - } + tci_args_rrrm(insn, &r0, &r1, &r2, &oi); + tmp64 = tci_uint64(regs[r1], regs[r0]); + taddr = (uint32_t)regs[r2]; + } + goto do_st_i64; + case INDEX_op_qemu_st_a64_i64: + if (TCG_TARGET_REG_BITS == 64) { + tci_args_rrm(insn, &r0, &r1, &oi); + tmp64 = regs[r0]; + taddr = regs[r1]; + } else { + tci_args_rrrrr(insn, &r0, &r1, &r2, &r3, &r4); tmp64 = tci_uint64(regs[r1], regs[r0]); + taddr = tci_uint64(regs[r3], regs[r2]); + oi = regs[r4]; } + do_st_i64: tci_qemu_st(env, taddr, tmp64, oi, tb_ptr); break; @@ -1359,15 +1272,21 @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) str_r(r3), str_r(r4), str_r(r5)); break; - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i64: - len = DIV_ROUND_UP(64, TCG_TARGET_REG_BITS); + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_st_a32_i32: + len = 1 + 1; + goto do_qemu_ldst; + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_st_a64_i32: + len = 1 + DIV_ROUND_UP(64, TCG_TARGET_REG_BITS); + goto do_qemu_ldst; + case INDEX_op_qemu_ld_a64_i64: + case INDEX_op_qemu_st_a64_i64: + len = 2 * DIV_ROUND_UP(64, TCG_TARGET_REG_BITS); goto do_qemu_ldst; - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_st_i32: - len = 1; do_qemu_ldst: - len += DIV_ROUND_UP(TARGET_LONG_BITS, TCG_TARGET_REG_BITS); switch (len) { case 2: tci_args_rrm(insn, &r0, &r1, &oi); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index 4cf03a579c..c9516a5e8b 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -156,22 +156,22 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_setcond2_i32: return C_O1_I4(r, r, r, r, r); - case INDEX_op_qemu_ld_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O1_I1(r, r) - : C_O1_I2(r, r, r)); - case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, r) - : C_O2_I2(r, r, r, r)); - case INDEX_op_qemu_st_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O0_I2(r, r) - : C_O0_I3(r, r, r)); - case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(r, r, r) - : C_O0_I4(r, r, r, r)); + case INDEX_op_qemu_ld_a32_i32: + return C_O1_I1(r, r); + case INDEX_op_qemu_ld_a64_i32: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O1_I2(r, r, r); + case INDEX_op_qemu_ld_a32_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O2_I1(r, r, r); + case INDEX_op_qemu_ld_a64_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O2_I2(r, r, r, r); + case INDEX_op_qemu_st_a32_i32: + return C_O0_I2(r, r); + case INDEX_op_qemu_st_a64_i32: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) : C_O0_I3(r, r, r); + case INDEX_op_qemu_st_a32_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) : C_O0_I3(r, r, r); + case INDEX_op_qemu_st_a64_i64: + return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) : C_O0_I4(r, r, r, r); default: g_assert_not_reached(); @@ -243,7 +243,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, return false; } -static void stack_bounds_check(TCGReg base, target_long offset) +static void stack_bounds_check(TCGReg base, intptr_t offset) { if (base == TCG_REG_CALL_STACK) { tcg_debug_assert(offset >= 0); @@ -849,21 +849,24 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_op_rrrr(s, opc, args[0], args[1], args[2], args[3]); break; - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_st_i32: - if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_st_a32_i32: + tcg_out_op_rrm(s, opc, args[0], args[1], args[2]); + break; + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_st_a32_i64: + if (TCG_TARGET_REG_BITS == 64) { tcg_out_op_rrm(s, opc, args[0], args[1], args[2]); } else { tcg_out_op_rrrm(s, opc, args[0], args[1], args[2], args[3]); } break; - - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i64: + case INDEX_op_qemu_ld_a64_i64: + case INDEX_op_qemu_st_a64_i64: if (TCG_TARGET_REG_BITS == 64) { tcg_out_op_rrm(s, opc, args[0], args[1], args[2]); - } else if (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS) { - tcg_out_op_rrrm(s, opc, args[0], args[1], args[2], args[3]); } else { tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, args[4]); tcg_out_op_rrrrr(s, opc, args[0], args[1], @@ -963,3 +966,8 @@ static void tcg_target_init(TCGContext *s) static inline void tcg_target_qemu_prologue(TCGContext *s) { } + +bool tcg_target_has_memory_bswap(MemOp memop) +{ + return true; +} diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index 7140a76a73..28dc6d5cfc 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -127,6 +127,8 @@ #define TCG_TARGET_HAS_mulu2_i32 1 #endif /* TCG_TARGET_REG_BITS == 64 */ +#define TCG_TARGET_HAS_qemu_ldst_i128 0 + /* Number of registers available. */ #define TCG_TARGET_NB_REGS 16 @@ -176,6 +178,4 @@ typedef enum { We prefer consistency across hosts on this. */ #define TCG_TARGET_DEFAULT_MO (0) -#define TCG_TARGET_HAS_MEMORY_BSWAP 1 - #endif /* TCG_TARGET_H */ diff --git a/tests/Makefile.include b/tests/Makefile.include index a4de0ad5a2..5b838ec438 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -89,7 +89,8 @@ distclean-tcg: $(DISTCLEAN_TCG_TARGET_RULES) # Build up our target list from the filtered list of ninja targets TARGETS=$(patsubst libqemu-%.fa, %, $(filter libqemu-%.fa, $(ninja-targets))) -TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv +TESTS_VENV_DIR=$(BUILD_DIR)/pyvenv +TESTS_VENV_TOKEN=$(BUILD_DIR)/pyvenv/tests.group TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results TESTS_PYTHON=$(TESTS_VENV_DIR)/bin/python3 @@ -111,8 +112,7 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \ $(TESTS_PYTHON) -m pip -q --disable-pip-version-check $1, \ "VENVPIP","$1") -$(TESTS_VENV_DIR): $(TESTS_VENV_REQ) - $(call quiet-command, $(PYTHON) -m venv $@, VENV, $@) +$(TESTS_VENV_TOKEN): $(TESTS_VENV_REQ) $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/") $(call quiet-venv-pip,install -r $(TESTS_VENV_REQ)) $(call quiet-command, touch $@) @@ -121,7 +121,7 @@ $(TESTS_RESULTS_DIR): $(call quiet-command, mkdir -p $@, \ MKDIR, $@) -check-venv: $(TESTS_VENV_DIR) +check-venv: $(TESTS_VENV_TOKEN) FEDORA_31_ARCHES_TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGETS))) FEDORA_31_ARCHES_CANDIDATES=$(patsubst ppc64,ppc64le,$(FEDORA_31_ARCHES_TARGETS)) @@ -167,7 +167,7 @@ check: check-build: run-ninja check-clean: - rm -rf $(TESTS_VENV_DIR) $(TESTS_RESULTS_DIR) + rm -rf $(TESTS_RESULTS_DIR) clean: check-clean clean-tcg distclean: distclean-tcg diff --git a/tests/avocado/machine_loongarch.py b/tests/avocado/machine_loongarch.py new file mode 100644 index 0000000000..7d8a3c1fa5 --- /dev/null +++ b/tests/avocado/machine_loongarch.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# LoongArch virt test. +# +# Copyright (c) 2023 Loongson Technology Corporation Limited +# + +from avocado_qemu import QemuSystemTest +from avocado_qemu import exec_command_and_wait_for_pattern +from avocado_qemu import wait_for_console_pattern + +class LoongArchMachine(QemuSystemTest): + KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' + + timeout = 120 + + def wait_for_console_pattern(self, success_message, vm=None): + wait_for_console_pattern(self, success_message, + failure_message='Kernel panic - not syncing', + vm=vm) + + def test_loongarch64_devices(self): + + """ + :avocado: tags=arch:loongarch64 + :avocado: tags=machine:virt + """ + + kernel_url = ('https://github.com/yangxiaojuan-loongson/qemu-binary/' + 'releases/download/binary-files/vmlinuz.efi') + kernel_hash = '951b485b16e3788b6db03a3e1793c067009e31a2' + kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash) + + initrd_url = ('https://github.com/yangxiaojuan-loongson/qemu-binary/' + 'releases/download/binary-files/ramdisk') + initrd_hash = 'c67658d9b2a447ce7db2f73ba3d373c9b2b90ab2' + initrd_path = self.fetch_asset(initrd_url, asset_hash=initrd_hash) + + bios_url = ('https://github.com/yangxiaojuan-loongson/qemu-binary/' + 'releases/download/binary-files/QEMU_EFI.fd') + bios_hash = ('dfc1bfba4853cd763b9d392d0031827e8addbca8') + bios_path = self.fetch_asset(bios_url, asset_hash=bios_hash) + + self.vm.set_console() + kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + + 'root=/dev/ram rdinit=/sbin/init console=ttyS0,115200') + self.vm.add_args('-nographic', + '-smp', '4', + '-m', '1024', + '-cpu', 'la464', + '-kernel', kernel_path, + '-initrd', initrd_path, + '-bios', bios_path, + '-append', kernel_command_line) + self.vm.launch() + self.wait_for_console_pattern('Run /sbin/init as init process') + exec_command_and_wait_for_pattern(self, 'cat /proc/cpuinfo', + 'processor : 3') diff --git a/tests/avocado/virtio-gpu.py b/tests/avocado/virtio-gpu.py index 2a249a3a2c..e3b58fe799 100644 --- a/tests/avocado/virtio-gpu.py +++ b/tests/avocado/virtio-gpu.py @@ -36,13 +36,13 @@ class VirtioGPUx86(QemuSystemTest): KERNEL_COMMAND_LINE = "printk.time=0 console=ttyS0 rdinit=/bin/bash" KERNEL_URL = ( - "https://archives.fedoraproject.org/pub/fedora" + "https://archives.fedoraproject.org/pub/archive/fedora" "/linux/releases/33/Everything/x86_64/os/images" "/pxeboot/vmlinuz" ) KERNEL_HASH = '1433cfe3f2ffaa44de4ecfb57ec25dc2399cdecf' INITRD_URL = ( - "https://archives.fedoraproject.org/pub/fedora" + "https://archives.fedoraproject.org/pub/archive/fedora" "/linux/releases/33/Everything/x86_64/os/images" "/pxeboot/initrd.img" ) diff --git a/tests/docker/dockerfiles/alpine.docker b/tests/docker/dockerfiles/alpine.docker index 81c70aeaf9..0097637dca 100644 --- a/tests/docker/dockerfiles/alpine.docker +++ b/tests/docker/dockerfiles/alpine.docker @@ -19,7 +19,6 @@ RUN apk update && \ ca-certificates \ capstone-dev \ ccache \ - cdrkit \ ceph-dev \ clang \ cmocka-dev \ @@ -67,6 +66,7 @@ RUN apk update && \ make \ mesa-dev \ meson \ + mtools \ multipath-tools \ musl-dev \ ncurses-dev \ @@ -108,6 +108,7 @@ RUN apk update && \ which \ xen-dev \ xfsprogs-dev \ + xorriso \ zlib-dev \ zlib-static \ zstd \ diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index 1a6a9087c1..78f454b782 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -36,7 +36,6 @@ RUN dnf distro-sync -y && \ fuse3-devel \ gcc \ gcc-c++ \ - genisoimage \ gettext \ git \ glib2-devel \ @@ -82,6 +81,7 @@ RUN dnf distro-sync -y && \ lzo-devel \ make \ mesa-libgbm-devel \ + mtools \ ncurses-devel \ nettle-devel \ ninja-build \ @@ -114,6 +114,7 @@ RUN dnf distro-sync -y && \ vte291-devel \ which \ xfsprogs-devel \ + xorriso \ zlib-devel \ zlib-static \ zstd && \ diff --git a/tests/docker/dockerfiles/debian-all-test-cross.docker b/tests/docker/dockerfiles/debian-all-test-cross.docker index 981e9bdc7b..f9f401544a 100644 --- a/tests/docker/dockerfiles/debian-all-test-cross.docker +++ b/tests/docker/dockerfiles/debian-all-test-cross.docker @@ -57,7 +57,8 @@ RUN DEBIAN_FRONTEND=noninteractive eatmydata \ gcc-sh4-linux-gnu \ libc6-dev-sh4-cross \ gcc-sparc64-linux-gnu \ - libc6-dev-sparc64-cross + libc6-dev-sparc64-cross \ + python3-venv ENV QEMU_CONFIGURE_OPTS --disable-system --disable-docs --disable-tools ENV DEF_TARGET_LIST aarch64-linux-user,alpha-linux-user,arm-linux-user,hppa-linux-user,i386-linux-user,m68k-linux-user,mips-linux-user,mips64-linux-user,mips64el-linux-user,mipsel-linux-user,ppc-linux-user,ppc64-linux-user,ppc64le-linux-user,riscv64-linux-user,s390x-linux-user,sh4-linux-user,sparc64-linux-user diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker index 2e7eb445f1..40a2b6acc4 100644 --- a/tests/docker/dockerfiles/debian-amd64-cross.docker +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -25,7 +25,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ findutils \ flex \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -37,6 +36,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ ncat \ ninja-build \ openssh-client \ @@ -57,6 +57,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ + xorriso \ zstd && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker index 28e2fa81b1..e39871c7bb 100644 --- a/tests/docker/dockerfiles/debian-amd64.docker +++ b/tests/docker/dockerfiles/debian-amd64.docker @@ -28,7 +28,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++ \ gcc \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -103,6 +102,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ multipath-tools \ ncat \ nettle-dev \ @@ -127,6 +127,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tesseract-ocr \ tesseract-ocr-eng \ xfslibs-dev \ + xorriso \ zlib1g-dev \ zstd && \ eatmydata apt-get autoremove -y && \ diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker index f558770f84..c99300bbfa 100644 --- a/tests/docker/dockerfiles/debian-arm64-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -25,7 +25,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ findutils \ flex \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -37,6 +36,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ ncat \ ninja-build \ openssh-client \ @@ -57,6 +57,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ + xorriso \ zstd && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ diff --git a/tests/docker/dockerfiles/debian-armel-cross.docker b/tests/docker/dockerfiles/debian-armel-cross.docker index f3d7e07cce..5db5c78b31 100644 --- a/tests/docker/dockerfiles/debian-armel-cross.docker +++ b/tests/docker/dockerfiles/debian-armel-cross.docker @@ -25,7 +25,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ findutils \ flex \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -37,6 +36,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ ncat \ ninja-build \ openssh-client \ @@ -57,6 +57,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ + xorriso \ zstd && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker index 531c556ad5..ae6600b25f 100644 --- a/tests/docker/dockerfiles/debian-armhf-cross.docker +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -25,7 +25,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ findutils \ flex \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -37,6 +36,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ ncat \ ninja-build \ openssh-client \ @@ -57,6 +57,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ + xorriso \ zstd && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ diff --git a/tests/docker/dockerfiles/debian-hexagon-cross.docker b/tests/docker/dockerfiles/debian-hexagon-cross.docker index b99d99f943..c2cfb6a5d0 100644 --- a/tests/docker/dockerfiles/debian-hexagon-cross.docker +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker @@ -20,7 +20,8 @@ RUN apt-get update && \ bison \ flex \ git \ - ninja-build && \ + ninja-build \ + python3-venv && \ # Install QEMU build deps for use in CI DEBIAN_FRONTEND=noninteractive eatmydata \ apt build-dep -yy --arch-only qemu diff --git a/tests/docker/dockerfiles/debian-mips64el-cross.docker b/tests/docker/dockerfiles/debian-mips64el-cross.docker index 816dbd2911..daa2d48e36 100644 --- a/tests/docker/dockerfiles/debian-mips64el-cross.docker +++ b/tests/docker/dockerfiles/debian-mips64el-cross.docker @@ -25,7 +25,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ findutils \ flex \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -37,6 +36,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ ncat \ ninja-build \ openssh-client \ @@ -57,6 +57,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ + xorriso \ zstd && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ diff --git a/tests/docker/dockerfiles/debian-mipsel-cross.docker b/tests/docker/dockerfiles/debian-mipsel-cross.docker index b115b29af3..5af04e2054 100644 --- a/tests/docker/dockerfiles/debian-mipsel-cross.docker +++ b/tests/docker/dockerfiles/debian-mipsel-cross.docker @@ -25,7 +25,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ findutils \ flex \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -37,6 +36,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ ncat \ ninja-build \ openssh-client \ @@ -57,6 +57,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ + xorriso \ zstd && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ diff --git a/tests/docker/dockerfiles/debian-ppc64el-cross.docker b/tests/docker/dockerfiles/debian-ppc64el-cross.docker index 301bddb536..1eeba7fcab 100644 --- a/tests/docker/dockerfiles/debian-ppc64el-cross.docker +++ b/tests/docker/dockerfiles/debian-ppc64el-cross.docker @@ -25,7 +25,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ findutils \ flex \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -37,6 +36,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ ncat \ ninja-build \ openssh-client \ @@ -57,6 +57,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ + xorriso \ zstd && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ diff --git a/tests/docker/dockerfiles/debian-riscv64-cross.docker b/tests/docker/dockerfiles/debian-riscv64-cross.docker index 803afb9573..081404e014 100644 --- a/tests/docker/dockerfiles/debian-riscv64-cross.docker +++ b/tests/docker/dockerfiles/debian-riscv64-cross.docker @@ -28,7 +28,8 @@ RUN DEBIAN_FRONTEND=noninteractive eatmydata apt install -yy \ libglib2.0-dev \ ninja-build \ pkg-config \ - python3 + python3 \ + python3-venv # Add ports and riscv64 architecture RUN echo "deb http://ftp.ports.debian.org/debian-ports/ sid main" >> /etc/apt/sources.list diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker index 5d27c91c17..52e89a6dab 100644 --- a/tests/docker/dockerfiles/debian-s390x-cross.docker +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -25,7 +25,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ findutils \ flex \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -37,6 +36,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ ncat \ ninja-build \ openssh-client \ @@ -57,6 +57,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ + xorriso \ zstd && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ diff --git a/tests/docker/dockerfiles/debian-tricore-cross.docker b/tests/docker/dockerfiles/debian-tricore-cross.docker index cfd2faf9a8..269bfa8d42 100644 --- a/tests/docker/dockerfiles/debian-tricore-cross.docker +++ b/tests/docker/dockerfiles/debian-tricore-cross.docker @@ -33,7 +33,8 @@ RUN apt update && \ pkgconf \ python3-pip \ python3-setuptools \ - python3-wheel + python3-wheel \ + python3-venv RUN curl -#SL https://github.com/bkoppelmann/package_940/releases/download/tricore-toolchain-9.40/tricore-toolchain-9.4.0.tar.gz \ | tar -xzC /usr/local/ diff --git a/tests/docker/dockerfiles/fedora-win32-cross.docker b/tests/docker/dockerfiles/fedora-win32-cross.docker index e7966ec7fd..dc72ae9cc9 100644 --- a/tests/docker/dockerfiles/fedora-win32-cross.docker +++ b/tests/docker/dockerfiles/fedora-win32-cross.docker @@ -30,7 +30,6 @@ exec "$@"\n' > /usr/bin/nosync && \ findutils \ flex \ gcovr \ - genisoimage \ git \ glib2-devel \ glibc-langpack-en \ @@ -38,6 +37,7 @@ exec "$@"\n' > /usr/bin/nosync && \ llvm \ make \ meson \ + mtools \ ninja-build \ nmap-ncat \ openssh-clients \ @@ -59,6 +59,7 @@ exec "$@"\n' > /usr/bin/nosync && \ tesseract-langpack-eng \ util-linux \ which \ + xorriso \ zstd && \ nosync dnf autoremove -y && \ nosync dnf clean all -y diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index 86c3a8f2ac..7eb4a5dba2 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -30,7 +30,6 @@ exec "$@"\n' > /usr/bin/nosync && \ findutils \ flex \ gcovr \ - genisoimage \ git \ glib2-devel \ glibc-langpack-en \ @@ -38,6 +37,7 @@ exec "$@"\n' > /usr/bin/nosync && \ llvm \ make \ meson \ + mtools \ ninja-build \ nmap-ncat \ openssh-clients \ @@ -59,6 +59,7 @@ exec "$@"\n' > /usr/bin/nosync && \ tesseract-langpack-eng \ util-linux \ which \ + xorriso \ zstd && \ nosync dnf autoremove -y && \ nosync dnf clean all -y diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index b698b7595d..3a69eefdda 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -43,7 +43,6 @@ exec "$@"\n' > /usr/bin/nosync && \ gcc \ gcc-c++ \ gcovr \ - genisoimage \ gettext \ git \ glib2-devel \ @@ -90,6 +89,7 @@ exec "$@"\n' > /usr/bin/nosync && \ make \ mesa-libgbm-devel \ meson \ + mtools \ ncurses-devel \ nettle-devel \ ninja-build \ @@ -128,6 +128,7 @@ exec "$@"\n' > /usr/bin/nosync && \ which \ xen-devel \ xfsprogs-devel \ + xorriso \ zlib-devel \ zlib-static \ zstd && \ diff --git a/tests/docker/dockerfiles/opensuse-leap.docker b/tests/docker/dockerfiles/opensuse-leap.docker index afb9f5419f..185abe57d8 100644 --- a/tests/docker/dockerfiles/opensuse-leap.docker +++ b/tests/docker/dockerfiles/opensuse-leap.docker @@ -81,7 +81,7 @@ RUN zypper update -y && \ lttng-ust-devel \ lzo-devel \ make \ - mkisofs \ + mtools \ ncat \ ncurses-devel \ ninja \ @@ -111,6 +111,7 @@ RUN zypper update -y && \ which \ xen-devel \ xfsprogs-devel \ + xorriso \ zlib-devel \ zlib-devel-static \ zstd && \ diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index aa2f5ca7b4..8f864d19e6 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -28,7 +28,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++ \ gcc \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -100,6 +99,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ llvm \ locales \ make \ + mtools \ multipath-tools \ ncat \ nettle-dev \ @@ -126,6 +126,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tesseract-ocr \ tesseract-ocr-eng \ xfslibs-dev \ + xorriso \ zlib1g-dev \ zstd && \ eatmydata apt-get autoremove -y && \ diff --git a/tests/docker/dockerfiles/ubuntu2204.docker b/tests/docker/dockerfiles/ubuntu2204.docker index 3f7d30e5d0..1d442cdfe6 100644 --- a/tests/docker/dockerfiles/ubuntu2204.docker +++ b/tests/docker/dockerfiles/ubuntu2204.docker @@ -28,7 +28,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++ \ gcc \ gcovr \ - genisoimage \ gettext \ git \ hostname \ @@ -103,6 +102,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ locales \ make \ meson \ + mtools \ multipath-tools \ ncat \ nettle-dev \ @@ -127,6 +127,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tesseract-ocr \ tesseract-ocr-eng \ xfslibs-dev \ + xorriso \ zlib1g-dev \ zstd && \ eatmydata apt-get autoremove -y && \ diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci -Subproject 85487e140415b2ac54b01a9a6b600fd7c21edc2 +Subproject c8971e90ac169ee2b539c747f74d96c876debdf diff --git a/tests/lcitool/projects/qemu.yml b/tests/lcitool/projects/qemu.yml index af3700379a..566db8313b 100644 --- a/tests/lcitool/projects/qemu.yml +++ b/tests/lcitool/projects/qemu.yml @@ -26,7 +26,6 @@ packages: - gcc - gcovr - gettext - - genisoimage - glib2 - glib2-native - glib2-static @@ -73,6 +72,7 @@ packages: - llvm - lttng-ust - lzo + - mtools - netcat - nettle - ninja @@ -116,6 +116,7 @@ packages: - which - xen - xfsprogs + - xorriso - zstdtools - zlib - zlib-static diff --git a/tests/qemu-iotests/227.out b/tests/qemu-iotests/227.out index 378c1b8fb1..a947b1a87d 100644 --- a/tests/qemu-iotests/227.out +++ b/tests/qemu-iotests/227.out @@ -17,6 +17,7 @@ Testing: -drive driver=null-co,read-zeroes=on,if=virtio "stats": { "unmap_operations": 0, "unmap_merged": 0, + "failed_zone_append_operations": 0, "flush_total_time_ns": 0, "wr_highest_offset": 0, "wr_total_time_ns": 0, @@ -27,6 +28,7 @@ Testing: -drive driver=null-co,read-zeroes=on,if=virtio "timed_stats": [ ], "failed_unmap_operations": 0, + "zone_append_merged": 0, "failed_flush_operations": 0, "account_invalid": true, "rd_total_time_ns": 0, @@ -39,7 +41,11 @@ Testing: -drive driver=null-co,read-zeroes=on,if=virtio "unmap_total_time_ns": 0, "invalid_flush_operations": 0, "account_failed": true, + "zone_append_total_time_ns": 0, + "zone_append_operations": 0, "rd_operations": 0, + "zone_append_bytes": 0, + "invalid_zone_append_operations": 0, "invalid_wr_operations": 0, "invalid_rd_operations": 0 }, @@ -82,6 +88,7 @@ Testing: -drive driver=null-co,if=none "stats": { "unmap_operations": 0, "unmap_merged": 0, + "failed_zone_append_operations": 0, "flush_total_time_ns": 0, "wr_highest_offset": 0, "wr_total_time_ns": 0, @@ -92,6 +99,7 @@ Testing: -drive driver=null-co,if=none "timed_stats": [ ], "failed_unmap_operations": 0, + "zone_append_merged": 0, "failed_flush_operations": 0, "account_invalid": true, "rd_total_time_ns": 0, @@ -104,7 +112,11 @@ Testing: -drive driver=null-co,if=none "unmap_total_time_ns": 0, "invalid_flush_operations": 0, "account_failed": true, + "zone_append_total_time_ns": 0, + "zone_append_operations": 0, "rd_operations": 0, + "zone_append_bytes": 0, + "invalid_zone_append_operations": 0, "invalid_wr_operations": 0, "invalid_rd_operations": 0 }, @@ -177,6 +189,7 @@ Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-b "stats": { "unmap_operations": 0, "unmap_merged": 0, + "failed_zone_append_operations": 0, "flush_total_time_ns": 0, "wr_highest_offset": 0, "wr_total_time_ns": 0, @@ -187,6 +200,7 @@ Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-b "timed_stats": [ ], "failed_unmap_operations": 0, + "zone_append_merged": 0, "failed_flush_operations": 0, "account_invalid": true, "rd_total_time_ns": 0, @@ -199,7 +213,11 @@ Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-b "unmap_total_time_ns": 0, "invalid_flush_operations": 0, "account_failed": true, + "zone_append_total_time_ns": 0, + "zone_append_operations": 0, "rd_operations": 0, + "zone_append_bytes": 0, + "invalid_zone_append_operations": 0, "invalid_wr_operations": 0, "invalid_rd_operations": 0 }, diff --git a/tests/qemu-iotests/meson.build b/tests/qemu-iotests/meson.build index 9735071a29..44761e1e4d 100644 --- a/tests/qemu-iotests/meson.build +++ b/tests/qemu-iotests/meson.build @@ -2,7 +2,7 @@ if not have_tools or targetos == 'windows' or get_option('gprof') subdir_done() endif -foreach cflag: config_host['QEMU_CFLAGS'].split() +foreach cflag: qemu_ldflags if cflag.startswith('-fsanitize') and \ not cflag.contains('safe-stack') and not cflag.contains('cfi-icall') message('Sanitizers are enabled ==> Disabled the qemu-iotests.') diff --git a/tests/qemu-iotests/tests/zoned b/tests/qemu-iotests/tests/zoned new file mode 100755 index 0000000000..3d23ce9cc1 --- /dev/null +++ b/tests/qemu-iotests/tests/zoned @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# +# Test zone management operations. +# + +seq="$(basename $0)" +echo "QA output created by $seq" +status=1 # failure is the default! + +_cleanup() +{ + _cleanup_test_img + sudo -n rmmod null_blk +} +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# get standard environment, filters and checks +. ../common.rc +. ../common.filter +. ../common.qemu + +# This test only runs on Linux hosts with raw image files. +_supported_fmt raw +_supported_proto file +_supported_os Linux + +sudo -n true || \ + _notrun 'Password-less sudo required' + +IMG="--image-opts -n driver=host_device,filename=/dev/nullb0" +QEMU_IO_OPTIONS=$QEMU_IO_OPTIONS_NO_FMT + +echo "Testing a null_blk device:" +echo "case 1: if the operations work" +sudo -n modprobe null_blk nr_devices=1 zoned=1 +sudo -n chmod 0666 /dev/nullb0 + +echo "(1) report the first zone:" +$QEMU_IO $IMG -c "zrp 0 1" +echo +echo "report the first 10 zones" +$QEMU_IO $IMG -c "zrp 0 10" +echo +echo "report the last zone:" +$QEMU_IO $IMG -c "zrp 0x3e70000000 2" # 0x3e70000000 / 512 = 0x1f380000 +echo +echo +echo "(2) opening the first zone" +$QEMU_IO $IMG -c "zo 0 268435456" # 268435456 / 512 = 524288 +echo "report after:" +$QEMU_IO $IMG -c "zrp 0 1" +echo +echo "opening the second zone" +$QEMU_IO $IMG -c "zo 268435456 268435456" # +echo "report after:" +$QEMU_IO $IMG -c "zrp 268435456 1" +echo +echo "opening the last zone" +$QEMU_IO $IMG -c "zo 0x3e70000000 268435456" +echo "report after:" +$QEMU_IO $IMG -c "zrp 0x3e70000000 2" +echo +echo +echo "(3) closing the first zone" +$QEMU_IO $IMG -c "zc 0 268435456" +echo "report after:" +$QEMU_IO $IMG -c "zrp 0 1" +echo +echo "closing the last zone" +$QEMU_IO $IMG -c "zc 0x3e70000000 268435456" +echo "report after:" +$QEMU_IO $IMG -c "zrp 0x3e70000000 2" +echo +echo +echo "(4) finishing the second zone" +$QEMU_IO $IMG -c "zf 268435456 268435456" +echo "After finishing a zone:" +$QEMU_IO $IMG -c "zrp 268435456 1" +echo +echo +echo "(5) resetting the second zone" +$QEMU_IO $IMG -c "zrs 268435456 268435456" +echo "After resetting a zone:" +$QEMU_IO $IMG -c "zrp 268435456 1" +echo +echo +echo "(6) append write" # the physical block size of the device is 4096 +$QEMU_IO $IMG -c "zrp 0 1" +$QEMU_IO $IMG -c "zap -p 0 0x1000 0x2000" +echo "After appending the first zone firstly:" +$QEMU_IO $IMG -c "zrp 0 1" +$QEMU_IO $IMG -c "zap -p 0 0x1000 0x2000" +echo "After appending the first zone secondly:" +$QEMU_IO $IMG -c "zrp 0 1" +$QEMU_IO $IMG -c "zap -p 268435456 0x1000 0x2000" +echo "After appending the second zone firstly:" +$QEMU_IO $IMG -c "zrp 268435456 1" +$QEMU_IO $IMG -c "zap -p 268435456 0x1000 0x2000" +echo "After appending the second zone secondly:" +$QEMU_IO $IMG -c "zrp 268435456 1" + +# success, all done +echo "*** done" +rm -f $seq.full +status=0 diff --git a/tests/qemu-iotests/tests/zoned.out b/tests/qemu-iotests/tests/zoned.out new file mode 100644 index 0000000000..fe53ba4744 --- /dev/null +++ b/tests/qemu-iotests/tests/zoned.out @@ -0,0 +1,69 @@ +QA output created by zoned +Testing a null_blk device: +case 1: if the operations work +(1) report the first zone: +start: 0x0, len 0x80000, cap 0x80000, wptr 0x0, zcond:1, [type: 2] + +report the first 10 zones +start: 0x0, len 0x80000, cap 0x80000, wptr 0x0, zcond:1, [type: 2] +start: 0x80000, len 0x80000, cap 0x80000, wptr 0x80000, zcond:1, [type: 2] +start: 0x100000, len 0x80000, cap 0x80000, wptr 0x100000, zcond:1, [type: 2] +start: 0x180000, len 0x80000, cap 0x80000, wptr 0x180000, zcond:1, [type: 2] +start: 0x200000, len 0x80000, cap 0x80000, wptr 0x200000, zcond:1, [type: 2] +start: 0x280000, len 0x80000, cap 0x80000, wptr 0x280000, zcond:1, [type: 2] +start: 0x300000, len 0x80000, cap 0x80000, wptr 0x300000, zcond:1, [type: 2] +start: 0x380000, len 0x80000, cap 0x80000, wptr 0x380000, zcond:1, [type: 2] +start: 0x400000, len 0x80000, cap 0x80000, wptr 0x400000, zcond:1, [type: 2] +start: 0x480000, len 0x80000, cap 0x80000, wptr 0x480000, zcond:1, [type: 2] + +report the last zone: +start: 0x1f380000, len 0x80000, cap 0x80000, wptr 0x1f380000, zcond:1, [type: 2] + + +(2) opening the first zone +report after: +start: 0x0, len 0x80000, cap 0x80000, wptr 0x0, zcond:3, [type: 2] + +opening the second zone +report after: +start: 0x80000, len 0x80000, cap 0x80000, wptr 0x80000, zcond:3, [type: 2] + +opening the last zone +report after: +start: 0x1f380000, len 0x80000, cap 0x80000, wptr 0x1f380000, zcond:3, [type: 2] + + +(3) closing the first zone +report after: +start: 0x0, len 0x80000, cap 0x80000, wptr 0x0, zcond:1, [type: 2] + +closing the last zone +report after: +start: 0x1f380000, len 0x80000, cap 0x80000, wptr 0x1f380000, zcond:1, [type: 2] + + +(4) finishing the second zone +After finishing a zone: +start: 0x80000, len 0x80000, cap 0x80000, wptr 0x100000, zcond:14, [type: 2] + + +(5) resetting the second zone +After resetting a zone: +start: 0x80000, len 0x80000, cap 0x80000, wptr 0x80000, zcond:1, [type: 2] + + +(6) append write +start: 0x0, len 0x80000, cap 0x80000, wptr 0x0, zcond:1, [type: 2] +After zap done, the append sector is 0x0 +After appending the first zone firstly: +start: 0x0, len 0x80000, cap 0x80000, wptr 0x18, zcond:2, [type: 2] +After zap done, the append sector is 0x18 +After appending the first zone secondly: +start: 0x0, len 0x80000, cap 0x80000, wptr 0x30, zcond:2, [type: 2] +After zap done, the append sector is 0x80000 +After appending the second zone firstly: +start: 0x80000, len 0x80000, cap 0x80000, wptr 0x80018, zcond:2, [type: 2] +After zap done, the append sector is 0x80018 +After appending the second zone secondly: +start: 0x80000, len 0x80000, cap 0x80000, wptr 0x80030, zcond:2, [type: 2] +*** done diff --git a/tests/qtest/ahci-test.c b/tests/qtest/ahci-test.c index 1967cd5898..abab761c26 100644 --- a/tests/qtest/ahci-test.c +++ b/tests/qtest/ahci-test.c @@ -36,9 +36,6 @@ #include "hw/pci/pci_ids.h" #include "hw/pci/pci_regs.h" -/* TODO actually test the results and get rid of this */ -#define qmp_discard_response(s, ...) qobject_unref(qtest_qmp(s, __VA_ARGS__)) - /* Test images sizes in MB */ #define TEST_IMAGE_SIZE_MB_LARGE (200 * 1024) #define TEST_IMAGE_SIZE_MB_SMALL 64 @@ -1595,9 +1592,9 @@ static void test_atapi_tray(void) rsp = qtest_qmp_receive(ahci->parent->qts); qobject_unref(rsp); - qmp_discard_response(ahci->parent->qts, - "{'execute': 'blockdev-remove-medium', " - "'arguments': {'id': 'cd0'}}"); + qtest_qmp_assert_success(ahci->parent->qts, + "{'execute': 'blockdev-remove-medium', " + "'arguments': {'id': 'cd0'}}"); /* Test the tray without a medium */ ahci_atapi_load(ahci, port); @@ -1607,16 +1604,18 @@ static void test_atapi_tray(void) atapi_wait_tray(ahci, true); /* Re-insert media */ - qmp_discard_response(ahci->parent->qts, - "{'execute': 'blockdev-add', " - "'arguments': {'node-name': 'node0', " - "'driver': 'raw', " - "'file': { 'driver': 'file', " - "'filename': %s }}}", iso); - qmp_discard_response(ahci->parent->qts, - "{'execute': 'blockdev-insert-medium'," - "'arguments': { 'id': 'cd0', " - "'node-name': 'node0' }}"); + qtest_qmp_assert_success( + ahci->parent->qts, + "{'execute': 'blockdev-add', " + "'arguments': {'node-name': 'node0', " + "'driver': 'raw', " + "'file': { 'driver': 'file', " + "'filename': %s }}}", iso); + qtest_qmp_assert_success( + ahci->parent->qts, + "{'execute': 'blockdev-insert-medium'," + "'arguments': { 'id': 'cd0', " + "'node-name': 'node0' }}"); /* Again, the event shows up first */ qtest_qmp_send(ahci->parent->qts, "{'execute': 'blockdev-close-tray', " diff --git a/tests/qtest/boot-order-test.c b/tests/qtest/boot-order-test.c index 0680d79d6d..8f2b6ef05a 100644 --- a/tests/qtest/boot-order-test.c +++ b/tests/qtest/boot-order-test.c @@ -16,9 +16,6 @@ #include "qapi/qmp/qdict.h" #include "standard-headers/linux/qemu_fw_cfg.h" -/* TODO actually test the results and get rid of this */ -#define qmp_discard_response(qs, ...) qobject_unref(qtest_qmp(qs, __VA_ARGS__)) - typedef struct { const char *args; uint64_t expected_boot; @@ -43,7 +40,7 @@ static void test_a_boot_order(const char *machine, machine ?: "", test_args); actual = read_boot_order(qts); g_assert_cmphex(actual, ==, expected_boot); - qmp_discard_response(qts, "{ 'execute': 'system_reset' }"); + qtest_qmp_assert_success(qts, "{ 'execute': 'system_reset' }"); /* * system_reset only requests reset. We get a RESET event after * the actual reset completes. Need to wait for that. diff --git a/tests/qtest/cdrom-test.c b/tests/qtest/cdrom-test.c index 26a2400181..2b7e10d920 100644 --- a/tests/qtest/cdrom-test.c +++ b/tests/qtest/cdrom-test.c @@ -17,7 +17,7 @@ static char isoimage[] = "cdrom-boot-iso-XXXXXX"; -static int exec_genisoimg(const char **args) +static int exec_xorrisofs(const char **args) { gchar *out_err = NULL; gint exit_status = -1; @@ -43,7 +43,7 @@ static int prepare_image(const char *arch, char *isoimage) char *codefile = NULL; int ifh, ret = -1; const char *args[] = { - "genisoimage", "-quiet", "-l", "-no-emul-boot", + "xorrisofs", "-quiet", "-l", "-no-emul-boot", "-b", NULL, "-o", isoimage, srcdir, NULL }; @@ -75,9 +75,9 @@ static int prepare_image(const char *arch, char *isoimage) } args[5] = strchr(codefile, '/') + 1; - ret = exec_genisoimg(args); + ret = exec_xorrisofs(args); if (ret) { - fprintf(stderr, "genisoimage failed: %i\n", ret); + fprintf(stderr, "xorrisofs failed: %i\n", ret); } unlink(codefile); @@ -130,6 +130,11 @@ static void test_cdboot(gconstpointer data) static void add_x86_tests(void) { + if (!qtest_has_accel("tcg") && !qtest_has_accel("kvm")) { + g_test_skip("No KVM or TCG accelerator available, skipping boot tests"); + return; + } + qtest_add_data_func("cdrom/boot/default", "-cdrom ", test_cdboot); qtest_add_data_func("cdrom/boot/virtio-scsi", "-device virtio-scsi -device scsi-cd,drive=cdr " @@ -176,6 +181,11 @@ static void add_x86_tests(void) static void add_s390x_tests(void) { + if (!qtest_has_accel("tcg") && !qtest_has_accel("kvm")) { + g_test_skip("No KVM or TCG accelerator available, skipping boot tests"); + return; + } + qtest_add_data_func("cdrom/boot/default", "-cdrom ", test_cdboot); qtest_add_data_func("cdrom/boot/virtio-scsi", "-device virtio-scsi -device scsi-cd,drive=cdr " @@ -201,12 +211,12 @@ int main(int argc, char **argv) { int ret; const char *arch = qtest_get_arch(); - const char *genisocheck[] = { "genisoimage", "-version", NULL }; + const char *xorrisocheck[] = { "xorrisofs", "-version", NULL }; g_test_init(&argc, &argv, NULL); - if (exec_genisoimg(genisocheck)) { - /* genisoimage not available - so can't run tests */ + if (exec_xorrisofs(xorrisocheck)) { + /* xorrisofs not available - so can't run tests */ return g_test_run(); } diff --git a/tests/qtest/fdc-test.c b/tests/qtest/fdc-test.c index 1f9b99ad6d..5e8fbda9df 100644 --- a/tests/qtest/fdc-test.c +++ b/tests/qtest/fdc-test.c @@ -28,9 +28,6 @@ #include "libqtest-single.h" #include "qapi/qmp/qdict.h" -/* TODO actually test the results and get rid of this */ -#define qmp_discard_response(...) qobject_unref(qmp(__VA_ARGS__)) - #define DRIVE_FLOPPY_BLANK \ "-drive if=floppy,file=null-co://,file.read-zeroes=on,format=raw,size=1440k" @@ -304,9 +301,10 @@ static void test_media_insert(void) /* Insert media in drive. DSKCHK should not be reset until a step pulse * is sent. */ - qmp_discard_response("{'execute':'blockdev-change-medium', 'arguments':{" - " 'id':'floppy0', 'filename': %s, 'format': 'raw' }}", - test_image); + qtest_qmp_assert_success(global_qtest, + "{'execute':'blockdev-change-medium', 'arguments':{" + " 'id':'floppy0', 'filename': %s, 'format': 'raw' }}", + test_image); dir = inb(FLOPPY_BASE + reg_dir); assert_bit_set(dir, DSKCHG); @@ -335,8 +333,9 @@ static void test_media_change(void) /* Eject the floppy and check that DSKCHG is set. Reading it out doesn't * reset the bit. */ - qmp_discard_response("{'execute':'eject', 'arguments':{" - " 'id':'floppy0' }}"); + qtest_qmp_assert_success(global_qtest, + "{'execute':'eject', 'arguments':{" + " 'id':'floppy0' }}"); dir = inb(FLOPPY_BASE + reg_dir); assert_bit_set(dir, DSKCHG); diff --git a/tests/qtest/ide-test.c b/tests/qtest/ide-test.c index dcb050bf9b..d6b4f6e36a 100644 --- a/tests/qtest/ide-test.c +++ b/tests/qtest/ide-test.c @@ -34,9 +34,6 @@ #include "hw/pci/pci_ids.h" #include "hw/pci/pci_regs.h" -/* TODO actually test the results and get rid of this */ -#define qmp_discard_response(q, ...) qobject_unref(qtest_qmp(q, __VA_ARGS__)) - #define TEST_IMAGE_SIZE 64 * 1024 * 1024 #define IDE_PCI_DEV 1 @@ -766,7 +763,7 @@ static void test_pci_retry_flush(void) qtest_qmp_eventwait(qts, "STOP"); /* Complete the command */ - qmp_discard_response(qts, "{'execute':'cont' }"); + qtest_qmp_assert_success(qts, "{'execute':'cont' }"); /* Check registers */ data = qpci_io_readb(dev, ide_bar, reg_device); diff --git a/tests/qtest/libqos/virtio-9p-client.c b/tests/qtest/libqos/virtio-9p-client.c index e4a368e036..b8adc8d4b9 100644 --- a/tests/qtest/libqos/virtio-9p-client.c +++ b/tests/qtest/libqos/virtio-9p-client.c @@ -594,6 +594,8 @@ void v9fs_rreaddir(P9Req *req, uint32_t *count, uint32_t *nentries, { uint32_t local_count; struct V9fsDirent *e = NULL; + /* only used to avoid a leak if entries was NULL */ + struct V9fsDirent *unused_entries = NULL; uint16_t slen; uint32_t n = 0; @@ -612,6 +614,8 @@ void v9fs_rreaddir(P9Req *req, uint32_t *count, uint32_t *nentries, e = g_new(struct V9fsDirent, 1); if (entries) { *entries = e; + } else { + unused_entries = e; } } else { e = e->next = g_new(struct V9fsDirent, 1); @@ -628,6 +632,7 @@ void v9fs_rreaddir(P9Req *req, uint32_t *count, uint32_t *nentries, *nentries = n; } + v9fs_free_dirents(unused_entries); v9fs_req_free(req); } diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build index 48cd35b5b2..ab422772d3 100644 --- a/tests/qtest/meson.build +++ b/tests/qtest/meson.build @@ -23,7 +23,7 @@ qtests_generic = [ 'readconfig-test', 'netdev-socket', ] -if config_host.has_key('CONFIG_MODULES') +if enable_modules qtests_generic += [ 'modules-test' ] endif diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c index 8a5df84624..b99b49a314 100644 --- a/tests/qtest/migration-test.c +++ b/tests/qtest/migration-test.c @@ -40,9 +40,6 @@ #include "linux/kvm.h" #endif -/* TODO actually test the results and get rid of this */ -#define qtest_qmp_discard_response(...) qobject_unref(qtest_qmp(__VA_ARGS__)) - unsigned start_address; unsigned end_address; static bool uffd_feature_thread_id; @@ -766,7 +763,7 @@ static void test_migrate_end(QTestState *from, QTestState *to, bool test_dest) usleep(1000 * 10); } while (dest_byte_a == dest_byte_b); - qtest_qmp_discard_response(to, "{ 'execute' : 'stop'}"); + qtest_qmp_assert_success(to, "{ 'execute' : 'stop'}"); /* With it stopped, check nothing changes */ qtest_memread(to, start_address, &dest_byte_c, 1); diff --git a/tests/qtest/netdev-socket.c b/tests/qtest/netdev-socket.c index 9cf1b0698e..097abc0230 100644 --- a/tests/qtest/netdev-socket.c +++ b/tests/qtest/netdev-socket.c @@ -189,28 +189,26 @@ static void wait_stream_disconnected(QTestState *qts, const char *id) qobject_unref(resp); } -static void test_stream_inet_reconnect(void) +static void test_stream_unix_reconnect(void) { QTestState *qts0, *qts1; - int port; SocketAddress *addr; + gchar *path; - port = inet_get_free_port(false); + path = g_strconcat(tmpdir, "/stream_unix_reconnect", NULL); qts0 = qtest_initf("-nodefaults -M none " - "-netdev stream,id=st0,server=true,addr.type=inet," - "addr.ipv4=on,addr.ipv6=off," - "addr.host=127.0.0.1,addr.port=%d", port); + "-netdev stream,id=st0,server=true,addr.type=unix," + "addr.path=%s", path); EXPECT_STATE(qts0, "st0: index=0,type=stream,\r\n", 0); qts1 = qtest_initf("-nodefaults -M none " - "-netdev stream,server=false,id=st0,addr.type=inet," - "addr.ipv4=on,addr.ipv6=off,reconnect=1," - "addr.host=127.0.0.1,addr.port=%d", port); + "-netdev stream,server=false,id=st0,addr.type=unix," + "addr.path=%s,reconnect=1", path); wait_stream_connected(qts0, "st0", &addr); - g_assert_cmpint(addr->type, ==, SOCKET_ADDRESS_TYPE_INET); - g_assert_cmpstr(addr->u.inet.host, ==, "127.0.0.1"); + g_assert_cmpint(addr->type, ==, SOCKET_ADDRESS_TYPE_UNIX); + g_assert_cmpstr(addr->u.q_unix.path, ==, path); qapi_free_SocketAddress(addr); /* kill server */ @@ -221,24 +219,23 @@ static void test_stream_inet_reconnect(void) /* restart server */ qts0 = qtest_initf("-nodefaults -M none " - "-netdev stream,id=st0,server=true,addr.type=inet," - "addr.ipv4=on,addr.ipv6=off," - "addr.host=127.0.0.1,addr.port=%d", port); + "-netdev stream,id=st0,server=true,addr.type=unix," + "addr.path=%s", path); /* wait connection events*/ wait_stream_connected(qts0, "st0", &addr); - g_assert_cmpint(addr->type, ==, SOCKET_ADDRESS_TYPE_INET); - g_assert_cmpstr(addr->u.inet.host, ==, "127.0.0.1"); + g_assert_cmpint(addr->type, ==, SOCKET_ADDRESS_TYPE_UNIX); + g_assert_cmpstr(addr->u.q_unix.path, ==, path); qapi_free_SocketAddress(addr); wait_stream_connected(qts1, "st0", &addr); - g_assert_cmpint(addr->type, ==, SOCKET_ADDRESS_TYPE_INET); - g_assert_cmpstr(addr->u.inet.host, ==, "127.0.0.1"); - g_assert_cmpint(atoi(addr->u.inet.port), ==, port); + g_assert_cmpint(addr->type, ==, SOCKET_ADDRESS_TYPE_UNIX); + g_assert_cmpstr(addr->u.q_unix.path, ==, path); qapi_free_SocketAddress(addr); qtest_quit(qts1); qtest_quit(qts0); + g_free(path); } static void test_stream_inet_ipv6(void) @@ -517,8 +514,6 @@ int main(int argc, char **argv) #ifndef _WIN32 qtest_add_func("/netdev/dgram/mcast", test_dgram_mcast); #endif - qtest_add_func("/netdev/stream/inet/reconnect", - test_stream_inet_reconnect); } if (has_ipv6) { qtest_add_func("/netdev/stream/inet/ipv6", test_stream_inet_ipv6); @@ -530,6 +525,8 @@ int main(int argc, char **argv) qtest_add_func("/netdev/dgram/unix", test_dgram_unix); #endif qtest_add_func("/netdev/stream/unix", test_stream_unix); + qtest_add_func("/netdev/stream/unix/reconnect", + test_stream_unix_reconnect); #ifdef CONFIG_LINUX qtest_add_func("/netdev/stream/unix/abstract", test_stream_unix_abstract); diff --git a/tests/qtest/test-filter-mirror.c b/tests/qtest/test-filter-mirror.c index 248fc88699..adeada3eb8 100644 --- a/tests/qtest/test-filter-mirror.c +++ b/tests/qtest/test-filter-mirror.c @@ -16,9 +16,6 @@ #include "qemu/error-report.h" #include "qemu/main-loop.h" -/* TODO actually test the results and get rid of this */ -#define qmp_discard_response(qs, ...) qobject_unref(qtest_qmp(qs, __VA_ARGS__)) - static void test_mirror(void) { int send_sock[2], recv_sock[2]; @@ -52,7 +49,7 @@ static void test_mirror(void) }; /* send a qmp command to guarantee that 'connected' is setting to true. */ - qmp_discard_response(qts, "{ 'execute' : 'query-status'}"); + qtest_qmp_assert_success(qts, "{ 'execute' : 'query-status'}"); ret = iov_send(send_sock[0], iov, 2, 0, sizeof(size) + sizeof(send_buf)); g_assert_cmpint(ret, ==, sizeof(send_buf) + sizeof(size)); close(send_sock[0]); diff --git a/tests/qtest/test-filter-redirector.c b/tests/qtest/test-filter-redirector.c index 24ca9280f8..e72e3b7873 100644 --- a/tests/qtest/test-filter-redirector.c +++ b/tests/qtest/test-filter-redirector.c @@ -58,9 +58,6 @@ #include "qemu/error-report.h" #include "qemu/main-loop.h" -/* TODO actually test the results and get rid of this */ -#define qmp_discard_response(qs, ...) qobject_unref(qtest_qmp(qs, __VA_ARGS__)) - static void test_redirector_tx(void) { int backend_sock[2], recv_sock; @@ -98,7 +95,7 @@ static void test_redirector_tx(void) g_assert_cmpint(recv_sock, !=, -1); /* send a qmp command to guarantee that 'connected' is setting to true. */ - qmp_discard_response(qts, "{ 'execute' : 'query-status'}"); + qtest_qmp_assert_success(qts, "{ 'execute' : 'query-status'}"); struct iovec iov[] = { { @@ -176,7 +173,7 @@ static void test_redirector_rx(void) send_sock = unix_connect(sock_path1, NULL); g_assert_cmpint(send_sock, !=, -1); /* send a qmp command to guarantee that 'connected' is setting to true. */ - qmp_discard_response(qts, "{ 'execute' : 'query-status'}"); + qtest_qmp_assert_success(qts, "{ 'execute' : 'query-status'}"); ret = iov_send(send_sock, iov, 2, 0, sizeof(size) + sizeof(send_buf)); g_assert_cmpint(ret, ==, sizeof(send_buf) + sizeof(size)); diff --git a/tests/qtest/virtio-blk-test.c b/tests/qtest/virtio-blk-test.c index 19c01f808b..98c906ebb4 100644 --- a/tests/qtest/virtio-blk-test.c +++ b/tests/qtest/virtio-blk-test.c @@ -17,9 +17,6 @@ #include "libqos/qgraph.h" #include "libqos/virtio-blk.h" -/* TODO actually test the results and get rid of this */ -#define qmp_discard_response(...) qobject_unref(qmp(__VA_ARGS__)) - #define TEST_IMAGE_SIZE (64 * 1024 * 1024) #define QVIRTIO_BLK_TIMEOUT_US (30 * 1000 * 1000) #define PCI_SLOT_HP 0x06 @@ -453,9 +450,10 @@ static void config(void *obj, void *data, QGuestAllocator *t_alloc) qvirtio_set_driver_ok(dev); - qmp_discard_response("{ 'execute': 'block_resize', " - " 'arguments': { 'device': 'drive0', " - " 'size': %d } }", n_size); + qtest_qmp_assert_success(global_qtest, + "{ 'execute': 'block_resize', " + " 'arguments': { 'device': 'drive0', " + " 'size': %d } }", n_size); qvirtio_wait_config_isr(dev, QVIRTIO_BLK_TIMEOUT_US); capacity = qvirtio_config_readq(dev, 0); @@ -502,9 +500,10 @@ static void msix(void *obj, void *u_data, QGuestAllocator *t_alloc) qvirtio_set_driver_ok(dev); - qmp_discard_response("{ 'execute': 'block_resize', " - " 'arguments': { 'device': 'drive0', " - " 'size': %d } }", n_size); + qtest_qmp_assert_success(global_qtest, + "{ 'execute': 'block_resize', " + " 'arguments': { 'device': 'drive0', " + " 'size': %d } }", n_size); qvirtio_wait_config_isr(dev, QVIRTIO_BLK_TIMEOUT_US); @@ -758,9 +757,10 @@ static void resize(void *obj, void *data, QGuestAllocator *t_alloc) vq = test_basic(dev, t_alloc); - qmp_discard_response("{ 'execute': 'block_resize', " - " 'arguments': { 'device': 'drive0', " - " 'size': %d } }", n_size); + qtest_qmp_assert_success(global_qtest, + "{ 'execute': 'block_resize', " + " 'arguments': { 'device': 'drive0', " + " 'size': %d } }", n_size); qvirtio_wait_queue_isr(qts, dev, vq, QVIRTIO_BLK_TIMEOUT_US); diff --git a/tests/requirements.txt b/tests/requirements.txt index a6f73da681..0e008b9aec 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,9 @@ # Add Python module requirements, one per line, to be installed -# in the tests/venv Python virtual environment. For more info, +# in the qemu build_dir/pyvenv Python virtual environment. For more info, # refer to: https://pip.pypa.io/en/stable/user_guide/#id1 -# Note that qemu.git/python/ is always implicitly installed. +# +# Note that qemu.git/python/ is implicitly installed to this venv when +# 'make check-venv' is run, and will persist until configure is run +# again. avocado-framework==101.0 pycdlib==1.11.0 diff --git a/tests/tcg/i386/test-avx.py b/tests/tcg/i386/test-avx.py index d9ca00a49e..641a2ef69e 100755 --- a/tests/tcg/i386/test-avx.py +++ b/tests/tcg/i386/test-avx.py @@ -49,7 +49,7 @@ imask = { 'VEXTRACT[FI]128': 0x01, 'VINSERT[FI]128': 0x01, 'VPBLENDD': 0xff, - 'VPERM2[FI]128': 0x33, + 'VPERM2[FI]128': 0xbb, 'VPERMPD': 0xff, 'VPERMQ': 0xff, 'VPERMILPS': 0xff, diff --git a/tests/tcg/multiarch/system/memory.c b/tests/tcg/multiarch/system/memory.c index 214f7d4f54..e29786ae55 100644 --- a/tests/tcg/multiarch/system/memory.c +++ b/tests/tcg/multiarch/system/memory.c @@ -40,18 +40,21 @@ static void pdot(int count) } /* - * Helper macros for shift/extract so we can keep our endian handling - * in one place. + * Helper macros for endian handling. */ -#define BYTE_SHIFT(b, pos) ((uint64_t)b << (pos * 8)) -#define BYTE_EXTRACT(b, pos) ((b >> (pos * 8)) & 0xff) +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define BYTE_SHIFT(b, pos) (b << (pos * 8)) +#define BYTE_NEXT(b) ((b)++) +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define BYTE_SHIFT(b, pos) (b << ((sizeof(b) - 1 - (pos)) * 8)) +#define BYTE_NEXT(b) (--(b)) +#else +#error Unsupported __BYTE_ORDER__ +#endif /* - * Fill the data with ascending value bytes. - * - * Currently we only support Little Endian machines so write in - * ascending address order. When we read higher address bytes should - * either be zero or higher than the lower bytes. + * Fill the data with ascending (for little-endian) or descending (for + * big-endian) value bytes. */ static void init_test_data_u8(int unused_offset) @@ -62,14 +65,14 @@ static void init_test_data_u8(int unused_offset) ml_printf("Filling test area with u8:"); for (i = 0; i < TEST_SIZE; i++) { - *ptr++ = count++; + *ptr++ = BYTE_NEXT(count); pdot(i); } ml_printf("done\n"); } /* - * Full the data with alternating positive and negative bytes. This + * Fill the data with alternating positive and negative bytes. This * should mean for reads larger than a byte all subsequent reads will * stay either negative or positive. We never write 0. */ @@ -119,7 +122,7 @@ static void init_test_data_u16(int offset) reset_start_data(offset); for (i = 0; i < max; i++) { - uint8_t low = count++, high = count++; + uint16_t low = BYTE_NEXT(count), high = BYTE_NEXT(count); word = BYTE_SHIFT(high, 1) | BYTE_SHIFT(low, 0); *ptr++ = word; pdot(i); @@ -139,9 +142,10 @@ static void init_test_data_u32(int offset) reset_start_data(offset); for (i = 0; i < max; i++) { - uint8_t b4 = count++, b3 = count++; - uint8_t b2 = count++, b1 = count++; - word = BYTE_SHIFT(b1, 3) | BYTE_SHIFT(b2, 2) | BYTE_SHIFT(b3, 1) | b4; + uint32_t b4 = BYTE_NEXT(count), b3 = BYTE_NEXT(count); + uint32_t b2 = BYTE_NEXT(count), b1 = BYTE_NEXT(count); + word = BYTE_SHIFT(b1, 3) | BYTE_SHIFT(b2, 2) | BYTE_SHIFT(b3, 1) | + BYTE_SHIFT(b4, 0); *ptr++ = word; pdot(i); } @@ -160,13 +164,13 @@ static void init_test_data_u64(int offset) reset_start_data(offset); for (i = 0; i < max; i++) { - uint8_t b8 = count++, b7 = count++; - uint8_t b6 = count++, b5 = count++; - uint8_t b4 = count++, b3 = count++; - uint8_t b2 = count++, b1 = count++; + uint64_t b8 = BYTE_NEXT(count), b7 = BYTE_NEXT(count); + uint64_t b6 = BYTE_NEXT(count), b5 = BYTE_NEXT(count); + uint64_t b4 = BYTE_NEXT(count), b3 = BYTE_NEXT(count); + uint64_t b2 = BYTE_NEXT(count), b1 = BYTE_NEXT(count); word = BYTE_SHIFT(b1, 7) | BYTE_SHIFT(b2, 6) | BYTE_SHIFT(b3, 5) | BYTE_SHIFT(b4, 4) | BYTE_SHIFT(b5, 3) | BYTE_SHIFT(b6, 2) | - BYTE_SHIFT(b7, 1) | b8; + BYTE_SHIFT(b7, 1) | BYTE_SHIFT(b8, 0); *ptr++ = word; pdot(i); } @@ -374,12 +378,20 @@ static bool read_test_data_s16(int offset, bool neg_first) ml_printf("Reading s16 from %#lx (offset %d, %s):", ptr, offset, neg_first ? "neg" : "pos"); + /* + * If the first byte is negative, then the last byte is positive. + * Therefore the logic below must be flipped for big-endian. + */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + neg_first = !neg_first; +#endif + for (i = 0; i < max; i++) { int32_t data = *ptr++; if (neg_first && data < 0) { pdot(i); - } else if (data > 0) { + } else if (!neg_first && data > 0) { pdot(i); } else { ml_printf("Error %d %c 0\n", data, neg_first ? '<' : '>'); @@ -399,12 +411,20 @@ static bool read_test_data_s32(int offset, bool neg_first) ml_printf("Reading s32 from %#lx (offset %d, %s):", ptr, offset, neg_first ? "neg" : "pos"); + /* + * If the first byte is negative, then the last byte is positive. + * Therefore the logic below must be flipped for big-endian. + */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + neg_first = !neg_first; +#endif + for (i = 0; i < max; i++) { int64_t data = *ptr++; if (neg_first && data < 0) { pdot(i); - } else if (data > 0) { + } else if (!neg_first && data > 0) { pdot(i); } else { ml_printf("Error %d %c 0\n", data, neg_first ? '<' : '>'); @@ -419,8 +439,7 @@ static bool read_test_data_s32(int offset, bool neg_first) * Read the test data and verify at various offsets * * For everything except bytes all our reads should be either positive - * or negative depending on what offset we are reading from. Currently - * we only handle LE systems. + * or negative depending on what offset we are reading from. */ read_sfn read_sfns[] = { read_test_data_s8, read_test_data_s16, diff --git a/tests/tcg/s390x/Makefile.softmmu-target b/tests/tcg/s390x/Makefile.softmmu-target index 192315dd20..44dfd71629 100644 --- a/tests/tcg/s390x/Makefile.softmmu-target +++ b/tests/tcg/s390x/Makefile.softmmu-target @@ -1,28 +1,40 @@ S390X_SRC=$(SRC_PATH)/tests/tcg/s390x VPATH+=$(S390X_SRC) -QEMU_OPTS=-action panic=exit-failure -kernel +QEMU_OPTS=-action panic=exit-failure -nographic -kernel LINK_SCRIPT=$(S390X_SRC)/softmmu.ld -LDFLAGS=-nostdlib -static -Wl,-T$(LINK_SCRIPT) -Wl,--build-id=none +CFLAGS+=-ggdb -O0 +LDFLAGS=-nostdlib -static %.o: %.S $(CC) -march=z13 -m64 -c $< -o $@ -%: %.o $(LINK_SCRIPT) +%.o: %.c + $(CC) $(CFLAGS) $(EXTRA_CFLAGS) -march=z13 -m64 -c $< -o $@ + +%: %.o $(CC) $< -o $@ $(LDFLAGS) -TESTS += unaligned-lowcore -TESTS += bal -TESTS += sam -TESTS += lpsw -TESTS += lpswe-early -TESTS += ssm-early -TESTS += stosm-early -TESTS += exrl-ssm-early +ASM_TESTS = \ + bal \ + exrl-ssm-early \ + sam \ + lpsw \ + lpswe-early \ + ssm-early \ + stosm-early \ + unaligned-lowcore include $(S390X_SRC)/pgm-specification.mak $(PGM_SPECIFICATION_TESTS): pgm-specification-softmmu.o $(PGM_SPECIFICATION_TESTS): LDFLAGS+=pgm-specification-softmmu.o -TESTS += $(PGM_SPECIFICATION_TESTS) +ASM_TESTS += $(PGM_SPECIFICATION_TESTS) + +$(ASM_TESTS): LDFLAGS += -Wl,-T$(LINK_SCRIPT) -Wl,--build-id=none +$(ASM_TESTS): $(LINK_SCRIPT) +TESTS += $(ASM_TESTS) -# We don't currently support the multiarch system tests -undefine MULTIARCH_TESTS +S390X_MULTIARCH_RUNTIME_OBJS = head64.o console.o $(MINILIB_OBJS) +$(MULTIARCH_TESTS): $(S390X_MULTIARCH_RUNTIME_OBJS) +$(MULTIARCH_TESTS): LDFLAGS += $(S390X_MULTIARCH_RUNTIME_OBJS) +$(MULTIARCH_TESTS): CFLAGS += $(MINILIB_INC) +memory: CFLAGS += -DCHECK_UNALIGNED=0 diff --git a/tests/tcg/s390x/Makefile.target b/tests/tcg/s390x/Makefile.target index 0031868b13..23dc8b6a63 100644 --- a/tests/tcg/s390x/Makefile.target +++ b/tests/tcg/s390x/Makefile.target @@ -34,6 +34,7 @@ TESTS+=cdsg TESTS+=chrl TESTS+=rxsbg TESTS+=ex-relative-long +TESTS+=ex-branch cdsg: CFLAGS+=-pthread cdsg: LDFLAGS+=-pthread diff --git a/tests/tcg/s390x/console.c b/tests/tcg/s390x/console.c new file mode 100644 index 0000000000..d43ce3f44b --- /dev/null +++ b/tests/tcg/s390x/console.c @@ -0,0 +1,12 @@ +/* + * Console code for multiarch tests. + * Reuses the pc-bios/s390-ccw implementation. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ +#include "../../../pc-bios/s390-ccw/sclp.c" + +void __sys_outc(char c) +{ + write(1, &c, sizeof(c)); +} diff --git a/tests/tcg/s390x/ex-branch.c b/tests/tcg/s390x/ex-branch.c new file mode 100644 index 0000000000..c606719152 --- /dev/null +++ b/tests/tcg/s390x/ex-branch.c @@ -0,0 +1,158 @@ +/* Check EXECUTE with relative branch instructions as targets. */ +#include <assert.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +struct test { + const char *name; + void (*func)(long *link, long *magic); + long exp_link; +}; + +/* Branch instructions and their expected effects. */ +#define LINK_64(test) ((long)test ## _exp_link) +#define LINK_NONE(test) -1L +#define FOR_EACH_INSN(F) \ + F(bras, "%[link]", LINK_64) \ + F(brasl, "%[link]", LINK_64) \ + F(brc, "0x8", LINK_NONE) \ + F(brcl, "0x8", LINK_NONE) \ + F(brct, "%%r0", LINK_NONE) \ + F(brctg, "%%r0", LINK_NONE) \ + F(brxh, "%%r2,%%r0", LINK_NONE) \ + F(brxhg, "%%r2,%%r0", LINK_NONE) \ + F(brxle, "%%r0,%%r1", LINK_NONE) \ + F(brxlg, "%%r0,%%r1", LINK_NONE) \ + F(crj, "%%r0,%%r0,8", LINK_NONE) \ + F(cgrj, "%%r0,%%r0,8", LINK_NONE) \ + F(cij, "%%r0,0,8", LINK_NONE) \ + F(cgij, "%%r0,0,8", LINK_NONE) \ + F(clrj, "%%r0,%%r0,8", LINK_NONE) \ + F(clgrj, "%%r0,%%r0,8", LINK_NONE) \ + F(clij, "%%r0,0,8", LINK_NONE) \ + F(clgij, "%%r0,0,8", LINK_NONE) + +#define INIT_TEST \ + "xgr %%r0,%%r0\n" /* %r0 = 0; %cc = 0 */ \ + "lghi %%r1,1\n" /* %r1 = 1 */ \ + "lghi %%r2,2\n" /* %r2 = 2 */ + +#define CLOBBERS_TEST "cc", "0", "1", "2" + +#define DEFINE_TEST(insn, args, exp_link) \ + extern char insn ## _exp_link[]; \ + static void test_ ## insn(long *link, long *magic) \ + { \ + asm(INIT_TEST \ + #insn " " args ",0f\n" \ + ".globl " #insn "_exp_link\n" \ + #insn "_exp_link:\n" \ + ".org . + 90\n" \ + "0: lgfi %[magic],0x12345678\n" \ + : [link] "+r" (*link) \ + , [magic] "+r" (*magic) \ + : : CLOBBERS_TEST); \ + } \ + extern char ex_ ## insn ## _exp_link[]; \ + static void test_ex_ ## insn(long *link, long *magic) \ + { \ + unsigned long target; \ + \ + asm(INIT_TEST \ + "larl %[target],0f\n" \ + "ex %%r0,0(%[target])\n" \ + ".globl ex_" #insn "_exp_link\n" \ + "ex_" #insn "_exp_link:\n" \ + ".org . + 60\n" \ + "0: " #insn " " args ",1f\n" \ + ".org . + 120\n" \ + "1: lgfi %[magic],0x12345678\n" \ + : [target] "=r" (target) \ + , [link] "+r" (*link) \ + , [magic] "+r" (*magic) \ + : : CLOBBERS_TEST); \ + } \ + extern char exrl_ ## insn ## _exp_link[]; \ + static void test_exrl_ ## insn(long *link, long *magic) \ + { \ + asm(INIT_TEST \ + "exrl %%r0,0f\n" \ + ".globl exrl_" #insn "_exp_link\n" \ + "exrl_" #insn "_exp_link:\n" \ + ".org . + 60\n" \ + "0: " #insn " " args ",1f\n" \ + ".org . + 120\n" \ + "1: lgfi %[magic],0x12345678\n" \ + : [link] "+r" (*link) \ + , [magic] "+r" (*magic) \ + : : CLOBBERS_TEST); \ + } + +/* Test functions. */ +FOR_EACH_INSN(DEFINE_TEST) + +/* Test definitions. */ +#define REGISTER_TEST(insn, args, _exp_link) \ + { \ + .name = #insn, \ + .func = test_ ## insn, \ + .exp_link = (_exp_link(insn)), \ + }, \ + { \ + .name = "ex " #insn, \ + .func = test_ex_ ## insn, \ + .exp_link = (_exp_link(ex_ ## insn)), \ + }, \ + { \ + .name = "exrl " #insn, \ + .func = test_exrl_ ## insn, \ + .exp_link = (_exp_link(exrl_ ## insn)), \ + }, + +static const struct test tests[] = { + FOR_EACH_INSN(REGISTER_TEST) +}; + +int main(int argc, char **argv) +{ + const struct test *test; + int ret = EXIT_SUCCESS; + bool verbose = false; + long link, magic; + size_t i; + + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-v") == 0) { + verbose = true; + } + } + + for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + test = &tests[i]; + if (verbose) { + fprintf(stderr, "[ RUN ] %s\n", test->name); + } + link = -1; + magic = -1; + test->func(&link, &magic); +#define ASSERT_EQ(expected, actual) do { \ + if (expected != actual) { \ + fprintf(stderr, "%s: " #expected " (0x%lx) != " #actual " (0x%lx)\n", \ + test->name, expected, actual); \ + ret = EXIT_FAILURE; \ + } \ +} while (0) + ASSERT_EQ(test->exp_link, link); + ASSERT_EQ(0x12345678L, magic); +#undef ASSERT_EQ + } + + if (verbose) { + fprintf(stderr, ret == EXIT_SUCCESS ? "[ PASSED ]\n" : + "[ FAILED ]\n"); + } + + return ret; +} diff --git a/tests/tcg/s390x/head64.S b/tests/tcg/s390x/head64.S new file mode 100644 index 0000000000..c6f36dfea4 --- /dev/null +++ b/tests/tcg/s390x/head64.S @@ -0,0 +1,31 @@ +/* + * Startup code for multiarch tests. + * Reuses the pc-bios/s390-ccw implementation. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ +#define main main_pre +#include "../../../pc-bios/s390-ccw/start.S" +#undef main + +main_pre: + aghi %r15,-160 /* reserve stack for C code */ + brasl %r14,sclp_setup + brasl %r14,main + larl %r1,success_psw /* check main() return code */ + ltgr %r2,%r2 + je 0f + larl %r1,failure_psw +0: + lpswe 0(%r1) + + .align 8 +success_psw: + .quad 0x2000180000000,0xfff /* see is_special_wait_psw() */ +failure_psw: + .quad 0x2000180000000,0 /* disabled wait */ + + .section .bss + .align 0x1000 +stack: + .skip 0x8000 diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 3bc78d8660..48ae66011b 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -147,7 +147,7 @@ if have_system # Some tests: test-char, test-qdev-global-props, and test-qga, # are not runnable under TSan due to a known issue. # https://github.com/google/sanitizers/issues/1116 - if 'CONFIG_TSAN' not in config_host + if not get_option('tsan') if 'CONFIG_POSIX' in config_host tests += { 'test-char': ['socket-helpers.c', qom, io, chardev] diff --git a/tests/vm/netbsd b/tests/vm/netbsd index 0b9536ca17..c7e3f1e735 100755 --- a/tests/vm/netbsd +++ b/tests/vm/netbsd @@ -30,6 +30,8 @@ class NetBSDVM(basevm.BaseVM): "git-base", "pkgconf", "xz", + "python310", + "py310-expat", "ninja-build", # gnu tools diff --git a/ui/console.c b/ui/console.c index 6e8a3cdc62..e173731e20 100644 --- a/ui/console.c +++ b/ui/console.c @@ -311,7 +311,7 @@ static bool png_save(int fd, pixman_image_t *image, Error **errp) png_struct *png_ptr; png_info *info_ptr; g_autoptr(pixman_image_t) linebuf = - qemu_pixman_linebuf_create(PIXMAN_a8r8g8b8, width); + qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, width); uint8_t *buf = (uint8_t *)pixman_image_get_data(linebuf); FILE *f = fdopen(fd, "wb"); int y; @@ -341,7 +341,7 @@ static bool png_save(int fd, pixman_image_t *image, Error **errp) png_init_io(png_ptr, f); png_set_IHDR(png_ptr, info_ptr, width, height, 8, - PNG_COLOR_TYPE_RGB_ALPHA, PNG_INTERLACE_NONE, + PNG_COLOR_TYPE_RGB, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_BASE, PNG_FILTER_TYPE_BASE); png_write_info(png_ptr, info_ptr); diff --git a/util/async-teardown.c b/util/async-teardown.c index 62cdeb0f20..3ab19c8740 100644 --- a/util/async-teardown.c +++ b/util/async-teardown.c @@ -12,6 +12,9 @@ */ #include "qemu/osdep.h" +#include "qemu/config-file.h" +#include "qemu/option.h" +#include "qemu/module.h" #include <dirent.h> #include <sys/prctl.h> #include <sched.h> @@ -144,3 +147,21 @@ void init_async_teardown(void) clone(async_teardown_fn, new_stack_for_clone(), CLONE_VM, NULL); sigprocmask(SIG_SETMASK, &old_signals, NULL); } + +static QemuOptsList qemu_run_with_opts = { + .name = "run-with", + .head = QTAILQ_HEAD_INITIALIZER(qemu_run_with_opts.head), + .desc = { + { + .name = "async-teardown", + .type = QEMU_OPT_BOOL, + }, + { /* end of list */ } + }, +}; + +static void register_teardown(void) +{ + qemu_add_opts(&qemu_run_with_opts); +} +opts_init(register_teardown); diff --git a/util/coroutine-win32.c b/util/coroutine-windows.c index 7db2e8f8c8..7db2e8f8c8 100644 --- a/util/coroutine-win32.c +++ b/util/coroutine-windows.c diff --git a/util/meson.build b/util/meson.build index 3c2cfc6ede..e1f1c39e10 100644 --- a/util/meson.build +++ b/util/meson.build @@ -26,7 +26,9 @@ util_ss.add(when: 'CONFIG_WIN32', if_true: files('oslib-win32.c')) util_ss.add(when: 'CONFIG_WIN32', if_true: files('qemu-thread-win32.c')) util_ss.add(when: 'CONFIG_WIN32', if_true: winmm) util_ss.add(when: 'CONFIG_WIN32', if_true: pathcch) -util_ss.add(when: 'HAVE_GLIB_WITH_SLICE_ALLOCATOR', if_true: files('qtree.c')) +if glib_has_gslice + util_ss.add(files('qtree.c')) +endif util_ss.add(files('envlist.c', 'path.c', 'module.c')) util_ss.add(files('host-utils.c')) util_ss.add(files('bitmap.c', 'bitops.c')) @@ -76,7 +78,7 @@ if have_block or have_ga util_ss.add(files('base64.c')) util_ss.add(files('main-loop.c')) util_ss.add(files('qemu-coroutine.c', 'qemu-coroutine-lock.c', 'qemu-coroutine-io.c')) - util_ss.add(files('coroutine-@0@.c'.format(config_host['CONFIG_COROUTINE_BACKEND']))) + util_ss.add(files(f'coroutine-@coroutine_backend@.c')) util_ss.add(files('thread-pool.c', 'qemu-timer.c')) util_ss.add(files('qemu-sockets.c')) endif |