diff options
574 files changed, 19545 insertions, 13064 deletions
diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml index d4f145fdb5..118371e377 100644 --- a/.gitlab-ci.d/buildtest-template.yml +++ b/.gitlab-ci.d/buildtest-template.yml @@ -115,3 +115,30 @@ - du -chs ${CI_PROJECT_DIR}/*-cache variables: QEMU_JOB_FUNCTIONAL: 1 + +.wasm_build_job_template: + extends: .base_job_template + stage: build + image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG + before_script: + - source scripts/ci/gitlab-ci-section + - section_start setup "Pre-script setup" + - JOBS=$(expr $(nproc) + 1) + - section_end setup + script: + - du -sh .git + - mkdir build + - cd build + - section_start configure "Running configure" + - emconfigure ../configure --disable-docs + ${TARGETS:+--target-list="$TARGETS"} + $CONFIGURE_ARGS || + { cat config.log meson-logs/meson-log.txt && exit 1; } + - if test -n "$LD_JOBS"; + then + pyvenv/bin/meson configure . -Dbackend_max_links="$LD_JOBS" ; + fi || exit 1; + - section_end configure + - section_start build "Building QEMU" + - emmake make -j"$JOBS" + - section_end build diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index 431bc07d8f..248aaed137 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -792,3 +792,12 @@ coverity: when: never # Always manual on forks even if $QEMU_CI == "2" - when: manual + +build-wasm: + extends: .wasm_build_job_template + timeout: 2h + needs: + job: wasm-emsdk-cross-container + variables: + IMAGE: emsdk-wasm32-cross + CONFIGURE_ARGS: --static --disable-tools --enable-debug --enable-tcg-interpreter diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 34c0e729ad..8d3be53b75 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -67,11 +67,8 @@ ppc64el-debian-cross-container: riscv64-debian-cross-container: extends: .container_job_template stage: containers - # as we are currently based on 'sid/unstable' we may break so... - allow_failure: true variables: NAME: debian-riscv64-cross - QEMU_JOB_OPTIONAL: 1 s390x-debian-cross-container: extends: .container_job_template @@ -94,3 +91,8 @@ win64-fedora-cross-container: extends: .container_job_template variables: NAME: fedora-win64-cross + +wasm-emsdk-cross-container: + extends: .container_job_template + variables: + NAME: emsdk-wasm32-cross diff --git a/.gitlab-ci.d/crossbuilds.yml b/.gitlab-ci.d/crossbuilds.yml index 7ae0f966f1..3f76c901ba 100644 --- a/.gitlab-ci.d/crossbuilds.yml +++ b/.gitlab-ci.d/crossbuilds.yml @@ -118,12 +118,8 @@ cross-ppc64el-kvm-only: IMAGE: debian-ppc64el-cross EXTRA_CONFIGURE_OPTS: --disable-tcg --without-default-devices -# The riscv64 cross-builds currently use a 'sid' container to get -# compilers and libraries. Until something more stable is found we -# allow_failure so as not to block CI. cross-riscv64-system: extends: .cross_system_build_job - allow_failure: true needs: job: riscv64-debian-cross-container variables: @@ -131,7 +127,6 @@ cross-riscv64-system: cross-riscv64-user: extends: .cross_user_build_job - allow_failure: true needs: job: riscv64-debian-cross-container variables: diff --git a/MAINTAINERS b/MAINTAINERS index b3f9f2680b..6dacd6d004 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -168,7 +168,7 @@ F: include/exec/helper*.h.inc F: include/exec/helper-info.c.inc F: include/exec/page-protection.h F: include/system/tcg.h -F: include/accel/tcg/cpu-ops.h +F: include/accel/tcg/ F: host/include/*/host/cpuinfo.h F: util/cpuinfo-*.c F: include/tcg/ @@ -493,7 +493,6 @@ M: Richard Henderson <richard.henderson@linaro.org> R: Paolo Bonzini <pbonzini@redhat.com> S: Maintained F: include/exec/cpu*.h -F: include/exec/exec-all.h F: include/exec/target_long.h F: include/qemu/accel.h F: include/system/accel-*.h @@ -526,6 +525,7 @@ R: Phil Dennis-Jordan <phil@philjordan.eu> W: https://wiki.qemu.org/Features/HVF S: Maintained F: accel/hvf/ +F: accel/stubs/hvf-stub.c F: include/system/hvf.h F: include/system/hvf_int.h @@ -628,6 +628,15 @@ F: .gitlab-ci.d/cirrus/macos-* F: */*.m F: scripts/entitlement.sh +WebAssembly +M: Kohei Tokunaga <ktokunaga.mail@gmail.com> +S: Maintained +F: include/system/os-wasm.h +F: os-wasm.c +F: util/coroutine-wasm.c +F: configs/meson/emscripten.txt +F: tests/docker/dockerfiles/emsdk-wasm32-cross.docker + Alpha Machines -------------- M: Richard Henderson <richard.henderson@linaro.org> @@ -843,6 +852,7 @@ F: include/hw/arm/fsl-imx8mp.h F: include/hw/misc/imx8mp_*.h F: include/hw/pci-host/fsl_imx8m_phy.h F: docs/system/arm/imx8mp-evk.rst +F: tests/functional/test_aarch64_imx8mp_evk.py F: tests/qtest/rs5c372-test.c MPS2 / MPS3 @@ -1174,6 +1184,7 @@ F: docs/system/arm/fby35.rst F: tests/*/*aspeed* F: tests/*/*ast2700* F: hw/arm/fby35.c +F: pc-bios/ast27x0_bootrom.bin NRF51 M: Joel Stanley <joel@jms.id.au> @@ -3538,6 +3549,7 @@ R: Li Zhijian <lizhijian@fujitsu.com> R: Peter Xu <peterx@redhat.com> S: Odd Fixes F: migration/rdma* +F: scripts/rdma-migration-helper.sh Migration dirty limit and dirty page rate M: Hyman Huang <yong.huang@smartx.com> diff --git a/accel/hvf/hvf-accel-ops.c b/accel/hvf/hvf-accel-ops.c index 5375de7bcf..b8b6116bc8 100644 --- a/accel/hvf/hvf-accel-ops.c +++ b/accel/hvf/hvf-accel-ops.c @@ -51,7 +51,6 @@ #include "qemu/error-report.h" #include "qemu/main-loop.h" #include "system/address-spaces.h" -#include "exec/exec-all.h" #include "gdbstub/enums.h" #include "hw/boards.h" #include "system/accel-ops.h" diff --git a/accel/hvf/hvf-all.c b/accel/hvf/hvf-all.c index d404e01ade..8c387fda24 100644 --- a/accel/hvf/hvf-all.c +++ b/accel/hvf/hvf-all.c @@ -12,6 +12,7 @@ #include "qemu/error-report.h" #include "system/hvf.h" #include "system/hvf_int.h" +#include "hw/core/cpu.h" const char *hvf_return_string(hv_return_t ret) { @@ -58,8 +59,13 @@ int hvf_sw_breakpoints_active(CPUState *cpu) return !QTAILQ_EMPTY(&hvf_state->hvf_sw_breakpoints); } -int hvf_update_guest_debug(CPUState *cpu) +static void do_hvf_update_guest_debug(CPUState *cpu, run_on_cpu_data arg) { hvf_arch_update_guest_debug(cpu); +} + +int hvf_update_guest_debug(CPUState *cpu) +{ + run_on_cpu(cpu, do_hvf_update_guest_debug, RUN_ON_CPU_NULL); return 0; } diff --git a/accel/stubs/hvf-stub.c b/accel/stubs/hvf-stub.c new file mode 100644 index 0000000000..42eadc5ca9 --- /dev/null +++ b/accel/stubs/hvf-stub.c @@ -0,0 +1,12 @@ +/* + * HVF stubs for QEMU + * + * Copyright (c) Linaro + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "system/hvf.h" + +bool hvf_allowed; diff --git a/accel/stubs/meson.build b/accel/stubs/meson.build index 91a2d21925..8ca1a4529e 100644 --- a/accel/stubs/meson.build +++ b/accel/stubs/meson.build @@ -2,5 +2,6 @@ system_stubs_ss = ss.source_set() system_stubs_ss.add(when: 'CONFIG_XEN', if_false: files('xen-stub.c')) system_stubs_ss.add(when: 'CONFIG_KVM', if_false: files('kvm-stub.c')) system_stubs_ss.add(when: 'CONFIG_TCG', if_false: files('tcg-stub.c')) +system_stubs_ss.add(when: 'CONFIG_HVF', if_false: files('hvf-stub.c')) specific_ss.add_all(when: ['CONFIG_SYSTEM_ONLY'], if_true: system_stubs_ss) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index 87eba83d7d..cc5f362305 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -22,8 +22,8 @@ #include "qapi/error.h" #include "qapi/type-helpers.h" #include "hw/core/cpu.h" -#include "accel/tcg/cpu-ldst.h" #include "accel/tcg/cpu-ops.h" +#include "accel/tcg/helper-retaddr.h" #include "trace.h" #include "disas/disas.h" #include "exec/cpu-common.h" @@ -36,7 +36,6 @@ #include "qemu/rcu.h" #include "exec/log.h" #include "qemu/main-loop.h" -#include "cpu.h" #include "exec/icount.h" #include "exec/replay-core.h" #include "system/tcg.h" @@ -46,7 +45,6 @@ #include "tb-context.h" #include "tb-internal.h" #include "internal-common.h" -#include "internal-target.h" /* -icount align implementation. */ @@ -151,12 +149,9 @@ static void init_delay_params(SyncClocks *sc, const CPUState *cpu) #endif /* CONFIG USER ONLY */ struct tb_desc { - vaddr pc; - uint64_t cs_base; + TCGTBCPUState s; CPUArchState *env; tb_page_addr_t page_addr0; - uint32_t flags; - uint32_t cflags; }; static bool tb_lookup_cmp(const void *p, const void *d) @@ -164,11 +159,11 @@ static bool tb_lookup_cmp(const void *p, const void *d) const TranslationBlock *tb = p; const struct tb_desc *desc = d; - if ((tb_cflags(tb) & CF_PCREL || tb->pc == desc->pc) && + if ((tb_cflags(tb) & CF_PCREL || tb->pc == desc->s.pc) && tb_page_addr0(tb) == desc->page_addr0 && - tb->cs_base == desc->cs_base && - tb->flags == desc->flags && - tb_cflags(tb) == desc->cflags) { + tb->cs_base == desc->s.cs_base && + tb->flags == desc->s.flags && + tb_cflags(tb) == desc->s.cflags) { /* check next page if needed */ tb_page_addr_t tb_phys_page1 = tb_page_addr1(tb); if (tb_phys_page1 == -1) { @@ -186,7 +181,7 @@ static bool tb_lookup_cmp(const void *p, const void *d) * is different for the new TB. Therefore any exception raised * here by the faulting lookup is not premature. */ - virt_page1 = TARGET_PAGE_ALIGN(desc->pc); + virt_page1 = TARGET_PAGE_ALIGN(desc->s.pc); phys_page1 = get_page_addr_code(desc->env, virt_page1); if (tb_phys_page1 == phys_page1) { return true; @@ -196,26 +191,21 @@ static bool tb_lookup_cmp(const void *p, const void *d) return false; } -static TranslationBlock *tb_htable_lookup(CPUState *cpu, vaddr pc, - uint64_t cs_base, uint32_t flags, - uint32_t cflags) +static TranslationBlock *tb_htable_lookup(CPUState *cpu, TCGTBCPUState s) { tb_page_addr_t phys_pc; struct tb_desc desc; uint32_t h; + desc.s = s; desc.env = cpu_env(cpu); - desc.cs_base = cs_base; - desc.flags = flags; - desc.cflags = cflags; - desc.pc = pc; - phys_pc = get_page_addr_code(desc.env, pc); + phys_pc = get_page_addr_code(desc.env, s.pc); if (phys_pc == -1) { return NULL; } desc.page_addr0 = phys_pc; - h = tb_hash_func(phys_pc, (cflags & CF_PCREL ? 0 : pc), - flags, cs_base, cflags); + h = tb_hash_func(phys_pc, (s.cflags & CF_PCREL ? 0 : s.pc), + s.flags, s.cs_base, s.cflags); return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp); } @@ -233,35 +223,33 @@ static TranslationBlock *tb_htable_lookup(CPUState *cpu, vaddr pc, * * Returns: an existing translation block or NULL. */ -static inline TranslationBlock *tb_lookup(CPUState *cpu, vaddr pc, - uint64_t cs_base, uint32_t flags, - uint32_t cflags) +static inline TranslationBlock *tb_lookup(CPUState *cpu, TCGTBCPUState s) { TranslationBlock *tb; CPUJumpCache *jc; uint32_t hash; /* we should never be trying to look up an INVALID tb */ - tcg_debug_assert(!(cflags & CF_INVALID)); + tcg_debug_assert(!(s.cflags & CF_INVALID)); - hash = tb_jmp_cache_hash_func(pc); + hash = tb_jmp_cache_hash_func(s.pc); jc = cpu->tb_jmp_cache; tb = qatomic_read(&jc->array[hash].tb); if (likely(tb && - jc->array[hash].pc == pc && - tb->cs_base == cs_base && - tb->flags == flags && - tb_cflags(tb) == cflags)) { + jc->array[hash].pc == s.pc && + tb->cs_base == s.cs_base && + tb->flags == s.flags && + tb_cflags(tb) == s.cflags)) { goto hit; } - tb = tb_htable_lookup(cpu, pc, cs_base, flags, cflags); + tb = tb_htable_lookup(cpu, s); if (tb == NULL) { return NULL; } - jc->array[hash].pc = pc; + jc->array[hash].pc = s.pc; qatomic_set(&jc->array[hash].tb, tb); hit: @@ -269,7 +257,7 @@ hit: * As long as tb is not NULL, the contents are consistent. Therefore, * the virtual PC has to match for non-CF_PCREL translations. */ - assert((tb_cflags(tb) & CF_PCREL) || tb->pc == pc); + assert((tb_cflags(tb) & CF_PCREL) || tb->pc == s.pc); return tb; } @@ -286,14 +274,11 @@ static void log_cpu_exec(vaddr pc, CPUState *cpu, if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) { FILE *logfile = qemu_log_trylock(); if (logfile) { - int flags = 0; + int flags = CPU_DUMP_CCOP; if (qemu_loglevel_mask(CPU_LOG_TB_FPU)) { flags |= CPU_DUMP_FPU; } -#if defined(TARGET_I386) - flags |= CPU_DUMP_CCOP; -#endif if (qemu_loglevel_mask(CPU_LOG_TB_VPU)) { flags |= CPU_DUMP_VPU; } @@ -389,9 +374,6 @@ const void *HELPER(lookup_tb_ptr)(CPUArchState *env) { CPUState *cpu = env_cpu(env); TranslationBlock *tb; - vaddr pc; - uint64_t cs_base; - uint32_t flags, cflags; /* * By definition we've just finished a TB, so I/O is OK. @@ -401,20 +383,21 @@ const void *HELPER(lookup_tb_ptr)(CPUArchState *env) * The next TB, if we chain to it, will clear the flag again. */ cpu->neg.can_do_io = true; - cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); - cflags = curr_cflags(cpu); - if (check_for_breakpoints(cpu, pc, &cflags)) { + TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu); + s.cflags = curr_cflags(cpu); + + if (check_for_breakpoints(cpu, s.pc, &s.cflags)) { cpu_loop_exit(cpu); } - tb = tb_lookup(cpu, pc, cs_base, flags, cflags); + tb = tb_lookup(cpu, s); if (tb == NULL) { return tcg_code_gen_epilogue; } if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) { - log_cpu_exec(pc, cpu, tb); + log_cpu_exec(s.pc, cpu, tb); } return tb->tc.ptr; @@ -564,11 +547,7 @@ static void cpu_exec_longjmp_cleanup(CPUState *cpu) void cpu_exec_step_atomic(CPUState *cpu) { - CPUArchState *env = cpu_env(cpu); TranslationBlock *tb; - vaddr pc; - uint64_t cs_base; - uint32_t flags, cflags; int tb_exit; if (sigsetjmp(cpu->jmp_env, 0) == 0) { @@ -577,13 +556,13 @@ void cpu_exec_step_atomic(CPUState *cpu) g_assert(!cpu->running); cpu->running = true; - cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); + TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu); + s.cflags = curr_cflags(cpu); - cflags = curr_cflags(cpu); /* Execute in a serial context. */ - cflags &= ~CF_PARALLEL; + s.cflags &= ~CF_PARALLEL; /* After 1 insn, return and release the exclusive lock. */ - cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | 1; + s.cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | 1; /* * No need to check_for_breakpoints here. * We only arrive in cpu_exec_step_atomic after beginning execution @@ -591,16 +570,16 @@ void cpu_exec_step_atomic(CPUState *cpu) * Any breakpoint for this insn will have been recognized earlier. */ - tb = tb_lookup(cpu, pc, cs_base, flags, cflags); + tb = tb_lookup(cpu, s); if (tb == NULL) { mmap_lock(); - tb = tb_gen_code(cpu, pc, cs_base, flags, cflags); + tb = tb_gen_code(cpu, s); mmap_unlock(); } cpu_exec_enter(cpu); /* execute the generated code */ - trace_exec_tb(tb, pc); + trace_exec_tb(tb, s.pc); cpu_tb_exec(cpu, tb, &tb_exit); cpu_exec_exit(cpu); } else { @@ -733,10 +712,10 @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret) * If user mode only, we simulate a fake exception which will be * handled outside the cpu execution loop. */ -#if defined(TARGET_I386) const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; - tcg_ops->fake_user_interrupt(cpu); -#endif /* TARGET_I386 */ + if (tcg_ops->fake_user_interrupt) { + tcg_ops->fake_user_interrupt(cpu); + } *ret = cpu->exception_index; cpu->exception_index = -1; return true; @@ -823,33 +802,22 @@ static inline bool cpu_handle_interrupt(CPUState *cpu, cpu->exception_index = EXCP_HLT; bql_unlock(); return true; - } -#if defined(TARGET_I386) - else if (interrupt_request & CPU_INTERRUPT_INIT) { - X86CPU *x86_cpu = X86_CPU(cpu); - CPUArchState *env = &x86_cpu->env; - replay_interrupt(); - cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0, 0); - do_cpu_init(x86_cpu); - cpu->exception_index = EXCP_HALTED; - bql_unlock(); - return true; - } -#else - else if (interrupt_request & CPU_INTERRUPT_RESET) { - replay_interrupt(); - cpu_reset(cpu); - bql_unlock(); - return true; - } -#endif /* !TARGET_I386 */ - /* The target hook has 3 exit conditions: - False when the interrupt isn't processed, - True when it is, and we should restart on a new TB, - and via longjmp via cpu_loop_exit. */ - else { + } else { const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; + if (interrupt_request & CPU_INTERRUPT_RESET) { + replay_interrupt(); + tcg_ops->cpu_exec_reset(cpu); + bql_unlock(); + return true; + } + + /* + * The target hook has 3 exit conditions: + * False when the interrupt isn't processed, + * True when it is, and we should restart on a new TB, + * and via longjmp via cpu_loop_exit. + */ if (tcg_ops->cpu_exec_interrupt(cpu, interrupt_request)) { if (!tcg_ops->need_replay_interrupt || tcg_ops->need_replay_interrupt(interrupt_request)) { @@ -956,11 +924,8 @@ cpu_exec_loop(CPUState *cpu, SyncClocks *sc) while (!cpu_handle_interrupt(cpu, &last_tb)) { TranslationBlock *tb; - vaddr pc; - uint64_t cs_base; - uint32_t flags, cflags; - - cpu_get_tb_cpu_state(cpu_env(cpu), &pc, &cs_base, &flags); + TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu); + s.cflags = cpu->cflags_next_tb; /* * When requested, use an exact setting for cflags for the next @@ -969,33 +934,32 @@ cpu_exec_loop(CPUState *cpu, SyncClocks *sc) * have CF_INVALID set, -1 is a convenient invalid value that * does not require tcg headers for cpu_common_reset. */ - cflags = cpu->cflags_next_tb; - if (cflags == -1) { - cflags = curr_cflags(cpu); + if (s.cflags == -1) { + s.cflags = curr_cflags(cpu); } else { cpu->cflags_next_tb = -1; } - if (check_for_breakpoints(cpu, pc, &cflags)) { + if (check_for_breakpoints(cpu, s.pc, &s.cflags)) { break; } - tb = tb_lookup(cpu, pc, cs_base, flags, cflags); + tb = tb_lookup(cpu, s); if (tb == NULL) { CPUJumpCache *jc; uint32_t h; mmap_lock(); - tb = tb_gen_code(cpu, pc, cs_base, flags, cflags); + tb = tb_gen_code(cpu, s); mmap_unlock(); /* * We add the TB in the virtual pc hash table * for the fast lookup */ - h = tb_jmp_cache_hash_func(pc); + h = tb_jmp_cache_hash_func(s.pc); jc = cpu->tb_jmp_cache; - jc->array[h].pc = pc; + jc->array[h].pc = s.pc; qatomic_set(&jc->array[h].tb, tb); } @@ -1015,7 +979,7 @@ cpu_exec_loop(CPUState *cpu, SyncClocks *sc) tb_add_jump(last_tb, tb_exit, tb); } - cpu_loop_exec_tb(cpu, tb, pc, &last_tb, &tb_exit); + cpu_loop_exec_tb(cpu, tb, s.pc, &last_tb, &tb_exit); /* Try to align the host and virtual clocks if the guest is in advance */ @@ -1074,8 +1038,10 @@ bool tcg_exec_realizefn(CPUState *cpu, Error **errp) #ifndef CONFIG_USER_ONLY assert(tcg_ops->cpu_exec_halt); assert(tcg_ops->cpu_exec_interrupt); + assert(tcg_ops->cpu_exec_reset); #endif /* !CONFIG_USER_ONLY */ assert(tcg_ops->translate_code); + assert(tcg_ops->get_tb_cpu_state); assert(tcg_ops->mmu_index); tcg_ops->initialize(); tcg_target_initialized = true; diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index d9fb68d719..5f6d7c601c 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -19,11 +19,14 @@ #include "qemu/osdep.h" #include "qemu/main-loop.h" +#include "qemu/target-info.h" #include "accel/tcg/cpu-ops.h" -#include "exec/exec-all.h" +#include "accel/tcg/iommu.h" +#include "accel/tcg/probe.h" #include "exec/page-protection.h" #include "system/memory.h" -#include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/cpu-ldst-common.h" +#include "accel/tcg/cpu-mmu-index.h" #include "exec/cputlb.h" #include "exec/tb-flush.h" #include "system/ram_addr.h" @@ -43,7 +46,6 @@ #include "tb-internal.h" #include "tlb-bounds.h" #include "internal-common.h" -#include "internal-target.h" #ifdef CONFIG_PLUGIN #include "qemu/plugin-memory.h" #endif @@ -771,19 +773,19 @@ void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr, assert_cpu_is_self(cpu); + /* If no page bits are significant, this devolves to tlb_flush. */ + if (bits < TARGET_PAGE_BITS) { + tlb_flush_by_mmuidx(cpu, idxmap); + return; + } /* * If all bits are significant, and len is small, * this devolves to tlb_flush_page. */ - if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) { + if (len <= TARGET_PAGE_SIZE && bits >= target_long_bits()) { tlb_flush_page_by_mmuidx(cpu, addr, idxmap); return; } - /* If no page bits are significant, this devolves to tlb_flush. */ - if (bits < TARGET_PAGE_BITS) { - tlb_flush_by_mmuidx(cpu, idxmap); - return; - } /* This should already be page aligned */ d.addr = addr & TARGET_PAGE_MASK; @@ -809,19 +811,19 @@ void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu, TLBFlushRangeData d, *p; CPUState *dst_cpu; + /* If no page bits are significant, this devolves to tlb_flush. */ + if (bits < TARGET_PAGE_BITS) { + tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, idxmap); + return; + } /* * If all bits are significant, and len is small, * this devolves to tlb_flush_page. */ - if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) { + if (len <= TARGET_PAGE_SIZE && bits >= target_long_bits()) { tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu, addr, idxmap); return; } - /* If no page bits are significant, this devolves to tlb_flush. */ - if (bits < TARGET_PAGE_BITS) { - tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, idxmap); - return; - } /* This should already be page aligned */ d.addr = addr & TARGET_PAGE_MASK; @@ -1340,7 +1342,7 @@ static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size); if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) { - tb_invalidate_phys_range_fast(ram_addr, size, retaddr); + tb_invalidate_phys_range_fast(cpu, ram_addr, size, retaddr); } /* diff --git a/accel/tcg/internal-common.h b/accel/tcg/internal-common.h index 2f00560d10..1dbc45dd95 100644 --- a/accel/tcg/internal-common.h +++ b/accel/tcg/internal-common.h @@ -11,6 +11,8 @@ #include "exec/cpu-common.h" #include "exec/translation-block.h" +#include "exec/mmap-lock.h" +#include "accel/tcg/tb-cpu-state.h" extern int64_t max_delay; extern int64_t max_advance; @@ -45,9 +47,7 @@ static inline bool cpu_plugin_mem_cbs_enabled(const CPUState *cpu) #endif } -TranslationBlock *tb_gen_code(CPUState *cpu, vaddr pc, - uint64_t cs_base, uint32_t flags, - int cflags); +TranslationBlock *tb_gen_code(CPUState *cpu, TCGTBCPUState s); void page_init(void); void tb_htable_init(void); void tb_reset_jump(TranslationBlock *tb, int n); @@ -108,4 +108,35 @@ static inline tb_page_addr_t get_page_addr_code(CPUArchState *env, return get_page_addr_code_hostp(env, addr, NULL); } +/* + * Access to the various translations structures need to be serialised + * via locks for consistency. In user-mode emulation access to the + * memory related structures are protected with mmap_lock. + * In !user-mode we use per-page locks. + */ +#ifdef CONFIG_USER_ONLY +#define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) +#else +#define assert_memory_lock() +#endif + +#if defined(CONFIG_SOFTMMU) && defined(CONFIG_DEBUG_TCG) +void assert_no_pages_locked(void); +#else +static inline void assert_no_pages_locked(void) { } +#endif + +#ifdef CONFIG_USER_ONLY +static inline void page_table_config_init(void) { } +#else +void page_table_config_init(void); +#endif + +#ifndef CONFIG_USER_ONLY +G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr); +#endif /* CONFIG_USER_ONLY */ + +void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr); +void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr); + #endif diff --git a/accel/tcg/internal-target.h b/accel/tcg/internal-target.h deleted file mode 100644 index 9a9cef3140..0000000000 --- a/accel/tcg/internal-target.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Internal execution defines for qemu (target specific) - * - * Copyright (c) 2003 Fabrice Bellard - * - * SPDX-License-Identifier: LGPL-2.1-or-later - */ - -#ifndef ACCEL_TCG_INTERNAL_TARGET_H -#define ACCEL_TCG_INTERNAL_TARGET_H - -#include "cpu-param.h" -#include "exec/exec-all.h" -#include "exec/translation-block.h" -#include "tb-internal.h" -#include "exec/mmap-lock.h" - -/* - * Access to the various translations structures need to be serialised - * via locks for consistency. In user-mode emulation access to the - * memory related structures are protected with mmap_lock. - * In !user-mode we use per-page locks. - */ -#ifdef CONFIG_USER_ONLY -#define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) -#else -#define assert_memory_lock() -#endif - -#if defined(CONFIG_SOFTMMU) && defined(CONFIG_DEBUG_TCG) -void assert_no_pages_locked(void); -#else -static inline void assert_no_pages_locked(void) { } -#endif - -#ifdef CONFIG_USER_ONLY -static inline void page_table_config_init(void) { } -#else -void page_table_config_init(void); -#endif - -#ifndef CONFIG_USER_ONLY -G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr); -#endif /* CONFIG_USER_ONLY */ - -#endif /* ACCEL_TCG_INTERNAL_H */ diff --git a/accel/tcg/ldst_common.c.inc b/accel/tcg/ldst_common.c.inc index 9791a4e9ef..57f3e06192 100644 --- a/accel/tcg/ldst_common.c.inc +++ b/accel/tcg/ldst_common.c.inc @@ -123,7 +123,7 @@ void helper_st_i128(CPUArchState *env, uint64_t addr, Int128 val, MemOpIdx oi) * Load helpers for cpu_ldst.h */ -static void plugin_load_cb(CPUArchState *env, abi_ptr addr, +static void plugin_load_cb(CPUArchState *env, vaddr addr, uint64_t value_low, uint64_t value_high, MemOpIdx oi) @@ -193,7 +193,7 @@ Int128 cpu_ld16_mmu(CPUArchState *env, vaddr addr, * Store helpers for cpu_ldst.h */ -static void plugin_store_cb(CPUArchState *env, abi_ptr addr, +static void plugin_store_cb(CPUArchState *env, vaddr addr, uint64_t value_low, uint64_t value_high, MemOpIdx oi) diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build index 047afa49a2..d6f533f9a1 100644 --- a/accel/tcg/meson.build +++ b/accel/tcg/meson.build @@ -5,9 +5,13 @@ endif tcg_ss = ss.source_set() tcg_ss.add(files( + 'cpu-exec.c', 'cpu-exec-common.c', 'tcg-runtime.c', 'tcg-runtime-gvec.c', + 'tb-maint.c', + 'tcg-all.c', + 'translate-all.c', 'translator.c', )) if get_option('plugins') @@ -17,25 +21,13 @@ endif libuser_ss.add_all(tcg_ss) libsystem_ss.add_all(tcg_ss) -tcg_specific_ss = ss.source_set() -tcg_specific_ss.add(files( - 'tcg-all.c', - 'cpu-exec.c', - 'tb-maint.c', - 'translate-all.c', -)) -tcg_specific_ss.add(when: 'CONFIG_USER_ONLY', if_true: files('user-exec.c')) -specific_ss.add_all(when: 'CONFIG_TCG', if_true: tcg_specific_ss) - -specific_ss.add(when: ['CONFIG_SYSTEM_ONLY', 'CONFIG_TCG'], if_true: files( - 'cputlb.c', -)) - libuser_ss.add(files( + 'user-exec.c', 'user-exec-stub.c', )) libsystem_ss.add(files( + 'cputlb.c', 'icount-common.c', 'monitor.c', 'tcg-accel-ops.c', diff --git a/accel/tcg/tb-hash.h b/accel/tcg/tb-hash.h index 3bc5042d9d..f7b159f04c 100644 --- a/accel/tcg/tb-hash.h +++ b/accel/tcg/tb-hash.h @@ -20,8 +20,7 @@ #ifndef EXEC_TB_HASH_H #define EXEC_TB_HASH_H -#include "exec/cpu-defs.h" -#include "exec/exec-all.h" +#include "exec/vaddr.h" #include "exec/target_page.h" #include "exec/translation-block.h" #include "qemu/xxhash.h" diff --git a/accel/tcg/tb-internal.h b/accel/tcg/tb-internal.h index 08538e2896..40439f03c3 100644 --- a/accel/tcg/tb-internal.h +++ b/accel/tcg/tb-internal.h @@ -45,11 +45,11 @@ void tb_unlock_pages(TranslationBlock *); #endif #ifdef CONFIG_SOFTMMU -void tb_invalidate_phys_range_fast(ram_addr_t ram_addr, - unsigned size, - uintptr_t retaddr); +void tb_invalidate_phys_range_fast(CPUState *cpu, ram_addr_t ram_addr, + unsigned size, uintptr_t retaddr); #endif /* CONFIG_SOFTMMU */ -bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc); +bool tb_invalidate_phys_page_unwind(CPUState *cpu, tb_page_addr_t addr, + uintptr_t pc); #endif diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c index d479f53ae0..0048316f99 100644 --- a/accel/tcg/tb-maint.c +++ b/accel/tcg/tb-maint.c @@ -20,14 +20,13 @@ #include "qemu/osdep.h" #include "qemu/interval-tree.h" #include "qemu/qtree.h" -#include "cpu.h" #include "exec/cputlb.h" #include "exec/log.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/mmap-lock.h" #include "exec/tb-flush.h" #include "exec/target_page.h" +#include "accel/tcg/cpu-ops.h" #include "tb-internal.h" #include "system/tcg.h" #include "tcg/tcg.h" @@ -35,7 +34,6 @@ #include "tb-context.h" #include "tb-internal.h" #include "internal-common.h" -#include "internal-target.h" #ifdef CONFIG_USER_ONLY #include "user/page-protection.h" #endif @@ -159,11 +157,7 @@ static PageForEachNext foreach_tb_next(PageForEachNext tb, /* * In system mode we want L1_MAP to be based on ram offsets. */ -#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS -# define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS -#else -# define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS -#endif +#define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS /* Size of the L2 (and L3, etc) page tables. */ #define V_L2_BITS 10 @@ -1012,7 +1006,8 @@ TranslationBlock *tb_link_page(TranslationBlock *tb) * Called with mmap_lock held for user-mode emulation. * NOTE: this function must not be called while a TB is running. */ -void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last) +void tb_invalidate_phys_range(CPUState *cpu, tb_page_addr_t start, + tb_page_addr_t last) { TranslationBlock *tb; PageForEachNext n; @@ -1035,17 +1030,16 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr) start = addr & TARGET_PAGE_MASK; last = addr | ~TARGET_PAGE_MASK; - tb_invalidate_phys_range(start, last); + tb_invalidate_phys_range(NULL, start, last); } /* * Called with mmap_lock held. If pc is not 0 then it indicates the * host PC of the faulting store instruction that caused this invalidate. - * Returns true if the caller needs to abort execution of the current - * TB (because it was modified by this store and the guest CPU has - * precise-SMC semantics). + * Returns true if the caller needs to abort execution of the current TB. */ -bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc) +bool tb_invalidate_phys_page_unwind(CPUState *cpu, tb_page_addr_t addr, + uintptr_t pc) { TranslationBlock *current_tb; bool current_tb_modified; @@ -1057,10 +1051,7 @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc) * Without precise smc semantics, or when outside of a TB, * we can skip to invalidate. */ -#ifndef TARGET_HAS_PRECISE_SMC - pc = 0; -#endif - if (!pc) { + if (!pc || !cpu || !cpu->cc->tcg_ops->precise_smc) { tb_invalidate_phys_page(addr); return false; } @@ -1083,15 +1074,14 @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc) * the CPU state. */ current_tb_modified = true; - cpu_restore_state_from_tb(current_cpu, current_tb, pc); + cpu_restore_state_from_tb(cpu, current_tb, pc); } tb_phys_invalidate__locked(tb); } if (current_tb_modified) { /* Force execution of one insn next time. */ - CPUState *cpu = current_cpu; - cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(current_cpu); + cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu); return true; } return false; @@ -1100,23 +1090,28 @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc) /* * @p must be non-NULL. * Call with all @pages locked. + * (@cpu, @retaddr) may be (NULL, 0) outside of a cpu context, + * in which case precise_smc need not be detected. */ static void -tb_invalidate_phys_page_range__locked(struct page_collection *pages, +tb_invalidate_phys_page_range__locked(CPUState *cpu, + struct page_collection *pages, PageDesc *p, tb_page_addr_t start, tb_page_addr_t last, uintptr_t retaddr) { TranslationBlock *tb; PageForEachNext n; -#ifdef TARGET_HAS_PRECISE_SMC bool current_tb_modified = false; - TranslationBlock *current_tb = retaddr ? tcg_tb_lookup(retaddr) : NULL; -#endif /* TARGET_HAS_PRECISE_SMC */ + TranslationBlock *current_tb = NULL; /* Range may not cross a page. */ tcg_debug_assert(((start ^ last) & TARGET_PAGE_MASK) == 0); + if (retaddr && cpu && cpu->cc->tcg_ops->precise_smc) { + current_tb = tcg_tb_lookup(retaddr); + } + /* * We remove all the TBs in the range [start, last]. * XXX: see if in some cases it could be faster to invalidate all the code @@ -1134,8 +1129,7 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages, tb_last = tb_start + (tb_last & ~TARGET_PAGE_MASK); } if (!(tb_last < start || tb_start > last)) { -#ifdef TARGET_HAS_PRECISE_SMC - if (current_tb == tb && + if (unlikely(current_tb == tb) && (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) { /* * If we are modifying the current TB, we must stop @@ -1145,9 +1139,8 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages, * restore the CPU state. */ current_tb_modified = true; - cpu_restore_state_from_tb(current_cpu, current_tb, retaddr); + cpu_restore_state_from_tb(cpu, current_tb, retaddr); } -#endif /* TARGET_HAS_PRECISE_SMC */ tb_phys_invalidate__locked(tb); } } @@ -1157,15 +1150,13 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages, tlb_unprotect_code(start); } -#ifdef TARGET_HAS_PRECISE_SMC - if (current_tb_modified) { + if (unlikely(current_tb_modified)) { page_collection_unlock(pages); /* Force execution of one insn next time. */ - current_cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(current_cpu); + cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu); mmap_unlock(); - cpu_loop_exit_noexc(current_cpu); + cpu_loop_exit_noexc(cpu); } -#endif } /* @@ -1175,7 +1166,8 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages, * access: the virtual CPU will exit the current TB if code is modified inside * this TB. */ -void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last) +void tb_invalidate_phys_range(CPUState *cpu, tb_page_addr_t start, + tb_page_addr_t last) { struct page_collection *pages; tb_page_addr_t index, index_last; @@ -1194,44 +1186,30 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last) page_start = index << TARGET_PAGE_BITS; page_last = page_start | ~TARGET_PAGE_MASK; page_last = MIN(page_last, last); - tb_invalidate_phys_page_range__locked(pages, pd, + tb_invalidate_phys_page_range__locked(cpu, pages, pd, page_start, page_last, 0); } page_collection_unlock(pages); } /* - * Call with all @pages in the range [@start, @start + len[ locked. - */ -static void tb_invalidate_phys_page_fast__locked(struct page_collection *pages, - tb_page_addr_t start, - unsigned len, uintptr_t ra) -{ - PageDesc *p; - - p = page_find(start >> TARGET_PAGE_BITS); - if (!p) { - return; - } - - assert_page_locked(p); - tb_invalidate_phys_page_range__locked(pages, p, start, start + len - 1, ra); -} - -/* * len must be <= 8 and start must be a multiple of len. * Called via softmmu_template.h when code areas are written to with * iothread mutex not held. */ -void tb_invalidate_phys_range_fast(ram_addr_t ram_addr, - unsigned size, - uintptr_t retaddr) +void tb_invalidate_phys_range_fast(CPUState *cpu, ram_addr_t start, + unsigned len, uintptr_t ra) { - struct page_collection *pages; + PageDesc *p = page_find(start >> TARGET_PAGE_BITS); - pages = page_collection_lock(ram_addr, ram_addr + size - 1); - tb_invalidate_phys_page_fast__locked(pages, ram_addr, size, retaddr); - page_collection_unlock(pages); + if (p) { + ram_addr_t last = start + len - 1; + struct page_collection *pages = page_collection_lock(start, last); + + tb_invalidate_phys_page_range__locked(cpu, pages, p, + start, last, ra); + page_collection_unlock(pages); + } } #endif /* CONFIG_USER_ONLY */ diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c index 0ce34ac912..6e5dc333d5 100644 --- a/accel/tcg/tcg-all.c +++ b/accel/tcg/tcg-all.c @@ -36,15 +36,11 @@ #include "qapi/qapi-builtin-visit.h" #include "qemu/units.h" #include "qemu/target-info.h" -#if defined(CONFIG_USER_ONLY) -#include "hw/qdev-core.h" -#else +#ifndef CONFIG_USER_ONLY #include "hw/boards.h" -#include "system/tcg.h" #endif #include "accel/tcg/cpu-ops.h" #include "internal-common.h" -#include "cpu-param.h" struct TCGState { diff --git a/accel/tcg/tlb-bounds.h b/accel/tcg/tlb-bounds.h index efd34d4793..f83d9ac9ee 100644 --- a/accel/tcg/tlb-bounds.h +++ b/accel/tcg/tlb-bounds.h @@ -7,26 +7,7 @@ #define ACCEL_TCG_TLB_BOUNDS_H #define CPU_TLB_DYN_MIN_BITS 6 +#define CPU_TLB_DYN_MAX_BITS (32 - TARGET_PAGE_BITS) #define CPU_TLB_DYN_DEFAULT_BITS 8 -# if HOST_LONG_BITS == 32 -/* Make sure we do not require a double-word shift for the TLB load */ -# define CPU_TLB_DYN_MAX_BITS (32 - TARGET_PAGE_BITS) -# else /* HOST_LONG_BITS == 64 */ -/* - * Assuming TARGET_PAGE_BITS==12, with 2**22 entries we can cover 2**(22+12) == - * 2**34 == 16G of address space. This is roughly what one would expect a - * TLB to cover in a modern (as of 2018) x86_64 CPU. For instance, Intel - * Skylake's Level-2 STLB has 16 1G entries. - * Also, make sure we do not size the TLB past the guest's address space. - */ -# ifdef TARGET_PAGE_BITS_VARY -# define CPU_TLB_DYN_MAX_BITS \ - MIN(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS) -# else -# define CPU_TLB_DYN_MAX_BITS \ - MIN_CONST(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS) -# endif -# endif - #endif /* ACCEL_TCG_TLB_BOUNDS_H */ diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index c007b9a190..451b383aa8 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -21,55 +21,23 @@ #include "trace.h" #include "disas/disas.h" -#include "exec/exec-all.h" #include "tcg/tcg.h" -#if defined(CONFIG_USER_ONLY) -#include "qemu.h" -#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) -#include <sys/param.h> -#if __FreeBSD_version >= 700104 -#define HAVE_KINFO_GETVMMAP -#define sigqueue sigqueue_freebsd /* avoid redefinition */ -#include <sys/proc.h> -#include <machine/profile.h> -#define _KERNEL -#include <sys/user.h> -#undef _KERNEL -#undef sigqueue -#include <libutil.h> -#endif -#endif -#else -#include "system/ram_addr.h" -#endif - -#include "cpu-param.h" -#include "exec/cputlb.h" -#include "exec/page-protection.h" #include "exec/mmap-lock.h" #include "tb-internal.h" #include "tlb-bounds.h" -#include "exec/translator.h" #include "exec/tb-flush.h" -#include "qemu/bitmap.h" -#include "qemu/qemu-print.h" -#include "qemu/main-loop.h" #include "qemu/cacheinfo.h" -#include "qemu/timer.h" +#include "qemu/target-info.h" #include "exec/log.h" #include "exec/icount.h" -#include "system/tcg.h" -#include "qapi/error.h" #include "accel/tcg/cpu-ops.h" #include "tb-jmp-cache.h" #include "tb-hash.h" #include "tb-context.h" #include "tb-internal.h" #include "internal-common.h" -#include "internal-target.h" #include "tcg/perf.h" #include "tcg/insn-start-words.h" -#include "cpu.h" TBContext tb_ctx; @@ -110,7 +78,7 @@ static int64_t decode_sleb128(const uint8_t **pp) val |= (int64_t)(byte & 0x7f) << shift; shift += 7; } while (byte & 0x80); - if (shift < TARGET_LONG_BITS && (byte & 0x40)) { + if (shift < 64 && (byte & 0x40)) { val |= -(int64_t)1 << shift; } @@ -121,7 +89,7 @@ static int64_t decode_sleb128(const uint8_t **pp) /* Encode the data collected about the instructions while compiling TB. Place the data at BLOCK, and return the number of bytes consumed. - The logical table consists of TARGET_INSN_START_WORDS target_ulong's, + The logical table consists of INSN_START_WORDS uint64_t's, which come from the target's insn_start data, followed by a uintptr_t which comes from the host pc of the end of the code implementing the insn. @@ -141,13 +109,13 @@ static int encode_search(TranslationBlock *tb, uint8_t *block) for (i = 0, n = tb->icount; i < n; ++i) { uint64_t prev, curr; - for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { + for (j = 0; j < INSN_START_WORDS; ++j) { if (i == 0) { prev = (!(tb_cflags(tb) & CF_PCREL) && j == 0 ? tb->pc : 0); } else { - prev = insn_data[(i - 1) * TARGET_INSN_START_WORDS + j]; + prev = insn_data[(i - 1) * INSN_START_WORDS + j]; } - curr = insn_data[i * TARGET_INSN_START_WORDS + j]; + curr = insn_data[i * INSN_START_WORDS + j]; p = encode_sleb128(p, curr - prev); } prev = (i == 0 ? 0 : insn_end_off[i - 1]); @@ -179,7 +147,7 @@ static int cpu_unwind_data_from_tb(TranslationBlock *tb, uintptr_t host_pc, return -1; } - memset(data, 0, sizeof(uint64_t) * TARGET_INSN_START_WORDS); + memset(data, 0, sizeof(uint64_t) * INSN_START_WORDS); if (!(tb_cflags(tb) & CF_PCREL)) { data[0] = tb->pc; } @@ -189,7 +157,7 @@ static int cpu_unwind_data_from_tb(TranslationBlock *tb, uintptr_t host_pc, * at which the end of the insn exceeds host_pc. */ for (i = 0; i < num_insns; ++i) { - for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { + for (j = 0; j < INSN_START_WORDS; ++j) { data[j] += decode_sleb128(&p); } iter_pc += decode_sleb128(&p); @@ -207,7 +175,7 @@ static int cpu_unwind_data_from_tb(TranslationBlock *tb, uintptr_t host_pc, void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, uintptr_t host_pc) { - uint64_t data[TARGET_INSN_START_WORDS]; + uint64_t data[INSN_START_WORDS]; int insns_left = cpu_unwind_data_from_tb(tb, host_pc, data); if (insns_left < 0) { @@ -291,9 +259,7 @@ static int setjmp_gen_code(CPUArchState *env, TranslationBlock *tb, } /* Called with mmap_lock held for user mode emulation. */ -TranslationBlock *tb_gen_code(CPUState *cpu, - vaddr pc, uint64_t cs_base, - uint32_t flags, int cflags) +TranslationBlock *tb_gen_code(CPUState *cpu, TCGTBCPUState s) { CPUArchState *env = cpu_env(cpu); TranslationBlock *tb, *existing_tb; @@ -306,14 +272,14 @@ TranslationBlock *tb_gen_code(CPUState *cpu, assert_memory_lock(); qemu_thread_jit_write(); - phys_pc = get_page_addr_code_hostp(env, pc, &host_pc); + phys_pc = get_page_addr_code_hostp(env, s.pc, &host_pc); if (phys_pc == -1) { /* Generate a one-shot TB with 1 insn in it */ - cflags = (cflags & ~CF_COUNT_MASK) | 1; + s.cflags = (s.cflags & ~CF_COUNT_MASK) | 1; } - max_insns = cflags & CF_COUNT_MASK; + max_insns = s.cflags & CF_COUNT_MASK; if (max_insns == 0) { max_insns = TCG_MAX_INSNS; } @@ -333,12 +299,12 @@ TranslationBlock *tb_gen_code(CPUState *cpu, gen_code_buf = tcg_ctx->code_gen_ptr; tb->tc.ptr = tcg_splitwx_to_rx(gen_code_buf); - if (!(cflags & CF_PCREL)) { - tb->pc = pc; + if (!(s.cflags & CF_PCREL)) { + tb->pc = s.pc; } - tb->cs_base = cs_base; - tb->flags = flags; - tb->cflags = cflags; + tb->cs_base = s.cs_base; + tb->flags = s.flags; + tb->cflags = s.cflags; tb_set_page_addr0(tb, phys_pc); tb_set_page_addr1(tb, -1); if (phys_pc != -1) { @@ -346,19 +312,18 @@ TranslationBlock *tb_gen_code(CPUState *cpu, } tcg_ctx->gen_tb = tb; - tcg_ctx->addr_type = TARGET_LONG_BITS == 32 ? TCG_TYPE_I32 : TCG_TYPE_I64; + tcg_ctx->addr_type = target_long_bits() == 32 ? TCG_TYPE_I32 : TCG_TYPE_I64; #ifdef CONFIG_SOFTMMU tcg_ctx->page_bits = TARGET_PAGE_BITS; tcg_ctx->page_mask = TARGET_PAGE_MASK; tcg_ctx->tlb_dyn_max_bits = CPU_TLB_DYN_MAX_BITS; #endif - tcg_ctx->insn_start_words = TARGET_INSN_START_WORDS; tcg_ctx->guest_mo = cpu->cc->tcg_ops->guest_default_memory_order; restart_translate: - trace_translate_block(tb, pc, tb->tc.ptr); + trace_translate_block(tb, s.pc, tb->tc.ptr); - gen_code_size = setjmp_gen_code(env, tb, pc, host_pc, &max_insns, &ti); + gen_code_size = setjmp_gen_code(env, tb, s.pc, host_pc, &max_insns, &ti); if (unlikely(gen_code_size < 0)) { switch (gen_code_size) { case -1: @@ -435,10 +400,10 @@ TranslationBlock *tb_gen_code(CPUState *cpu, * For CF_PCREL, attribute all executions of the generated code * to its first mapping. */ - perf_report_code(pc, tb, tcg_splitwx_to_rx(gen_code_buf)); + perf_report_code(s.pc, tb, tcg_splitwx_to_rx(gen_code_buf)); if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && - qemu_log_in_addr_range(pc)) { + qemu_log_in_addr_range(s.pc)) { FILE *logfile = qemu_log_trylock(); if (logfile) { int code_size, data_size; @@ -460,7 +425,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, fprintf(logfile, "OUT: [size=%d]\n", gen_code_size); fprintf(logfile, " -- guest addr 0x%016" PRIx64 " + tb prologue\n", - tcg_ctx->gen_insn_data[insn * TARGET_INSN_START_WORDS]); + tcg_ctx->gen_insn_data[insn * INSN_START_WORDS]); chunk_start = tcg_ctx->gen_insn_end_off[insn]; disas(logfile, tb->tc.ptr, chunk_start); @@ -473,7 +438,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, size_t chunk_end = tcg_ctx->gen_insn_end_off[insn]; if (chunk_end > chunk_start) { fprintf(logfile, " -- guest addr 0x%016" PRIx64 "\n", - tcg_ctx->gen_insn_data[insn * TARGET_INSN_START_WORDS]); + tcg_ctx->gen_insn_data[insn * INSN_START_WORDS]); disas(logfile, tb->tc.ptr + chunk_start, chunk_end - chunk_start); chunk_start = chunk_end; @@ -591,15 +556,11 @@ void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr) /* The exception probably happened in a helper. The CPU state should have been saved before calling it. Fetch the PC from there. */ CPUArchState *env = cpu_env(cpu); - vaddr pc; - uint64_t cs_base; - tb_page_addr_t addr; - uint32_t flags; + TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu); + tb_page_addr_t addr = get_page_addr_code(env, s.pc); - cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); - addr = get_page_addr_code(env, pc); if (addr != -1) { - tb_invalidate_phys_range(addr, addr); + tb_invalidate_phys_range(cpu, addr, addr); } } } diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 5eef8e7f18..f25d80e2dc 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -19,19 +19,20 @@ #include "qemu/osdep.h" #include "accel/tcg/cpu-ops.h" #include "disas/disas.h" -#include "cpu.h" #include "exec/vaddr.h" -#include "exec/exec-all.h" #include "exec/tlb-flags.h" #include "tcg/tcg.h" #include "qemu/bitops.h" #include "qemu/rcu.h" -#include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/cpu-ldst-common.h" +#include "accel/tcg/helper-retaddr.h" +#include "accel/tcg/probe.h" #include "user/cpu_loop.h" +#include "user/guest-host.h" #include "qemu/main-loop.h" #include "user/page-protection.h" #include "exec/page-protection.h" -#include "exec/helper-proto.h" +#include "exec/helper-proto-common.h" #include "qemu/atomic128.h" #include "qemu/bswap.h" #include "qemu/int128.h" @@ -39,7 +40,6 @@ #include "tcg/tcg-ldst.h" #include "backend-ldst.h" #include "internal-common.h" -#include "internal-target.h" #include "tb-internal.h" __thread uintptr_t helper_retaddr; @@ -126,9 +126,9 @@ MMUAccessType adjust_signal_pc(uintptr_t *pc, bool is_write) * guest, we'd end up in an infinite loop of retrying the faulting access. */ bool handle_sigsegv_accerr_write(CPUState *cpu, sigset_t *old_set, - uintptr_t host_pc, abi_ptr guest_addr) + uintptr_t host_pc, vaddr guest_addr) { - switch (page_unprotect(guest_addr, host_pc)) { + switch (page_unprotect(cpu, guest_addr, host_pc)) { case 0: /* * Fault not caused by a page marked unwritable to protect @@ -162,7 +162,7 @@ typedef struct PageFlagsNode { static IntervalTreeRoot pageflags_root; -static PageFlagsNode *pageflags_find(target_ulong start, target_ulong last) +static PageFlagsNode *pageflags_find(vaddr start, vaddr last) { IntervalTreeNode *n; @@ -170,8 +170,7 @@ static PageFlagsNode *pageflags_find(target_ulong start, target_ulong last) return n ? container_of(n, PageFlagsNode, itree) : NULL; } -static PageFlagsNode *pageflags_next(PageFlagsNode *p, target_ulong start, - target_ulong last) +static PageFlagsNode *pageflags_next(PageFlagsNode *p, vaddr start, vaddr last) { IntervalTreeNode *n; @@ -200,13 +199,22 @@ int walk_memory_regions(void *priv, walk_memory_regions_fn fn) return rc; } -static int dump_region(void *priv, target_ulong start, - target_ulong end, unsigned long prot) +static int dump_region(void *opaque, vaddr start, vaddr end, int prot) { - FILE *f = (FILE *)priv; + FILE *f = opaque; + uint64_t mask; + int width; - fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx" "TARGET_FMT_lx" %c%c%c\n", - start, end, end - start, + if (guest_addr_max <= UINT32_MAX) { + mask = UINT32_MAX, width = 8; + } else { + mask = UINT64_MAX, width = 16; + } + + fprintf(f, "%0*" PRIx64 "-%0*" PRIx64 " %0*" PRIx64 " %c%c%c\n", + width, start & mask, + width, end & mask, + width, (end - start) & mask, ((prot & PAGE_READ) ? 'r' : '-'), ((prot & PAGE_WRITE) ? 'w' : '-'), ((prot & PAGE_EXEC) ? 'x' : '-')); @@ -216,14 +224,14 @@ static int dump_region(void *priv, target_ulong start, /* dump memory mappings */ void page_dump(FILE *f) { - const int length = sizeof(target_ulong) * 2; + int width = guest_addr_max <= UINT32_MAX ? 8 : 16; fprintf(f, "%-*s %-*s %-*s %s\n", - length, "start", length, "end", length, "size", "prot"); + width, "start", width, "end", width, "size", "prot"); walk_memory_regions(f, dump_region); } -int page_get_flags(target_ulong address) +int page_get_flags(vaddr address) { PageFlagsNode *p = pageflags_find(address, address); @@ -246,7 +254,7 @@ int page_get_flags(target_ulong address) } /* A subroutine of page_set_flags: insert a new node for [start,last]. */ -static void pageflags_create(target_ulong start, target_ulong last, int flags) +static void pageflags_create(vaddr start, vaddr last, int flags) { PageFlagsNode *p = g_new(PageFlagsNode, 1); @@ -257,13 +265,13 @@ static void pageflags_create(target_ulong start, target_ulong last, int flags) } /* A subroutine of page_set_flags: remove everything in [start,last]. */ -static bool pageflags_unset(target_ulong start, target_ulong last) +static bool pageflags_unset(vaddr start, vaddr last) { bool inval_tb = false; while (true) { PageFlagsNode *p = pageflags_find(start, last); - target_ulong p_last; + vaddr p_last; if (!p) { break; @@ -302,8 +310,7 @@ static bool pageflags_unset(target_ulong start, target_ulong last) * A subroutine of page_set_flags: nothing overlaps [start,last], * but check adjacent mappings and maybe merge into a single range. */ -static void pageflags_create_merge(target_ulong start, target_ulong last, - int flags) +static void pageflags_create_merge(vaddr start, vaddr last, int flags) { PageFlagsNode *next = NULL, *prev = NULL; @@ -354,11 +361,11 @@ static void pageflags_create_merge(target_ulong start, target_ulong last, #define PAGE_STICKY (PAGE_ANON | PAGE_PASSTHROUGH | PAGE_TARGET_STICKY) /* A subroutine of page_set_flags: add flags to [start,last]. */ -static bool pageflags_set_clear(target_ulong start, target_ulong last, +static bool pageflags_set_clear(vaddr start, vaddr last, int set_flags, int clear_flags) { PageFlagsNode *p; - target_ulong p_start, p_last; + vaddr p_start, p_last; int p_flags, merge_flags; bool inval_tb = false; @@ -493,7 +500,7 @@ static bool pageflags_set_clear(target_ulong start, target_ulong last, return inval_tb; } -void page_set_flags(target_ulong start, target_ulong last, int flags) +void page_set_flags(vaddr start, vaddr last, int flags) { bool reset = false; bool inval_tb = false; @@ -502,7 +509,7 @@ void page_set_flags(target_ulong start, target_ulong last, int flags) guest address space. If this assert fires, it probably indicates a missing call to h2g_valid. */ assert(start <= last); - assert(last <= GUEST_ADDR_MAX); + assert(last <= guest_addr_max); /* Only set PAGE_ANON with new mappings. */ assert(!(flags & PAGE_ANON) || (flags & PAGE_RESET)); assert_memory_lock(); @@ -529,13 +536,13 @@ void page_set_flags(target_ulong start, target_ulong last, int flags) ~(reset ? 0 : PAGE_STICKY)); } if (inval_tb) { - tb_invalidate_phys_range(start, last); + tb_invalidate_phys_range(NULL, start, last); } } -bool page_check_range(target_ulong start, target_ulong len, int flags) +bool page_check_range(vaddr start, vaddr len, int flags) { - target_ulong last; + vaddr last; int locked; /* tri-state: =0: unlocked, +1: global, -1: local */ bool ret; @@ -584,7 +591,7 @@ bool page_check_range(target_ulong start, target_ulong len, int flags) break; } /* Asking about writable, but has been protected: undo. */ - if (!page_unprotect(start, 0)) { + if (!page_unprotect(NULL, start, 0)) { ret = false; break; } @@ -611,20 +618,19 @@ bool page_check_range(target_ulong start, target_ulong len, int flags) return ret; } -bool page_check_range_empty(target_ulong start, target_ulong last) +bool page_check_range_empty(vaddr start, vaddr last) { assert(last >= start); assert_memory_lock(); return pageflags_find(start, last) == NULL; } -target_ulong page_find_range_empty(target_ulong min, target_ulong max, - target_ulong len, target_ulong align) +vaddr page_find_range_empty(vaddr min, vaddr max, vaddr len, vaddr align) { - target_ulong len_m1, align_m1; + vaddr len_m1, align_m1; assert(min <= max); - assert(max <= GUEST_ADDR_MAX); + assert(max <= guest_addr_max); assert(len != 0); assert(is_power_of_2(align)); assert_memory_lock(); @@ -662,7 +668,7 @@ target_ulong page_find_range_empty(target_ulong min, target_ulong max, void tb_lock_page0(tb_page_addr_t address) { PageFlagsNode *p; - target_ulong start, last; + vaddr start, last; int host_page_size = qemu_real_host_page_size(); int prot; @@ -704,11 +710,13 @@ void tb_lock_page0(tb_page_addr_t address) * immediately exited. (We can only return 2 if the 'pc' argument is * non-zero.) */ -int page_unprotect(tb_page_addr_t address, uintptr_t pc) +int page_unprotect(CPUState *cpu, tb_page_addr_t address, uintptr_t pc) { PageFlagsNode *p; bool current_tb_invalidated; + assert((cpu == NULL) == (pc == 0)); + /* * Technically this isn't safe inside a signal handler. However we * know this only ever happens in a synchronous SEGV handler, so in @@ -731,15 +739,15 @@ int page_unprotect(tb_page_addr_t address, uintptr_t pc) * this thread raced with another one which got here first and * set the page to PAGE_WRITE and did the TB invalidate for us. */ -#ifdef TARGET_HAS_PRECISE_SMC - TranslationBlock *current_tb = tcg_tb_lookup(pc); - if (current_tb) { - current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID; + if (pc && cpu->cc->tcg_ops->precise_smc) { + TranslationBlock *current_tb = tcg_tb_lookup(pc); + if (current_tb) { + current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID; + } } -#endif } else { int host_page_size = qemu_real_host_page_size(); - target_ulong start, len, i; + vaddr start, len, i; int prot; if (host_page_size <= TARGET_PAGE_SIZE) { @@ -747,14 +755,15 @@ int page_unprotect(tb_page_addr_t address, uintptr_t pc) len = TARGET_PAGE_SIZE; prot = p->flags | PAGE_WRITE; pageflags_set_clear(start, start + len - 1, PAGE_WRITE, 0); - current_tb_invalidated = tb_invalidate_phys_page_unwind(start, pc); + current_tb_invalidated = + tb_invalidate_phys_page_unwind(cpu, start, pc); } else { start = address & -host_page_size; len = host_page_size; prot = 0; for (i = 0; i < len; i += TARGET_PAGE_SIZE) { - target_ulong addr = start + i; + vaddr addr = start + i; p = pageflags_find(addr, addr); if (p) { @@ -770,7 +779,7 @@ int page_unprotect(tb_page_addr_t address, uintptr_t pc) * the corresponding translated code. */ current_tb_invalidated |= - tb_invalidate_phys_page_unwind(addr, pc); + tb_invalidate_phys_page_unwind(cpu, addr, pc); } } if (prot & PAGE_EXEC) { @@ -850,6 +859,12 @@ void *probe_access(CPUArchState *env, vaddr addr, int size, return size ? g2h(env_cpu(env), addr) : NULL; } +void *tlb_vaddr_to_host(CPUArchState *env, vaddr addr, + MMUAccessType access_type, int mmu_idx) +{ + return g2h(env_cpu(env), addr); +} + tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, vaddr addr, void **hostp) { @@ -864,7 +879,6 @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, vaddr addr, return addr; } -#ifdef TARGET_PAGE_DATA_SIZE /* * Allocate chunks of target data together. For the only current user, * if we allocate one hunk per page, we have overhead of 40/128 or 40%. @@ -880,10 +894,16 @@ typedef struct TargetPageDataNode { } TargetPageDataNode; static IntervalTreeRoot targetdata_root; +static size_t target_page_data_size; -void page_reset_target_data(target_ulong start, target_ulong last) +void page_reset_target_data(vaddr start, vaddr last) { IntervalTreeNode *n, *next; + size_t size = target_page_data_size; + + if (likely(size == 0)) { + return; + } assert_memory_lock(); @@ -895,7 +915,7 @@ void page_reset_target_data(target_ulong start, target_ulong last) n != NULL; n = next, next = next ? interval_tree_iter_next(n, start, last) : NULL) { - target_ulong n_start, n_last, p_ofs, p_len; + vaddr n_start, n_last, p_ofs, p_len; TargetPageDataNode *t = container_of(n, TargetPageDataNode, itree); if (n->start >= start && n->last <= last) { @@ -914,16 +934,21 @@ void page_reset_target_data(target_ulong start, target_ulong last) n_last = MIN(last, n->last); p_len = (n_last + 1 - n_start) >> TARGET_PAGE_BITS; - memset(t->data + p_ofs * TARGET_PAGE_DATA_SIZE, 0, - p_len * TARGET_PAGE_DATA_SIZE); + memset(t->data + p_ofs * size, 0, p_len * size); } } -void *page_get_target_data(target_ulong address) +void *page_get_target_data(vaddr address, size_t size) { IntervalTreeNode *n; TargetPageDataNode *t; - target_ulong page, region, p_ofs; + vaddr page, region, p_ofs; + + /* Remember the size from the first call, and it should be constant. */ + if (unlikely(target_page_data_size != size)) { + assert(target_page_data_size == 0); + target_page_data_size = size; + } page = address & TARGET_PAGE_MASK; region = address & TBD_MASK; @@ -939,8 +964,7 @@ void *page_get_target_data(target_ulong address) mmap_lock(); n = interval_tree_iter_first(&targetdata_root, page, page); if (!n) { - t = g_malloc0(sizeof(TargetPageDataNode) - + TPD_PAGES * TARGET_PAGE_DATA_SIZE); + t = g_malloc0(sizeof(TargetPageDataNode) + TPD_PAGES * size); n = &t->itree; n->start = region; n->last = region | ~TBD_MASK; @@ -951,11 +975,8 @@ void *page_get_target_data(target_ulong address) t = container_of(n, TargetPageDataNode, itree); p_ofs = (page - region) >> TARGET_PAGE_BITS; - return t->data + p_ofs * TARGET_PAGE_DATA_SIZE; + return t->data + p_ofs * size; } -#else -void page_reset_target_data(target_ulong start, target_ulong last) { } -#endif /* TARGET_PAGE_DATA_SIZE */ /* The system-mode versions of these helpers are in cputlb.c. */ @@ -1017,7 +1038,7 @@ int cpu_memory_rw_debug(CPUState *cpu, vaddr addr, * be under mmap_lock() in order to prevent the creation of * another TranslationBlock in between. */ - tb_invalidate_phys_range(addr, addr + l - 1); + tb_invalidate_phys_range(NULL, addr, addr + l - 1); written = pwrite(fd, buf, l, (off_t)(uintptr_t)g2h_untagged(addr)); if (written != l) { @@ -1123,7 +1144,7 @@ static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, return ret; } -static Int128 do_ld16_mmu(CPUState *cpu, abi_ptr addr, +static Int128 do_ld16_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, uintptr_t ra) { void *haddr; diff --git a/backends/meson.build b/backends/meson.build index da714b93d1..9b88d22685 100644 --- a/backends/meson.build +++ b/backends/meson.build @@ -12,8 +12,10 @@ system_ss.add([files( if host_os != 'windows' system_ss.add(files('rng-random.c')) - system_ss.add(files('hostmem-file.c')) - system_ss.add([files('hostmem-shm.c'), rt]) + if host_os != 'emscripten' + system_ss.add(files('hostmem-file.c')) + system_ss.add([files('hostmem-shm.c'), rt]) + endif endif if host_os == 'linux' system_ss.add(files('hostmem-memfd.c')) diff --git a/block.c b/block.c index 0ece805e41..f222e1a50a 100644 --- a/block.c +++ b/block.c @@ -3153,7 +3153,7 @@ bdrv_attach_child_common(BlockDriverState *child_bs, * stop new requests from coming in. This is fine, we don't care about the * old requests here, they are not for this child. If another place enters a * drain section for the same parent, but wants it to be fully quiesced, it - * will not run most of the the code in .drained_begin() again (which is not + * will not run most of the code in .drained_begin() again (which is not * a problem, we already did this), but it will still poll until the parent * is fully quiesced, so it will not be negatively affected either. */ diff --git a/block/file-posix.c b/block/file-posix.c index 0d6e12f880..ef52ed9169 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -110,6 +110,10 @@ #include <sys/diskslice.h> #endif +#ifdef EMSCRIPTEN +#include <sys/ioctl.h> +#endif + /* OS X does not have O_DSYNC */ #ifndef O_DSYNC #ifdef O_SYNC @@ -1573,7 +1577,7 @@ static void raw_refresh_limits(BlockDriverState *bs, Error **errp) int ret; ret = hdev_get_pdiscard_alignment(&st, &dalign); - if (ret == 0) { + if (ret == 0 && dalign != 0) { uint32_t ralign = bs->bl.request_alignment; /* Probably never happens, but handle it just in case */ @@ -2076,8 +2080,11 @@ static int handle_aiocb_write_zeroes_unmap(void *opaque) } #ifndef HAVE_COPY_FILE_RANGE -static off_t copy_file_range(int in_fd, off_t *in_off, int out_fd, - off_t *out_off, size_t len, unsigned int flags) +#ifndef EMSCRIPTEN +static +#endif +ssize_t copy_file_range(int in_fd, off_t *in_off, int out_fd, + off_t *out_off, size_t len, unsigned int flags) { #ifdef __NR_copy_file_range return syscall(__NR_copy_file_range, in_fd, in_off, out_fd, diff --git a/block/nvme.c b/block/nvme.c index bbf7c23dcd..8df53ee4ca 100644 --- a/block/nvme.c +++ b/block/nvme.c @@ -18,6 +18,7 @@ #include "qobject/qstring.h" #include "qemu/defer-call.h" #include "qemu/error-report.h" +#include "qemu/host-pci-mmio.h" #include "qemu/main-loop.h" #include "qemu/module.h" #include "qemu/cutils.h" @@ -60,7 +61,7 @@ typedef struct { uint8_t *queue; uint64_t iova; /* Hardware MMIO register */ - volatile uint32_t *doorbell; + uint32_t *doorbell; } NVMeQueue; typedef struct { @@ -100,7 +101,7 @@ struct BDRVNVMeState { QEMUVFIOState *vfio; void *bar0_wo_map; /* Memory mapped registers */ - volatile struct { + struct { uint32_t sq_tail; uint32_t cq_head; } *doorbells; @@ -292,7 +293,7 @@ static void nvme_kick(NVMeQueuePair *q) assert(!(q->sq.tail & 0xFF00)); /* Fence the write to submission queue entry before notifying the device. */ smp_wmb(); - *q->sq.doorbell = cpu_to_le32(q->sq.tail); + host_pci_stl_le_p(q->sq.doorbell, q->sq.tail); q->inflight += q->need_kick; q->need_kick = 0; } @@ -441,7 +442,7 @@ static bool nvme_process_completion(NVMeQueuePair *q) if (progress) { /* Notify the device so it can post more completions. */ smp_mb_release(); - *q->cq.doorbell = cpu_to_le32(q->cq.head); + host_pci_stl_le_p(q->cq.doorbell, q->cq.head); nvme_wake_free_req_locked(q); } @@ -460,7 +461,7 @@ static void nvme_process_completion_bh(void *opaque) * so notify the device that it has space to fill in more completions now. */ smp_mb_release(); - *q->cq.doorbell = cpu_to_le32(q->cq.head); + host_pci_stl_le_p(q->cq.doorbell, q->cq.head); nvme_wake_free_req_locked(q); nvme_process_completion(q); @@ -749,9 +750,10 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, int ret; uint64_t cap; uint32_t ver; + uint32_t cc; uint64_t timeout_ms; uint64_t deadline, now; - volatile NvmeBar *regs = NULL; + NvmeBar *regs = NULL; qemu_co_mutex_init(&s->dma_map_lock); qemu_co_queue_init(&s->dma_flush_queue); @@ -779,7 +781,7 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, /* Perform initialize sequence as described in NVMe spec "7.6.1 * Initialization". */ - cap = le64_to_cpu(regs->cap); + cap = host_pci_ldq_le_p(®s->cap); trace_nvme_controller_capability_raw(cap); trace_nvme_controller_capability("Maximum Queue Entries Supported", 1 + NVME_CAP_MQES(cap)); @@ -805,16 +807,17 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, bs->bl.request_alignment = s->page_size; timeout_ms = MIN(500 * NVME_CAP_TO(cap), 30000); - ver = le32_to_cpu(regs->vs); + ver = host_pci_ldl_le_p(®s->vs); trace_nvme_controller_spec_version(extract32(ver, 16, 16), extract32(ver, 8, 8), extract32(ver, 0, 8)); /* Reset device to get a clean state. */ - regs->cc = cpu_to_le32(le32_to_cpu(regs->cc) & 0xFE); + cc = host_pci_ldl_le_p(®s->cc); + host_pci_stl_le_p(®s->cc, cc & 0xFE); /* Wait for CSTS.RDY = 0. */ deadline = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ms * SCALE_MS; - while (NVME_CSTS_RDY(le32_to_cpu(regs->csts))) { + while (NVME_CSTS_RDY(host_pci_ldl_le_p(®s->csts))) { if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) { error_setg(errp, "Timeout while waiting for device to reset (%" PRId64 " ms)", @@ -843,19 +846,21 @@ static int nvme_init(BlockDriverState *bs, const char *device, int namespace, s->queues[INDEX_ADMIN] = q; s->queue_count = 1; QEMU_BUILD_BUG_ON((NVME_QUEUE_SIZE - 1) & 0xF000); - regs->aqa = cpu_to_le32(((NVME_QUEUE_SIZE - 1) << AQA_ACQS_SHIFT) | - ((NVME_QUEUE_SIZE - 1) << AQA_ASQS_SHIFT)); - regs->asq = cpu_to_le64(q->sq.iova); - regs->acq = cpu_to_le64(q->cq.iova); + host_pci_stl_le_p(®s->aqa, + ((NVME_QUEUE_SIZE - 1) << AQA_ACQS_SHIFT) | + ((NVME_QUEUE_SIZE - 1) << AQA_ASQS_SHIFT)); + host_pci_stq_le_p(®s->asq, q->sq.iova); + host_pci_stq_le_p(®s->acq, q->cq.iova); /* After setting up all control registers we can enable device now. */ - regs->cc = cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES) << CC_IOCQES_SHIFT) | - (ctz32(NVME_SQ_ENTRY_BYTES) << CC_IOSQES_SHIFT) | - CC_EN_MASK); + host_pci_stl_le_p(®s->cc, + (ctz32(NVME_CQ_ENTRY_BYTES) << CC_IOCQES_SHIFT) | + (ctz32(NVME_SQ_ENTRY_BYTES) << CC_IOSQES_SHIFT) | + CC_EN_MASK); /* Wait for CSTS.RDY = 1. */ now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); deadline = now + timeout_ms * SCALE_MS; - while (!NVME_CSTS_RDY(le32_to_cpu(regs->csts))) { + while (!NVME_CSTS_RDY(host_pci_ldl_le_p(®s->csts))) { if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) { error_setg(errp, "Timeout while waiting for device to start (%" PRId64 " ms)", diff --git a/bsd-user/main.c b/bsd-user/main.c index fdb160bed0..7c0a059c3b 100644 --- a/bsd-user/main.c +++ b/bsd-user/main.c @@ -36,7 +36,6 @@ #include "qemu/help_option.h" #include "qemu/module.h" #include "qemu/plugin.h" -#include "exec/exec-all.h" #include "user/guest-base.h" #include "user/page-protection.h" #include "tcg/startup.h" @@ -90,6 +89,7 @@ bool have_guest_base; #endif unsigned long reserved_va; +unsigned long guest_addr_max; const char *interp_prefix = CONFIG_QEMU_INTERP_PREFIX; const char *qemu_uname_release; @@ -175,6 +175,9 @@ static void usage(void) "-strace log system calls\n" "-trace [[enable=]<pattern>][,events=<file>][,file=<file>]\n" " specify tracing options\n" +#ifdef CONFIG_PLUGIN + "-plugin [file=]<file>[,<argname>=<argvalue>]\n" +#endif "\n" "Environment variables:\n" "QEMU_STRACE Print system calls and arguments similar to the\n" @@ -225,6 +228,8 @@ static void init_task_state(TaskState *ts) }; } +static QemuPluginList plugins = QTAILQ_HEAD_INITIALIZER(plugins); + void gemu_log(const char *fmt, ...) { va_list ap; @@ -307,6 +312,7 @@ int main(int argc, char **argv) cpu_model = NULL; qemu_add_opts(&qemu_trace_opts); + qemu_plugin_add_opts(); optind = 1; for (;;) { @@ -399,6 +405,11 @@ int main(int argc, char **argv) do_strace = 1; } else if (!strcmp(r, "trace")) { trace_opt_parse(optarg); +#ifdef CONFIG_PLUGIN + } else if (!strcmp(r, "plugin")) { + r = argv[optind++]; + qemu_plugin_opt_parse(r, &plugins); +#endif } else if (!strcmp(r, "0")) { argv0 = argv[optind++]; } else { @@ -433,6 +444,7 @@ int main(int argc, char **argv) exit(1); } trace_init_file(); + qemu_plugin_load_list(&plugins, &error_fatal); /* Zero out regs */ memset(regs, 0, sizeof(struct target_pt_regs)); @@ -501,6 +513,13 @@ int main(int argc, char **argv) /* MAX_RESERVED_VA + 1 is a large power of 2, so is aligned. */ reserved_va = max_reserved_va; } + if (reserved_va != 0) { + guest_addr_max = reserved_va; + } else if (MIN(TARGET_VIRT_ADDR_SPACE_BITS, TARGET_ABI_BITS) <= 32) { + guest_addr_max = UINT32_MAX; + } else { + guest_addr_max = ~0ul; + } if (getenv("QEMU_STRACE")) { do_strace = 1; diff --git a/bsd-user/qemu.h b/bsd-user/qemu.h index 244670dd24..93388e7c34 100644 --- a/bsd-user/qemu.h +++ b/bsd-user/qemu.h @@ -23,7 +23,6 @@ #include "cpu.h" #include "qemu/units.h" #include "accel/tcg/cpu-ldst.h" -#include "exec/exec-all.h" #include "user/abitypes.h" #include "user/cpu_loop.h" diff --git a/bsd-user/signal.c b/bsd-user/signal.c index 1aa0fd79d6..dadcc037dc 100644 --- a/bsd-user/signal.c +++ b/bsd-user/signal.c @@ -1030,7 +1030,7 @@ void process_pending_signals(CPUArchState *env) ts->in_sigsuspend = false; } -void cpu_loop_exit_sigsegv(CPUState *cpu, target_ulong addr, +void cpu_loop_exit_sigsegv(CPUState *cpu, vaddr addr, MMUAccessType access_type, bool maperr, uintptr_t ra) { const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; @@ -1046,7 +1046,7 @@ void cpu_loop_exit_sigsegv(CPUState *cpu, target_ulong addr, cpu_loop_exit_restore(cpu, ra); } -void cpu_loop_exit_sigbus(CPUState *cpu, target_ulong addr, +void cpu_loop_exit_sigbus(CPUState *cpu, vaddr addr, MMUAccessType access_type, uintptr_t ra) { const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; diff --git a/configs/meson/emscripten.txt b/configs/meson/emscripten.txt new file mode 100644 index 0000000000..4230e88005 --- /dev/null +++ b/configs/meson/emscripten.txt @@ -0,0 +1,8 @@ +[built-in options] +c_args = ['-pthread'] +cpp_args = ['-pthread'] +objc_args = ['-pthread'] +# -sPROXY_TO_PTHREAD link time flag always requires -pthread even during +# configuration so explicitly add the flag here. +c_link_args = ['-pthread','-sASYNCIFY=1','-sPROXY_TO_PTHREAD=1','-sFORCE_FILESYSTEM','-sALLOW_TABLE_GROWTH','-sTOTAL_MEMORY=2GB','-sWASM_BIGINT','-sEXPORT_ES6=1','-sASYNCIFY_IMPORTS=ffi_call_js','-sEXPORTED_RUNTIME_METHODS=addFunction,removeFunction,TTY,FS'] +cpp_link_args = ['-pthread','-sASYNCIFY=1','-sPROXY_TO_PTHREAD=1','-sFORCE_FILESYSTEM','-sALLOW_TABLE_GROWTH','-sTOTAL_MEMORY=2GB','-sWASM_BIGINT','-sEXPORT_ES6=1','-sASYNCIFY_IMPORTS=ffi_call_js','-sEXPORTED_RUNTIME_METHODS=addFunction,removeFunction,TTY,FS'] diff --git a/configure b/configure index 000309cf61..2ce8d29fac 100755 --- a/configure +++ b/configure @@ -360,6 +360,10 @@ elif check_define __NetBSD__; then host_os=netbsd elif check_define __APPLE__; then host_os=darwin +elif check_define EMSCRIPTEN ; then + host_os=emscripten + cpu=wasm32 + cross_compile="yes" else # This is a fatal error, but don't report it yet, because we # might be going to just print the --help text, or it might @@ -526,6 +530,9 @@ case "$cpu" in linux_arch=x86 CPU_CFLAGS="-m64" ;; + wasm32) + CPU_CFLAGS="-m32" + ;; esac if test -n "$host_arch" && { @@ -540,17 +547,17 @@ if test -n "$linux_arch" && ! test -d "$source_path/linux-headers/asm-$linux_arc fi check_py_version() { - # We require python >= 3.8. + # We require python >= 3.9. # NB: a True python conditional creates a non-zero return code (Failure) - "$1" -c 'import sys; sys.exit(sys.version_info < (3,8))' + "$1" -c 'import sys; sys.exit(sys.version_info < (3,9))' } first_python= if test -z "${PYTHON}"; then # A bare 'python' is traditionally python 2.x, but some distros # have it as python 3.x, so check in both places. - for binary in python3 python python3.12 python3.11 \ - python3.10 python3.9 python3.8; do + for binary in python3 python python3.13 python3.12 python3.11 \ + python3.10 python3.9 ; do if has "$binary"; then python=$(command -v "$binary") if check_py_version "$python"; then @@ -933,7 +940,7 @@ then # If first_python is set, there was a binary somewhere even though # it was not suitable. Use it for the error message. if test -n "$first_python"; then - error_exit "Cannot use '$first_python', Python >= 3.8 is required." \ + error_exit "Cannot use '$first_python', Python >= 3.9 is required." \ "Use --python=/path/to/python to specify a supported Python." else error_exit "Python not found. Use --python=/path/to/python" @@ -941,11 +948,11 @@ then fi if ! check_py_version "$python"; then - error_exit "Cannot use '$python', Python >= 3.8 is required." \ + error_exit "Cannot use '$python', Python >= 3.9 is required." \ "Use --python=/path/to/python to specify a supported Python." \ "Maybe try:" \ " openSUSE Leap 15.3+: zypper install python39" \ - " CentOS 8: dnf install python38" + " CentOS: dnf install python3.12" fi # Resolve PATH diff --git a/docs/about/build-platforms.rst b/docs/about/build-platforms.rst index 52521552c8..8ecbd6b26f 100644 --- a/docs/about/build-platforms.rst +++ b/docs/about/build-platforms.rst @@ -101,7 +101,7 @@ Python runtime option of the ``configure`` script to point QEMU to a supported version of the Python runtime. - As of QEMU |version|, the minimum supported version of Python is 3.8. + As of QEMU |version|, the minimum supported version of Python is 3.9. Python build dependencies Some of QEMU's build dependencies are written in Python. Usually these @@ -118,9 +118,14 @@ Rust build dependencies include bindgen or have an older version, it is recommended to install a newer version using ``cargo install bindgen-cli``. - Developers may want to use Cargo-based tools in the QEMU source tree; - this requires Cargo 1.74.0. Note that Cargo is not required in order - to build QEMU. + QEMU requires Rust 1.77.0. This is available on all supported platforms + with one exception, namely the ``mips64el`` architecture on Debian bookworm. + For all other architectures, Debian bookworm provides a new-enough Rust + compiler in the ``rustc-web`` package. + + Also, on Ubuntu 22.04 or 24.04 this requires the ``rustc-1.77`` + (or newer) package. The path to ``rustc`` and ``rustdoc`` must be + provided manually to the configure script. Optional build dependencies Build components whose absence does not affect the ability to build QEMU diff --git a/docs/about/deprecated.rst b/docs/about/deprecated.rst index b0d3ae6b32..44d3427e98 100644 --- a/docs/about/deprecated.rst +++ b/docs/about/deprecated.rst @@ -309,6 +309,13 @@ CPU implementation for a while before removing all support. System emulator machines ------------------------ +Versioned machine types (aarch64, arm, i386, m68k, ppc64, s390x, x86_64) +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +In accordance with our versioned machine type deprecation policy, all machine +types with version |VER_MACHINE_DEPRECATION_VERSION|, or older, have been +deprecated. + Arm ``virt`` machine ``dtb-kaslr-seed`` property (since 7.1) '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' diff --git a/docs/about/removed-features.rst b/docs/about/removed-features.rst index 790a5e481c..063284d4f8 100644 --- a/docs/about/removed-features.rst +++ b/docs/about/removed-features.rst @@ -981,10 +981,12 @@ from Linux in 2021, and is not supported anymore by QEMU either. System emulator machines ------------------------ -Note: Versioned machine types that have been introduced in a QEMU version -that has initially been released more than 6 years before are considered -obsolete and will be removed without further notice in this document. -Please use newer machine types instead. +Versioned machine types (aarch64, arm, i386, m68k, ppc64, s390x, x86_64) +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +In accordance with our versioned machine type deprecation policy, all machine +types with version |VER_MACHINE_DELETION_VERSION|, or older, have been +removed. ``s390-virtio`` (removed in 2.6) '''''''''''''''''''''''''''''''' diff --git a/docs/conf.py b/docs/conf.py index 7b5712e122..f892a6e1da 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -117,6 +117,32 @@ finally: else: version = release = "unknown version" +bits = version.split(".") + +major = int(bits[0]) +minor = int(bits[1]) +micro = int(bits[2]) + +# Check for a dev snapshot, so we can adjust to next +# predicted release version. +# +# This assumes we do 3 releases per year, so must bump +# major if minor == 2 +if micro >= 50: + micro = 0 + if minor == 2: + major += 1 + minor = 0 + else: + minor += 1 + +# These thresholds must match the constants +# MACHINE_VER_DELETION_MAJOR & MACHINE_VER_DEPRECATION_MAJOR +# defined in include/hw/boards.h and the introductory text in +# docs/about/deprecated.rst +ver_machine_deprecation_version = "%d.%d.0" % (major - 3, minor) +ver_machine_deletion_version = "%d.%d.0" % (major - 6, minor) + # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # @@ -145,7 +171,18 @@ suppress_warnings = ["ref.option"] # environment variable is not set is for the benefit of readthedocs # style document building; our Makefile always sets the variable. confdir = os.getenv('CONFDIR', "/etc/qemu") -rst_epilog = ".. |CONFDIR| replace:: ``" + confdir + "``\n" + +vars = { + "CONFDIR": confdir, + "VER_MACHINE_DEPRECATION_VERSION": ver_machine_deprecation_version, + "VER_MACHINE_DELETION_VERSION": ver_machine_deletion_version, +} + +rst_epilog = "".join([ + ".. |" + key + "| replace:: ``" + vars[key] + "``\n" + for key in vars.keys() +]) + # We slurp in the defs.rst.inc and literally include it into rst_epilog, # because Sphinx's include:: directive doesn't work with absolute paths # and there isn't any one single relative path that will work for all diff --git a/docs/devel/build-environment.rst b/docs/devel/build-environment.rst index f133ef2e01..661f6ea850 100644 --- a/docs/devel/build-environment.rst +++ b/docs/devel/build-environment.rst @@ -97,11 +97,11 @@ build QEMU in MSYS2 itself. :: - pacman -S wget + pacman -S wget base-devel git wget https://raw.githubusercontent.com/msys2/MINGW-packages/refs/heads/master/mingw-w64-qemu/PKGBUILD # Some packages may be missing for your environment, installation will still # be done though. - makepkg -s PKGBUILD || true + makepkg --syncdeps --nobuild PKGBUILD || true Build on windows-aarch64 ++++++++++++++++++++++++ diff --git a/docs/devel/build-system.rst b/docs/devel/build-system.rst index 258cfad3fe..2c884197a2 100644 --- a/docs/devel/build-system.rst +++ b/docs/devel/build-system.rst @@ -168,7 +168,7 @@ The required versions of the packages are stored in a configuration file ``pythondeps.toml``. The format is custom to QEMU, but it is documented at the top of the file itself and it should be easy to understand. The requirements should make it possible to use the version that is packaged -that is provided by supported distros. +by QEMU's supported distros. When dependencies are downloaded, instead, ``configure`` uses a "known good" version that is also listed in ``pythondeps.toml``. In this diff --git a/docs/devel/codebase.rst b/docs/devel/codebase.rst index 40273e7d31..2a3143787a 100644 --- a/docs/devel/codebase.rst +++ b/docs/devel/codebase.rst @@ -116,7 +116,7 @@ yet, so sometimes the source code is all you have. * `monitor <https://gitlab.com/qemu-project/qemu/-/tree/master/monitor>`_: `Monitor <QEMU monitor>` implementation (HMP & QMP). * `nbd <https://gitlab.com/qemu-project/qemu/-/tree/master/nbd>`_: - QEMU `NBD (Network Block Device) <nbd>` server. + QEMU NBD (Network Block Device) server. * `net <https://gitlab.com/qemu-project/qemu/-/tree/master/net>`_: Network (host) support. * `pc-bios <https://gitlab.com/qemu-project/qemu/-/tree/master/pc-bios>`_: diff --git a/docs/devel/rust.rst b/docs/devel/rust.rst index 88bdec1eb2..4de8637502 100644 --- a/docs/devel/rust.rst +++ b/docs/devel/rust.rst @@ -71,36 +71,9 @@ Building Rust code with ``--enable-modules`` is not supported yet. Supported tools ''''''''''''''' -QEMU supports rustc version 1.63.0 and newer. Notably, the following features +QEMU supports rustc version 1.77.0 and newer. Notably, the following features are missing: -* ``core::ffi`` (1.64.0). Use ``std::os::raw`` and ``std::ffi`` instead. - -* ``cast_mut()``/``cast_const()`` (1.65.0). Use ``as`` instead. - -* "let ... else" (1.65.0). Use ``if let`` instead. This is currently patched - in QEMU's vendored copy of the bilge crate. - -* Generic Associated Types (1.65.0) - -* ``CStr::from_bytes_with_nul()`` as a ``const`` function (1.72.0). - -* "Return position ``impl Trait`` in Traits" (1.75.0, blocker for including - the pinned-init create). - -* ``MaybeUninit::zeroed()`` as a ``const`` function (1.75.0). QEMU's - ``Zeroable`` trait can be implemented without ``MaybeUninit::zeroed()``, - so this would be just a cleanup. - -* ``c"" literals`` (stable in 1.77.0). QEMU provides a ``c_str!()`` macro - to define ``CStr`` constants easily - -* ``offset_of!`` (stable in 1.77.0). QEMU uses ``offset_of!()`` heavily; it - provides a replacement in the ``qemu_api`` crate, but it does not support - lifetime parameters and therefore ``&'a Something`` fields in the struct - may have to be replaced by ``NonNull<Something>``. *Nested* ``offset_of!`` - was only stabilized in Rust 1.82.0, but it is not used. - * inline const expression (stable in 1.79.0), currently worked around with associated constants in the ``FnCall`` trait. @@ -125,12 +98,6 @@ are missing: __ https://github.com/rust-lang/rust/pull/125258 -It is expected that QEMU will advance its minimum supported version of -rustc to 1.77.0 as soon as possible; as of January 2025, blockers -for that right now are Debian bookworm and 32-bit MIPS processors. -This unfortunately means that references to statics in constants will -remain an issue. - QEMU also supports version 0.60.x of bindgen, which is missing option ``--generate-cstr``. This option requires version 0.66.x and will be adopted as soon as supporting these older versions is not necessary @@ -153,8 +120,7 @@ QEMU includes four crates: .. [#issues] The ``pl011`` crate is synchronized with ``hw/char/pl011.c`` as of commit 02b1f7f61928. The ``hpet`` crate is synchronized as of - commit f32352ff9e. Both are lacking tracing functionality; ``hpet`` - is also lacking support for migration. + commit 1433e38cc8. Both are lacking tracing functionality. This section explains how to work with them. @@ -184,7 +150,6 @@ module status ``bitops`` complete ``callbacks`` complete ``cell`` stable -``c_str`` complete ``errno`` complete ``irq`` complete ``memory`` stable @@ -441,7 +406,7 @@ Adding dependencies Generally, the set of dependent crates is kept small. Think twice before adding a new external crate, especially if it comes with a large set of dependencies itself. Sometimes QEMU only needs a small subset of the -functionality; see for example QEMU's ``assertions`` or ``c_str`` modules. +functionality; see for example QEMU's ``assertions`` module. On top of this recommendation, adding external crates to QEMU is a slightly complicated process, mostly due to the need to teach Meson how diff --git a/docs/devel/tcg-ops.rst b/docs/devel/tcg-ops.rst index 688984fd39..f26b837a30 100644 --- a/docs/devel/tcg-ops.rst +++ b/docs/devel/tcg-ops.rst @@ -239,7 +239,7 @@ Jumps/Labels - | Jump to label. - * - brcond_i32/i64 *t0*, *t1*, *cond*, *label* + * - brcond *t0*, *t1*, *cond*, *label* - | Conditional jump if *t0* *cond* *t1* is true. *cond* can be: | @@ -261,98 +261,117 @@ Arithmetic .. list-table:: - * - add_i32/i64 *t0*, *t1*, *t2* + * - add *t0*, *t1*, *t2* - | *t0* = *t1* + *t2* - * - sub_i32/i64 *t0*, *t1*, *t2* + * - sub *t0*, *t1*, *t2* - | *t0* = *t1* - *t2* - * - neg_i32/i64 *t0*, *t1* + * - neg *t0*, *t1* - | *t0* = -*t1* (two's complement) - * - mul_i32/i64 *t0*, *t1*, *t2* + * - mul *t0*, *t1*, *t2* - | *t0* = *t1* * *t2* - * - div_i32/i64 *t0*, *t1*, *t2* + * - divs *t0*, *t1*, *t2* - | *t0* = *t1* / *t2* (signed) | Undefined behavior if division by zero or overflow. - * - divu_i32/i64 *t0*, *t1*, *t2* + * - divu *t0*, *t1*, *t2* - | *t0* = *t1* / *t2* (unsigned) | Undefined behavior if division by zero. - * - rem_i32/i64 *t0*, *t1*, *t2* + * - rems *t0*, *t1*, *t2* - | *t0* = *t1* % *t2* (signed) | Undefined behavior if division by zero or overflow. - * - remu_i32/i64 *t0*, *t1*, *t2* + * - remu *t0*, *t1*, *t2* - | *t0* = *t1* % *t2* (unsigned) | Undefined behavior if division by zero. + * - divs2 *q*, *r*, *nl*, *nh*, *d* + + - | *q* = *nh:nl* / *d* (signed) + | *r* = *nh:nl* % *d* + | Undefined behaviour if division by zero, or the double-word + numerator divided by the single-word divisor does not fit + within the single-word quotient. The code generator will + pass *nh* as a simple sign-extension of *nl*, so the only + overflow should be *INT_MIN* / -1. + + * - divu2 *q*, *r*, *nl*, *nh*, *d* + + - | *q* = *nh:nl* / *d* (unsigned) + | *r* = *nh:nl* % *d* + | Undefined behaviour if division by zero, or the double-word + numerator divided by the single-word divisor does not fit + within the single-word quotient. The code generator will + pass 0 to *nh* to make a simple zero-extension of *nl*, + so overflow should never occur. Logical ------- .. list-table:: - * - and_i32/i64 *t0*, *t1*, *t2* + * - and *t0*, *t1*, *t2* - | *t0* = *t1* & *t2* - * - or_i32/i64 *t0*, *t1*, *t2* + * - or *t0*, *t1*, *t2* - | *t0* = *t1* | *t2* - * - xor_i32/i64 *t0*, *t1*, *t2* + * - xor *t0*, *t1*, *t2* - | *t0* = *t1* ^ *t2* - * - not_i32/i64 *t0*, *t1* + * - not *t0*, *t1* - | *t0* = ~\ *t1* - * - andc_i32/i64 *t0*, *t1*, *t2* + * - andc *t0*, *t1*, *t2* - | *t0* = *t1* & ~\ *t2* - * - eqv_i32/i64 *t0*, *t1*, *t2* + * - eqv *t0*, *t1*, *t2* - | *t0* = ~(*t1* ^ *t2*), or equivalently, *t0* = *t1* ^ ~\ *t2* - * - nand_i32/i64 *t0*, *t1*, *t2* + * - nand *t0*, *t1*, *t2* - | *t0* = ~(*t1* & *t2*) - * - nor_i32/i64 *t0*, *t1*, *t2* + * - nor *t0*, *t1*, *t2* - | *t0* = ~(*t1* | *t2*) - * - orc_i32/i64 *t0*, *t1*, *t2* + * - orc *t0*, *t1*, *t2* - | *t0* = *t1* | ~\ *t2* - * - clz_i32/i64 *t0*, *t1*, *t2* + * - clz *t0*, *t1*, *t2* - | *t0* = *t1* ? clz(*t1*) : *t2* - * - ctz_i32/i64 *t0*, *t1*, *t2* + * - ctz *t0*, *t1*, *t2* - | *t0* = *t1* ? ctz(*t1*) : *t2* - * - ctpop_i32/i64 *t0*, *t1* + * - ctpop *t0*, *t1* - | *t0* = number of bits set in *t1* | - | With *ctpop* short for "count population", matching - | the function name used in ``include/qemu/host-utils.h``. + | The name *ctpop* is short for "count population", and matches + the function name used in ``include/qemu/host-utils.h``. Shifts/Rotates @@ -360,30 +379,30 @@ Shifts/Rotates .. list-table:: - * - shl_i32/i64 *t0*, *t1*, *t2* + * - shl *t0*, *t1*, *t2* - | *t0* = *t1* << *t2* - | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + | Unspecified behavior for negative or out-of-range shifts. - * - shr_i32/i64 *t0*, *t1*, *t2* + * - shr *t0*, *t1*, *t2* - | *t0* = *t1* >> *t2* (unsigned) - | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + | Unspecified behavior for negative or out-of-range shifts. - * - sar_i32/i64 *t0*, *t1*, *t2* + * - sar *t0*, *t1*, *t2* - | *t0* = *t1* >> *t2* (signed) - | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + | Unspecified behavior for negative or out-of-range shifts. - * - rotl_i32/i64 *t0*, *t1*, *t2* + * - rotl *t0*, *t1*, *t2* - | Rotation of *t2* bits to the left - | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + | Unspecified behavior for negative or out-of-range shifts. - * - rotr_i32/i64 *t0*, *t1*, *t2* + * - rotr *t0*, *t1*, *t2* - | Rotation of *t2* bits to the right. - | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + | Unspecified behavior for negative or out-of-range shifts. Misc @@ -391,26 +410,12 @@ Misc .. list-table:: - * - mov_i32/i64 *t0*, *t1* + * - mov *t0*, *t1* - | *t0* = *t1* - | Move *t1* to *t0* (both operands must have the same type). - - * - ext8s_i32/i64 *t0*, *t1* - - ext8u_i32/i64 *t0*, *t1* - - ext16s_i32/i64 *t0*, *t1* - - ext16u_i32/i64 *t0*, *t1* + | Move *t1* to *t0*. - ext32s_i64 *t0*, *t1* - - ext32u_i64 *t0*, *t1* - - - | 8, 16 or 32 bit sign/zero extension (both operands must have the same type) - - * - bswap16_i32/i64 *t0*, *t1*, *flags* + * - bswap16 *t0*, *t1*, *flags* - | 16 bit byte swap on the low bits of a 32/64 bit input. | @@ -420,24 +425,24 @@ Misc | | If neither ``TCG_BSWAP_OZ`` nor ``TCG_BSWAP_OS`` are set, then the bits of *t0* above bit 15 may contain any value. - * - bswap32_i64 *t0*, *t1*, *flags* - - - | 32 bit byte swap on a 64-bit value. The flags are the same as for bswap16, - except they apply from bit 31 instead of bit 15. + * - bswap32 *t0*, *t1*, *flags* - * - bswap32_i32 *t0*, *t1*, *flags* + - | 32 bit byte swap. The flags are the same as for bswap16, except + they apply from bit 31 instead of bit 15. On TCG_TYPE_I32, the + flags should be zero. - bswap64_i64 *t0*, *t1*, *flags* + * - bswap64 *t0*, *t1*, *flags* - - | 32/64 bit byte swap. The flags are ignored, but still present - for consistency with the other bswap opcodes. + - | 64 bit byte swap. The flags are ignored, but still present + for consistency with the other bswap opcodes. For future + compatibility, the flags should be zero. * - discard_i32/i64 *t0* - | Indicate that the value of *t0* won't be used later. It is useful to force dead code elimination. - * - deposit_i32/i64 *dest*, *t1*, *t2*, *pos*, *len* + * - deposit *dest*, *t1*, *t2*, *pos*, *len* - | Deposit *t2* as a bitfield into *t1*, placing the result in *dest*. | @@ -446,14 +451,16 @@ Misc | *len* - the length of the bitfield | *pos* - the position of the first bit, counting from the LSB | - | For example, "deposit_i32 dest, t1, t2, 8, 4" indicates a 4-bit field + | For example, "deposit dest, t1, t2, 8, 4" indicates a 4-bit field at bit 8. This operation would be equivalent to | | *dest* = (*t1* & ~0x0f00) | ((*t2* << 8) & 0x0f00) + | + | on TCG_TYPE_I32. - * - extract_i32/i64 *dest*, *t1*, *pos*, *len* + * - extract *dest*, *t1*, *pos*, *len* - sextract_i32/i64 *dest*, *t1*, *pos*, *len* + sextract *dest*, *t1*, *pos*, *len* - | Extract a bitfield from *t1*, placing the result in *dest*. | @@ -462,16 +469,16 @@ Misc to the left with zeros; for sextract_*, the result will be extended to the left with copies of the bitfield sign bit at *pos* + *len* - 1. | - | For example, "sextract_i32 dest, t1, 8, 4" indicates a 4-bit field + | For example, "sextract dest, t1, 8, 4" indicates a 4-bit field at bit 8. This operation would be equivalent to | | *dest* = (*t1* << 20) >> 28 | - | (using an arithmetic right shift). + | (using an arithmetic right shift) on TCG_TYPE_I32. - * - extract2_i32/i64 *dest*, *t1*, *t2*, *pos* + * - extract2 *dest*, *t1*, *t2*, *pos* - - | For N = {32,64}, extract an N-bit quantity from the concatenation + - | For TCG_TYPE_I{N}, extract an N-bit quantity from the concatenation of *t2*:*t1*, beginning at *pos*. The tcg_gen_extract2_{i32,i64} expander accepts 0 <= *pos* <= N as inputs. The backend code generator will not see either 0 or N as inputs for these opcodes. @@ -494,19 +501,19 @@ Conditional moves .. list-table:: - * - setcond_i32/i64 *dest*, *t1*, *t2*, *cond* + * - setcond *dest*, *t1*, *t2*, *cond* - | *dest* = (*t1* *cond* *t2*) | | Set *dest* to 1 if (*t1* *cond* *t2*) is true, otherwise set to 0. - * - negsetcond_i32/i64 *dest*, *t1*, *t2*, *cond* + * - negsetcond *dest*, *t1*, *t2*, *cond* - | *dest* = -(*t1* *cond* *t2*) | | Set *dest* to -1 if (*t1* *cond* *t2*) is true, otherwise set to 0. - * - movcond_i32/i64 *dest*, *c1*, *c2*, *v1*, *v2*, *cond* + * - movcond *dest*, *c1*, *c2*, *v1*, *v2*, *cond* - | *dest* = (*c1* *cond* *c2* ? *v1* : *v2*) | @@ -586,26 +593,79 @@ Multiword arithmetic support .. list-table:: - * - add2_i32/i64 *t0_low*, *t0_high*, *t1_low*, *t1_high*, *t2_low*, *t2_high* + * - addco *t0*, *t1*, *t2* + + - | Compute *t0* = *t1* + *t2* and in addition output to the + carry bit provided by the host architecture. + + * - addci *t0, *t1*, *t2* - sub2_i32/i64 *t0_low*, *t0_high*, *t1_low*, *t1_high*, *t2_low*, *t2_high* + - | Compute *t0* = *t1* + *t2* + *C*, where *C* is the + input carry bit provided by the host architecture. + The output carry bit need not be computed. - - | Similar to add/sub, except that the double-word inputs *t1* and *t2* are - formed from two single-word arguments, and the double-word output *t0* - is returned in two single-word outputs. + * - addcio *t0, *t1*, *t2* - * - mulu2_i32/i64 *t0_low*, *t0_high*, *t1*, *t2* + - | Compute *t0* = *t1* + *t2* + *C*, where *C* is the + input carry bit provided by the host architecture, + and also compute the output carry bit. + + * - addc1o *t0, *t1*, *t2* + + - | Compute *t0* = *t1* + *t2* + 1, and in addition output to the + carry bit provided by the host architecture. This is akin to + *addcio* with a fixed carry-in value of 1. + | This is intended to be used by the optimization pass, + intermediate to complete folding of the addition chain. + In some cases complete folding is not possible and this + opcode will remain until output. If this happens, the + code generator will use ``tcg_out_set_carry`` and then + the output routine for *addcio*. + + * - subbo *t0*, *t1*, *t2* + + - | Compute *t0* = *t1* - *t2* and in addition output to the + borrow bit provided by the host architecture. + | Depending on the host architecture, the carry bit may or may not be + identical to the borrow bit. Thus the addc\* and subb\* + opcodes must not be mixed. + + * - subbi *t0, *t1*, *t2* + + - | Compute *t0* = *t1* - *t2* - *B*, where *B* is the + input borrow bit provided by the host architecture. + The output borrow bit need not be computed. + + * - subbio *t0, *t1*, *t2* + + - | Compute *t0* = *t1* - *t2* - *B*, where *B* is the + input borrow bit provided by the host architecture, + and also compute the output borrow bit. + + * - subb1o *t0, *t1*, *t2* + + - | Compute *t0* = *t1* - *t2* - 1, and in addition output to the + borrow bit provided by the host architecture. This is akin to + *subbio* with a fixed borrow-in value of 1. + | This is intended to be used by the optimization pass, + intermediate to complete folding of the subtraction chain. + In some cases complete folding is not possible and this + opcode will remain until output. If this happens, the + code generator will use ``tcg_out_set_borrow`` and then + the output routine for *subbio*. + + * - mulu2 *t0_low*, *t0_high*, *t1*, *t2* - | Similar to mul, except two unsigned inputs *t1* and *t2* yielding the full double-word product *t0*. The latter is returned in two single-word outputs. - * - muls2_i32/i64 *t0_low*, *t0_high*, *t1*, *t2* + * - muls2 *t0_low*, *t0_high*, *t1*, *t2* - | Similar to mulu2, except the two inputs *t1* and *t2* are signed. - * - mulsh_i32/i64 *t0*, *t1*, *t2* + * - mulsh *t0*, *t1*, *t2* - muluh_i32/i64 *t0*, *t1*, *t2* + muluh *t0*, *t1*, *t2* - | Provide the high part of a signed or unsigned multiply, respectively. | @@ -684,8 +744,6 @@ QEMU specific operations qemu_st_i32/i64/i128 *t0*, *t1*, *flags*, *memidx* - qemu_st8_i32 *t0*, *t1*, *flags*, *memidx* - - | Load data at the guest address *t1* into *t0*, or store data in *t0* at guest address *t1*. The _i32/_i64/_i128 size applies to the size of the input/output register *t0* only. The address *t1* is always sized according to the guest, @@ -703,10 +761,6 @@ QEMU specific operations 64-bit memory access specified in *flags*. | | For qemu_ld/st_i128, these are only supported for a 64-bit host. - | - | For i386, qemu_st8_i32 is exactly like qemu_st_i32, except the size of - the memory operation is known to be 8-bit. This allows the backend to - provide a different set of register constraints. Host vector operations @@ -884,9 +938,9 @@ Assumptions The target word size (``TCG_TARGET_REG_BITS``) is expected to be 32 bit or 64 bit. It is expected that the pointer has the same size as the word. -On a 32 bit target, all 64 bit operations are converted to 32 bits. A -few specific operations must be implemented to allow it (see add2_i32, -sub2_i32, brcond2_i32). +On a 32 bit target, all 64 bit operations are converted to 32 bits. +A few specific operations must be implemented to allow it +(see brcond2_i32, setcond2_i32). On a 64 bit target, the values are transferred between 32 and 64-bit registers using the following ops: diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst index 8030cb4299..9e56dd1b11 100644 --- a/docs/devel/testing/functional.rst +++ b/docs/devel/testing/functional.rst @@ -274,7 +274,7 @@ speed mode in the meson.build file, while the "quick" speed mode is fine for functional tests that can be run without downloading files. ``make check`` then only runs the quick functional tests along with the other quick tests from the other test suites. If you choose to -run only run ``make check-functional``, the "thorough" tests will be +run only ``make check-functional``, the "thorough" tests will be executed, too. And to run all functional tests along with the others, you can use something like:: diff --git a/docs/igd-assign.txt b/docs/igd-assign.txt index 3aed7956d5..af4e8391fc 100644 --- a/docs/igd-assign.txt +++ b/docs/igd-assign.txt @@ -47,6 +47,7 @@ Intel document [1] shows how to dump VBIOS to file. For UEFI Option ROM, see QEMU also provides a "Legacy" mode that implicitly enables full functionality on IGD, it is automatically enabled when +* IGD generation is 6 to 9 (Sandy Bridge to Comet Lake) * Machine type is i440fx * IGD is assigned to guest BDF 00:02.0 * ROM BAR or romfile is present @@ -101,7 +102,7 @@ digital formats work well. Options ======= -* x-igd-opregion=[on|*off*] +* x-igd-opregion=[*on*|off] Copy host IGD OpRegion and expose it to guest with fw_cfg * x-igd-lpc=[on|*off*] @@ -123,7 +124,7 @@ Examples * Adding IGD with OpRegion and LPC ID hack, but without VGA ranges (For UEFI guests) - -device vfio-pci,host=00:02.0,id=hostdev0,addr=2.0,x-igd-legacy-mode=off,x-igd-opregion=on,x-igd-lpc=on,romfile=efi_oprom.rom + -device vfio-pci,host=00:02.0,id=hostdev0,addr=2.0,x-igd-legacy-mode=off,x-igd-lpc=on,romfile=efi_oprom.rom Guest firmware @@ -156,6 +157,12 @@ fw_cfg requirements on the VM firmware: it's expected that this fw_cfg file is only relevant to a single PCI class VGA device with Intel vendor ID, appearing at PCI bus address 00:02.0. + Starting from Meteor Lake, IGD devices access stolen memory via its MMIO + BAR2 (LMEMBAR) and removed the BDSM register in config space. There is + no need for guest firmware to allocate data stolen memory in guest address + space and write it to BDSM register. Value of this fw_cfg file is 0 in + such case. + Upstream Seabios has OpRegion and BDSM (pre-Gen11 device only) support. However, the support is not accepted by upstream EDK2/OVMF. A recommended solution is to create a virtual OpRom with following DXE drivers: diff --git a/docs/system/arm/aspeed.rst b/docs/system/arm/aspeed.rst index 97fd6a0e7f..58a8020eec 100644 --- a/docs/system/arm/aspeed.rst +++ b/docs/system/arm/aspeed.rst @@ -1,12 +1,11 @@ -Aspeed family boards (``ast2500-evb``, ``ast2600-evb``, ``ast2700-evb``, ``bletchley-bmc``, ``fuji-bmc``, ``fby35-bmc``, ``fp5280g2-bmc``, ``g220a-bmc``, ``palmetto-bmc``, ``qcom-dc-scm-v1-bmc``, ``qcom-firework-bmc``, ``quanta-q71l-bmc``, ``rainier-bmc``, ``romulus-bmc``, ``sonorapass-bmc``, ``supermicrox11-bmc``, ``supermicrox11spi-bmc``, ``tiogapass-bmc``, ``witherspoon-bmc``, ``yosemitev2-bmc``) -================================================================================================================================================================================================================================================================================================================================================================================================================== +Aspeed family boards (``ast2500-evb``, ``ast2600-evb``, ``ast2700-evb``, ``ast2700fc``, ``bletchley-bmc``, ``fuji-bmc``, ``fby35-bmc``, ``fp5280g2-bmc``, ``g220a-bmc``, ``palmetto-bmc``, ``qcom-dc-scm-v1-bmc``, ``qcom-firework-bmc``, ``quanta-q71l-bmc``, ``rainier-bmc``, ``romulus-bmc``, ``sonorapass-bmc``, ``supermicrox11-bmc``, ``supermicrox11spi-bmc``, ``tiogapass-bmc``, ``witherspoon-bmc``, ``yosemitev2-bmc``) +================================================================================================================================================================================================================================================================================================================================================================================================================================= The QEMU Aspeed machines model BMCs of various OpenPOWER systems and Aspeed evaluation boards. They are based on different releases of the Aspeed SoC : the AST2400 integrating an ARM926EJ-S CPU (400MHz), the AST2500 with an ARM1176JZS CPU (800MHz), the AST2600 -with dual cores ARM Cortex-A7 CPUs (1.2GHz) and more recently the AST2700 -with quad cores ARM Cortex-A35 64 bits CPUs (1.6GHz) +with dual cores ARM Cortex-A7 CPUs (1.2GHz). The SoC comes with RAM, Gigabit ethernet, USB, SD/MMC, USB, SPI, I2C, etc. @@ -39,10 +38,6 @@ AST2600 SoC based machines : - ``qcom-dc-scm-v1-bmc`` Qualcomm DC-SCM V1 BMC - ``qcom-firework-bmc`` Qualcomm Firework BMC -AST2700 SoC based machines : - -- ``ast2700-evb`` Aspeed AST2700 Evaluation board (Cortex-A35) - Supported devices ----------------- @@ -247,10 +242,78 @@ under Linux), use : -M ast2500-evb,bmc-console=uart3 +Aspeed 2700 family boards (``ast2700-evb``) +================================================================== + +The QEMU Aspeed machines model BMCs of Aspeed evaluation boards. +They are based on different releases of the Aspeed SoC : +the AST2700 with quad cores ARM Cortex-A35 64 bits CPUs (1.6GHz). + +The SoC comes with RAM, Gigabit ethernet, USB, SD/MMC, USB, SPI, I2C, +etc. + +AST2700 SoC based machines : + +- ``ast2700-evb`` Aspeed AST2700 Evaluation board (Cortex-A35) +- ``ast2700fc`` Aspeed AST2700 Evaluation board (Cortex-A35 + Cortex-M4) + +Supported devices +----------------- + * Interrupt Controller + * Timer Controller + * RTC Controller + * I2C Controller + * System Control Unit (SCU) + * SRAM mapping + * X-DMA Controller (basic interface) + * Static Memory Controller (SMC or FMC) - Only SPI Flash support + * SPI Memory Controller + * USB 2.0 Controller + * SD/MMC storage controllers + * SDRAM controller (dummy interface for basic settings and training) + * Watchdog Controller + * GPIO Controller (Master only) + * UART + * Ethernet controllers + * Front LEDs (PCA9552 on I2C bus) + * LPC Peripheral Controller (a subset of subdevices are supported) + * Hash/Crypto Engine (HACE) - Hash support only. TODO: Crypto + * ADC + * eMMC Boot Controller (dummy) + * PECI Controller (minimal) + * I3C Controller + * Internal Bridge Controller (SLI dummy) + +Missing devices +--------------- + * PWM and Fan Controller + * Slave GPIO Controller + * Super I/O Controller + * PCI-Express 1 Controller + * Graphic Display Controller + * MCTP Controller + * Mailbox Controller + * Virtual UART + * eSPI Controller + +Boot options +------------ + +Images can be downloaded from the ASPEED Forked OpenBMC GitHub release repository : + + https://github.com/AspeedTech-BMC/openbmc/releases + Booting the ast2700-evb machine ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Boot the AST2700 machine from the flash image, use an MTD drive : +Boot the AST2700 machine from the flash image. + +There are two supported methods for booting the AST2700 machine with a flash image: + +Manual boot using ``-device loader``: + +It causes all 4 CPU cores to start execution from address ``0x430000000``, which +corresponds to the BL31 image load address. .. code-block:: bash @@ -270,6 +333,89 @@ Boot the AST2700 machine from the flash image, use an MTD drive : -drive file=${IMGDIR}/image-bmc,format=raw,if=mtd \ -nographic +Boot using a virtual boot ROM (``-bios``): + +If users do not specify the ``-bios option``, QEMU will attempt to load the +default vbootrom image ``ast27x0_bootrom.bin`` from either the current working +directory or the ``pc-bios`` directory within the QEMU source tree. + +.. code-block:: bash + + $ qemu-system-aarch64 -M ast2700-evb \ + -drive file=image-bmc,format=raw,if=mtd \ + -nographic + +The ``-bios`` option allows users to specify a custom path for the vbootrom +image to be loaded during boot. This will load the vbootrom image from the +specified path in the ${HOME} directory. + +.. code-block:: bash + + -bios ${HOME}/ast27x0_bootrom.bin + +Booting the ast2700fc machine +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +AST2700 features four Cortex-A35 primary processors and two Cortex-M4 coprocessors. +**ast2700-evb** machine focuses on emulating the four Cortex-A35 primary processors, +**ast2700fc** machine extends **ast2700-evb** by adding support for the two Cortex-M4 coprocessors. + +Steps to boot the AST2700fc machine: + +1. Ensure you have the following AST2700A1 binaries available in a directory + + * u-boot-nodtb.bin + * u-boot.dtb + * bl31.bin + * optee/tee-raw.bin + * image-bmc + * zephyr-aspeed-ssp.elf (for SSP firmware, CPU 5) + * zephyr-aspeed-tsp.elf (for TSP firmware, CPU 6) + +2. Execute the following command to start ``ast2700fc`` machine: + +.. code-block:: bash + + IMGDIR=ast2700-default + UBOOT_SIZE=$(stat --format=%s -L ${IMGDIR}/u-boot-nodtb.bin) + + $ qemu-system-aarch64 -M ast2700fc \ + -device loader,force-raw=on,addr=0x400000000,file=${IMGDIR}/u-boot-nodtb.bin \ + -device loader,force-raw=on,addr=$((0x400000000 + ${UBOOT_SIZE})),file=${IMGDIR}/u-boot.dtb \ + -device loader,force-raw=on,addr=0x430000000,file=${IMGDIR}/bl31.bin \ + -device loader,force-raw=on,addr=0x430080000,file=${IMGDIR}/optee/tee-raw.bin \ + -device loader,cpu-num=0,addr=0x430000000 \ + -device loader,cpu-num=1,addr=0x430000000 \ + -device loader,cpu-num=2,addr=0x430000000 \ + -device loader,cpu-num=3,addr=0x430000000 \ + -drive file=${IMGDIR}/image-bmc,if=mtd,format=raw \ + -device loader,file=${IMGDIR}/zephyr-aspeed-ssp.elf,cpu-num=4 \ + -device loader,file=${IMGDIR}/zephyr-aspeed-tsp.elf,cpu-num=5 \ + -serial pty -serial pty -serial pty \ + -snapshot \ + -S -nographic + +After launching QEMU, serial devices will be automatically redirected. +Example output: + +.. code-block:: bash + + char device redirected to /dev/pts/55 (label serial0) + char device redirected to /dev/pts/56 (label serial1) + char device redirected to /dev/pts/57 (label serial2) + +- serial0: Console for the four Cortex-A35 primary processors. +- serial1 and serial2: Consoles for the two Cortex-M4 coprocessors. + +Use ``tio`` or another terminal emulator to connect to the consoles: + +.. code-block:: bash + + $ tio /dev/pts/55 + $ tio /dev/pts/56 + $ tio /dev/pts/57 + + Aspeed minibmc family boards (``ast1030-evb``) ================================================================== diff --git a/docs/system/gdb.rst b/docs/system/gdb.rst index 4228cb56bb..d50470b135 100644 --- a/docs/system/gdb.rst +++ b/docs/system/gdb.rst @@ -20,7 +20,7 @@ connection, use the ``-gdb dev`` option instead of ``-s``. See .. parsed-literal:: - |qemu_system| -s -S -kernel bzImage -hda rootdisk.img -append "root=/dev/hda" + |qemu_system| -s -S -kernel bzImage -drive file=rootdisk.img,format=raw -append "root=/dev/sda" QEMU will launch but will silently wait for gdb to connect. diff --git a/docs/system/linuxboot.rst b/docs/system/linuxboot.rst index 5db2e560dc..2328b4a73d 100644 --- a/docs/system/linuxboot.rst +++ b/docs/system/linuxboot.rst @@ -11,7 +11,7 @@ The syntax is: .. parsed-literal:: - |qemu_system| -kernel bzImage -hda rootdisk.img -append "root=/dev/hda" + |qemu_system| -kernel bzImage -drive file=rootdisk.img,format=raw -append "root=/dev/sda" Use ``-kernel`` to provide the Linux kernel image and ``-append`` to give the kernel command line arguments. The ``-initrd`` option can be @@ -23,8 +23,8 @@ virtual serial port and the QEMU monitor to the console with the .. parsed-literal:: - |qemu_system| -kernel bzImage -hda rootdisk.img \ - -append "root=/dev/hda console=ttyS0" -nographic + |qemu_system| -kernel bzImage -drive file=rootdisk.img,format=raw \ + -append "root=/dev/sda console=ttyS0" -nographic Use Ctrl-a c to switch between the serial console and the monitor (see :ref:`GUI_keys`). diff --git a/docs/system/qemu-block-drivers.rst.inc b/docs/system/qemu-block-drivers.rst.inc index cfe1acb78a..384e95ba76 100644 --- a/docs/system/qemu-block-drivers.rst.inc +++ b/docs/system/qemu-block-drivers.rst.inc @@ -500,8 +500,6 @@ What you should *never* do: - expect it to work when loadvm'ing - write to the FAT directory on the host system while accessing it with the guest system -.. _nbd: - NBD access ~~~~~~~~~~ diff --git a/docs/system/target-mips.rst b/docs/system/target-mips.rst index 83239fb9df..9028c3b304 100644 --- a/docs/system/target-mips.rst +++ b/docs/system/target-mips.rst @@ -112,5 +112,5 @@ https://mipsdistros.mips.com/LinuxDistro/nanomips/kernels/v4.15.18-432-gb2eb9a8b Start system emulation of Malta board with nanoMIPS I7200 CPU:: qemu-system-mipsel -cpu I7200 -kernel <kernel_image_file> \ - -M malta -serial stdio -m <memory_size> -hda <disk_image_file> \ + -M malta -serial stdio -m <memory_size> -drive file=<disk_image_file>,format=raw \ -append "mem=256m@0x0 rw console=ttyS0 vga=cirrus vesa=0x111 root=/dev/sda" diff --git a/fsdev/file-op-9p.h b/fsdev/file-op-9p.h index 4997677460..b9dae8c84c 100644 --- a/fsdev/file-op-9p.h +++ b/fsdev/file-op-9p.h @@ -129,6 +129,8 @@ struct FileOperations { int (*chown)(FsContext *, V9fsPath *, FsCred *); int (*mknod)(FsContext *, V9fsPath *, const char *, FsCred *); int (*utimensat)(FsContext *, V9fsPath *, const struct timespec *); + int (*futimens)(FsContext *ctx, int fid_type, V9fsFidOpenState *fs, + const struct timespec *times); int (*remove)(FsContext *, const char *); int (*symlink)(FsContext *, const char *, V9fsPath *, const char *, FsCred *); @@ -152,6 +154,8 @@ struct FileOperations { int (*fstat)(FsContext *, int, V9fsFidOpenState *, struct stat *); int (*rename)(FsContext *, const char *, const char *); int (*truncate)(FsContext *, V9fsPath *, off_t); + int (*ftruncate)(FsContext *ctx, int fid_type, V9fsFidOpenState *fs, + off_t size); int (*fsync)(FsContext *, int, V9fsFidOpenState *, int); int (*statfs)(FsContext *s, V9fsPath *path, struct statfs *stbuf); ssize_t (*lgetxattr)(FsContext *, V9fsPath *, @@ -164,6 +168,7 @@ struct FileOperations { int (*renameat)(FsContext *ctx, V9fsPath *olddir, const char *old_name, V9fsPath *newdir, const char *new_name); int (*unlinkat)(FsContext *ctx, V9fsPath *dir, const char *name, int flags); + bool (*has_valid_file_handle)(int fid_type, V9fsFidOpenState *fs); }; #endif diff --git a/hw/9pfs/9p-local.c b/hw/9pfs/9p-local.c index 928523afcc..31e216227c 100644 --- a/hw/9pfs/9p-local.c +++ b/hw/9pfs/9p-local.c @@ -766,16 +766,19 @@ out: return err; } -static int local_fstat(FsContext *fs_ctx, int fid_type, - V9fsFidOpenState *fs, struct stat *stbuf) +static int local_fid_fd(int fid_type, V9fsFidOpenState *fs) { - int err, fd; - if (fid_type == P9_FID_DIR) { - fd = dirfd(fs->dir.stream); + return dirfd(fs->dir.stream); } else { - fd = fs->fd; + return fs->fd; } +} + +static int local_fstat(FsContext *fs_ctx, int fid_type, + V9fsFidOpenState *fs, struct stat *stbuf) +{ + int err, fd = local_fid_fd(fid_type, fs); err = fstat(fd, stbuf); if (err) { @@ -1039,6 +1042,14 @@ static int local_truncate(FsContext *ctx, V9fsPath *fs_path, off_t size) return ret; } +static int local_ftruncate(FsContext *ctx, int fid_type, V9fsFidOpenState *fs, + off_t size) +{ + int fd = local_fid_fd(fid_type, fs); + + return ftruncate(fd, size); +} + static int local_chown(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp) { char *dirpath = g_path_get_dirname(fs_path->data); @@ -1089,6 +1100,14 @@ out: return ret; } +static int local_futimens(FsContext *s, int fid_type, V9fsFidOpenState *fs, + const struct timespec *times) +{ + int fd = local_fid_fd(fid_type, fs); + + return qemu_futimens(fd, times); +} + static int local_unlinkat_common(FsContext *ctx, int dirfd, const char *name, int flags) { @@ -1167,13 +1186,7 @@ out: static int local_fsync(FsContext *ctx, int fid_type, V9fsFidOpenState *fs, int datasync) { - int fd; - - if (fid_type == P9_FID_DIR) { - fd = dirfd(fs->dir.stream); - } else { - fd = fs->fd; - } + int fd = local_fid_fd(fid_type, fs); if (datasync) { return qemu_fdatasync(fd); @@ -1575,6 +1588,13 @@ static int local_parse_opts(QemuOpts *opts, FsDriverEntry *fse, Error **errp) return 0; } +static bool local_has_valid_file_handle(int fid_type, V9fsFidOpenState *fs) +{ + return + (fid_type == P9_FID_FILE && fs->fd != -1) || + (fid_type == P9_FID_DIR && fs->dir.stream != NULL); +} + FileOperations local_ops = { .parse_opts = local_parse_opts, .init = local_init, @@ -1612,4 +1632,7 @@ FileOperations local_ops = { .name_to_path = local_name_to_path, .renameat = local_renameat, .unlinkat = local_unlinkat, + .has_valid_file_handle = local_has_valid_file_handle, + .ftruncate = local_ftruncate, + .futimens = local_futimens, }; diff --git a/hw/9pfs/9p-synth.c b/hw/9pfs/9p-synth.c index 2abaf3a291..9cd1884224 100644 --- a/hw/9pfs/9p-synth.c +++ b/hw/9pfs/9p-synth.c @@ -356,6 +356,13 @@ static int synth_truncate(FsContext *ctx, V9fsPath *path, off_t offset) return -1; } +static int synth_ftruncate(FsContext *ctx, int fid_type, V9fsFidOpenState *fs, + off_t size) +{ + errno = ENOSYS; + return -1; +} + static int synth_chmod(FsContext *fs_ctx, V9fsPath *path, FsCred *credp) { errno = EPERM; @@ -417,6 +424,13 @@ static int synth_utimensat(FsContext *fs_ctx, V9fsPath *path, return 0; } +static int synth_futimens(FsContext *fs_ctx, int fid_type, V9fsFidOpenState *fs, + const struct timespec *buf) +{ + errno = ENOSYS; + return -1; +} + static int synth_remove(FsContext *ctx, const char *path) { errno = EPERM; @@ -615,6 +629,11 @@ static int synth_init(FsContext *ctx, Error **errp) return 0; } +static bool synth_has_valid_file_handle(int fid_type, V9fsFidOpenState *fs) +{ + return false; +} + FileOperations synth_ops = { .init = synth_init, .lstat = synth_lstat, @@ -650,4 +669,7 @@ FileOperations synth_ops = { .name_to_path = synth_name_to_path, .renameat = synth_renameat, .unlinkat = synth_unlinkat, + .has_valid_file_handle = synth_has_valid_file_handle, + .ftruncate = synth_ftruncate, + .futimens = synth_futimens, }; diff --git a/hw/9pfs/9p-util.h b/hw/9pfs/9p-util.h index 7bc4ec8e85..a1924fe3f0 100644 --- a/hw/9pfs/9p-util.h +++ b/hw/9pfs/9p-util.h @@ -103,6 +103,7 @@ static inline int errno_to_dotl(int err) { #define qemu_renameat renameat #define qemu_utimensat utimensat #define qemu_unlinkat unlinkat +#define qemu_futimens futimens static inline void close_preserve_errno(int fd) { diff --git a/hw/9pfs/9p.c b/hw/9pfs/9p.c index 7cad2bce62..8b001b9112 100644 --- a/hw/9pfs/9p.c +++ b/hw/9pfs/9p.c @@ -434,16 +434,24 @@ void coroutine_fn v9fs_reclaim_fd(V9fsPDU *pdu) V9fsFidState *f; GHashTableIter iter; gpointer fid; + int err; + int nclosed = 0; + + /* prevent multiple coroutines running this function simultaniously */ + if (s->reclaiming) { + return; + } + s->reclaiming = true; g_hash_table_iter_init(&iter, s->fids); QSLIST_HEAD(, V9fsFidState) reclaim_list = QSLIST_HEAD_INITIALIZER(reclaim_list); + /* Pick FIDs to be closed, collect them on reclaim_list. */ while (g_hash_table_iter_next(&iter, &fid, (gpointer *) &f)) { /* - * Unlink fids cannot be reclaimed. Check - * for them and skip them. Also skip fids + * Unlinked fids cannot be reclaimed, skip those, and also skip fids * currently being operated on. */ if (f->ref || f->flags & FID_NON_RECLAIMABLE) { @@ -493,23 +501,42 @@ void coroutine_fn v9fs_reclaim_fd(V9fsPDU *pdu) } } /* - * Now close the fid in reclaim list. Free them if they - * are already clunked. + * Close the picked FIDs altogether on a background I/O driver thread. Do + * this all at once to keep latency (i.e. amount of thread hops between main + * thread <-> fs driver background thread) as low as possible. */ + v9fs_co_run_in_worker({ + QSLIST_FOREACH(f, &reclaim_list, reclaim_next) { + err = (f->fid_type == P9_FID_DIR) ? + s->ops->closedir(&s->ctx, &f->fs_reclaim) : + s->ops->close(&s->ctx, &f->fs_reclaim); + + /* 'man 2 close' suggests to ignore close() errors except of EBADF */ + if (unlikely(err && errno == EBADF)) { + /* + * unexpected case as FIDs were picked above by having a valid + * file descriptor + */ + error_report("9pfs: v9fs_reclaim_fd() WARNING: close() failed with EBADF"); + } else { + /* total_open_fd must only be mutated on main thread */ + nclosed++; + } + } + }); + total_open_fd -= nclosed; + /* Free the closed FIDs. */ while (!QSLIST_EMPTY(&reclaim_list)) { f = QSLIST_FIRST(&reclaim_list); QSLIST_REMOVE(&reclaim_list, f, V9fsFidState, reclaim_next); - if (f->fid_type == P9_FID_FILE) { - v9fs_co_close(pdu, &f->fs_reclaim); - } else if (f->fid_type == P9_FID_DIR) { - v9fs_co_closedir(pdu, &f->fs_reclaim); - } /* * Now drop the fid reference, free it * if clunked. */ put_fid(pdu, f); } + + s->reclaiming = false; } /* @@ -1574,6 +1601,11 @@ out_nofid: pdu_complete(pdu, err); } +static bool fid_has_valid_file_handle(V9fsState *s, V9fsFidState *fidp) +{ + return s->ops->has_valid_file_handle(fidp->fid_type, &fidp->fs); +} + static void coroutine_fn v9fs_getattr(void *opaque) { int32_t fid; @@ -1596,9 +1628,7 @@ static void coroutine_fn v9fs_getattr(void *opaque) retval = -ENOENT; goto out_nofid; } - if ((fidp->fid_type == P9_FID_FILE && fidp->fs.fd != -1) || - (fidp->fid_type == P9_FID_DIR && fidp->fs.dir.stream)) - { + if (fid_has_valid_file_handle(pdu->s, fidp)) { retval = v9fs_co_fstat(pdu, fidp, &stbuf); } else { retval = v9fs_co_lstat(pdu, &fidp->path, &stbuf); @@ -1705,7 +1735,11 @@ static void coroutine_fn v9fs_setattr(void *opaque) } else { times[1].tv_nsec = UTIME_OMIT; } - err = v9fs_co_utimensat(pdu, &fidp->path, times); + if (fid_has_valid_file_handle(pdu->s, fidp)) { + err = v9fs_co_futimens(pdu, fidp, times); + } else { + err = v9fs_co_utimensat(pdu, &fidp->path, times); + } if (err < 0) { goto out; } @@ -1730,7 +1764,11 @@ static void coroutine_fn v9fs_setattr(void *opaque) } } if (v9iattr.valid & (P9_ATTR_SIZE)) { - err = v9fs_co_truncate(pdu, &fidp->path, v9iattr.size); + if (fid_has_valid_file_handle(pdu->s, fidp)) { + err = v9fs_co_ftruncate(pdu, fidp, v9iattr.size); + } else { + err = v9fs_co_truncate(pdu, &fidp->path, v9iattr.size); + } if (err < 0) { goto out; } @@ -4324,6 +4362,8 @@ int v9fs_device_realize_common(V9fsState *s, const V9fsTransport *t, s->ctx.fst = &fse->fst; fsdev_throttle_init(s->ctx.fst); + s->reclaiming = false; + rc = 0; out: if (rc) { diff --git a/hw/9pfs/9p.h b/hw/9pfs/9p.h index 5e041e1f60..259ad32ed1 100644 --- a/hw/9pfs/9p.h +++ b/hw/9pfs/9p.h @@ -362,6 +362,7 @@ struct V9fsState { uint64_t qp_ndevices; /* Amount of entries in qpd_table. */ uint16_t qp_affix_next; uint64_t qp_fullpath_next; + bool reclaiming; }; /* 9p2000.L open flags */ diff --git a/hw/9pfs/codir.c b/hw/9pfs/codir.c index 2068a4779d..bce7dd96e9 100644 --- a/hw/9pfs/codir.c +++ b/hw/9pfs/codir.c @@ -20,6 +20,7 @@ #include "fsdev/qemu-fsdev.h" #include "qemu/thread.h" #include "qemu/main-loop.h" +#include "qemu/error-report.h" #include "coth.h" #include "9p-xattr.h" #include "9p-util.h" @@ -353,7 +354,11 @@ int coroutine_fn v9fs_co_closedir(V9fsPDU *pdu, V9fsFidOpenState *fs) err = -errno; } }); - if (!err) { + /* 'man 2 close' suggests to ignore close() errors except of EBADF */ + if (unlikely(err && errno == EBADF)) { + /* unexpected case as we should have checked for a valid file handle */ + error_report("9pfs: WARNING: v9fs_co_closedir() failed with EBADF"); + } else { total_open_fd--; } return err; diff --git a/hw/9pfs/cofile.c b/hw/9pfs/cofile.c index 71174c3e4a..6e775c8e41 100644 --- a/hw/9pfs/cofile.c +++ b/hw/9pfs/cofile.c @@ -20,6 +20,7 @@ #include "fsdev/qemu-fsdev.h" #include "qemu/thread.h" #include "qemu/main-loop.h" +#include "qemu/error-report.h" #include "coth.h" int coroutine_fn v9fs_co_st_gen(V9fsPDU *pdu, V9fsPath *path, mode_t st_mode, @@ -197,7 +198,11 @@ int coroutine_fn v9fs_co_close(V9fsPDU *pdu, V9fsFidOpenState *fs) err = -errno; } }); - if (!err) { + /* 'man 2 close' suggests to ignore close() errors except of EBADF */ + if (unlikely(err && errno == EBADF)) { + /* unexpected case as we should have checked for a valid file handle */ + error_report("9pfs: WARNING: v9fs_co_close() failed with EBADF"); + } else { total_open_fd--; } return err; diff --git a/hw/9pfs/cofs.c b/hw/9pfs/cofs.c index 67e3ae5c5c..12fa8c9fe9 100644 --- a/hw/9pfs/cofs.c +++ b/hw/9pfs/cofs.c @@ -139,6 +139,25 @@ int coroutine_fn v9fs_co_utimensat(V9fsPDU *pdu, V9fsPath *path, return err; } +int coroutine_fn v9fs_co_futimens(V9fsPDU *pdu, V9fsFidState *fidp, + struct timespec times[2]) +{ + int err; + V9fsState *s = pdu->s; + + if (v9fs_request_cancelled(pdu)) { + return -EINTR; + } + v9fs_co_run_in_worker( + { + err = s->ops->futimens(&s->ctx, fidp->fid_type, &fidp->fs, times); + if (err < 0) { + err = -errno; + } + }); + return err; +} + int coroutine_fn v9fs_co_chown(V9fsPDU *pdu, V9fsPath *path, uid_t uid, gid_t gid) { @@ -184,6 +203,24 @@ int coroutine_fn v9fs_co_truncate(V9fsPDU *pdu, V9fsPath *path, off_t size) return err; } +int coroutine_fn v9fs_co_ftruncate(V9fsPDU *pdu, V9fsFidState *fidp, off_t size) +{ + int err; + V9fsState *s = pdu->s; + + if (v9fs_request_cancelled(pdu)) { + return -EINTR; + } + v9fs_co_run_in_worker( + { + err = s->ops->ftruncate(&s->ctx, fidp->fid_type, &fidp->fs, size); + if (err < 0) { + err = -errno; + } + }); + return err; +} + int coroutine_fn v9fs_co_mknod(V9fsPDU *pdu, V9fsFidState *fidp, V9fsString *name, uid_t uid, gid_t gid, dev_t dev, mode_t mode, struct stat *stbuf) diff --git a/hw/9pfs/coth.h b/hw/9pfs/coth.h index 2c54249b35..7906fa7782 100644 --- a/hw/9pfs/coth.h +++ b/hw/9pfs/coth.h @@ -71,8 +71,12 @@ int coroutine_fn v9fs_co_statfs(V9fsPDU *, V9fsPath *, struct statfs *); int coroutine_fn v9fs_co_lstat(V9fsPDU *, V9fsPath *, struct stat *); int coroutine_fn v9fs_co_chmod(V9fsPDU *, V9fsPath *, mode_t); int coroutine_fn v9fs_co_utimensat(V9fsPDU *, V9fsPath *, struct timespec [2]); +int coroutine_fn v9fs_co_futimens(V9fsPDU *pdu, V9fsFidState *fidp, + struct timespec times[2]); int coroutine_fn v9fs_co_chown(V9fsPDU *, V9fsPath *, uid_t, gid_t); int coroutine_fn v9fs_co_truncate(V9fsPDU *, V9fsPath *, off_t); +int coroutine_fn v9fs_co_ftruncate(V9fsPDU *pdu, V9fsFidState *fidp, + off_t size); int coroutine_fn v9fs_co_llistxattr(V9fsPDU *, V9fsPath *, void *, size_t); int coroutine_fn v9fs_co_lgetxattr(V9fsPDU *, V9fsPath *, V9fsString *, void *, size_t); diff --git a/hw/acpi/acpi-pci-hotplug-stub.c b/hw/acpi/acpi-pci-hotplug-stub.c index b67b4a92da..b7bc6e40a1 100644 --- a/hw/acpi/acpi-pci-hotplug-stub.c +++ b/hw/acpi/acpi-pci-hotplug-stub.c @@ -34,7 +34,7 @@ void acpi_pcihp_reset(AcpiPciHpState *s) { } -bool acpi_pcihp_is_hotpluggbale_bus(AcpiPciHpState *s, BusState *bus) +bool acpi_pcihp_is_hotpluggable_bus(AcpiPciHpState *s, BusState *bus) { return true; } diff --git a/hw/acpi/generic_event_device.c b/hw/acpi/generic_event_device.c index d8adfea648..7a62f8d5bc 100644 --- a/hw/acpi/generic_event_device.c +++ b/hw/acpi/generic_event_device.c @@ -458,11 +458,11 @@ static void acpi_ged_initfn(Object *obj) * container for memory hotplug IO and expose it as GED sysbus * MMIO so that boards can map it separately. */ - memory_region_init(&s->container_memhp, OBJECT(dev), "memhp container", - MEMORY_HOTPLUG_IO_LEN); - sysbus_init_mmio(sbd, &s->container_memhp); - acpi_memory_hotplug_init(&s->container_memhp, OBJECT(dev), - &s->memhp_state, 0); + memory_region_init(&s->container_memhp, OBJECT(dev), "memhp container", + MEMORY_HOTPLUG_IO_LEN); + sysbus_init_mmio(sbd, &s->container_memhp); + acpi_memory_hotplug_init(&s->container_memhp, OBJECT(dev), + &s->memhp_state, 0); memory_region_init_io(&ged_st->regs, obj, &ged_regs_ops, ged_st, TYPE_ACPI_GED "-regs", ACPI_GED_REG_COUNT); diff --git a/hw/acpi/ich9.c b/hw/acpi/ich9.c index c7a735bf64..967b67485e 100644 --- a/hw/acpi/ich9.c +++ b/hw/acpi/ich9.c @@ -41,15 +41,6 @@ #include "hw/mem/pc-dimm.h" #include "hw/mem/nvdimm.h" -//#define DEBUG - -#ifdef DEBUG -#define ICH9_DEBUG(fmt, ...) \ -do { printf("%s "fmt, __func__, ## __VA_ARGS__); } while (0) -#else -#define ICH9_DEBUG(fmt, ...) do { } while (0) -#endif - static void ich9_pm_update_sci_fn(ACPIREGS *regs) { ICH9LPCPMRegs *pm = container_of(regs, ICH9LPCPMRegs, acpi_regs); @@ -135,8 +126,6 @@ static const MemoryRegionOps ich9_smi_ops = { void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base) { - ICH9_DEBUG("to 0x%x\n", pm_io_base); - assert((pm_io_base & ICH9_PMIO_MASK) == 0); pm->pm_io_base = pm_io_base; @@ -570,7 +559,7 @@ void ich9_pm_device_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, bool ich9_pm_is_hotpluggable_bus(HotplugHandler *hotplug_dev, BusState *bus) { ICH9LPCState *lpc = ICH9_LPC_DEVICE(hotplug_dev); - return acpi_pcihp_is_hotpluggbale_bus(&lpc->pm.acpi_pci_hotplug, bus); + return acpi_pcihp_is_hotpluggable_bus(&lpc->pm.acpi_pci_hotplug, bus); } void ich9_pm_ospm_status(AcpiDeviceIf *adev, ACPIOSTInfoList ***list) diff --git a/hw/acpi/pcihp.c b/hw/acpi/pcihp.c index 5f79c9016b..aac90013d4 100644 --- a/hw/acpi/pcihp.c +++ b/hw/acpi/pcihp.c @@ -371,7 +371,7 @@ void acpi_pcihp_device_unplug_request_cb(HotplugHandler *hotplug_dev, acpi_send_event(DEVICE(hotplug_dev), ACPI_PCI_HOTPLUG_STATUS); } -bool acpi_pcihp_is_hotpluggbale_bus(AcpiPciHpState *s, BusState *bus) +bool acpi_pcihp_is_hotpluggable_bus(AcpiPciHpState *s, BusState *bus) { Object *o = OBJECT(bus->parent); diff --git a/hw/acpi/piix4.c b/hw/acpi/piix4.c index b16d45f03e..d98b80df6d 100644 --- a/hw/acpi/piix4.c +++ b/hw/acpi/piix4.c @@ -406,7 +406,7 @@ static bool piix4_is_hotpluggable_bus(HotplugHandler *hotplug_dev, BusState *bus) { PIIX4PMState *s = PIIX4_PM(hotplug_dev); - return acpi_pcihp_is_hotpluggbale_bus(&s->acpi_pci_hotplug, bus); + return acpi_pcihp_is_hotpluggable_bus(&s->acpi_pci_hotplug, bus); } static void piix4_pm_machine_ready(Notifier *n, void *opaque) diff --git a/hw/arm/aspeed.c b/hw/arm/aspeed.c index 20f418fb63..d0b333646e 100644 --- a/hw/arm/aspeed.c +++ b/hw/arm/aspeed.c @@ -27,6 +27,7 @@ #include "system/reset.h" #include "hw/loader.h" #include "qemu/error-report.h" +#include "qemu/datadir.h" #include "qemu/units.h" #include "hw/qdev-clock.h" #include "system/system.h" @@ -305,6 +306,33 @@ static void aspeed_install_boot_rom(AspeedMachineState *bmc, BlockBackend *blk, rom_size, &error_abort); } +#define VBOOTROM_FILE_NAME "ast27x0_bootrom.bin" + +/* + * This function locates the vbootrom image file specified via the command line + * using the -bios option. It loads the specified image into the vbootrom + * memory region and handles errors if the file cannot be found or loaded. + */ +static void aspeed_load_vbootrom(AspeedMachineState *bmc, const char *bios_name, + Error **errp) +{ + g_autofree char *filename = NULL; + AspeedSoCState *soc = bmc->soc; + int ret; + + filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name); + if (!filename) { + error_setg(errp, "Could not find vbootrom image '%s'", bios_name); + return; + } + + ret = load_image_mr(filename, &soc->vbootrom); + if (ret < 0) { + error_setg(errp, "Failed to load vbootrom image '%s'", bios_name); + return; + } +} + void aspeed_board_init_flashes(AspeedSMCState *s, const char *flashtype, unsigned int count, int unit0) { @@ -380,6 +408,7 @@ static void aspeed_machine_init(MachineState *machine) AspeedMachineClass *amc = ASPEED_MACHINE_GET_CLASS(machine); AspeedSoCClass *sc; int i; + const char *bios_name = NULL; DriveInfo *emmc0 = NULL; bool boot_emmc; @@ -482,6 +511,11 @@ static void aspeed_machine_init(MachineState *machine) } } + if (amc->vbootrom) { + bios_name = machine->firmware ?: VBOOTROM_FILE_NAME; + aspeed_load_vbootrom(bmc, bios_name, &error_abort); + } + arm_load_kernel(ARM_CPU(first_cpu), machine, &aspeed_board_binfo); } @@ -1701,6 +1735,7 @@ static void aspeed_machine_ast2700a0_evb_class_init(ObjectClass *oc, amc->macs_mask = ASPEED_MAC0_ON | ASPEED_MAC1_ON | ASPEED_MAC2_ON; amc->uart_default = ASPEED_DEV_UART12; amc->i2c_init = ast2700_evb_i2c_init; + amc->vbootrom = true; mc->auto_create_sdcard = true; mc->default_ram_size = 1 * GiB; aspeed_machine_class_init_cpus_defaults(mc); @@ -1722,6 +1757,7 @@ static void aspeed_machine_ast2700a1_evb_class_init(ObjectClass *oc, amc->macs_mask = ASPEED_MAC0_ON | ASPEED_MAC1_ON | ASPEED_MAC2_ON; amc->uart_default = ASPEED_DEV_UART12; amc->i2c_init = ast2700_evb_i2c_init; + amc->vbootrom = true; mc->auto_create_sdcard = true; mc->default_ram_size = 1 * GiB; aspeed_machine_class_init_cpus_defaults(mc); diff --git a/hw/arm/aspeed_ast27x0-fc.c b/hw/arm/aspeed_ast27x0-fc.c new file mode 100644 index 0000000000..125a3ade40 --- /dev/null +++ b/hw/arm/aspeed_ast27x0-fc.c @@ -0,0 +1,192 @@ +/* + * ASPEED SoC 2700 family + * + * Copyright (C) 2025 ASPEED Technology Inc. + * + * This code is licensed under the GPL version 2 or later. See + * the COPYING file in the top-level directory. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "qemu/units.h" +#include "qapi/error.h" +#include "system/block-backend.h" +#include "system/system.h" +#include "hw/arm/aspeed.h" +#include "hw/boards.h" +#include "hw/qdev-clock.h" +#include "hw/arm/aspeed_soc.h" +#include "hw/loader.h" +#include "hw/arm/boot.h" +#include "hw/block/flash.h" + + +#define TYPE_AST2700A1FC MACHINE_TYPE_NAME("ast2700fc") +OBJECT_DECLARE_SIMPLE_TYPE(Ast2700FCState, AST2700A1FC); + +static struct arm_boot_info ast2700fc_board_info = { + .board_id = -1, /* device-tree-only board */ +}; + +struct Ast2700FCState { + MachineState parent_obj; + + MemoryRegion ca35_memory; + MemoryRegion ca35_dram; + MemoryRegion ssp_memory; + MemoryRegion tsp_memory; + + Clock *ssp_sysclk; + Clock *tsp_sysclk; + + Aspeed27x0SoCState ca35; + Aspeed27x0SSPSoCState ssp; + Aspeed27x0TSPSoCState tsp; + + bool mmio_exec; +}; + +#define AST2700FC_BMC_RAM_SIZE (2 * GiB) +#define AST2700FC_CM4_DRAM_SIZE (32 * MiB) + +#define AST2700FC_HW_STRAP1 0x000000C0 +#define AST2700FC_HW_STRAP2 0x00000003 +#define AST2700FC_FMC_MODEL "w25q01jvq" +#define AST2700FC_SPI_MODEL "w25q512jv" + +static void ast2700fc_ca35_init(MachineState *machine) +{ + Ast2700FCState *s = AST2700A1FC(machine); + AspeedSoCState *soc; + AspeedSoCClass *sc; + + object_initialize_child(OBJECT(s), "ca35", &s->ca35, "ast2700-a1"); + soc = ASPEED_SOC(&s->ca35); + sc = ASPEED_SOC_GET_CLASS(soc); + + memory_region_init(&s->ca35_memory, OBJECT(&s->ca35), "ca35-memory", + UINT64_MAX); + + if (!memory_region_init_ram(&s->ca35_dram, OBJECT(&s->ca35), "ca35-dram", + AST2700FC_BMC_RAM_SIZE, &error_abort)) { + return; + } + if (!object_property_set_link(OBJECT(&s->ca35), "memory", + OBJECT(&s->ca35_memory), + &error_abort)) { + return; + }; + if (!object_property_set_link(OBJECT(&s->ca35), "dram", + OBJECT(&s->ca35_dram), &error_abort)) { + return; + } + if (!object_property_set_int(OBJECT(&s->ca35), "ram-size", + AST2700FC_BMC_RAM_SIZE, &error_abort)) { + return; + } + if (!object_property_set_int(OBJECT(&s->ca35), "hw-strap1", + AST2700FC_HW_STRAP1, &error_abort)) { + return; + } + if (!object_property_set_int(OBJECT(&s->ca35), "hw-strap2", + AST2700FC_HW_STRAP2, &error_abort)) { + return; + } + aspeed_soc_uart_set_chr(soc, ASPEED_DEV_UART12, serial_hd(0)); + if (!qdev_realize(DEVICE(&s->ca35), NULL, &error_abort)) { + return; + } + + /* + * AST2700 EVB has a LM75 temperature sensor on I2C bus 0 at address 0x4d. + */ + i2c_slave_create_simple(aspeed_i2c_get_bus(&soc->i2c, 0), "tmp105", 0x4d); + + aspeed_board_init_flashes(&soc->fmc, AST2700FC_FMC_MODEL, 2, 0); + aspeed_board_init_flashes(&soc->spi[0], AST2700FC_SPI_MODEL, 1, 2); + + ast2700fc_board_info.ram_size = machine->ram_size; + ast2700fc_board_info.loader_start = sc->memmap[ASPEED_DEV_SDRAM]; + + arm_load_kernel(ARM_CPU(first_cpu), machine, &ast2700fc_board_info); +} + +static void ast2700fc_ssp_init(MachineState *machine) +{ + AspeedSoCState *soc; + Ast2700FCState *s = AST2700A1FC(machine); + s->ssp_sysclk = clock_new(OBJECT(s), "SSP_SYSCLK"); + clock_set_hz(s->ssp_sysclk, 200000000ULL); + + object_initialize_child(OBJECT(s), "ssp", &s->ssp, TYPE_ASPEED27X0SSP_SOC); + memory_region_init(&s->ssp_memory, OBJECT(&s->ssp), "ssp-memory", + UINT64_MAX); + + qdev_connect_clock_in(DEVICE(&s->ssp), "sysclk", s->ssp_sysclk); + if (!object_property_set_link(OBJECT(&s->ssp), "memory", + OBJECT(&s->ssp_memory), &error_abort)) { + return; + } + + soc = ASPEED_SOC(&s->ssp); + aspeed_soc_uart_set_chr(soc, ASPEED_DEV_UART4, serial_hd(1)); + if (!qdev_realize(DEVICE(&s->ssp), NULL, &error_abort)) { + return; + } +} + +static void ast2700fc_tsp_init(MachineState *machine) +{ + AspeedSoCState *soc; + Ast2700FCState *s = AST2700A1FC(machine); + s->tsp_sysclk = clock_new(OBJECT(s), "TSP_SYSCLK"); + clock_set_hz(s->tsp_sysclk, 200000000ULL); + + object_initialize_child(OBJECT(s), "tsp", &s->tsp, TYPE_ASPEED27X0TSP_SOC); + memory_region_init(&s->tsp_memory, OBJECT(&s->tsp), "tsp-memory", + UINT64_MAX); + + qdev_connect_clock_in(DEVICE(&s->tsp), "sysclk", s->tsp_sysclk); + if (!object_property_set_link(OBJECT(&s->tsp), "memory", + OBJECT(&s->tsp_memory), &error_abort)) { + return; + } + + soc = ASPEED_SOC(&s->tsp); + aspeed_soc_uart_set_chr(soc, ASPEED_DEV_UART7, serial_hd(2)); + if (!qdev_realize(DEVICE(&s->tsp), NULL, &error_abort)) { + return; + } +} + +static void ast2700fc_init(MachineState *machine) +{ + ast2700fc_ca35_init(machine); + ast2700fc_ssp_init(machine); + ast2700fc_tsp_init(machine); +} + +static void ast2700fc_class_init(ObjectClass *oc, const void *data) +{ + MachineClass *mc = MACHINE_CLASS(oc); + + mc->alias = "ast2700fc"; + mc->desc = "ast2700 full core support"; + mc->init = ast2700fc_init; + mc->no_floppy = 1; + mc->no_cdrom = 1; + mc->min_cpus = mc->max_cpus = mc->default_cpus = 6; +} + +static const TypeInfo ast2700fc_types[] = { + { + .name = MACHINE_TYPE_NAME("ast2700fc"), + .parent = TYPE_MACHINE, + .class_init = ast2700fc_class_init, + .instance_size = sizeof(Ast2700FCState), + }, +}; + +DEFINE_TYPES(ast2700fc_types) diff --git a/hw/arm/aspeed_ast27x0-ssp.c b/hw/arm/aspeed_ast27x0-ssp.c new file mode 100644 index 0000000000..80ec5996c1 --- /dev/null +++ b/hw/arm/aspeed_ast27x0-ssp.c @@ -0,0 +1,294 @@ +/* + * ASPEED Ast27x0 SSP SoC + * + * Copyright (C) 2025 ASPEED Technology Inc. + * + * This code is licensed under the GPL version 2 or later. See + * the COPYING file in the top-level directory. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "qapi/error.h" +#include "hw/qdev-clock.h" +#include "hw/misc/unimp.h" +#include "hw/arm/aspeed_soc.h" + +#define AST2700_SSP_RAM_SIZE (32 * MiB) + +static const hwaddr aspeed_soc_ast27x0ssp_memmap[] = { + [ASPEED_DEV_SRAM] = 0x00000000, + [ASPEED_DEV_INTC] = 0x72100000, + [ASPEED_DEV_SCU] = 0x72C02000, + [ASPEED_DEV_SCUIO] = 0x74C02000, + [ASPEED_DEV_UART0] = 0x74C33000, + [ASPEED_DEV_UART1] = 0x74C33100, + [ASPEED_DEV_UART2] = 0x74C33200, + [ASPEED_DEV_UART3] = 0x74C33300, + [ASPEED_DEV_UART4] = 0x72C1A000, + [ASPEED_DEV_INTCIO] = 0x74C18000, + [ASPEED_DEV_IPC0] = 0x72C1C000, + [ASPEED_DEV_IPC1] = 0x74C39000, + [ASPEED_DEV_UART5] = 0x74C33400, + [ASPEED_DEV_UART6] = 0x74C33500, + [ASPEED_DEV_UART7] = 0x74C33600, + [ASPEED_DEV_UART8] = 0x74C33700, + [ASPEED_DEV_UART9] = 0x74C33800, + [ASPEED_DEV_UART10] = 0x74C33900, + [ASPEED_DEV_UART11] = 0x74C33A00, + [ASPEED_DEV_UART12] = 0x74C33B00, + [ASPEED_DEV_TIMER1] = 0x72C10000, +}; + +static const int aspeed_soc_ast27x0ssp_irqmap[] = { + [ASPEED_DEV_SCU] = 12, + [ASPEED_DEV_UART0] = 164, + [ASPEED_DEV_UART1] = 164, + [ASPEED_DEV_UART2] = 164, + [ASPEED_DEV_UART3] = 164, + [ASPEED_DEV_UART4] = 8, + [ASPEED_DEV_UART5] = 164, + [ASPEED_DEV_UART6] = 164, + [ASPEED_DEV_UART7] = 164, + [ASPEED_DEV_UART8] = 164, + [ASPEED_DEV_UART9] = 164, + [ASPEED_DEV_UART10] = 164, + [ASPEED_DEV_UART11] = 164, + [ASPEED_DEV_UART12] = 164, + [ASPEED_DEV_TIMER1] = 16, +}; + +/* SSPINT 164 */ +static const int ast2700_ssp132_ssp164_intcmap[] = { + [ASPEED_DEV_UART0] = 7, + [ASPEED_DEV_UART1] = 8, + [ASPEED_DEV_UART2] = 9, + [ASPEED_DEV_UART3] = 10, + [ASPEED_DEV_UART5] = 11, + [ASPEED_DEV_UART6] = 12, + [ASPEED_DEV_UART7] = 13, + [ASPEED_DEV_UART8] = 14, + [ASPEED_DEV_UART9] = 15, + [ASPEED_DEV_UART10] = 16, + [ASPEED_DEV_UART11] = 17, + [ASPEED_DEV_UART12] = 18, +}; + +struct nvic_intc_irq_info { + int irq; + int intc_idx; + int orgate_idx; + const int *ptr; +}; + +static struct nvic_intc_irq_info ast2700_ssp_intcmap[] = { + {160, 1, 0, NULL}, + {161, 1, 1, NULL}, + {162, 1, 2, NULL}, + {163, 1, 3, NULL}, + {164, 1, 4, ast2700_ssp132_ssp164_intcmap}, + {165, 1, 5, NULL}, + {166, 1, 6, NULL}, + {167, 1, 7, NULL}, + {168, 1, 8, NULL}, + {169, 1, 9, NULL}, + {128, 0, 1, NULL}, + {129, 0, 2, NULL}, + {130, 0, 3, NULL}, + {131, 0, 4, NULL}, + {132, 0, 5, ast2700_ssp132_ssp164_intcmap}, + {133, 0, 6, NULL}, + {134, 0, 7, NULL}, + {135, 0, 8, NULL}, + {136, 0, 9, NULL}, +}; + +static qemu_irq aspeed_soc_ast27x0ssp_get_irq(AspeedSoCState *s, int dev) +{ + Aspeed27x0SSPSoCState *a = ASPEED27X0SSP_SOC(s); + AspeedSoCClass *sc = ASPEED_SOC_GET_CLASS(s); + + int or_idx; + int idx; + int i; + + for (i = 0; i < ARRAY_SIZE(ast2700_ssp_intcmap); i++) { + if (sc->irqmap[dev] == ast2700_ssp_intcmap[i].irq) { + assert(ast2700_ssp_intcmap[i].ptr); + or_idx = ast2700_ssp_intcmap[i].orgate_idx; + idx = ast2700_ssp_intcmap[i].intc_idx; + return qdev_get_gpio_in(DEVICE(&a->intc[idx].orgates[or_idx]), + ast2700_ssp_intcmap[i].ptr[dev]); + } + } + + return qdev_get_gpio_in(DEVICE(&a->armv7m), sc->irqmap[dev]); +} + +static void aspeed_soc_ast27x0ssp_init(Object *obj) +{ + Aspeed27x0SSPSoCState *a = ASPEED27X0SSP_SOC(obj); + AspeedSoCState *s = ASPEED_SOC(obj); + AspeedSoCClass *sc = ASPEED_SOC_GET_CLASS(s); + int i; + + object_initialize_child(obj, "armv7m", &a->armv7m, TYPE_ARMV7M); + object_initialize_child(obj, "scu", &s->scu, TYPE_ASPEED_2700_SCU); + s->sysclk = qdev_init_clock_in(DEVICE(s), "sysclk", NULL, NULL, 0); + qdev_prop_set_uint32(DEVICE(&s->scu), "silicon-rev", sc->silicon_rev); + + for (i = 0; i < sc->uarts_num; i++) { + object_initialize_child(obj, "uart[*]", &s->uart[i], TYPE_SERIAL_MM); + } + + object_initialize_child(obj, "intc0", &a->intc[0], + TYPE_ASPEED_2700SSP_INTC); + object_initialize_child(obj, "intc1", &a->intc[1], + TYPE_ASPEED_2700SSP_INTCIO); + + object_initialize_child(obj, "timerctrl", &s->timerctrl, + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "ipc0", &a->ipc[0], + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "ipc1", &a->ipc[1], + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "scuio", &a->scuio, + TYPE_UNIMPLEMENTED_DEVICE); +} + +static void aspeed_soc_ast27x0ssp_realize(DeviceState *dev_soc, Error **errp) +{ + Aspeed27x0SSPSoCState *a = ASPEED27X0SSP_SOC(dev_soc); + AspeedSoCState *s = ASPEED_SOC(dev_soc); + AspeedSoCClass *sc = ASPEED_SOC_GET_CLASS(s); + DeviceState *armv7m; + g_autofree char *sram_name = NULL; + int i; + + if (!clock_has_source(s->sysclk)) { + error_setg(errp, "sysclk clock must be wired up by the board code"); + return; + } + + /* AST27X0 SSP Core */ + armv7m = DEVICE(&a->armv7m); + qdev_prop_set_uint32(armv7m, "num-irq", 256); + qdev_prop_set_string(armv7m, "cpu-type", aspeed_soc_cpu_type(sc)); + qdev_connect_clock_in(armv7m, "cpuclk", s->sysclk); + object_property_set_link(OBJECT(&a->armv7m), "memory", + OBJECT(s->memory), &error_abort); + sysbus_realize(SYS_BUS_DEVICE(&a->armv7m), &error_abort); + + sram_name = g_strdup_printf("aspeed.dram.%d", + CPU(a->armv7m.cpu)->cpu_index); + + if (!memory_region_init_ram(&s->sram, OBJECT(s), sram_name, sc->sram_size, + errp)) { + return; + } + memory_region_add_subregion(s->memory, + sc->memmap[ASPEED_DEV_SRAM], + &s->sram); + + /* SCU */ + if (!sysbus_realize(SYS_BUS_DEVICE(&s->scu), errp)) { + return; + } + aspeed_mmio_map(s, SYS_BUS_DEVICE(&s->scu), 0, sc->memmap[ASPEED_DEV_SCU]); + + /* INTC */ + if (!sysbus_realize(SYS_BUS_DEVICE(&a->intc[0]), errp)) { + return; + } + + aspeed_mmio_map(s, SYS_BUS_DEVICE(&a->intc[0]), 0, + sc->memmap[ASPEED_DEV_INTC]); + + /* INTCIO */ + if (!sysbus_realize(SYS_BUS_DEVICE(&a->intc[1]), errp)) { + return; + } + + aspeed_mmio_map(s, SYS_BUS_DEVICE(&a->intc[1]), 0, + sc->memmap[ASPEED_DEV_INTCIO]); + + /* irq source orgates -> INTC0 */ + for (i = 0; i < ASPEED_INTC_GET_CLASS(&a->intc[0])->num_inpins; i++) { + qdev_connect_gpio_out(DEVICE(&a->intc[0].orgates[i]), 0, + qdev_get_gpio_in(DEVICE(&a->intc[0]), i)); + } + for (i = 0; i < ASPEED_INTC_GET_CLASS(&a->intc[0])->num_outpins; i++) { + assert(i < ARRAY_SIZE(ast2700_ssp_intcmap)); + sysbus_connect_irq(SYS_BUS_DEVICE(&a->intc[0]), i, + qdev_get_gpio_in(DEVICE(&a->armv7m), + ast2700_ssp_intcmap[i].irq)); + } + /* irq source orgates -> INTCIO */ + for (i = 0; i < ASPEED_INTC_GET_CLASS(&a->intc[1])->num_inpins; i++) { + qdev_connect_gpio_out(DEVICE(&a->intc[1].orgates[i]), 0, + qdev_get_gpio_in(DEVICE(&a->intc[1]), i)); + } + /* INTCIO -> INTC */ + for (i = 0; i < ASPEED_INTC_GET_CLASS(&a->intc[1])->num_outpins; i++) { + sysbus_connect_irq(SYS_BUS_DEVICE(&a->intc[1]), i, + qdev_get_gpio_in(DEVICE(&a->intc[0].orgates[0]), i)); + } + /* UART */ + if (!aspeed_soc_uart_realize(s, errp)) { + return; + } + + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&s->timerctrl), + "aspeed.timerctrl", + sc->memmap[ASPEED_DEV_TIMER1], 0x200); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&a->ipc[0]), + "aspeed.ipc0", + sc->memmap[ASPEED_DEV_IPC0], 0x1000); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&a->ipc[1]), + "aspeed.ipc1", + sc->memmap[ASPEED_DEV_IPC1], 0x1000); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&a->scuio), + "aspeed.scuio", + sc->memmap[ASPEED_DEV_SCUIO], 0x1000); +} + +static void aspeed_soc_ast27x0ssp_class_init(ObjectClass *klass, const void *data) +{ + static const char * const valid_cpu_types[] = { + ARM_CPU_TYPE_NAME("cortex-m4"), /* TODO: cortex-m4f */ + NULL + }; + DeviceClass *dc = DEVICE_CLASS(klass); + AspeedSoCClass *sc = ASPEED_SOC_CLASS(dc); + + /* Reason: The Aspeed SoC can only be instantiated from a board */ + dc->user_creatable = false; + dc->realize = aspeed_soc_ast27x0ssp_realize; + + sc->valid_cpu_types = valid_cpu_types; + sc->silicon_rev = AST2700_A1_SILICON_REV; + sc->sram_size = AST2700_SSP_RAM_SIZE; + sc->spis_num = 0; + sc->ehcis_num = 0; + sc->wdts_num = 0; + sc->macs_num = 0; + sc->uarts_num = 13; + sc->uarts_base = ASPEED_DEV_UART0; + sc->irqmap = aspeed_soc_ast27x0ssp_irqmap; + sc->memmap = aspeed_soc_ast27x0ssp_memmap; + sc->num_cpus = 1; + sc->get_irq = aspeed_soc_ast27x0ssp_get_irq; +} + +static const TypeInfo aspeed_soc_ast27x0ssp_types[] = { + { + .name = TYPE_ASPEED27X0SSP_SOC, + .parent = TYPE_ASPEED_SOC, + .instance_size = sizeof(Aspeed27x0SSPSoCState), + .instance_init = aspeed_soc_ast27x0ssp_init, + .class_init = aspeed_soc_ast27x0ssp_class_init, + }, +}; + +DEFINE_TYPES(aspeed_soc_ast27x0ssp_types) diff --git a/hw/arm/aspeed_ast27x0-tsp.c b/hw/arm/aspeed_ast27x0-tsp.c new file mode 100644 index 0000000000..4e0efaef07 --- /dev/null +++ b/hw/arm/aspeed_ast27x0-tsp.c @@ -0,0 +1,294 @@ +/* + * ASPEED Ast27x0 TSP SoC + * + * Copyright (C) 2025 ASPEED Technology Inc. + * + * This code is licensed under the GPL version 2 or later. See + * the COPYING file in the top-level directory. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "qapi/error.h" +#include "hw/qdev-clock.h" +#include "hw/misc/unimp.h" +#include "hw/arm/aspeed_soc.h" + +#define AST2700_TSP_RAM_SIZE (32 * MiB) + +static const hwaddr aspeed_soc_ast27x0tsp_memmap[] = { + [ASPEED_DEV_SRAM] = 0x00000000, + [ASPEED_DEV_INTC] = 0x72100000, + [ASPEED_DEV_SCU] = 0x72C02000, + [ASPEED_DEV_SCUIO] = 0x74C02000, + [ASPEED_DEV_UART0] = 0x74C33000, + [ASPEED_DEV_UART1] = 0x74C33100, + [ASPEED_DEV_UART2] = 0x74C33200, + [ASPEED_DEV_UART3] = 0x74C33300, + [ASPEED_DEV_UART4] = 0x72C1A000, + [ASPEED_DEV_INTCIO] = 0x74C18000, + [ASPEED_DEV_IPC0] = 0x72C1C000, + [ASPEED_DEV_IPC1] = 0x74C39000, + [ASPEED_DEV_UART5] = 0x74C33400, + [ASPEED_DEV_UART6] = 0x74C33500, + [ASPEED_DEV_UART7] = 0x74C33600, + [ASPEED_DEV_UART8] = 0x74C33700, + [ASPEED_DEV_UART9] = 0x74C33800, + [ASPEED_DEV_UART10] = 0x74C33900, + [ASPEED_DEV_UART11] = 0x74C33A00, + [ASPEED_DEV_UART12] = 0x74C33B00, + [ASPEED_DEV_TIMER1] = 0x72C10000, +}; + +static const int aspeed_soc_ast27x0tsp_irqmap[] = { + [ASPEED_DEV_SCU] = 12, + [ASPEED_DEV_UART0] = 164, + [ASPEED_DEV_UART1] = 164, + [ASPEED_DEV_UART2] = 164, + [ASPEED_DEV_UART3] = 164, + [ASPEED_DEV_UART4] = 8, + [ASPEED_DEV_UART5] = 164, + [ASPEED_DEV_UART6] = 164, + [ASPEED_DEV_UART7] = 164, + [ASPEED_DEV_UART8] = 164, + [ASPEED_DEV_UART9] = 164, + [ASPEED_DEV_UART10] = 164, + [ASPEED_DEV_UART11] = 164, + [ASPEED_DEV_UART12] = 164, + [ASPEED_DEV_TIMER1] = 16, +}; + +/* TSPINT 164 */ +static const int ast2700_tsp132_tsp164_intcmap[] = { + [ASPEED_DEV_UART0] = 7, + [ASPEED_DEV_UART1] = 8, + [ASPEED_DEV_UART2] = 9, + [ASPEED_DEV_UART3] = 10, + [ASPEED_DEV_UART5] = 11, + [ASPEED_DEV_UART6] = 12, + [ASPEED_DEV_UART7] = 13, + [ASPEED_DEV_UART8] = 14, + [ASPEED_DEV_UART9] = 15, + [ASPEED_DEV_UART10] = 16, + [ASPEED_DEV_UART11] = 17, + [ASPEED_DEV_UART12] = 18, +}; + +struct nvic_intc_irq_info { + int irq; + int intc_idx; + int orgate_idx; + const int *ptr; +}; + +static struct nvic_intc_irq_info ast2700_tsp_intcmap[] = { + {160, 1, 0, NULL}, + {161, 1, 1, NULL}, + {162, 1, 2, NULL}, + {163, 1, 3, NULL}, + {164, 1, 4, ast2700_tsp132_tsp164_intcmap}, + {165, 1, 5, NULL}, + {166, 1, 6, NULL}, + {167, 1, 7, NULL}, + {168, 1, 8, NULL}, + {169, 1, 9, NULL}, + {128, 0, 1, NULL}, + {129, 0, 2, NULL}, + {130, 0, 3, NULL}, + {131, 0, 4, NULL}, + {132, 0, 5, ast2700_tsp132_tsp164_intcmap}, + {133, 0, 6, NULL}, + {134, 0, 7, NULL}, + {135, 0, 8, NULL}, + {136, 0, 9, NULL}, +}; + +static qemu_irq aspeed_soc_ast27x0tsp_get_irq(AspeedSoCState *s, int dev) +{ + Aspeed27x0TSPSoCState *a = ASPEED27X0TSP_SOC(s); + AspeedSoCClass *sc = ASPEED_SOC_GET_CLASS(s); + + int or_idx; + int idx; + int i; + + for (i = 0; i < ARRAY_SIZE(ast2700_tsp_intcmap); i++) { + if (sc->irqmap[dev] == ast2700_tsp_intcmap[i].irq) { + assert(ast2700_tsp_intcmap[i].ptr); + or_idx = ast2700_tsp_intcmap[i].orgate_idx; + idx = ast2700_tsp_intcmap[i].intc_idx; + return qdev_get_gpio_in(DEVICE(&a->intc[idx].orgates[or_idx]), + ast2700_tsp_intcmap[i].ptr[dev]); + } + } + + return qdev_get_gpio_in(DEVICE(&a->armv7m), sc->irqmap[dev]); +} + +static void aspeed_soc_ast27x0tsp_init(Object *obj) +{ + Aspeed27x0TSPSoCState *a = ASPEED27X0TSP_SOC(obj); + AspeedSoCState *s = ASPEED_SOC(obj); + AspeedSoCClass *sc = ASPEED_SOC_GET_CLASS(s); + int i; + + object_initialize_child(obj, "armv7m", &a->armv7m, TYPE_ARMV7M); + object_initialize_child(obj, "scu", &s->scu, TYPE_ASPEED_2700_SCU); + s->sysclk = qdev_init_clock_in(DEVICE(s), "sysclk", NULL, NULL, 0); + qdev_prop_set_uint32(DEVICE(&s->scu), "silicon-rev", sc->silicon_rev); + + for (i = 0; i < sc->uarts_num; i++) { + object_initialize_child(obj, "uart[*]", &s->uart[i], TYPE_SERIAL_MM); + } + + object_initialize_child(obj, "intc0", &a->intc[0], + TYPE_ASPEED_2700TSP_INTC); + object_initialize_child(obj, "intc1", &a->intc[1], + TYPE_ASPEED_2700TSP_INTCIO); + + object_initialize_child(obj, "timerctrl", &s->timerctrl, + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "ipc0", &a->ipc[0], + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "ipc1", &a->ipc[1], + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "scuio", &a->scuio, + TYPE_UNIMPLEMENTED_DEVICE); +} + +static void aspeed_soc_ast27x0tsp_realize(DeviceState *dev_soc, Error **errp) +{ + Aspeed27x0TSPSoCState *a = ASPEED27X0TSP_SOC(dev_soc); + AspeedSoCState *s = ASPEED_SOC(dev_soc); + AspeedSoCClass *sc = ASPEED_SOC_GET_CLASS(s); + DeviceState *armv7m; + g_autofree char *sram_name = NULL; + int i; + + if (!clock_has_source(s->sysclk)) { + error_setg(errp, "sysclk clock must be wired up by the board code"); + return; + } + + /* AST27X0 TSP Core */ + armv7m = DEVICE(&a->armv7m); + qdev_prop_set_uint32(armv7m, "num-irq", 256); + qdev_prop_set_string(armv7m, "cpu-type", aspeed_soc_cpu_type(sc)); + qdev_connect_clock_in(armv7m, "cpuclk", s->sysclk); + object_property_set_link(OBJECT(&a->armv7m), "memory", + OBJECT(s->memory), &error_abort); + sysbus_realize(SYS_BUS_DEVICE(&a->armv7m), &error_abort); + + sram_name = g_strdup_printf("aspeed.dram.%d", + CPU(a->armv7m.cpu)->cpu_index); + + if (!memory_region_init_ram(&s->sram, OBJECT(s), sram_name, sc->sram_size, + errp)) { + return; + } + memory_region_add_subregion(s->memory, + sc->memmap[ASPEED_DEV_SRAM], + &s->sram); + + /* SCU */ + if (!sysbus_realize(SYS_BUS_DEVICE(&s->scu), errp)) { + return; + } + aspeed_mmio_map(s, SYS_BUS_DEVICE(&s->scu), 0, sc->memmap[ASPEED_DEV_SCU]); + + /* INTC */ + if (!sysbus_realize(SYS_BUS_DEVICE(&a->intc[0]), errp)) { + return; + } + + aspeed_mmio_map(s, SYS_BUS_DEVICE(&a->intc[0]), 0, + sc->memmap[ASPEED_DEV_INTC]); + + /* INTCIO */ + if (!sysbus_realize(SYS_BUS_DEVICE(&a->intc[1]), errp)) { + return; + } + + aspeed_mmio_map(s, SYS_BUS_DEVICE(&a->intc[1]), 0, + sc->memmap[ASPEED_DEV_INTCIO]); + + /* irq source orgates -> INTC */ + for (i = 0; i < ASPEED_INTC_GET_CLASS(&a->intc[0])->num_inpins; i++) { + qdev_connect_gpio_out(DEVICE(&a->intc[0].orgates[i]), 0, + qdev_get_gpio_in(DEVICE(&a->intc[0]), i)); + } + for (i = 0; i < ASPEED_INTC_GET_CLASS(&a->intc[0])->num_outpins; i++) { + assert(i < ARRAY_SIZE(ast2700_tsp_intcmap)); + sysbus_connect_irq(SYS_BUS_DEVICE(&a->intc[0]), i, + qdev_get_gpio_in(DEVICE(&a->armv7m), + ast2700_tsp_intcmap[i].irq)); + } + /* irq source orgates -> INTC */ + for (i = 0; i < ASPEED_INTC_GET_CLASS(&a->intc[1])->num_inpins; i++) { + qdev_connect_gpio_out(DEVICE(&a->intc[1].orgates[i]), 0, + qdev_get_gpio_in(DEVICE(&a->intc[1]), i)); + } + /* INTCIO -> INTC */ + for (i = 0; i < ASPEED_INTC_GET_CLASS(&a->intc[1])->num_outpins; i++) { + sysbus_connect_irq(SYS_BUS_DEVICE(&a->intc[1]), i, + qdev_get_gpio_in(DEVICE(&a->intc[0].orgates[0]), i)); + } + /* UART */ + if (!aspeed_soc_uart_realize(s, errp)) { + return; + } + + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&s->timerctrl), + "aspeed.timerctrl", + sc->memmap[ASPEED_DEV_TIMER1], 0x200); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&a->ipc[0]), + "aspeed.ipc0", + sc->memmap[ASPEED_DEV_IPC0], 0x1000); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&a->ipc[1]), + "aspeed.ipc1", + sc->memmap[ASPEED_DEV_IPC1], 0x1000); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&a->scuio), + "aspeed.scuio", + sc->memmap[ASPEED_DEV_SCUIO], 0x1000); +} + +static void aspeed_soc_ast27x0tsp_class_init(ObjectClass *klass, const void *data) +{ + static const char * const valid_cpu_types[] = { + ARM_CPU_TYPE_NAME("cortex-m4"), /* TODO cortex-m4f */ + NULL + }; + DeviceClass *dc = DEVICE_CLASS(klass); + AspeedSoCClass *sc = ASPEED_SOC_CLASS(dc); + + /* Reason: The Aspeed SoC can only be instantiated from a board */ + dc->user_creatable = false; + dc->realize = aspeed_soc_ast27x0tsp_realize; + + sc->valid_cpu_types = valid_cpu_types; + sc->silicon_rev = AST2700_A1_SILICON_REV; + sc->sram_size = AST2700_TSP_RAM_SIZE; + sc->spis_num = 0; + sc->ehcis_num = 0; + sc->wdts_num = 0; + sc->macs_num = 0; + sc->uarts_num = 13; + sc->uarts_base = ASPEED_DEV_UART0; + sc->irqmap = aspeed_soc_ast27x0tsp_irqmap; + sc->memmap = aspeed_soc_ast27x0tsp_memmap; + sc->num_cpus = 1; + sc->get_irq = aspeed_soc_ast27x0tsp_get_irq; +} + +static const TypeInfo aspeed_soc_ast27x0tsp_types[] = { + { + .name = TYPE_ASPEED27X0TSP_SOC, + .parent = TYPE_ASPEED_SOC, + .instance_size = sizeof(Aspeed27x0TSPSoCState), + .instance_init = aspeed_soc_ast27x0tsp_init, + .class_init = aspeed_soc_ast27x0tsp_class_init, + }, +}; + +DEFINE_TYPES(aspeed_soc_ast27x0tsp_types) diff --git a/hw/arm/aspeed_ast27x0.c b/hw/arm/aspeed_ast27x0.c index 63a366f7e8..1974a25766 100644 --- a/hw/arm/aspeed_ast27x0.c +++ b/hw/arm/aspeed_ast27x0.c @@ -23,8 +23,19 @@ #include "qobject/qlist.h" #include "qemu/log.h" +#define AST2700_SOC_IO_SIZE 0x01000000 +#define AST2700_SOC_IOMEM_SIZE 0x01000000 +#define AST2700_SOC_DPMCU_SIZE 0x00040000 +#define AST2700_SOC_LTPI_SIZE 0x01000000 + static const hwaddr aspeed_soc_ast2700_memmap[] = { + [ASPEED_DEV_IOMEM] = 0x00000000, + [ASPEED_DEV_VBOOTROM] = 0x00000000, [ASPEED_DEV_SRAM] = 0x10000000, + [ASPEED_DEV_DPMCU] = 0x11000000, + [ASPEED_DEV_IOMEM0] = 0x12000000, + [ASPEED_DEV_EHCI1] = 0x12061000, + [ASPEED_DEV_EHCI2] = 0x12063000, [ASPEED_DEV_HACE] = 0x12070000, [ASPEED_DEV_EMMC] = 0x12090000, [ASPEED_DEV_INTC] = 0x12100000, @@ -35,7 +46,8 @@ static const hwaddr aspeed_soc_ast2700_memmap[] = { [ASPEED_DEV_RTC] = 0x12C0F000, [ASPEED_DEV_TIMER1] = 0x12C10000, [ASPEED_DEV_SLI] = 0x12C17000, - [ASPEED_DEV_UART4] = 0X12C1A000, + [ASPEED_DEV_UART4] = 0x12C1A000, + [ASPEED_DEV_IOMEM1] = 0x14000000, [ASPEED_DEV_FMC] = 0x14000000, [ASPEED_DEV_SPI0] = 0x14010000, [ASPEED_DEV_SPI1] = 0x14020000, @@ -47,27 +59,30 @@ static const hwaddr aspeed_soc_ast2700_memmap[] = { [ASPEED_DEV_ETH2] = 0x14060000, [ASPEED_DEV_ETH3] = 0x14070000, [ASPEED_DEV_SDHCI] = 0x14080000, + [ASPEED_DEV_EHCI3] = 0x14121000, + [ASPEED_DEV_EHCI4] = 0x14123000, [ASPEED_DEV_ADC] = 0x14C00000, [ASPEED_DEV_SCUIO] = 0x14C02000, [ASPEED_DEV_GPIO] = 0x14C0B000, [ASPEED_DEV_I2C] = 0x14C0F000, [ASPEED_DEV_INTCIO] = 0x14C18000, [ASPEED_DEV_SLIIO] = 0x14C1E000, - [ASPEED_DEV_VUART] = 0X14C30000, - [ASPEED_DEV_UART0] = 0X14C33000, - [ASPEED_DEV_UART1] = 0X14C33100, - [ASPEED_DEV_UART2] = 0X14C33200, - [ASPEED_DEV_UART3] = 0X14C33300, - [ASPEED_DEV_UART5] = 0X14C33400, - [ASPEED_DEV_UART6] = 0X14C33500, - [ASPEED_DEV_UART7] = 0X14C33600, - [ASPEED_DEV_UART8] = 0X14C33700, - [ASPEED_DEV_UART9] = 0X14C33800, - [ASPEED_DEV_UART10] = 0X14C33900, - [ASPEED_DEV_UART11] = 0X14C33A00, - [ASPEED_DEV_UART12] = 0X14C33B00, + [ASPEED_DEV_VUART] = 0x14C30000, + [ASPEED_DEV_UART0] = 0x14C33000, + [ASPEED_DEV_UART1] = 0x14C33100, + [ASPEED_DEV_UART2] = 0x14C33200, + [ASPEED_DEV_UART3] = 0x14C33300, + [ASPEED_DEV_UART5] = 0x14C33400, + [ASPEED_DEV_UART6] = 0x14C33500, + [ASPEED_DEV_UART7] = 0x14C33600, + [ASPEED_DEV_UART8] = 0x14C33700, + [ASPEED_DEV_UART9] = 0x14C33800, + [ASPEED_DEV_UART10] = 0x14C33900, + [ASPEED_DEV_UART11] = 0x14C33A00, + [ASPEED_DEV_UART12] = 0x14C33B00, [ASPEED_DEV_WDT] = 0x14C37000, [ASPEED_DEV_SPI_BOOT] = 0x100000000, + [ASPEED_DEV_LTPI] = 0x300000000, [ASPEED_DEV_SDRAM] = 0x400000000, }; @@ -91,6 +106,8 @@ static const int aspeed_soc_ast2700a0_irqmap[] = { [ASPEED_DEV_TIMER7] = 22, [ASPEED_DEV_TIMER8] = 23, [ASPEED_DEV_DP] = 28, + [ASPEED_DEV_EHCI1] = 33, + [ASPEED_DEV_EHCI2] = 37, [ASPEED_DEV_LPC] = 128, [ASPEED_DEV_IBT] = 128, [ASPEED_DEV_KCS] = 128, @@ -137,6 +154,8 @@ static const int aspeed_soc_ast2700a1_irqmap[] = { [ASPEED_DEV_TIMER7] = 22, [ASPEED_DEV_TIMER8] = 23, [ASPEED_DEV_DP] = 28, + [ASPEED_DEV_EHCI1] = 33, + [ASPEED_DEV_EHCI2] = 37, [ASPEED_DEV_LPC] = 192, [ASPEED_DEV_IBT] = 192, [ASPEED_DEV_KCS] = 192, @@ -212,6 +231,8 @@ static const int ast2700_gic132_gic196_intcmap[] = { [ASPEED_DEV_UART10] = 16, [ASPEED_DEV_UART11] = 17, [ASPEED_DEV_UART12] = 18, + [ASPEED_DEV_EHCI3] = 28, + [ASPEED_DEV_EHCI4] = 29, }; /* GICINT 133 */ @@ -434,6 +455,11 @@ static void aspeed_soc_ast2700_init(Object *obj) object_initialize_child(obj, "spi[*]", &s->spi[i], typename); } + for (i = 0; i < sc->ehcis_num; i++) { + object_initialize_child(obj, "ehci[*]", &s->ehci[i], + TYPE_PLATFORM_EHCI); + } + snprintf(typename, sizeof(typename), "aspeed.sdmc-%s", socname); object_initialize_child(obj, "sdmc", &s->sdmc, typename); object_property_add_alias(obj, "ram-size", OBJECT(&s->sdmc), @@ -491,6 +517,16 @@ static void aspeed_soc_ast2700_init(Object *obj) snprintf(typename, sizeof(typename), "aspeed.hace-%s", socname); object_initialize_child(obj, "hace", &s->hace, typename); + object_initialize_child(obj, "dpmcu", &s->dpmcu, + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "ltpi", &s->ltpi, + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "iomem", &s->iomem, + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "iomem0", &s->iomem0, + TYPE_UNIMPLEMENTED_DEVICE); + object_initialize_child(obj, "iomem1", &s->iomem1, + TYPE_UNIMPLEMENTED_DEVICE); } /* @@ -526,8 +562,11 @@ static bool aspeed_soc_ast2700_gic_realize(DeviceState *dev, Error **errp) if (!sysbus_realize(gicbusdev, errp)) { return false; } - sysbus_mmio_map(gicbusdev, 0, sc->memmap[ASPEED_GIC_DIST]); - sysbus_mmio_map(gicbusdev, 1, sc->memmap[ASPEED_GIC_REDIST]); + + aspeed_mmio_map(s, SYS_BUS_DEVICE(&a->gic), 0, + sc->memmap[ASPEED_GIC_DIST]); + aspeed_mmio_map(s, SYS_BUS_DEVICE(&a->gic), 1, + sc->memmap[ASPEED_GIC_REDIST]); for (i = 0; i < sc->num_cpus; i++) { DeviceState *cpudev = DEVICE(&a->cpu[i]); @@ -577,7 +616,7 @@ static void aspeed_soc_ast2700_realize(DeviceState *dev, Error **errp) AspeedSoCClass *sc = ASPEED_SOC_GET_CLASS(s); AspeedINTCClass *ic = ASPEED_INTC_GET_CLASS(&a->intc[0]); AspeedINTCClass *icio = ASPEED_INTC_GET_CLASS(&a->intc[1]); - g_autofree char *sram_name = NULL; + g_autofree char *name = NULL; qemu_irq irq; /* Default boot region (SPI memory or ROMs) */ @@ -649,14 +688,22 @@ static void aspeed_soc_ast2700_realize(DeviceState *dev, Error **errp) } /* SRAM */ - sram_name = g_strdup_printf("aspeed.sram.%d", CPU(&a->cpu[0])->cpu_index); - if (!memory_region_init_ram(&s->sram, OBJECT(s), sram_name, sc->sram_size, - errp)) { + name = g_strdup_printf("aspeed.sram.%d", CPU(&a->cpu[0])->cpu_index); + if (!memory_region_init_ram(&s->sram, OBJECT(s), name, sc->sram_size, + errp)) { return; } memory_region_add_subregion(s->memory, sc->memmap[ASPEED_DEV_SRAM], &s->sram); + /* VBOOTROM */ + if (!memory_region_init_ram(&s->vbootrom, OBJECT(s), "aspeed.vbootrom", + 0x20000, errp)) { + return; + } + memory_region_add_subregion(s->memory, + sc->memmap[ASPEED_DEV_VBOOTROM], &s->vbootrom); + /* SCU */ if (!sysbus_realize(SYS_BUS_DEVICE(&s->scu), errp)) { return; @@ -709,6 +756,17 @@ static void aspeed_soc_ast2700_realize(DeviceState *dev, Error **errp) ASPEED_SMC_GET_CLASS(&s->spi[i])->flash_window_base); } + /* EHCI */ + for (i = 0; i < sc->ehcis_num; i++) { + if (!sysbus_realize(SYS_BUS_DEVICE(&s->ehci[i]), errp)) { + return; + } + aspeed_mmio_map(s, SYS_BUS_DEVICE(&s->ehci[i]), 0, + sc->memmap[ASPEED_DEV_EHCI1 + i]); + sysbus_connect_irq(SYS_BUS_DEVICE(&s->ehci[i]), 0, + aspeed_soc_get_irq(s, ASPEED_DEV_EHCI1 + i)); + } + /* * SDMC - SDRAM Memory Controller * The SDMC controller is unlocked at SPL stage. @@ -876,11 +934,26 @@ static void aspeed_soc_ast2700_realize(DeviceState *dev, Error **errp) sysbus_connect_irq(SYS_BUS_DEVICE(&s->hace), 0, aspeed_soc_get_irq(s, ASPEED_DEV_HACE)); - create_unimplemented_device("ast2700.dpmcu", 0x11000000, 0x40000); - create_unimplemented_device("ast2700.iomem0", 0x12000000, 0x01000000); - create_unimplemented_device("ast2700.iomem1", 0x14000000, 0x01000000); - create_unimplemented_device("ast2700.ltpi", 0x30000000, 0x1000000); - create_unimplemented_device("ast2700.io", 0x0, 0x4000000); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&s->dpmcu), + "aspeed.dpmcu", + sc->memmap[ASPEED_DEV_DPMCU], + AST2700_SOC_DPMCU_SIZE); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&s->ltpi), + "aspeed.ltpi", + sc->memmap[ASPEED_DEV_LTPI], + AST2700_SOC_LTPI_SIZE); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&s->iomem), + "aspeed.io", + sc->memmap[ASPEED_DEV_IOMEM], + AST2700_SOC_IO_SIZE); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&s->iomem0), + "aspeed.iomem0", + sc->memmap[ASPEED_DEV_IOMEM0], + AST2700_SOC_IOMEM_SIZE); + aspeed_mmio_map_unimplemented(s, SYS_BUS_DEVICE(&s->iomem1), + "aspeed.iomem1", + sc->memmap[ASPEED_DEV_IOMEM1], + AST2700_SOC_IOMEM_SIZE); } static void aspeed_soc_ast2700a0_class_init(ObjectClass *oc, const void *data) @@ -900,6 +973,7 @@ static void aspeed_soc_ast2700a0_class_init(ObjectClass *oc, const void *data) sc->silicon_rev = AST2700_A0_SILICON_REV; sc->sram_size = 0x20000; sc->spis_num = 3; + sc->ehcis_num = 2; sc->wdts_num = 8; sc->macs_num = 1; sc->uarts_num = 13; @@ -927,6 +1001,7 @@ static void aspeed_soc_ast2700a1_class_init(ObjectClass *oc, const void *data) sc->silicon_rev = AST2700_A1_SILICON_REV; sc->sram_size = 0x20000; sc->spis_num = 3; + sc->ehcis_num = 4; sc->wdts_num = 8; sc->macs_num = 3; sc->uarts_num = 13; diff --git a/hw/arm/meson.build b/hw/arm/meson.build index 09b1cfe5b5..5098795f61 100644 --- a/hw/arm/meson.build +++ b/hw/arm/meson.build @@ -44,10 +44,14 @@ arm_ss.add(when: 'CONFIG_ASPEED_SOC', if_true: files( 'aspeed_soc_common.c', 'aspeed_ast2400.c', 'aspeed_ast2600.c', + 'aspeed_ast27x0-ssp.c', + 'aspeed_ast27x0-tsp.c', 'aspeed_ast10x0.c', 'aspeed_eeprom.c', 'fby35.c')) -arm_common_ss.add(when: ['CONFIG_ASPEED_SOC', 'TARGET_AARCH64'], if_true: files('aspeed_ast27x0.c')) +arm_common_ss.add(when: ['CONFIG_ASPEED_SOC', 'TARGET_AARCH64'], if_true: files( + 'aspeed_ast27x0.c', + 'aspeed_ast27x0-fc.c',)) arm_common_ss.add(when: 'CONFIG_MPS2', if_true: files('mps2.c')) arm_common_ss.add(when: 'CONFIG_MPS2', if_true: files('mps2-tz.c')) arm_common_ss.add(when: 'CONFIG_MSF2', if_true: files('msf2-soc.c')) diff --git a/hw/arm/npcm8xx.c b/hw/arm/npcm8xx.c index 5cc67b132f..d7ee306de7 100644 --- a/hw/arm/npcm8xx.c +++ b/hw/arm/npcm8xx.c @@ -67,6 +67,9 @@ /* SDHCI Modules */ #define NPCM8XX_MMC_BA 0xf0842000 +/* PSPI Modules */ +#define NPCM8XX_PSPI_BA 0xf0201000 + /* Run PLL1 at 1600 MHz */ #define NPCM8XX_PLLCON1_FIXUP_VAL 0x00402101 /* Run the CPU from PLL1 and UART from PLL2 */ @@ -83,6 +86,7 @@ enum NPCM8xxInterrupt { NPCM8XX_PECI_IRQ = 6, NPCM8XX_KCS_HIB_IRQ = 9, NPCM8XX_MMC_IRQ = 26, + NPCM8XX_PSPI_IRQ = 28, NPCM8XX_TIMER0_IRQ = 32, /* Timer Module 0 */ NPCM8XX_TIMER1_IRQ, NPCM8XX_TIMER2_IRQ, @@ -441,6 +445,7 @@ static void npcm8xx_init(Object *obj) } object_initialize_child(obj, "mmc", &s->mmc, TYPE_NPCM7XX_SDHCI); + object_initialize_child(obj, "pspi", &s->pspi, TYPE_NPCM_PSPI); } static void npcm8xx_realize(DeviceState *dev, Error **errp) @@ -705,6 +710,11 @@ static void npcm8xx_realize(DeviceState *dev, Error **errp) sysbus_connect_irq(SYS_BUS_DEVICE(&s->mmc), 0, npcm8xx_irq(s, NPCM8XX_MMC_IRQ)); + /* PSPI */ + sysbus_realize(SYS_BUS_DEVICE(&s->pspi), &error_abort); + sysbus_mmio_map(SYS_BUS_DEVICE(&s->pspi), 0, NPCM8XX_PSPI_BA); + sysbus_connect_irq(SYS_BUS_DEVICE(&s->pspi), 0, + npcm8xx_irq(s, NPCM8XX_PSPI_IRQ)); create_unimplemented_device("npcm8xx.shm", 0xc0001000, 4 * KiB); create_unimplemented_device("npcm8xx.gicextra", 0xdfffa000, 24 * KiB); @@ -720,7 +730,6 @@ static void npcm8xx_realize(DeviceState *dev, Error **errp) create_unimplemented_device("npcm8xx.siox[1]", 0xf0101000, 4 * KiB); create_unimplemented_device("npcm8xx.siox[2]", 0xf0102000, 4 * KiB); create_unimplemented_device("npcm8xx.tmps", 0xf0188000, 4 * KiB); - create_unimplemented_device("npcm8xx.pspi", 0xf0201000, 4 * KiB); create_unimplemented_device("npcm8xx.viru1", 0xf0204000, 4 * KiB); create_unimplemented_device("npcm8xx.viru2", 0xf0205000, 4 * KiB); create_unimplemented_device("npcm8xx.jtm1", 0xf0208000, 4 * KiB); diff --git a/hw/arm/npcm8xx_boards.c b/hw/arm/npcm8xx_boards.c index 9d9f6d0c9a..3bf3e1f8f1 100644 --- a/hw/arm/npcm8xx_boards.c +++ b/hw/arm/npcm8xx_boards.c @@ -213,7 +213,7 @@ static void npcm8xx_machine_class_init(ObjectClass *oc, const void *data) { MachineClass *mc = MACHINE_CLASS(oc); static const char * const valid_cpu_types[] = { - ARM_CPU_TYPE_NAME("cortex-a9"), + ARM_CPU_TYPE_NAME("cortex-a35"), NULL }; diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c index 3ac8f8e178..7e8e0f0298 100644 --- a/hw/arm/virt-acpi-build.c +++ b/hw/arm/virt-acpi-build.c @@ -537,15 +537,12 @@ build_srat(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) static void build_gtdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) { - VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(vms); /* * Table 5-117 Flag Definitions * set only "Timer interrupt Mode" and assume "Timer Interrupt * polarity" bit as '0: Interrupt is Active high' */ - uint32_t irqflags = vmc->claim_edge_triggered_timers ? - 1 : /* Interrupt is Edge triggered */ - 0; /* Interrupt is Level triggered */ + const uint32_t irqflags = 0; /* Interrupt is Level triggered */ AcpiTable table = { .sig = "GTDT", .rev = 3, .oem_id = vms->oem_id, .oem_table_id = vms->oem_table_id }; @@ -670,7 +667,6 @@ static void build_madt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) { int i; - VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(vms); const MemMapEntry *memmap = vms->memmap; AcpiTable table = { .sig = "APIC", .rev = 4, .oem_id = vms->oem_id, .oem_table_id = vms->oem_table_id }; @@ -741,7 +737,7 @@ build_madt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) memmap[VIRT_HIGH_GIC_REDIST2].size); } - if (its_class_name() && !vmc->no_its) { + if (its_class_name()) { /* * ACPI spec, Revision 6.0 Errata A * (original 6.0 definition has invalid Length) @@ -973,7 +969,7 @@ void virt_acpi_build(VirtMachineState *vms, AcpiBuildTables *tables) vms->oem_table_id); } - if (its_class_name() && !vmc->no_its) { + if (its_class_name()) { acpi_add_table(table_offsets, tables_blob); build_iort(tables_blob, tables->linker, vms); } diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 177f3dd22c..9a6cd085a3 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -370,14 +370,9 @@ static void fdt_add_timer_nodes(const VirtMachineState *vms) * the correct information. */ ARMCPU *armcpu; - VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(vms); uint32_t irqflags = GIC_FDT_IRQ_FLAGS_LEVEL_HI; MachineState *ms = MACHINE(vms); - if (vmc->claim_edge_triggered_timers) { - irqflags = GIC_FDT_IRQ_FLAGS_EDGE_LO_HI; - } - if (vms->gic_version == VIRT_GIC_VERSION_2) { irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START, GIC_FDT_IRQ_PPI_CPU_WIDTH, @@ -1704,7 +1699,6 @@ static void virt_build_smbios(VirtMachineState *vms) { MachineClass *mc = MACHINE_GET_CLASS(vms); MachineState *ms = MACHINE(vms); - VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(vms); uint8_t *smbios_tables, *smbios_anchor; size_t smbios_tables_len, smbios_anchor_len; struct smbios_phys_mem_area mem_array; @@ -1714,8 +1708,7 @@ static void virt_build_smbios(VirtMachineState *vms) product = "KVM Virtual Machine"; } - smbios_set_defaults("QEMU", product, - vmc->smbios_old_sys_ver ? "1.0" : mc->name); + smbios_set_defaults("QEMU", product, mc->name); /* build the array of physical mem area from base_memmap */ mem_array.address = vms->memmap[VIRT_MEM].base; @@ -1770,24 +1763,18 @@ void virt_machine_done(Notifier *notifier, void *data) static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx) { - uint8_t clustersz = ARM_DEFAULT_CPUS_PER_CLUSTER; - VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(vms); + uint8_t clustersz; - if (!vmc->disallow_affinity_adjustment) { - /* Adjust MPIDR like 64-bit KVM hosts, which incorporate the - * GIC's target-list limitations. 32-bit KVM hosts currently - * always create clusters of 4 CPUs, but that is expected to - * change when they gain support for gicv3. When KVM is enabled - * it will override the changes we make here, therefore our - * purposes are to make TCG consistent (with 64-bit KVM hosts) - * and to improve SGI efficiency. - */ - if (vms->gic_version == VIRT_GIC_VERSION_2) { - clustersz = GIC_TARGETLIST_BITS; - } else { - clustersz = GICV3_TARGETLIST_BITS; - } + /* + * Adjust MPIDR to make TCG consistent (with 64-bit KVM hosts) + * and to improve SGI efficiency. + */ + if (vms->gic_version == VIRT_GIC_VERSION_2) { + clustersz = GIC_TARGETLIST_BITS; + } else { + clustersz = GICV3_TARGETLIST_BITS; } + return arm_build_mp_affinity(idx, clustersz); } @@ -2273,10 +2260,6 @@ static void machvirt_init(MachineState *machine) object_property_set_bool(cpuobj, "kvm-steal-time", false, NULL); } - if (vmc->no_pmu && object_property_find(cpuobj, "pmu")) { - object_property_set_bool(cpuobj, "pmu", false, NULL); - } - if (vmc->no_tcg_lpa2 && object_property_find(cpuobj, "lpa2")) { object_property_set_bool(cpuobj, "lpa2", false, NULL); } @@ -3348,21 +3331,17 @@ static void virt_instance_init(Object *obj) vms->highmem_compact = !vmc->no_highmem_compact; vms->gic_version = VIRT_GIC_VERSION_NOSEL; - vms->highmem_ecam = !vmc->no_highmem_ecam; + vms->highmem_ecam = true; vms->highmem_mmio = true; vms->highmem_redists = true; - if (vmc->no_its) { - vms->its = false; - } else { - /* Default allows ITS instantiation */ - vms->its = true; + /* Default allows ITS instantiation */ + vms->its = true; - if (vmc->no_tcg_its) { - vms->tcg_its = false; - } else { - vms->tcg_its = true; - } + if (vmc->no_tcg_its) { + vms->tcg_its = false; + } else { + vms->tcg_its = true; } /* Default disallows iommu instantiation */ @@ -3583,99 +3562,3 @@ static void virt_machine_4_1_options(MachineClass *mc) mc->auto_enable_numa_with_memhp = false; } DEFINE_VIRT_MACHINE(4, 1) - -static void virt_machine_4_0_options(MachineClass *mc) -{ - virt_machine_4_1_options(mc); - compat_props_add(mc->compat_props, hw_compat_4_0, hw_compat_4_0_len); -} -DEFINE_VIRT_MACHINE(4, 0) - -static void virt_machine_3_1_options(MachineClass *mc) -{ - virt_machine_4_0_options(mc); - compat_props_add(mc->compat_props, hw_compat_3_1, hw_compat_3_1_len); -} -DEFINE_VIRT_MACHINE(3, 1) - -static void virt_machine_3_0_options(MachineClass *mc) -{ - virt_machine_3_1_options(mc); - compat_props_add(mc->compat_props, hw_compat_3_0, hw_compat_3_0_len); -} -DEFINE_VIRT_MACHINE(3, 0) - -static void virt_machine_2_12_options(MachineClass *mc) -{ - VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc)); - - virt_machine_3_0_options(mc); - compat_props_add(mc->compat_props, hw_compat_2_12, hw_compat_2_12_len); - vmc->no_highmem_ecam = true; - mc->max_cpus = 255; -} -DEFINE_VIRT_MACHINE(2, 12) - -static void virt_machine_2_11_options(MachineClass *mc) -{ - VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc)); - - virt_machine_2_12_options(mc); - compat_props_add(mc->compat_props, hw_compat_2_11, hw_compat_2_11_len); - vmc->smbios_old_sys_ver = true; -} -DEFINE_VIRT_MACHINE(2, 11) - -static void virt_machine_2_10_options(MachineClass *mc) -{ - virt_machine_2_11_options(mc); - compat_props_add(mc->compat_props, hw_compat_2_10, hw_compat_2_10_len); - /* before 2.11 we never faulted accesses to bad addresses */ - mc->ignore_memory_transaction_failures = true; -} -DEFINE_VIRT_MACHINE(2, 10) - -static void virt_machine_2_9_options(MachineClass *mc) -{ - virt_machine_2_10_options(mc); - compat_props_add(mc->compat_props, hw_compat_2_9, hw_compat_2_9_len); -} -DEFINE_VIRT_MACHINE(2, 9) - -static void virt_machine_2_8_options(MachineClass *mc) -{ - VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc)); - - virt_machine_2_9_options(mc); - compat_props_add(mc->compat_props, hw_compat_2_8, hw_compat_2_8_len); - /* For 2.8 and earlier we falsely claimed in the DT that - * our timers were edge-triggered, not level-triggered. - */ - vmc->claim_edge_triggered_timers = true; -} -DEFINE_VIRT_MACHINE(2, 8) - -static void virt_machine_2_7_options(MachineClass *mc) -{ - VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc)); - - virt_machine_2_8_options(mc); - compat_props_add(mc->compat_props, hw_compat_2_7, hw_compat_2_7_len); - /* ITS was introduced with 2.8 */ - vmc->no_its = true; - /* Stick with 1K pages for migration compatibility */ - mc->minimum_page_bits = 0; -} -DEFINE_VIRT_MACHINE(2, 7) - -static void virt_machine_2_6_options(MachineClass *mc) -{ - VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc)); - - virt_machine_2_7_options(mc); - compat_props_add(mc->compat_props, hw_compat_2_6, hw_compat_2_6_len); - vmc->disallow_affinity_adjustment = true; - /* Disable PMU for 2.6 as PMU support was first introduced in 2.7 */ - vmc->no_pmu = true; -} -DEFINE_VIRT_MACHINE(2, 6) diff --git a/hw/char/diva-gsp.c b/hw/char/diva-gsp.c index 60f933191d..e1f0713cb7 100644 --- a/hw/char/diva-gsp.c +++ b/hw/char/diva-gsp.c @@ -51,7 +51,6 @@ typedef struct PCIDivaSerialState { SerialState state[PCI_SERIAL_MAX_PORTS]; uint32_t level[PCI_SERIAL_MAX_PORTS]; qemu_irq *irqs; - uint8_t prog_if; bool disable; } PCIDivaSerialState; @@ -124,8 +123,8 @@ static void diva_pci_realize(PCIDevice *dev, Error **errp) size_t i, offset = 0; size_t portmask = di.omask; - pci->dev.config[PCI_CLASS_PROG] = pci->prog_if; - pci->dev.config[PCI_INTERRUPT_PIN] = 0x01; + pci->dev.config[PCI_CLASS_PROG] = 2; /* 16550 compatible */ + pci->dev.config[PCI_INTERRUPT_PIN] = 1; memory_region_init(&pci->membar, OBJECT(pci), "serial_ports", 4096); pci_register_bar(&pci->dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &pci->membar); pci->irqs = qemu_allocate_irqs(multi_serial_irq_mux, pci, di.nports); @@ -178,7 +177,6 @@ static const Property diva_serial_properties[] = { DEFINE_PROP_CHR("chardev2", PCIDivaSerialState, state[1].chr), DEFINE_PROP_CHR("chardev3", PCIDivaSerialState, state[2].chr), DEFINE_PROP_CHR("chardev4", PCIDivaSerialState, state[3].chr), - DEFINE_PROP_UINT8("prog_if", PCIDivaSerialState, prog_if, 0x02), DEFINE_PROP_UINT32("subvendor", PCIDivaSerialState, subvendor, PCI_DEVICE_ID_HP_DIVA_TOSCA1), }; diff --git a/hw/char/serial-pci-multi.c b/hw/char/serial-pci-multi.c index fb184c2e6d..13df272691 100644 --- a/hw/char/serial-pci-multi.c +++ b/hw/char/serial-pci-multi.c @@ -46,7 +46,6 @@ typedef struct PCIMultiSerialState { SerialState state[PCI_SERIAL_MAX_PORTS]; uint32_t level[PCI_SERIAL_MAX_PORTS]; IRQState irqs[PCI_SERIAL_MAX_PORTS]; - uint8_t prog_if; } PCIMultiSerialState; static void multi_serial_pci_exit(PCIDevice *dev) @@ -97,8 +96,8 @@ static void multi_serial_pci_realize(PCIDevice *dev, Error **errp) SerialState *s; size_t i, nports = multi_serial_get_port_count(pc); - pci->dev.config[PCI_CLASS_PROG] = pci->prog_if; - pci->dev.config[PCI_INTERRUPT_PIN] = 0x01; + pci->dev.config[PCI_CLASS_PROG] = 2; /* 16550 compatible */ + pci->dev.config[PCI_INTERRUPT_PIN] = 1; memory_region_init(&pci->iobar, OBJECT(pci), "multiserial", 8 * nports); pci_register_bar(&pci->dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &pci->iobar); @@ -133,7 +132,6 @@ static const VMStateDescription vmstate_pci_multi_serial = { static const Property multi_2x_serial_pci_properties[] = { DEFINE_PROP_CHR("chardev1", PCIMultiSerialState, state[0].chr), DEFINE_PROP_CHR("chardev2", PCIMultiSerialState, state[1].chr), - DEFINE_PROP_UINT8("prog_if", PCIMultiSerialState, prog_if, 0x02), }; static const Property multi_4x_serial_pci_properties[] = { @@ -141,7 +139,6 @@ static const Property multi_4x_serial_pci_properties[] = { DEFINE_PROP_CHR("chardev2", PCIMultiSerialState, state[1].chr), DEFINE_PROP_CHR("chardev3", PCIMultiSerialState, state[2].chr), DEFINE_PROP_CHR("chardev4", PCIMultiSerialState, state[3].chr), - DEFINE_PROP_UINT8("prog_if", PCIMultiSerialState, prog_if, 0x02), }; static void multi_2x_serial_pci_class_initfn(ObjectClass *klass, diff --git a/hw/char/serial-pci.c b/hw/char/serial-pci.c index 8707e81914..46efabc4cb 100644 --- a/hw/char/serial-pci.c +++ b/hw/char/serial-pci.c @@ -38,7 +38,6 @@ struct PCISerialState { PCIDevice dev; SerialState state; - uint8_t prog_if; }; #define TYPE_PCI_SERIAL "pci-serial" @@ -53,8 +52,8 @@ static void serial_pci_realize(PCIDevice *dev, Error **errp) return; } - pci->dev.config[PCI_CLASS_PROG] = pci->prog_if; - pci->dev.config[PCI_INTERRUPT_PIN] = 0x01; + pci->dev.config[PCI_CLASS_PROG] = 2; /* 16550 compatible */ + pci->dev.config[PCI_INTERRUPT_PIN] = 1; s->irq = pci_allocate_irq(&pci->dev); memory_region_init_io(&s->io, OBJECT(pci), &serial_io_ops, s, "serial", 8); @@ -81,10 +80,6 @@ static const VMStateDescription vmstate_pci_serial = { } }; -static const Property serial_pci_properties[] = { - DEFINE_PROP_UINT8("prog_if", PCISerialState, prog_if, 0x02), -}; - static void serial_pci_class_initfn(ObjectClass *klass, const void *data) { DeviceClass *dc = DEVICE_CLASS(klass); @@ -96,7 +91,6 @@ static void serial_pci_class_initfn(ObjectClass *klass, const void *data) pc->revision = 1; pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL; dc->vmsd = &vmstate_pci_serial; - device_class_set_props(dc, serial_pci_properties); set_bit(DEVICE_CATEGORY_INPUT, dc->categories); } diff --git a/hw/core/machine.c b/hw/core/machine.c index ed01798d37..b8ae155dfa 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -41,7 +41,7 @@ GlobalProperty hw_compat_10_0[] = {}; const size_t hw_compat_10_0_len = G_N_ELEMENTS(hw_compat_10_0); GlobalProperty hw_compat_9_2[] = { - {"arm-cpu", "backcompat-pauth-default-use-qarma5", "true"}, + { "arm-cpu", "backcompat-pauth-default-use-qarma5", "true"}, { "virtio-balloon-pci", "vectors", "0" }, { "virtio-balloon-pci-transitional", "vectors", "0" }, { "virtio-balloon-pci-non-transitional", "vectors", "0" }, @@ -58,12 +58,12 @@ GlobalProperty hw_compat_9_1[] = { const size_t hw_compat_9_1_len = G_N_ELEMENTS(hw_compat_9_1); GlobalProperty hw_compat_9_0[] = { - {"arm-cpu", "backcompat-cntfrq", "true" }, + { "arm-cpu", "backcompat-cntfrq", "true" }, { "scsi-hd", "migrate-emulated-scsi-request", "false" }, { "scsi-cd", "migrate-emulated-scsi-request", "false" }, - {"vfio-pci", "skip-vsc-check", "false" }, + { "vfio-pci", "skip-vsc-check", "false" }, { "virtio-pci", "x-pcie-pm-no-soft-reset", "off" }, - {"sd-card", "spec_version", "2" }, + { "sd-card", "spec_version", "2" }, }; const size_t hw_compat_9_0_len = G_N_ELEMENTS(hw_compat_9_0); diff --git a/hw/display/apple-gfx.m b/hw/display/apple-gfx.m index 2ff1c90df7..8dde1f138d 100644 --- a/hw/display/apple-gfx.m +++ b/hw/display/apple-gfx.m @@ -69,7 +69,7 @@ struct PGTask_s { mach_vm_address_t address; uint64_t len; /* - * All unique MemoryRegions for which a mapping has been created in in this + * All unique MemoryRegions for which a mapping has been created in this * task, and on which we have thus called memory_region_ref(). There are * usually very few regions of system RAM in total, so we expect this array * to be very short. Therefore, no need for sorting or fancy search diff --git a/hw/display/vhost-user-gpu.c b/hw/display/vhost-user-gpu.c index 06c4e7e190..43d4c08a2e 100644 --- a/hw/display/vhost-user-gpu.c +++ b/hw/display/vhost-user-gpu.c @@ -18,6 +18,7 @@ #include "chardev/char-fe.h" #include "qapi/error.h" #include "migration/blocker.h" +#include "standard-headers/drm/drm_fourcc.h" typedef enum VhostUserGpuRequest { VHOST_USER_GPU_NONE = 0, @@ -249,7 +250,9 @@ vhost_user_gpu_handle_display(VhostUserGPU *g, VhostUserGpuMsg *msg) case VHOST_USER_GPU_DMABUF_SCANOUT: { VhostUserGpuDMABUFScanout *m = &msg->payload.dmabuf_scanout; int fd = qemu_chr_fe_get_msgfd(&g->vhost_chr); - uint64_t modifier = 0; + uint32_t offset = 0; + uint32_t stride = m->fd_stride; + uint64_t modifier = DRM_FORMAT_MOD_INVALID; QemuDmaBuf *dmabuf; if (m->scanout_id >= g->parent_obj.conf.max_outputs) { @@ -282,10 +285,10 @@ vhost_user_gpu_handle_display(VhostUserGPU *g, VhostUserGpuMsg *msg) } dmabuf = qemu_dmabuf_new(m->width, m->height, - m->fd_stride, 0, 0, + &offset, &stride, 0, 0, m->fd_width, m->fd_height, m->fd_drm_fourcc, modifier, - fd, false, m->fd_flags & + &fd, 1, false, m->fd_flags & VIRTIO_GPU_RESOURCE_FLAG_Y_0_TOP); dpy_gl_scanout_dmabuf(con, dmabuf); diff --git a/hw/display/virtio-gpu-udmabuf.c b/hw/display/virtio-gpu-udmabuf.c index 0510577475..d804f321aa 100644 --- a/hw/display/virtio-gpu-udmabuf.c +++ b/hw/display/virtio-gpu-udmabuf.c @@ -25,6 +25,7 @@ #include <linux/memfd.h> #include "qemu/memfd.h" #include "standard-headers/linux/udmabuf.h" +#include "standard-headers/drm/drm_fourcc.h" static void virtio_gpu_create_udmabuf(struct virtio_gpu_simple_resource *res) { @@ -176,16 +177,19 @@ static VGPUDMABuf struct virtio_gpu_rect *r) { VGPUDMABuf *dmabuf; + uint32_t offset = 0; if (res->dmabuf_fd < 0) { return NULL; } dmabuf = g_new0(VGPUDMABuf, 1); - dmabuf->buf = qemu_dmabuf_new(r->width, r->height, fb->stride, + dmabuf->buf = qemu_dmabuf_new(r->width, r->height, + &offset, &fb->stride, r->x, r->y, fb->width, fb->height, qemu_pixman_to_drm_format(fb->format), - 0, res->dmabuf_fd, true, false); + DRM_FORMAT_MOD_INVALID, &res->dmabuf_fd, + 1, true, false); dmabuf->scanout_id = scanout_id; QTAILQ_INSERT_HEAD(&g->dmabuf.bufs, dmabuf, next); diff --git a/hw/gpio/imx_gpio.c b/hw/gpio/imx_gpio.c index f23c52af26..450ece4548 100644 --- a/hw/gpio/imx_gpio.c +++ b/hw/gpio/imx_gpio.c @@ -72,7 +72,7 @@ static void imx_gpio_update_int(IMXGPIOState *s) static void imx_gpio_set_int_line(IMXGPIOState *s, int line, IMXGPIOLevel level) { /* if this signal isn't configured as an input signal, nothing to do */ - if (!extract32(s->gdir, line, 1)) { + if (extract32(s->gdir, line, 1)) { return; } diff --git a/hw/hyperv/hyperv.c b/hw/hyperv/hyperv.c index 0271cfd271..e4d0688dbf 100644 --- a/hw/hyperv/hyperv.c +++ b/hw/hyperv/hyperv.c @@ -13,6 +13,8 @@ #include "qapi/error.h" #include "system/address-spaces.h" #include "system/memory.h" +#include "exec/target_page.h" +#include "linux/kvm.h" #include "system/kvm.h" #include "qemu/bitops.h" #include "qemu/error-report.h" @@ -23,7 +25,6 @@ #include "hw/hyperv/hyperv.h" #include "qom/object.h" #include "target/i386/kvm/hyperv-proto.h" -#include "target/i386/cpu.h" #include "exec/target_page.h" struct SynICState { diff --git a/hw/hyperv/meson.build b/hw/hyperv/meson.build index d3d2668c71..d1cf781f04 100644 --- a/hw/hyperv/meson.build +++ b/hw/hyperv/meson.build @@ -1,5 +1,6 @@ -specific_ss.add(when: 'CONFIG_HYPERV', if_true: files('hyperv.c')) -specific_ss.add(when: 'CONFIG_HYPERV_TESTDEV', if_true: files('hyperv_testdev.c')) -specific_ss.add(when: 'CONFIG_VMBUS', if_true: files('vmbus.c')) -specific_ss.add(when: 'CONFIG_SYNDBG', if_true: files('syndbg.c')) -specific_ss.add(when: 'CONFIG_HV_BALLOON', if_true: files('hv-balloon.c', 'hv-balloon-page_range_tree.c', 'hv-balloon-our_range_memslots.c'), if_false: files('hv-balloon-stub.c')) +system_ss.add(when: 'CONFIG_HYPERV', if_true: files('hyperv.c')) +system_ss.add(when: 'CONFIG_HYPERV_TESTDEV', if_true: files('hyperv_testdev.c')) +system_ss.add(when: 'CONFIG_VMBUS', if_true: files('vmbus.c')) +system_ss.add(when: 'CONFIG_SYNDBG', if_true: files('syndbg.c')) +system_ss.add(when: 'CONFIG_HV_BALLOON', if_true: files('hv-balloon.c', 'hv-balloon-page_range_tree.c', 'hv-balloon-our_range_memslots.c')) +system_ss.add(when: 'CONFIG_HV_BALLOON', if_false: files('hv-balloon-stub.c')) diff --git a/hw/hyperv/syndbg.c b/hw/hyperv/syndbg.c index ca291826a0..8b8a14750d 100644 --- a/hw/hyperv/syndbg.c +++ b/hw/hyperv/syndbg.c @@ -10,11 +10,11 @@ #include "qemu/error-report.h" #include "qemu/main-loop.h" #include "qemu/sockets.h" +#include "qemu/units.h" #include "qapi/error.h" #include "migration/vmstate.h" #include "hw/qdev-properties.h" #include "hw/loader.h" -#include "cpu.h" #include "exec/target_page.h" #include "hw/hyperv/hyperv.h" #include "hw/hyperv/vmbus-bridge.h" @@ -184,12 +184,15 @@ static bool create_udp_pkt(HvSynDbg *syndbg, void *pkt, uint32_t pkt_len, return true; } +#define MSG_BUFSZ (4 * KiB) + static uint16_t handle_recv_msg(HvSynDbg *syndbg, uint64_t outgpa, uint32_t count, bool is_raw, uint32_t options, uint64_t timeout, uint32_t *retrieved_count) { uint16_t ret; - uint8_t data_buf[TARGET_PAGE_SIZE - UDP_PKT_HEADER_SIZE]; + g_assert(MSG_BUFSZ >= qemu_target_page_size()); + uint8_t data_buf[MSG_BUFSZ]; hwaddr out_len; void *out_data; ssize_t recv_byte_count; @@ -202,7 +205,7 @@ static uint16_t handle_recv_msg(HvSynDbg *syndbg, uint64_t outgpa, recv_byte_count = 0; } else { recv_byte_count = recv(syndbg->socket, data_buf, - MIN(sizeof(data_buf), count), MSG_WAITALL); + MIN(MSG_BUFSZ, count), MSG_WAITALL); if (recv_byte_count == -1) { return HV_STATUS_INVALID_PARAMETER; } diff --git a/hw/hyperv/vmbus.c b/hw/hyperv/vmbus.c index b147ea06d8..961406cdd6 100644 --- a/hw/hyperv/vmbus.c +++ b/hw/hyperv/vmbus.c @@ -19,7 +19,7 @@ #include "hw/hyperv/vmbus.h" #include "hw/hyperv/vmbus-bridge.h" #include "hw/sysbus.h" -#include "cpu.h" +#include "exec/target_page.h" #include "trace.h" enum { diff --git a/hw/i2c/imx_i2c.c b/hw/i2c/imx_i2c.c index 91f84c2ad7..d26177c85d 100644 --- a/hw/i2c/imx_i2c.c +++ b/hw/i2c/imx_i2c.c @@ -79,13 +79,12 @@ static void imx_i2c_reset(DeviceState *dev) static inline void imx_i2c_raise_interrupt(IMXI2CState *s) { - /* - * raise an interrupt if the device is enabled and it is configured - * to generate some interrupts. - */ - if (imx_i2c_is_enabled(s) && imx_i2c_interrupt_is_enabled(s)) { + if (imx_i2c_is_enabled(s)) { s->i2sr |= I2SR_IIF; - qemu_irq_raise(s->irq); + + if (imx_i2c_interrupt_is_enabled(s)) { + qemu_irq_raise(s->irq); + } } } diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index 3fffa4a332..f40ad062f9 100644 --- a/hw/i386/acpi-build.c +++ b/hw/i386/acpi-build.c @@ -139,7 +139,7 @@ static void init_common_fadt_data(MachineState *ms, Object *o, /* * "ICH9-LPC" or "PIIX4_PM" has "smm-compat" property to keep the old * behavior for compatibility irrelevant to smm_enabled, which doesn't - * comforms to ACPI spec. + * conform to the ACPI spec. */ bool smm_enabled = object_property_get_bool(o, "smm-compat", NULL) ? true : x86_machine_is_smm_enabled(x86ms); @@ -589,8 +589,8 @@ void build_append_pci_bus_devices(Aml *parent_scope, PCIBus *bus) } } -static bool build_append_notfication_callback(Aml *parent_scope, - const PCIBus *bus) +static bool build_append_notification_callback(Aml *parent_scope, + const PCIBus *bus) { Aml *method; PCIBus *sec; @@ -604,7 +604,7 @@ static bool build_append_notfication_callback(Aml *parent_scope, continue; } nr_notifiers = nr_notifiers + - build_append_notfication_callback(br_scope, sec); + build_append_notification_callback(br_scope, sec); /* * add new child scope to parent * and keep track of bus that have PCNT, @@ -1773,7 +1773,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, PCIBus *b = PCI_HOST_BRIDGE(pci_host)->bus; scope = aml_scope("\\_SB.PCI0"); - has_pcnt = build_append_notfication_callback(scope, b); + has_pcnt = build_append_notification_callback(scope, b); if (has_pcnt) { aml_append(dsdt, scope); } diff --git a/hw/i386/acpi-build.h b/hw/i386/acpi-build.h index 0dce155c8c..275ec058a1 100644 --- a/hw/i386/acpi-build.h +++ b/hw/i386/acpi-build.h @@ -5,7 +5,7 @@ extern const struct AcpiGenericAddress x86_nvdimm_acpi_dsmio; -/* PCI Hot-plug registers bases. See docs/spec/acpi_pci_hotplug.txt */ +/* PCI Hot-plug registers' base. See docs/specs/acpi_pci_hotplug.rst */ #define ACPI_PCIHP_SEJ_BASE 0x8 #define ACPI_PCIHP_BNMR_BASE 0x10 diff --git a/hw/intc/aspeed_intc.c b/hw/intc/aspeed_intc.c index be7f516a3b..33fcbe729c 100644 --- a/hw/intc/aspeed_intc.c +++ b/hw/intc/aspeed_intc.c @@ -62,6 +62,95 @@ REG32(GICINT196_STATUS, 0x44) REG32(GICINT197_EN, 0x50) REG32(GICINT197_STATUS, 0x54) +/* + * SSP INTC Registers + */ +REG32(SSPINT128_EN, 0x2000) +REG32(SSPINT128_STATUS, 0x2004) +REG32(SSPINT129_EN, 0x2100) +REG32(SSPINT129_STATUS, 0x2104) +REG32(SSPINT130_EN, 0x2200) +REG32(SSPINT130_STATUS, 0x2204) +REG32(SSPINT131_EN, 0x2300) +REG32(SSPINT131_STATUS, 0x2304) +REG32(SSPINT132_EN, 0x2400) +REG32(SSPINT132_STATUS, 0x2404) +REG32(SSPINT133_EN, 0x2500) +REG32(SSPINT133_STATUS, 0x2504) +REG32(SSPINT134_EN, 0x2600) +REG32(SSPINT134_STATUS, 0x2604) +REG32(SSPINT135_EN, 0x2700) +REG32(SSPINT135_STATUS, 0x2704) +REG32(SSPINT136_EN, 0x2800) +REG32(SSPINT136_STATUS, 0x2804) +REG32(SSPINT137_EN, 0x2900) +REG32(SSPINT137_STATUS, 0x2904) +REG32(SSPINT138_EN, 0x2A00) +REG32(SSPINT138_STATUS, 0x2A04) +REG32(SSPINT160_169_EN, 0x2B00) +REG32(SSPINT160_169_STATUS, 0x2B04) + +/* + * SSP INTCIO Registers + */ +REG32(SSPINT160_EN, 0x180) +REG32(SSPINT160_STATUS, 0x184) +REG32(SSPINT161_EN, 0x190) +REG32(SSPINT161_STATUS, 0x194) +REG32(SSPINT162_EN, 0x1A0) +REG32(SSPINT162_STATUS, 0x1A4) +REG32(SSPINT163_EN, 0x1B0) +REG32(SSPINT163_STATUS, 0x1B4) +REG32(SSPINT164_EN, 0x1C0) +REG32(SSPINT164_STATUS, 0x1C4) +REG32(SSPINT165_EN, 0x1D0) +REG32(SSPINT165_STATUS, 0x1D4) + +/* + * TSP INTC Registers + */ +REG32(TSPINT128_EN, 0x3000) +REG32(TSPINT128_STATUS, 0x3004) +REG32(TSPINT129_EN, 0x3100) +REG32(TSPINT129_STATUS, 0x3104) +REG32(TSPINT130_EN, 0x3200) +REG32(TSPINT130_STATUS, 0x3204) +REG32(TSPINT131_EN, 0x3300) +REG32(TSPINT131_STATUS, 0x3304) +REG32(TSPINT132_EN, 0x3400) +REG32(TSPINT132_STATUS, 0x3404) +REG32(TSPINT133_EN, 0x3500) +REG32(TSPINT133_STATUS, 0x3504) +REG32(TSPINT134_EN, 0x3600) +REG32(TSPINT134_STATUS, 0x3604) +REG32(TSPINT135_EN, 0x3700) +REG32(TSPINT135_STATUS, 0x3704) +REG32(TSPINT136_EN, 0x3800) +REG32(TSPINT136_STATUS, 0x3804) +REG32(TSPINT137_EN, 0x3900) +REG32(TSPINT137_STATUS, 0x3904) +REG32(TSPINT138_EN, 0x3A00) +REG32(TSPINT138_STATUS, 0x3A04) +REG32(TSPINT160_169_EN, 0x3B00) +REG32(TSPINT160_169_STATUS, 0x3B04) + +/* + * TSP INTCIO Registers + */ + +REG32(TSPINT160_EN, 0x200) +REG32(TSPINT160_STATUS, 0x204) +REG32(TSPINT161_EN, 0x210) +REG32(TSPINT161_STATUS, 0x214) +REG32(TSPINT162_EN, 0x220) +REG32(TSPINT162_STATUS, 0x224) +REG32(TSPINT163_EN, 0x230) +REG32(TSPINT163_STATUS, 0x234) +REG32(TSPINT164_EN, 0x240) +REG32(TSPINT164_STATUS, 0x244) +REG32(TSPINT165_EN, 0x250) +REG32(TSPINT165_STATUS, 0x254) + static const AspeedINTCIRQ *aspeed_intc_get_irq(AspeedINTCClass *aic, uint32_t reg) { @@ -450,6 +539,90 @@ static void aspeed_intc_write(void *opaque, hwaddr offset, uint64_t data, } } +static void aspeed_ssp_intc_write(void *opaque, hwaddr offset, uint64_t data, + unsigned size) +{ + AspeedINTCState *s = ASPEED_INTC(opaque); + const char *name = object_get_typename(OBJECT(s)); + uint32_t reg = offset >> 2; + + trace_aspeed_intc_write(name, offset, size, data); + + switch (reg) { + case R_SSPINT128_EN: + case R_SSPINT129_EN: + case R_SSPINT130_EN: + case R_SSPINT131_EN: + case R_SSPINT132_EN: + case R_SSPINT133_EN: + case R_SSPINT134_EN: + case R_SSPINT135_EN: + case R_SSPINT136_EN: + case R_SSPINT160_169_EN: + aspeed_intc_enable_handler(s, offset, data); + break; + case R_SSPINT128_STATUS: + case R_SSPINT129_STATUS: + case R_SSPINT130_STATUS: + case R_SSPINT131_STATUS: + case R_SSPINT132_STATUS: + case R_SSPINT133_STATUS: + case R_SSPINT134_STATUS: + case R_SSPINT135_STATUS: + case R_SSPINT136_STATUS: + aspeed_intc_status_handler(s, offset, data); + break; + case R_SSPINT160_169_STATUS: + aspeed_intc_status_handler_multi_outpins(s, offset, data); + break; + default: + s->regs[reg] = data; + break; + } +} + +static void aspeed_tsp_intc_write(void *opaque, hwaddr offset, uint64_t data, + unsigned size) +{ + AspeedINTCState *s = ASPEED_INTC(opaque); + const char *name = object_get_typename(OBJECT(s)); + uint32_t reg = offset >> 2; + + trace_aspeed_intc_write(name, offset, size, data); + + switch (reg) { + case R_TSPINT128_EN: + case R_TSPINT129_EN: + case R_TSPINT130_EN: + case R_TSPINT131_EN: + case R_TSPINT132_EN: + case R_TSPINT133_EN: + case R_TSPINT134_EN: + case R_TSPINT135_EN: + case R_TSPINT136_EN: + case R_TSPINT160_169_EN: + aspeed_intc_enable_handler(s, offset, data); + break; + case R_TSPINT128_STATUS: + case R_TSPINT129_STATUS: + case R_TSPINT130_STATUS: + case R_TSPINT131_STATUS: + case R_TSPINT132_STATUS: + case R_TSPINT133_STATUS: + case R_TSPINT134_STATUS: + case R_TSPINT135_STATUS: + case R_TSPINT136_STATUS: + aspeed_intc_status_handler(s, offset, data); + break; + case R_TSPINT160_169_STATUS: + aspeed_intc_status_handler_multi_outpins(s, offset, data); + break; + default: + s->regs[reg] = data; + break; + } +} + static uint64_t aspeed_intcio_read(void *opaque, hwaddr offset, unsigned int size) { @@ -496,6 +669,69 @@ static void aspeed_intcio_write(void *opaque, hwaddr offset, uint64_t data, } } +static void aspeed_ssp_intcio_write(void *opaque, hwaddr offset, uint64_t data, + unsigned size) +{ + AspeedINTCState *s = ASPEED_INTC(opaque); + const char *name = object_get_typename(OBJECT(s)); + uint32_t reg = offset >> 2; + + trace_aspeed_intc_write(name, offset, size, data); + + switch (reg) { + case R_SSPINT160_EN: + case R_SSPINT161_EN: + case R_SSPINT162_EN: + case R_SSPINT163_EN: + case R_SSPINT164_EN: + case R_SSPINT165_EN: + aspeed_intc_enable_handler(s, offset, data); + break; + case R_SSPINT160_STATUS: + case R_SSPINT161_STATUS: + case R_SSPINT162_STATUS: + case R_SSPINT163_STATUS: + case R_SSPINT164_STATUS: + case R_SSPINT165_STATUS: + aspeed_intc_status_handler(s, offset, data); + break; + default: + s->regs[reg] = data; + break; + } +} + +static void aspeed_tsp_intcio_write(void *opaque, hwaddr offset, uint64_t data, + unsigned size) +{ + AspeedINTCState *s = ASPEED_INTC(opaque); + const char *name = object_get_typename(OBJECT(s)); + uint32_t reg = offset >> 2; + + trace_aspeed_intc_write(name, offset, size, data); + + switch (reg) { + case R_TSPINT160_EN: + case R_TSPINT161_EN: + case R_TSPINT162_EN: + case R_TSPINT163_EN: + case R_TSPINT164_EN: + case R_TSPINT165_EN: + aspeed_intc_enable_handler(s, offset, data); + break; + case R_TSPINT160_STATUS: + case R_TSPINT161_STATUS: + case R_TSPINT162_STATUS: + case R_TSPINT163_STATUS: + case R_TSPINT164_STATUS: + case R_TSPINT165_STATUS: + aspeed_intc_status_handler(s, offset, data); + break; + default: + s->regs[reg] = data; + break; + } +} static const MemoryRegionOps aspeed_intc_ops = { .read = aspeed_intc_read, @@ -517,6 +753,46 @@ static const MemoryRegionOps aspeed_intcio_ops = { } }; +static const MemoryRegionOps aspeed_ssp_intc_ops = { + .read = aspeed_intc_read, + .write = aspeed_ssp_intc_write, + .endianness = DEVICE_LITTLE_ENDIAN, + .valid = { + .min_access_size = 4, + .max_access_size = 4, + } +}; + +static const MemoryRegionOps aspeed_ssp_intcio_ops = { + .read = aspeed_intcio_read, + .write = aspeed_ssp_intcio_write, + .endianness = DEVICE_LITTLE_ENDIAN, + .valid = { + .min_access_size = 4, + .max_access_size = 4, + } +}; + +static const MemoryRegionOps aspeed_tsp_intc_ops = { + .read = aspeed_intc_read, + .write = aspeed_tsp_intc_write, + .endianness = DEVICE_LITTLE_ENDIAN, + .valid = { + .min_access_size = 4, + .max_access_size = 4, + } +}; + +static const MemoryRegionOps aspeed_tsp_intcio_ops = { + .read = aspeed_intcio_read, + .write = aspeed_tsp_intcio_write, + .endianness = DEVICE_LITTLE_ENDIAN, + .valid = { + .min_access_size = 4, + .max_access_size = 4, + } +}; + static void aspeed_intc_instance_init(Object *obj) { AspeedINTCState *s = ASPEED_INTC(obj); @@ -674,11 +950,151 @@ static const TypeInfo aspeed_2700_intcio_info = { .class_init = aspeed_2700_intcio_class_init, }; +static AspeedINTCIRQ aspeed_2700ssp_intc_irqs[ASPEED_INTC_MAX_INPINS] = { + {0, 0, 10, R_SSPINT160_169_EN, R_SSPINT160_169_STATUS}, + {1, 10, 1, R_SSPINT128_EN, R_SSPINT128_STATUS}, + {2, 11, 1, R_SSPINT129_EN, R_SSPINT129_STATUS}, + {3, 12, 1, R_SSPINT130_EN, R_SSPINT130_STATUS}, + {4, 13, 1, R_SSPINT131_EN, R_SSPINT131_STATUS}, + {5, 14, 1, R_SSPINT132_EN, R_SSPINT132_STATUS}, + {6, 15, 1, R_SSPINT133_EN, R_SSPINT133_STATUS}, + {7, 16, 1, R_SSPINT134_EN, R_SSPINT134_STATUS}, + {8, 17, 1, R_SSPINT135_EN, R_SSPINT135_STATUS}, + {9, 18, 1, R_SSPINT136_EN, R_SSPINT136_STATUS}, +}; + +static void aspeed_2700ssp_intc_class_init(ObjectClass *klass, const void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + AspeedINTCClass *aic = ASPEED_INTC_CLASS(klass); + + dc->desc = "ASPEED 2700 SSP INTC Controller"; + aic->num_lines = 32; + aic->num_inpins = 10; + aic->num_outpins = 19; + aic->mem_size = 0x4000; + aic->nr_regs = 0x2B08 >> 2; + aic->reg_offset = 0x0; + aic->reg_ops = &aspeed_ssp_intc_ops; + aic->irq_table = aspeed_2700ssp_intc_irqs; + aic->irq_table_count = ARRAY_SIZE(aspeed_2700ssp_intc_irqs); +} + +static const TypeInfo aspeed_2700ssp_intc_info = { + .name = TYPE_ASPEED_2700SSP_INTC, + .parent = TYPE_ASPEED_INTC, + .class_init = aspeed_2700ssp_intc_class_init, +}; + +static AspeedINTCIRQ aspeed_2700ssp_intcio_irqs[ASPEED_INTC_MAX_INPINS] = { + {0, 0, 1, R_SSPINT160_EN, R_SSPINT160_STATUS}, + {1, 1, 1, R_SSPINT161_EN, R_SSPINT161_STATUS}, + {2, 2, 1, R_SSPINT162_EN, R_SSPINT162_STATUS}, + {3, 3, 1, R_SSPINT163_EN, R_SSPINT163_STATUS}, + {4, 4, 1, R_SSPINT164_EN, R_SSPINT164_STATUS}, + {5, 5, 1, R_SSPINT165_EN, R_SSPINT165_STATUS}, +}; + +static void aspeed_2700ssp_intcio_class_init(ObjectClass *klass, const void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + AspeedINTCClass *aic = ASPEED_INTC_CLASS(klass); + + dc->desc = "ASPEED 2700 SSP INTC IO Controller"; + aic->num_lines = 32; + aic->num_inpins = 6; + aic->num_outpins = 6; + aic->mem_size = 0x400; + aic->nr_regs = 0x1d8 >> 2; + aic->reg_offset = 0; + aic->reg_ops = &aspeed_ssp_intcio_ops; + aic->irq_table = aspeed_2700ssp_intcio_irqs; + aic->irq_table_count = ARRAY_SIZE(aspeed_2700ssp_intcio_irqs); +} + +static const TypeInfo aspeed_2700ssp_intcio_info = { + .name = TYPE_ASPEED_2700SSP_INTCIO, + .parent = TYPE_ASPEED_INTC, + .class_init = aspeed_2700ssp_intcio_class_init, +}; + +static AspeedINTCIRQ aspeed_2700tsp_intc_irqs[ASPEED_INTC_MAX_INPINS] = { + {0, 0, 10, R_TSPINT160_169_EN, R_TSPINT160_169_STATUS}, + {1, 10, 1, R_TSPINT128_EN, R_TSPINT128_STATUS}, + {2, 11, 1, R_TSPINT129_EN, R_TSPINT129_STATUS}, + {3, 12, 1, R_TSPINT130_EN, R_TSPINT130_STATUS}, + {4, 13, 1, R_TSPINT131_EN, R_TSPINT131_STATUS}, + {5, 14, 1, R_TSPINT132_EN, R_TSPINT132_STATUS}, + {6, 15, 1, R_TSPINT133_EN, R_TSPINT133_STATUS}, + {7, 16, 1, R_TSPINT134_EN, R_TSPINT134_STATUS}, + {8, 17, 1, R_TSPINT135_EN, R_TSPINT135_STATUS}, + {9, 18, 1, R_TSPINT136_EN, R_TSPINT136_STATUS}, +}; + +static void aspeed_2700tsp_intc_class_init(ObjectClass *klass, const void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + AspeedINTCClass *aic = ASPEED_INTC_CLASS(klass); + + dc->desc = "ASPEED 2700 TSP INTC Controller"; + aic->num_lines = 32; + aic->num_inpins = 10; + aic->num_outpins = 19; + aic->mem_size = 0x4000; + aic->nr_regs = 0x3B08 >> 2; + aic->reg_offset = 0; + aic->reg_ops = &aspeed_tsp_intc_ops; + aic->irq_table = aspeed_2700tsp_intc_irqs; + aic->irq_table_count = ARRAY_SIZE(aspeed_2700tsp_intc_irqs); +} + +static const TypeInfo aspeed_2700tsp_intc_info = { + .name = TYPE_ASPEED_2700TSP_INTC, + .parent = TYPE_ASPEED_INTC, + .class_init = aspeed_2700tsp_intc_class_init, +}; + +static AspeedINTCIRQ aspeed_2700tsp_intcio_irqs[ASPEED_INTC_MAX_INPINS] = { + {0, 0, 1, R_TSPINT160_EN, R_TSPINT160_STATUS}, + {1, 1, 1, R_TSPINT161_EN, R_TSPINT161_STATUS}, + {2, 2, 1, R_TSPINT162_EN, R_TSPINT162_STATUS}, + {3, 3, 1, R_TSPINT163_EN, R_TSPINT163_STATUS}, + {4, 4, 1, R_TSPINT164_EN, R_TSPINT164_STATUS}, + {5, 5, 1, R_TSPINT165_EN, R_TSPINT165_STATUS}, +}; + +static void aspeed_2700tsp_intcio_class_init(ObjectClass *klass, const void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + AspeedINTCClass *aic = ASPEED_INTC_CLASS(klass); + + dc->desc = "ASPEED 2700 TSP INTC IO Controller"; + aic->num_lines = 32; + aic->num_inpins = 6; + aic->num_outpins = 6; + aic->mem_size = 0x400; + aic->nr_regs = 0x258 >> 2; + aic->reg_offset = 0x0; + aic->reg_ops = &aspeed_tsp_intcio_ops; + aic->irq_table = aspeed_2700tsp_intcio_irqs; + aic->irq_table_count = ARRAY_SIZE(aspeed_2700tsp_intcio_irqs); +} + +static const TypeInfo aspeed_2700tsp_intcio_info = { + .name = TYPE_ASPEED_2700TSP_INTCIO, + .parent = TYPE_ASPEED_INTC, + .class_init = aspeed_2700tsp_intcio_class_init, +}; + static void aspeed_intc_register_types(void) { type_register_static(&aspeed_intc_info); type_register_static(&aspeed_2700_intc_info); type_register_static(&aspeed_2700_intcio_info); + type_register_static(&aspeed_2700ssp_intc_info); + type_register_static(&aspeed_2700ssp_intcio_info); + type_register_static(&aspeed_2700tsp_intc_info); + type_register_static(&aspeed_2700tsp_intcio_info); } type_init(aspeed_intc_register_types); diff --git a/hw/intc/loongarch_extioi.c b/hw/intc/loongarch_extioi.c index f4fe961a98..7c38c4c9b7 100644 --- a/hw/intc/loongarch_extioi.c +++ b/hw/intc/loongarch_extioi.c @@ -377,11 +377,13 @@ static void loongarch_extioi_unrealize(DeviceState *dev) g_free(s->cpu); } -static void loongarch_extioi_reset(DeviceState *d) +static void loongarch_extioi_reset_hold(Object *obj, ResetType type) { - LoongArchExtIOICommonState *s = LOONGARCH_EXTIOI_COMMON(d); + LoongArchExtIOIClass *lec = LOONGARCH_EXTIOI_GET_CLASS(obj); - s->status = 0; + if (lec->parent_phases.hold) { + lec->parent_phases.hold(obj, type); + } } static int vmstate_extioi_post_load(void *opaque, int version_id) @@ -406,12 +408,14 @@ static void loongarch_extioi_class_init(ObjectClass *klass, const void *data) DeviceClass *dc = DEVICE_CLASS(klass); LoongArchExtIOIClass *lec = LOONGARCH_EXTIOI_CLASS(klass); LoongArchExtIOICommonClass *lecc = LOONGARCH_EXTIOI_COMMON_CLASS(klass); + ResettableClass *rc = RESETTABLE_CLASS(klass); device_class_set_parent_realize(dc, loongarch_extioi_realize, &lec->parent_realize); device_class_set_parent_unrealize(dc, loongarch_extioi_unrealize, &lec->parent_unrealize); - device_class_set_legacy_reset(dc, loongarch_extioi_reset); + resettable_class_set_parent_phases(rc, NULL, loongarch_extioi_reset_hold, + NULL, &lec->parent_phases); lecc->post_load = vmstate_extioi_post_load; } diff --git a/hw/intc/loongarch_extioi_common.c b/hw/intc/loongarch_extioi_common.c index 9e1589060c..4a904b3bc1 100644 --- a/hw/intc/loongarch_extioi_common.c +++ b/hw/intc/loongarch_extioi_common.c @@ -108,6 +108,43 @@ static void loongarch_extioi_common_realize(DeviceState *dev, Error **errp) } } +static void loongarch_extioi_common_reset_hold(Object *obj, ResetType type) +{ + LoongArchExtIOICommonClass *lecc = LOONGARCH_EXTIOI_COMMON_GET_CLASS(obj); + LoongArchExtIOICommonState *s = LOONGARCH_EXTIOI_COMMON(obj); + ExtIOICore *core; + int i; + + if (lecc->parent_phases.hold) { + lecc->parent_phases.hold(obj, type); + } + + /* Clear HW registers for the board */ + memset(s->nodetype, 0, sizeof(s->nodetype)); + memset(s->bounce, 0, sizeof(s->bounce)); + memset(s->isr, 0, sizeof(s->isr)); + memset(s->enable, 0, sizeof(s->enable)); + memset(s->ipmap, 0, sizeof(s->ipmap)); + memset(s->coremap, 0, sizeof(s->coremap)); + memset(s->sw_pending, 0, sizeof(s->sw_pending)); + memset(s->sw_ipmap, 0, sizeof(s->sw_ipmap)); + memset(s->sw_coremap, 0, sizeof(s->sw_coremap)); + + for (i = 0; i < s->num_cpu; i++) { + core = s->cpu + i; + /* EXTIOI with targeted CPU available however not present */ + if (!core->cpu) { + continue; + } + + /* Clear HW registers for CPUs */ + memset(core->coreisr, 0, sizeof(core->coreisr)); + memset(core->sw_isr, 0, sizeof(core->sw_isr)); + } + + s->status = 0; +} + static int loongarch_extioi_common_pre_save(void *opaque) { LoongArchExtIOICommonState *s = (LoongArchExtIOICommonState *)opaque; @@ -180,9 +217,13 @@ static void loongarch_extioi_common_class_init(ObjectClass *klass, DeviceClass *dc = DEVICE_CLASS(klass); LoongArchExtIOICommonClass *lecc = LOONGARCH_EXTIOI_COMMON_CLASS(klass); HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass); + ResettableClass *rc = RESETTABLE_CLASS(klass); device_class_set_parent_realize(dc, loongarch_extioi_common_realize, &lecc->parent_realize); + resettable_class_set_parent_phases(rc, NULL, + loongarch_extioi_common_reset_hold, + NULL, &lecc->parent_phases); device_class_set_props(dc, extioi_properties); dc->vmsd = &vmstate_loongarch_extioi; hc->plug = loongarch_extioi_cpu_plug; diff --git a/hw/intc/loongarch_ipi.c b/hw/intc/loongarch_ipi.c index 2f8bb57828..74372a2039 100644 --- a/hw/intc/loongarch_ipi.c +++ b/hw/intc/loongarch_ipi.c @@ -93,6 +93,32 @@ static void loongarch_ipi_realize(DeviceState *dev, Error **errp) } } +static void loongarch_ipi_reset_hold(Object *obj, ResetType type) +{ + int i; + LoongarchIPIClass *lic = LOONGARCH_IPI_GET_CLASS(obj); + LoongsonIPICommonState *lics = LOONGSON_IPI_COMMON(obj); + IPICore *core; + + if (lic->parent_phases.hold) { + lic->parent_phases.hold(obj, type); + } + + for (i = 0; i < lics->num_cpu; i++) { + core = lics->cpu + i; + /* IPI with targeted CPU available however not present */ + if (!core->cpu) { + continue; + } + + core->status = 0; + core->en = 0; + core->set = 0; + core->clear = 0; + memset(core->buf, 0, sizeof(core->buf)); + } +} + static void loongarch_ipi_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp) { @@ -145,10 +171,13 @@ static void loongarch_ipi_class_init(ObjectClass *klass, const void *data) LoongsonIPICommonClass *licc = LOONGSON_IPI_COMMON_CLASS(klass); HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass); LoongarchIPIClass *lic = LOONGARCH_IPI_CLASS(klass); + ResettableClass *rc = RESETTABLE_CLASS(klass); DeviceClass *dc = DEVICE_CLASS(klass); device_class_set_parent_realize(dc, loongarch_ipi_realize, &lic->parent_realize); + resettable_class_set_parent_phases(rc, NULL, loongarch_ipi_reset_hold, + NULL, &lic->parent_phases); licc->get_iocsr_as = get_iocsr_as; licc->cpu_by_arch_id = loongarch_cpu_by_arch_id; hc->plug = loongarch_ipi_cpu_plug; diff --git a/hw/intc/loongarch_pch_pic.c b/hw/intc/loongarch_pch_pic.c index 6c2b6de3f0..834096265a 100644 --- a/hw/intc/loongarch_pch_pic.c +++ b/hw/intc/loongarch_pch_pic.c @@ -354,25 +354,13 @@ static const MemoryRegionOps loongarch_pch_pic_reg8_ops = { .endianness = DEVICE_LITTLE_ENDIAN, }; -static void loongarch_pch_pic_reset(DeviceState *d) +static void loongarch_pic_reset_hold(Object *obj, ResetType type) { - LoongArchPICCommonState *s = LOONGARCH_PIC_COMMON(d); - int i; - - s->int_mask = -1; - s->htmsi_en = 0x0; - s->intedge = 0x0; - s->intclr = 0x0; - s->auto_crtl0 = 0x0; - s->auto_crtl1 = 0x0; - for (i = 0; i < 64; i++) { - s->route_entry[i] = 0x1; - s->htmsi_vector[i] = 0x0; + LoongarchPICClass *lpc = LOONGARCH_PIC_GET_CLASS(obj); + + if (lpc->parent_phases.hold) { + lpc->parent_phases.hold(obj, type); } - s->intirr = 0x0; - s->intisr = 0x0; - s->last_intirr = 0x0; - s->int_polarity = 0x0; } static void loongarch_pic_realize(DeviceState *dev, Error **errp) @@ -408,8 +396,10 @@ static void loongarch_pic_class_init(ObjectClass *klass, const void *data) { DeviceClass *dc = DEVICE_CLASS(klass); LoongarchPICClass *lpc = LOONGARCH_PIC_CLASS(klass); + ResettableClass *rc = RESETTABLE_CLASS(klass); - device_class_set_legacy_reset(dc, loongarch_pch_pic_reset); + resettable_class_set_parent_phases(rc, NULL, loongarch_pic_reset_hold, + NULL, &lpc->parent_phases); device_class_set_parent_realize(dc, loongarch_pic_realize, &lpc->parent_realize); } diff --git a/hw/intc/loongarch_pic_common.c b/hw/intc/loongarch_pic_common.c index fdb250c418..6dccacc741 100644 --- a/hw/intc/loongarch_pic_common.c +++ b/hw/intc/loongarch_pic_common.c @@ -44,6 +44,27 @@ static void loongarch_pic_common_realize(DeviceState *dev, Error **errp) } } +static void loongarch_pic_common_reset_hold(Object *obj, ResetType type) +{ + LoongArchPICCommonState *s = LOONGARCH_PIC_COMMON(obj); + int i; + + s->int_mask = UINT64_MAX; + s->htmsi_en = 0x0; + s->intedge = 0x0; + s->intclr = 0x0; + s->auto_crtl0 = 0x0; + s->auto_crtl1 = 0x0; + for (i = 0; i < 64; i++) { + s->route_entry[i] = 0x1; + s->htmsi_vector[i] = 0x0; + } + s->intirr = 0x0; + s->intisr = 0x0; + s->last_intirr = 0x0; + s->int_polarity = 0x0; +} + static const Property loongarch_pic_common_properties[] = { DEFINE_PROP_UINT32("pch_pic_irq_num", LoongArchPICCommonState, irq_num, 0), }; @@ -76,9 +97,13 @@ static void loongarch_pic_common_class_init(ObjectClass *klass, { DeviceClass *dc = DEVICE_CLASS(klass); LoongArchPICCommonClass *lpcc = LOONGARCH_PIC_COMMON_CLASS(klass); + ResettableClass *rc = RESETTABLE_CLASS(klass); device_class_set_parent_realize(dc, loongarch_pic_common_realize, &lpcc->parent_realize); + resettable_class_set_parent_phases(rc, NULL, + loongarch_pic_common_reset_hold, + NULL, &lpcc->parent_phases); device_class_set_props(dc, loongarch_pic_common_properties); dc->vmsd = &vmstate_loongarch_pic_common; } diff --git a/hw/isa/lpc_ich9.c b/hw/isa/lpc_ich9.c index 71afb45b63..304dffac32 100644 --- a/hw/isa/lpc_ich9.c +++ b/hw/isa/lpc_ich9.c @@ -182,7 +182,6 @@ static uint64_t ich9_cc_read(void *opaque, hwaddr addr, } /* IRQ routing */ -/* */ static void ich9_lpc_rout(uint8_t pirq_rout, int *pic_irq, int *pic_dis) { *pic_irq = pirq_rout & ICH9_LPC_PIRQ_ROUT_MASK; diff --git a/hw/loongarch/boot.c b/hw/loongarch/boot.c index 354cf458c8..0324d6adcb 100644 --- a/hw/loongarch/boot.c +++ b/hw/loongarch/boot.c @@ -245,6 +245,7 @@ static int64_t load_kernel_info(struct loongarch_boot_info *info) &kernel_entry, &kernel_low, &kernel_high, NULL, ELFDATA2LSB, EM_LOONGARCH, 1, 0); + kernel_entry = cpu_loongarch_virt_to_phys(NULL, kernel_entry); if (kernel_size < 0) { kernel_size = load_loongarch_linux_image(info->kernel_filename, &kernel_entry, &kernel_low, diff --git a/hw/loongarch/virt-acpi-build.c b/hw/loongarch/virt-acpi-build.c index fced6c445a..073b6de75c 100644 --- a/hw/loongarch/virt-acpi-build.c +++ b/hw/loongarch/virt-acpi-build.c @@ -514,7 +514,7 @@ static void acpi_build(AcpiBuildTables *tables, MachineState *machine) LoongArchVirtMachineState *lvms = LOONGARCH_VIRT_MACHINE(machine); GArray *table_offsets; AcpiFadtData fadt_data; - unsigned facs, rsdt, dsdt; + unsigned facs, xsdt, dsdt; uint8_t *u; GArray *tables_blob = tables->table_data; @@ -600,17 +600,17 @@ static void acpi_build(AcpiBuildTables *tables, MachineState *machine) } /* RSDT is pointed to by RSDP */ - rsdt = tables_blob->len; - build_rsdt(tables_blob, tables->linker, table_offsets, + xsdt = tables_blob->len; + build_xsdt(tables_blob, tables->linker, table_offsets, lvms->oem_id, lvms->oem_table_id); /* RSDP is in FSEG memory, so allocate it separately */ { AcpiRsdpData rsdp_data = { - .revision = 0, + .revision = 2, .oem_id = lvms->oem_id, - .xsdt_tbl_offset = NULL, - .rsdt_tbl_offset = &rsdt, + .xsdt_tbl_offset = &xsdt, + .rsdt_tbl_offset = NULL, }; build_rsdp(tables->rsdp, tables->linker, &rsdp_data); } diff --git a/hw/loongarch/virt.c b/hw/loongarch/virt.c index 779544fada..7ad7fb68ff 100644 --- a/hw/loongarch/virt.c +++ b/hw/loongarch/virt.c @@ -773,6 +773,48 @@ static void virt_set_acpi(Object *obj, Visitor *v, const char *name, visit_type_OnOffAuto(v, name, &lvms->acpi, errp); } +static char *virt_get_oem_id(Object *obj, Error **errp) +{ + LoongArchVirtMachineState *lvms = LOONGARCH_VIRT_MACHINE(obj); + + return g_strdup(lvms->oem_id); +} + +static void virt_set_oem_id(Object *obj, const char *value, Error **errp) +{ + LoongArchVirtMachineState *lvms = LOONGARCH_VIRT_MACHINE(obj); + size_t len = strlen(value); + + if (len > 6) { + error_setg(errp, + "User specified oem-id value is bigger than 6 bytes in size"); + return; + } + + strncpy(lvms->oem_id, value, 6); +} + +static char *virt_get_oem_table_id(Object *obj, Error **errp) +{ + LoongArchVirtMachineState *lvms = LOONGARCH_VIRT_MACHINE(obj); + + return g_strdup(lvms->oem_table_id); +} + +static void virt_set_oem_table_id(Object *obj, const char *value, + Error **errp) +{ + LoongArchVirtMachineState *lvms = LOONGARCH_VIRT_MACHINE(obj); + size_t len = strlen(value); + + if (len > 8) { + error_setg(errp, + "User specified oem-table-id value is bigger than 8 bytes in size"); + return; + } + strncpy(lvms->oem_table_id, value, 8); +} + static void virt_initfn(Object *obj) { LoongArchVirtMachineState *lvms = LOONGARCH_VIRT_MACHINE(obj); @@ -1177,6 +1219,22 @@ static void virt_class_init(ObjectClass *oc, const void *data) #ifdef CONFIG_TPM machine_class_allow_dynamic_sysbus_dev(mc, TYPE_TPM_TIS_SYSBUS); #endif + object_class_property_add_str(oc, "x-oem-id", + virt_get_oem_id, + virt_set_oem_id); + object_class_property_set_description(oc, "x-oem-id", + "Override the default value of field OEMID " + "in ACPI table header." + "The string may be up to 6 bytes in size"); + + + object_class_property_add_str(oc, "x-oem-table-id", + virt_get_oem_table_id, + virt_set_oem_table_id); + object_class_property_set_description(oc, "x-oem-table-id", + "Override the default value of field OEM Table ID " + "in ACPI table header." + "The string may be up to 8 bytes in size"); } static const TypeInfo virt_machine_types[] = { diff --git a/hw/net/e1000x_regs.h b/hw/net/e1000x_regs.h index cd896fc0ca..e9a74de6f4 100644 --- a/hw/net/e1000x_regs.h +++ b/hw/net/e1000x_regs.h @@ -900,7 +900,7 @@ struct e1000_context_desc { uint16_t tucse; /* TCP checksum end */ } tcp_fields; } upper_setup; - uint32_t cmd_and_length; /* */ + uint32_t cmd_and_length; union { uint32_t data; struct { diff --git a/hw/pci-host/designware.c b/hw/pci-host/designware.c index 183f838392..f6e49ce9b8 100644 --- a/hw/pci-host/designware.c +++ b/hw/pci-host/designware.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "qapi/error.h" -#include "qemu/module.h" #include "qemu/log.h" #include "qemu/bitops.h" #include "hw/pci/msi.h" @@ -349,14 +348,14 @@ static void designware_pcie_root_config_write(PCIDevice *d, uint32_t address, case DESIGNWARE_PCIE_ATU_LOWER_BASE: case DESIGNWARE_PCIE_ATU_UPPER_BASE: - viewport->base = deposit64(root->msi.base, + viewport->base = deposit64(viewport->base, address == DESIGNWARE_PCIE_ATU_LOWER_BASE ? 0 : 32, 32, val); break; case DESIGNWARE_PCIE_ATU_LOWER_TARGET: case DESIGNWARE_PCIE_ATU_UPPER_TARGET: - viewport->target = deposit64(root->msi.base, + viewport->target = deposit64(viewport->target, address == DESIGNWARE_PCIE_ATU_LOWER_TARGET ? 0 : 32, 32, val); break; diff --git a/hw/pci-host/gpex-acpi.c b/hw/pci-host/gpex-acpi.c index e8b4c64c5f..0aba47c71c 100644 --- a/hw/pci-host/gpex-acpi.c +++ b/hw/pci-host/gpex-acpi.c @@ -182,7 +182,7 @@ void acpi_dsdt_add_gpex(Aml *scope, struct GPEXConfig *cfg) /* * Resources defined for PXBs are composed of the following parts: - * 1. The resources the pci-brige/pcie-root-port need. + * 1. The resources the pci-bridge/pcie-root-port need. * 2. The resources the devices behind pxb need. */ crs = build_crs(PCI_HOST_BRIDGE(BUS(bus)->parent), &crs_range_set, diff --git a/hw/pci/pci.c b/hw/pci/pci.c index fe38c4c028..352b3d12c8 100644 --- a/hw/pci/pci.c +++ b/hw/pci/pci.c @@ -54,13 +54,6 @@ #include "hw/xen/xen.h" #include "hw/i386/kvm/xen_evtchn.h" -//#define DEBUG_PCI -#ifdef DEBUG_PCI -# define PCI_DPRINTF(format, ...) printf(format, ## __VA_ARGS__) -#else -# define PCI_DPRINTF(format, ...) do { } while (0) -#endif - bool pci_available = true; static char *pcibus_get_dev_path(DeviceState *dev); @@ -2439,12 +2432,12 @@ static void pci_patch_ids(PCIDevice *pdev, uint8_t *ptr, uint32_t size) /* Only a valid rom will be patched. */ rom_magic = pci_get_word(ptr); if (rom_magic != 0xaa55) { - PCI_DPRINTF("Bad ROM magic %04x\n", rom_magic); + trace_pci_bad_rom_magic(rom_magic, 0xaa55); return; } pcir_offset = pci_get_word(ptr + 0x18); if (pcir_offset + 8 >= size || memcmp(ptr + pcir_offset, "PCIR", 4)) { - PCI_DPRINTF("Bad PCIR offset 0x%x or signature\n", pcir_offset); + trace_pci_bad_pcir_offset(pcir_offset); return; } @@ -2453,8 +2446,8 @@ static void pci_patch_ids(PCIDevice *pdev, uint8_t *ptr, uint32_t size) rom_vendor_id = pci_get_word(ptr + pcir_offset + 4); rom_device_id = pci_get_word(ptr + pcir_offset + 6); - PCI_DPRINTF("%s: ROM id %04x%04x / PCI id %04x%04x\n", pdev->romfile, - vendor_id, device_id, rom_vendor_id, rom_device_id); + trace_pci_rom_and_pci_ids(pdev->romfile, vendor_id, device_id, + rom_vendor_id, rom_device_id); checksum = ptr[6]; @@ -2462,7 +2455,7 @@ static void pci_patch_ids(PCIDevice *pdev, uint8_t *ptr, uint32_t size) /* Patch vendor id and checksum (at offset 6 for etherboot roms). */ checksum += (uint8_t)rom_vendor_id + (uint8_t)(rom_vendor_id >> 8); checksum -= (uint8_t)vendor_id + (uint8_t)(vendor_id >> 8); - PCI_DPRINTF("ROM checksum %02x / %02x\n", ptr[6], checksum); + trace_pci_rom_checksum_change(ptr[6], checksum); ptr[6] = checksum; pci_set_word(ptr + pcir_offset + 4, vendor_id); } @@ -2471,7 +2464,7 @@ static void pci_patch_ids(PCIDevice *pdev, uint8_t *ptr, uint32_t size) /* Patch device id and checksum (at offset 6 for etherboot roms). */ checksum += (uint8_t)rom_device_id + (uint8_t)(rom_device_id >> 8); checksum -= (uint8_t)device_id + (uint8_t)(device_id >> 8); - PCI_DPRINTF("ROM checksum %02x / %02x\n", ptr[6], checksum); + trace_pci_rom_checksum_change(ptr[6], checksum); ptr[6] = checksum; pci_set_word(ptr + pcir_offset + 6, device_id); } diff --git a/hw/pci/pcie_port.c b/hw/pci/pcie_port.c index 54f639e3d4..f3841a2656 100644 --- a/hw/pci/pcie_port.c +++ b/hw/pci/pcie_port.c @@ -188,7 +188,7 @@ int pcie_count_ds_ports(PCIBus *bus) return dsp_count; } -static bool pcie_slot_is_hotpluggbale_bus(HotplugHandler *plug_handler, +static bool pcie_slot_is_hotpluggable_bus(HotplugHandler *plug_handler, BusState *bus) { PCIESlot *s = PCIE_SLOT(bus->parent); @@ -221,7 +221,7 @@ static void pcie_slot_class_init(ObjectClass *oc, const void *data) hc->plug = pcie_cap_slot_plug_cb; hc->unplug = pcie_cap_slot_unplug_cb; hc->unplug_request = pcie_cap_slot_unplug_request_cb; - hc->is_hotpluggable_bus = pcie_slot_is_hotpluggbale_bus; + hc->is_hotpluggable_bus = pcie_slot_is_hotpluggable_bus; } static const TypeInfo pcie_slot_type_info = { diff --git a/hw/pci/trace-events b/hw/pci/trace-events index 6a9968962f..02c80d3ec8 100644 --- a/hw/pci/trace-events +++ b/hw/pci/trace-events @@ -6,6 +6,10 @@ pci_pm_transition(const char *dev, uint32_t bus, uint32_t slot, uint32_t func, u pci_update_mappings_del(const char *dev, uint32_t bus, uint32_t slot, uint32_t func, int bar, uint64_t addr, uint64_t size) "%s %02x:%02x.%x %d,0x%"PRIx64"+0x%"PRIx64 pci_update_mappings_add(const char *dev, uint32_t bus, uint32_t slot, uint32_t func, int bar, uint64_t addr, uint64_t size) "%s %02x:%02x.%x %d,0x%"PRIx64"+0x%"PRIx64 pci_route_irq(int dev_irq, const char *dev_path, int parent_irq, const char *parent_path) "IRQ %d @%s -> IRQ %d @%s" +pci_bad_rom_magic(uint16_t bad_rom_magic, uint16_t good_rom_magic) "Bad ROM magic number: %04"PRIX16". Should be: %04"PRIX16 +pci_bad_pcir_offset(uint16_t pcir_offset) "Bad PCIR offset 0x%"PRIx16" or signature" +pci_rom_and_pci_ids(char *romfile, uint16_t vendor_id, uint16_t device_id, uint16_t rom_vendor_id, uint16_t rom_device_id) "%s: ROM ID %04"PRIx16":%04"PRIx16" | PCI ID %04"PRIx16":%04"PRIx16 +pci_rom_checksum_change(uint8_t old_checksum, uint8_t new_checksum) "ROM checksum changed from %02"PRIx8" to %02"PRIx8 # pci_host.c pci_cfg_read(const char *dev, uint32_t bus, uint32_t slot, uint32_t func, unsigned offs, unsigned val) "%s %02x:%02x.%x @0x%x -> 0x%x" diff --git a/hw/ppc/spapr_nested.c b/hw/ppc/spapr_nested.c index 820f752b9e..10cf634da1 100644 --- a/hw/ppc/spapr_nested.c +++ b/hw/ppc/spapr_nested.c @@ -1,6 +1,5 @@ #include "qemu/osdep.h" #include "qemu/cutils.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "exec/target_long.h" #include "helper_regs.h" diff --git a/hw/riscv/riscv-iommu-sys.c b/hw/riscv/riscv-iommu-sys.c index be2e3944dc..74e76b94a5 100644 --- a/hw/riscv/riscv-iommu-sys.c +++ b/hw/riscv/riscv-iommu-sys.c @@ -26,7 +26,6 @@ #include "qemu/host-utils.h" #include "qemu/module.h" #include "qom/object.h" -#include "exec/exec-all.h" #include "trace.h" #include "riscv-iommu.h" diff --git a/hw/rtc/mc146818rtc.c b/hw/rtc/mc146818rtc.c index 6f787be7af..f9f5cf396f 100644 --- a/hw/rtc/mc146818rtc.c +++ b/hw/rtc/mc146818rtc.c @@ -819,7 +819,7 @@ static const VMStateDescription vmstate_rtc_irq_reinject_on_ack_count = { static const VMStateDescription vmstate_rtc = { .name = "mc146818rtc", .version_id = 3, - .minimum_version_id = 1, + .minimum_version_id = 3, .pre_save = rtc_pre_save, .post_load = rtc_post_load, .fields = (const VMStateField[]) { @@ -829,13 +829,13 @@ static const VMStateDescription vmstate_rtc = { VMSTATE_TIMER_PTR(periodic_timer, MC146818RtcState), VMSTATE_INT64(next_periodic_time, MC146818RtcState), VMSTATE_UNUSED(3*8), - VMSTATE_UINT32_V(irq_coalesced, MC146818RtcState, 2), - VMSTATE_UINT32_V(period, MC146818RtcState, 2), - VMSTATE_UINT64_V(base_rtc, MC146818RtcState, 3), - VMSTATE_UINT64_V(last_update, MC146818RtcState, 3), - VMSTATE_INT64_V(offset, MC146818RtcState, 3), - VMSTATE_TIMER_PTR_V(update_timer, MC146818RtcState, 3), - VMSTATE_UINT64_V(next_alarm_time, MC146818RtcState, 3), + VMSTATE_UINT32(irq_coalesced, MC146818RtcState), + VMSTATE_UINT32(period, MC146818RtcState), + VMSTATE_UINT64(base_rtc, MC146818RtcState), + VMSTATE_UINT64(last_update, MC146818RtcState), + VMSTATE_INT64(offset, MC146818RtcState), + VMSTATE_TIMER_PTR(update_timer, MC146818RtcState), + VMSTATE_UINT64(next_alarm_time, MC146818RtcState), VMSTATE_END_OF_LIST() }, .subsections = (const VMStateDescription * const []) { @@ -929,8 +929,6 @@ static void rtc_realizefn(DeviceState *dev, Error **errp) memory_region_add_subregion(&s->io, 0, &s->coalesced_io); memory_region_add_coalescing(&s->coalesced_io, 0, 1); - qdev_set_legacy_instance_id(dev, s->io_base, 3); - object_property_add_tm(OBJECT(s), "date", rtc_get_date); qdev_init_gpio_out(dev, &s->irq, 1); diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c index 716a6b7869..2f082396c7 100644 --- a/hw/s390x/ipl.c +++ b/hw/s390x/ipl.c @@ -26,7 +26,6 @@ #include "hw/s390x/vfio-ccw.h" #include "hw/s390x/css.h" #include "hw/s390x/ebcdic.h" -#include "target/s390x/kvm/pv.h" #include "hw/scsi/scsi.h" #include "hw/virtio/virtio-net.h" #include "ipl.h" @@ -676,7 +675,7 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu) cpu_physical_memory_unmap(addr, len, 1, len); } -int s390_ipl_prepare_pv_header(Error **errp) +int s390_ipl_prepare_pv_header(struct S390PVResponse *pv_resp, Error **errp) { IplParameterBlock *ipib = s390_ipl_get_iplb_pv(); IPLBlockPV *ipib_pv = &ipib->pv; @@ -685,12 +684,13 @@ int s390_ipl_prepare_pv_header(Error **errp) cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr, ipib_pv->pv_header_len); - rc = s390_pv_set_sec_parms((uintptr_t)hdr, ipib_pv->pv_header_len, errp); + rc = s390_pv_set_sec_parms((uintptr_t)hdr, ipib_pv->pv_header_len, + pv_resp, errp); g_free(hdr); return rc; } -int s390_ipl_pv_unpack(void) +int s390_ipl_pv_unpack(struct S390PVResponse *pv_resp) { IplParameterBlock *ipib = s390_ipl_get_iplb_pv(); IPLBlockPV *ipib_pv = &ipib->pv; @@ -699,7 +699,8 @@ int s390_ipl_pv_unpack(void) for (i = 0; i < ipib_pv->num_comp; i++) { rc = s390_pv_unpack(ipib_pv->components[i].addr, TARGET_PAGE_ALIGN(ipib_pv->components[i].size), - ipib_pv->components[i].tweak_pref); + ipib_pv->components[i].tweak_pref, + pv_resp); if (rc) { break; } diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h index cb55101f06..505cded490 100644 --- a/hw/s390x/ipl.h +++ b/hw/s390x/ipl.h @@ -20,6 +20,7 @@ #include "hw/qdev-core.h" #include "hw/s390x/ipl/qipl.h" #include "qom/object.h" +#include "target/s390x/kvm/pv.h" #define DIAG308_FLAGS_LP_VALID 0x80 #define MAX_BOOT_DEVS 8 /* Max number of devices that may have a bootindex */ @@ -28,8 +29,9 @@ void s390_ipl_convert_loadparm(char *ascii_lp, uint8_t *ebcdic_lp); void s390_ipl_fmt_loadparm(uint8_t *loadparm, char *str, Error **errp); void s390_rebuild_iplb(uint16_t index, IplParameterBlock *iplb); void s390_ipl_update_diag308(IplParameterBlock *iplb); -int s390_ipl_prepare_pv_header(Error **errp); -int s390_ipl_pv_unpack(void); +int s390_ipl_prepare_pv_header(struct S390PVResponse *pv_resp, + Error **errp); +int s390_ipl_pv_unpack(struct S390PVResponse *pv_resp); void s390_ipl_prepare_cpu(S390CPU *cpu); IplParameterBlock *s390_ipl_get_iplb(void); IplParameterBlock *s390_ipl_get_iplb_pv(void); diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index 94edd42dd2..d5658afed9 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -365,7 +365,8 @@ static void s390_machine_unprotect(S390CcwMachineState *ms) ram_block_discard_disable(false); } -static int s390_machine_protect(S390CcwMachineState *ms) +static int s390_machine_protect(S390CcwMachineState *ms, + struct S390PVResponse *pv_resp) { Error *local_err = NULL; int rc; @@ -408,19 +409,19 @@ static int s390_machine_protect(S390CcwMachineState *ms) } /* Set SE header and unpack */ - rc = s390_ipl_prepare_pv_header(&local_err); + rc = s390_ipl_prepare_pv_header(pv_resp, &local_err); if (rc) { goto out_err; } /* Decrypt image */ - rc = s390_ipl_pv_unpack(); + rc = s390_ipl_pv_unpack(pv_resp); if (rc) { goto out_err; } /* Verify integrity */ - rc = s390_pv_verify(); + rc = s390_pv_verify(pv_resp); if (rc) { goto out_err; } @@ -452,6 +453,7 @@ static void s390_pv_prepare_reset(S390CcwMachineState *ms) static void s390_machine_reset(MachineState *machine, ResetType type) { S390CcwMachineState *ms = S390_CCW_MACHINE(machine); + struct S390PVResponse pv_resp; enum s390_reset reset_type; CPUState *cs, *t; S390CPU *cpu; @@ -540,8 +542,8 @@ static void s390_machine_reset(MachineState *machine, ResetType type) } run_on_cpu(cs, s390_do_cpu_reset, RUN_ON_CPU_NULL); - if (s390_machine_protect(ms)) { - s390_pv_inject_reset_error(cs); + if (s390_machine_protect(ms, &pv_resp)) { + s390_pv_inject_reset_error(cs, pv_resp); /* * Continue after the diag308 so the guest knows something * went wrong. diff --git a/hw/sh4/sh7750.c b/hw/sh4/sh7750.c index 41306fb600..300eabc595 100644 --- a/hw/sh4/sh7750.c +++ b/hw/sh4/sh7750.c @@ -36,7 +36,6 @@ #include "sh7750_regnames.h" #include "hw/sh4/sh_intc.h" #include "hw/timer/tmu012.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "trace.h" diff --git a/hw/ssi/aspeed_smc.c b/hw/ssi/aspeed_smc.c index 0d38f95c7a..614528b8ef 100644 --- a/hw/ssi/aspeed_smc.c +++ b/hw/ssi/aspeed_smc.c @@ -359,7 +359,7 @@ static const MemoryRegionOps aspeed_smc_flash_default_ops = { .endianness = DEVICE_LITTLE_ENDIAN, .valid = { .min_access_size = 1, - .max_access_size = 4, + .max_access_size = 8, }, }; @@ -670,7 +670,7 @@ static const MemoryRegionOps aspeed_smc_flash_ops = { .endianness = DEVICE_LITTLE_ENDIAN, .valid = { .min_access_size = 1, - .max_access_size = 4, + .max_access_size = 8, }, }; diff --git a/hw/vfio/ap.c b/hw/vfio/ap.c index 1207c08d8d..785c0a0197 100644 --- a/hw/vfio/ap.c +++ b/hw/vfio/ap.c @@ -74,10 +74,10 @@ static bool vfio_ap_register_irq_notifier(VFIOAPDevice *vapdev, unsigned int irq, Error **errp) { int fd; - size_t argsz; + int ret; IOHandler *fd_read; EventNotifier *notifier; - g_autofree struct vfio_irq_info *irq_info = NULL; + struct vfio_irq_info irq_info; VFIODevice *vdev = &vapdev->vdev; switch (irq) { @@ -96,14 +96,15 @@ static bool vfio_ap_register_irq_notifier(VFIOAPDevice *vapdev, return false; } - argsz = sizeof(*irq_info); - irq_info = g_malloc0(argsz); - irq_info->index = irq; - irq_info->argsz = argsz; + ret = vfio_device_get_irq_info(vdev, irq, &irq_info); + + if (ret < 0) { + error_setg_errno(errp, -ret, "vfio: Error getting irq info"); + return false; + } - if (ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, - irq_info) < 0 || irq_info->count < 1) { - error_setg_errno(errp, errno, "vfio: Error getting irq info"); + if (irq_info.count < 1) { + error_setg(errp, "vfio: Error getting irq info, count=0"); return false; } diff --git a/hw/vfio/ccw.c b/hw/vfio/ccw.c index fde0c3fbef..cea9d6e005 100644 --- a/hw/vfio/ccw.c +++ b/hw/vfio/ccw.c @@ -376,8 +376,8 @@ static bool vfio_ccw_register_irq_notifier(VFIOCCWDevice *vcdev, Error **errp) { VFIODevice *vdev = &vcdev->vdev; - g_autofree struct vfio_irq_info *irq_info = NULL; - size_t argsz; + struct vfio_irq_info irq_info; + int ret; int fd; EventNotifier *notifier; IOHandler *fd_read; @@ -406,13 +406,15 @@ static bool vfio_ccw_register_irq_notifier(VFIOCCWDevice *vcdev, return false; } - argsz = sizeof(*irq_info); - irq_info = g_malloc0(argsz); - irq_info->index = irq; - irq_info->argsz = argsz; - if (ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, - irq_info) < 0 || irq_info->count < 1) { - error_setg_errno(errp, errno, "vfio: Error getting irq info"); + ret = vfio_device_get_irq_info(vdev, irq, &irq_info); + + if (ret < 0) { + error_setg_errno(errp, -ret, "vfio: Error getting irq info"); + return false; + } + + if (irq_info.count < 1) { + error_setg(errp, "vfio: Error getting irq info, count=0"); return false; } @@ -502,7 +504,6 @@ static bool vfio_ccw_get_region(VFIOCCWDevice *vcdev, Error **errp) vcdev->io_region_offset = info->offset; vcdev->io_region = g_malloc0(info->size); - g_free(info); /* check for the optional async command region */ ret = vfio_device_get_region_info_type(vdev, VFIO_REGION_TYPE_CCW, @@ -515,7 +516,6 @@ static bool vfio_ccw_get_region(VFIOCCWDevice *vcdev, Error **errp) } vcdev->async_cmd_region_offset = info->offset; vcdev->async_cmd_region = g_malloc0(info->size); - g_free(info); } ret = vfio_device_get_region_info_type(vdev, VFIO_REGION_TYPE_CCW, @@ -528,7 +528,6 @@ static bool vfio_ccw_get_region(VFIOCCWDevice *vcdev, Error **errp) } vcdev->schib_region_offset = info->offset; vcdev->schib_region = g_malloc(info->size); - g_free(info); } ret = vfio_device_get_region_info_type(vdev, VFIO_REGION_TYPE_CCW, @@ -542,7 +541,6 @@ static bool vfio_ccw_get_region(VFIOCCWDevice *vcdev, Error **errp) } vcdev->crw_region_offset = info->offset; vcdev->crw_region = g_malloc(info->size); - g_free(info); } return true; @@ -552,7 +550,6 @@ out_err: g_free(vcdev->schib_region); g_free(vcdev->async_cmd_region); g_free(vcdev->io_region); - g_free(info); return false; } diff --git a/hw/vfio/container-base.c b/hw/vfio/container-base.c index 09340fd97a..1c6ca94b60 100644 --- a/hw/vfio/container-base.c +++ b/hw/vfio/container-base.c @@ -85,12 +85,12 @@ int vfio_container_dma_map(VFIOContainerBase *bcontainer, int vfio_container_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova, ram_addr_t size, - IOMMUTLBEntry *iotlb) + IOMMUTLBEntry *iotlb, bool unmap_all) { VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(bcontainer); g_assert(vioc->dma_unmap); - return vioc->dma_unmap(bcontainer, iova, size, iotlb); + return vioc->dma_unmap(bcontainer, iova, size, iotlb, unmap_all); } bool vfio_container_add_section_window(VFIOContainerBase *bcontainer, @@ -198,11 +198,7 @@ static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova, feature->flags = VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT; - if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { - return -errno; - } - - return 0; + return vbasedev->io_ops->device_feature(vbasedev, feature); } static int vfio_container_iommu_query_dirty_bitmap(const VFIOContainerBase *bcontainer, diff --git a/hw/vfio/container.c b/hw/vfio/container.c index 77ff56b43f..a9f0dbaec4 100644 --- a/hw/vfio/container.c +++ b/hw/vfio/container.c @@ -119,12 +119,9 @@ unmap_exit: return ret; } -/* - * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86 - */ -static int vfio_legacy_dma_unmap(const VFIOContainerBase *bcontainer, - hwaddr iova, ram_addr_t size, - IOMMUTLBEntry *iotlb) +static int vfio_legacy_dma_unmap_one(const VFIOContainerBase *bcontainer, + hwaddr iova, ram_addr_t size, + IOMMUTLBEntry *iotlb) { const VFIOContainer *container = container_of(bcontainer, VFIOContainer, bcontainer); @@ -181,6 +178,34 @@ static int vfio_legacy_dma_unmap(const VFIOContainerBase *bcontainer, return 0; } +/* + * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86 + */ +static int vfio_legacy_dma_unmap(const VFIOContainerBase *bcontainer, + hwaddr iova, ram_addr_t size, + IOMMUTLBEntry *iotlb, bool unmap_all) +{ + int ret; + + if (unmap_all) { + /* The unmap ioctl doesn't accept a full 64-bit span. */ + Int128 llsize = int128_rshift(int128_2_64(), 1); + + ret = vfio_legacy_dma_unmap_one(bcontainer, 0, int128_get64(llsize), + iotlb); + + if (ret == 0) { + ret = vfio_legacy_dma_unmap_one(bcontainer, int128_get64(llsize), + int128_get64(llsize), iotlb); + } + + } else { + ret = vfio_legacy_dma_unmap_one(bcontainer, iova, size, iotlb); + } + + return ret; +} + static int vfio_legacy_dma_map(const VFIOContainerBase *bcontainer, hwaddr iova, ram_addr_t size, void *vaddr, bool readonly) { @@ -205,7 +230,7 @@ static int vfio_legacy_dma_map(const VFIOContainerBase *bcontainer, hwaddr iova, */ if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 || (errno == EBUSY && - vfio_legacy_dma_unmap(bcontainer, iova, size, NULL) == 0 && + vfio_legacy_dma_unmap(bcontainer, iova, size, NULL, false) == 0 && ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) { return 0; } @@ -511,16 +536,10 @@ static bool vfio_legacy_setup(VFIOContainerBase *bcontainer, Error **errp) return true; } -static bool vfio_container_connect(VFIOGroup *group, AddressSpace *as, - Error **errp) +static bool vfio_container_attach_discard_disable(VFIOContainer *container, + VFIOGroup *group, Error **errp) { - VFIOContainer *container; - VFIOContainerBase *bcontainer; - int ret, fd; - VFIOAddressSpace *space; - VFIOIOMMUClass *vioc; - - space = vfio_address_space_get(as); + int ret; /* * VFIO is currently incompatible with discarding of RAM insofar as the @@ -553,97 +572,118 @@ static bool vfio_container_connect(VFIOGroup *group, AddressSpace *as, * details once we know which type of IOMMU we are using. */ + ret = vfio_ram_block_discard_disable(container, true); + if (ret) { + error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken"); + if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) { + error_report("vfio: error disconnecting group %d from" + " container", group->groupid); + } + } + return !ret; +} + +static bool vfio_container_group_add(VFIOContainer *container, VFIOGroup *group, + Error **errp) +{ + if (!vfio_container_attach_discard_disable(container, group, errp)) { + return false; + } + group->container = container; + QLIST_INSERT_HEAD(&container->group_list, group, container_next); + vfio_group_add_kvm_device(group); + return true; +} + +static void vfio_container_group_del(VFIOContainer *container, VFIOGroup *group) +{ + QLIST_REMOVE(group, container_next); + group->container = NULL; + vfio_group_del_kvm_device(group); + vfio_ram_block_discard_disable(container, false); +} + +static bool vfio_container_connect(VFIOGroup *group, AddressSpace *as, + Error **errp) +{ + VFIOContainer *container; + VFIOContainerBase *bcontainer; + int ret, fd = -1; + VFIOAddressSpace *space; + VFIOIOMMUClass *vioc = NULL; + bool new_container = false; + bool group_was_added = false; + + space = vfio_address_space_get(as); + QLIST_FOREACH(bcontainer, &space->containers, next) { container = container_of(bcontainer, VFIOContainer, bcontainer); if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) { - ret = vfio_ram_block_discard_disable(container, true); - if (ret) { - error_setg_errno(errp, -ret, - "Cannot set discarding of RAM broken"); - if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, - &container->fd)) { - error_report("vfio: error disconnecting group %d from" - " container", group->groupid); - } - return false; - } - group->container = container; - QLIST_INSERT_HEAD(&container->group_list, group, container_next); - vfio_group_add_kvm_device(group); - return true; + return vfio_container_group_add(container, group, errp); } } fd = qemu_open("/dev/vfio/vfio", O_RDWR, errp); if (fd < 0) { - goto put_space_exit; + goto fail; } ret = ioctl(fd, VFIO_GET_API_VERSION); if (ret != VFIO_API_VERSION) { error_setg(errp, "supported vfio version: %d, " "reported version: %d", VFIO_API_VERSION, ret); - goto close_fd_exit; + goto fail; } container = vfio_create_container(fd, group, errp); if (!container) { - goto close_fd_exit; + goto fail; } + new_container = true; bcontainer = &container->bcontainer; if (!vfio_cpr_register_container(bcontainer, errp)) { - goto free_container_exit; - } - - ret = vfio_ram_block_discard_disable(container, true); - if (ret) { - error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken"); - goto unregister_container_exit; + goto fail; } vioc = VFIO_IOMMU_GET_CLASS(bcontainer); assert(vioc->setup); if (!vioc->setup(bcontainer, errp)) { - goto enable_discards_exit; + goto fail; } - vfio_group_add_kvm_device(group); - vfio_address_space_insert(space, bcontainer); - group->container = container; - QLIST_INSERT_HEAD(&container->group_list, group, container_next); + if (!vfio_container_group_add(container, group, errp)) { + goto fail; + } + group_was_added = true; if (!vfio_listener_register(bcontainer, errp)) { - goto listener_release_exit; + goto fail; } bcontainer->initialized = true; return true; -listener_release_exit: - QLIST_REMOVE(group, container_next); - vfio_group_del_kvm_device(group); + +fail: vfio_listener_unregister(bcontainer); - if (vioc->release) { + + if (group_was_added) { + vfio_container_group_del(container, group); + } + if (vioc && vioc->release) { vioc->release(bcontainer); } - -enable_discards_exit: - vfio_ram_block_discard_disable(container, false); - -unregister_container_exit: - vfio_cpr_unregister_container(bcontainer); - -free_container_exit: - object_unref(container); - -close_fd_exit: - close(fd); - -put_space_exit: + if (new_container) { + vfio_cpr_unregister_container(bcontainer); + object_unref(container); + } + if (fd >= 0) { + close(fd); + } vfio_address_space_put(space); return false; @@ -811,18 +851,14 @@ static bool vfio_device_get(VFIOGroup *group, const char *name, } } + vfio_device_prepare(vbasedev, &group->container->bcontainer, info); + vbasedev->fd = fd; vbasedev->group = group; QLIST_INSERT_HEAD(&group->device_list, vbasedev, next); - vbasedev->num_irqs = info->num_irqs; - vbasedev->num_regions = info->num_regions; - vbasedev->flags = info->flags; - trace_vfio_device_get(name, info->flags, info->num_regions, info->num_irqs); - vbasedev->reset_works = !!(info->flags & VFIO_DEVICE_FLAGS_RESET); - return true; } @@ -875,7 +911,6 @@ static bool vfio_legacy_attach_device(const char *name, VFIODevice *vbasedev, int groupid = vfio_device_get_groupid(vbasedev, errp); VFIODevice *vbasedev_iter; VFIOGroup *group; - VFIOContainerBase *bcontainer; if (groupid < 0) { return false; @@ -904,11 +939,6 @@ static bool vfio_legacy_attach_device(const char *name, VFIODevice *vbasedev, goto device_put_exit; } - bcontainer = &group->container->bcontainer; - vbasedev->bcontainer = bcontainer; - QLIST_INSERT_HEAD(&bcontainer->device_list, vbasedev, container_next); - QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next); - return true; device_put_exit: @@ -922,10 +952,10 @@ static void vfio_legacy_detach_device(VFIODevice *vbasedev) { VFIOGroup *group = vbasedev->group; - QLIST_REMOVE(vbasedev, global_next); - QLIST_REMOVE(vbasedev, container_next); - vbasedev->bcontainer = NULL; trace_vfio_device_detach(vbasedev->name, group->groupid); + + vfio_device_unprepare(vbasedev); + object_unref(vbasedev->hiod); vfio_device_put(vbasedev); vfio_group_put(group); diff --git a/hw/vfio/device.c b/hw/vfio/device.c index d625a7c4db..9fba2c7272 100644 --- a/hw/vfio/device.c +++ b/hw/vfio/device.c @@ -82,7 +82,7 @@ void vfio_device_irq_disable(VFIODevice *vbasedev, int index) .count = 0, }; - ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); + vbasedev->io_ops->set_irqs(vbasedev, &irq_set); } void vfio_device_irq_unmask(VFIODevice *vbasedev, int index) @@ -95,7 +95,7 @@ void vfio_device_irq_unmask(VFIODevice *vbasedev, int index) .count = 1, }; - ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); + vbasedev->io_ops->set_irqs(vbasedev, &irq_set); } void vfio_device_irq_mask(VFIODevice *vbasedev, int index) @@ -108,7 +108,7 @@ void vfio_device_irq_mask(VFIODevice *vbasedev, int index) .count = 1, }; - ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); + vbasedev->io_ops->set_irqs(vbasedev, &irq_set); } static inline const char *action_to_str(int action) @@ -167,7 +167,7 @@ bool vfio_device_irq_set_signaling(VFIODevice *vbasedev, int index, int subindex pfd = (int32_t *)&irq_set->data; *pfd = fd; - if (!ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set)) { + if (!vbasedev->io_ops->set_irqs(vbasedev, irq_set)) { return true; } @@ -185,10 +185,28 @@ bool vfio_device_irq_set_signaling(VFIODevice *vbasedev, int index, int subindex return false; } +int vfio_device_get_irq_info(VFIODevice *vbasedev, int index, + struct vfio_irq_info *info) +{ + memset(info, 0, sizeof(*info)); + + info->argsz = sizeof(*info); + info->index = index; + + return vbasedev->io_ops->get_irq_info(vbasedev, info); +} + int vfio_device_get_region_info(VFIODevice *vbasedev, int index, struct vfio_region_info **info) { size_t argsz = sizeof(struct vfio_region_info); + int ret; + + /* check cache */ + if (vbasedev->reginfo[index] != NULL) { + *info = vbasedev->reginfo[index]; + return 0; + } *info = g_malloc0(argsz); @@ -196,10 +214,11 @@ int vfio_device_get_region_info(VFIODevice *vbasedev, int index, retry: (*info)->argsz = argsz; - if (ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, *info)) { + ret = vbasedev->io_ops->get_region_info(vbasedev, *info); + if (ret != 0) { g_free(*info); *info = NULL; - return -errno; + return ret; } if ((*info)->argsz > argsz) { @@ -209,6 +228,9 @@ retry: goto retry; } + /* fill cache */ + vbasedev->reginfo[index] = *info; + return 0; } @@ -227,7 +249,6 @@ int vfio_device_get_region_info_type(VFIODevice *vbasedev, uint32_t type, hdr = vfio_get_region_info_cap(*info, VFIO_REGION_INFO_CAP_TYPE); if (!hdr) { - g_free(*info); continue; } @@ -239,8 +260,6 @@ int vfio_device_get_region_info_type(VFIODevice *vbasedev, uint32_t type, if (cap_type->type == type && cap_type->subtype == subtype) { return 0; } - - g_free(*info); } *info = NULL; @@ -249,7 +268,7 @@ int vfio_device_get_region_info_type(VFIODevice *vbasedev, uint32_t type, bool vfio_device_has_region_cap(VFIODevice *vbasedev, int region, uint16_t cap_type) { - g_autofree struct vfio_region_info *info = NULL; + struct vfio_region_info *info = NULL; bool ret = false; if (!vfio_device_get_region_info(vbasedev, region, &info)) { @@ -305,11 +324,14 @@ void vfio_device_set_fd(VFIODevice *vbasedev, const char *str, Error **errp) vbasedev->fd = fd; } +static VFIODeviceIOOps vfio_device_io_ops_ioctl; + void vfio_device_init(VFIODevice *vbasedev, int type, VFIODeviceOps *ops, DeviceState *dev, bool ram_discard) { vbasedev->type = type; vbasedev->ops = ops; + vbasedev->io_ops = &vfio_device_io_ops_ioctl; vbasedev->dev = dev; vbasedev->fd = -1; @@ -370,27 +392,35 @@ bool vfio_device_hiod_create_and_realize(VFIODevice *vbasedev, VFIODevice *vfio_get_vfio_device(Object *obj) { if (object_dynamic_cast(obj, TYPE_VFIO_PCI)) { - return &VFIO_PCI(obj)->vbasedev; + return &VFIO_PCI_BASE(obj)->vbasedev; } else { return NULL; } } -bool vfio_device_attach(char *name, VFIODevice *vbasedev, - AddressSpace *as, Error **errp) +bool vfio_device_attach_by_iommu_type(const char *iommu_type, char *name, + VFIODevice *vbasedev, AddressSpace *as, + Error **errp) { const VFIOIOMMUClass *ops = - VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_LEGACY)); - - if (vbasedev->iommufd) { - ops = VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD)); - } + VFIO_IOMMU_CLASS(object_class_by_name(iommu_type)); assert(ops); return ops->attach_device(name, vbasedev, as, errp); } +bool vfio_device_attach(char *name, VFIODevice *vbasedev, + AddressSpace *as, Error **errp) +{ + const char *iommu_type = vbasedev->iommufd ? + TYPE_VFIO_IOMMU_IOMMUFD : + TYPE_VFIO_IOMMU_LEGACY; + + return vfio_device_attach_by_iommu_type(iommu_type, name, vbasedev, + as, errp); +} + void vfio_device_detach(VFIODevice *vbasedev) { if (!vbasedev->bcontainer) { @@ -398,3 +428,120 @@ void vfio_device_detach(VFIODevice *vbasedev) } VFIO_IOMMU_GET_CLASS(vbasedev->bcontainer)->detach_device(vbasedev); } + +void vfio_device_prepare(VFIODevice *vbasedev, VFIOContainerBase *bcontainer, + struct vfio_device_info *info) +{ + vbasedev->num_irqs = info->num_irqs; + vbasedev->num_regions = info->num_regions; + vbasedev->flags = info->flags; + vbasedev->reset_works = !!(info->flags & VFIO_DEVICE_FLAGS_RESET); + + vbasedev->bcontainer = bcontainer; + QLIST_INSERT_HEAD(&bcontainer->device_list, vbasedev, container_next); + + QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next); + + vbasedev->reginfo = g_new0(struct vfio_region_info *, + vbasedev->num_regions); +} + +void vfio_device_unprepare(VFIODevice *vbasedev) +{ + int i; + + for (i = 0; i < vbasedev->num_regions; i++) { + g_free(vbasedev->reginfo[i]); + } + g_free(vbasedev->reginfo); + vbasedev->reginfo = NULL; + + QLIST_REMOVE(vbasedev, container_next); + QLIST_REMOVE(vbasedev, global_next); + vbasedev->bcontainer = NULL; +} + +/* + * Traditional ioctl() based io + */ + +static int vfio_device_io_device_feature(VFIODevice *vbasedev, + struct vfio_device_feature *feature) +{ + int ret; + + ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); + + return ret < 0 ? -errno : ret; +} + +static int vfio_device_io_get_region_info(VFIODevice *vbasedev, + struct vfio_region_info *info) +{ + int ret; + + ret = ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, info); + + return ret < 0 ? -errno : ret; +} + +static int vfio_device_io_get_irq_info(VFIODevice *vbasedev, + struct vfio_irq_info *info) +{ + int ret; + + ret = ioctl(vbasedev->fd, VFIO_DEVICE_GET_IRQ_INFO, info); + + return ret < 0 ? -errno : ret; +} + +static int vfio_device_io_set_irqs(VFIODevice *vbasedev, + struct vfio_irq_set *irqs) +{ + int ret; + + ret = ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irqs); + + return ret < 0 ? -errno : ret; +} + +static int vfio_device_io_region_read(VFIODevice *vbasedev, uint8_t index, + off_t off, uint32_t size, void *data) +{ + struct vfio_region_info *info; + int ret; + + ret = vfio_device_get_region_info(vbasedev, index, &info); + if (ret != 0) { + return ret; + } + + ret = pread(vbasedev->fd, data, size, info->offset + off); + + return ret < 0 ? -errno : ret; +} + +static int vfio_device_io_region_write(VFIODevice *vbasedev, uint8_t index, + off_t off, uint32_t size, void *data) +{ + struct vfio_region_info *info; + int ret; + + ret = vfio_device_get_region_info(vbasedev, index, &info); + if (ret != 0) { + return ret; + } + + ret = pwrite(vbasedev->fd, data, size, info->offset + off); + + return ret < 0 ? -errno : ret; +} + +static VFIODeviceIOOps vfio_device_io_ops_ioctl = { + .device_feature = vfio_device_io_device_feature, + .get_region_info = vfio_device_io_get_region_info, + .get_irq_info = vfio_device_io_get_irq_info, + .set_irqs = vfio_device_io_set_irqs, + .region_read = vfio_device_io_region_read, + .region_write = vfio_device_io_region_write, +}; diff --git a/hw/vfio/display.c b/hw/vfio/display.c index f3e6581f15..9c6f5aa265 100644 --- a/hw/vfio/display.c +++ b/hw/vfio/display.c @@ -213,6 +213,7 @@ static VFIODMABuf *vfio_display_get_dmabuf(VFIOPCIDevice *vdev, struct vfio_device_gfx_plane_info plane; VFIODMABuf *dmabuf; int fd, ret; + uint32_t offset = 0; memset(&plane, 0, sizeof(plane)); plane.argsz = sizeof(plane); @@ -245,10 +246,10 @@ static VFIODMABuf *vfio_display_get_dmabuf(VFIOPCIDevice *vdev, dmabuf = g_new0(VFIODMABuf, 1); dmabuf->dmabuf_id = plane.dmabuf_id; - dmabuf->buf = qemu_dmabuf_new(plane.width, plane.height, - plane.stride, 0, 0, plane.width, + dmabuf->buf = qemu_dmabuf_new(plane.width, plane.height, &offset, + &plane.stride, 0, 0, plane.width, plane.height, plane.drm_format, - plane.drm_format_mod, fd, false, false); + plane.drm_format_mod, &fd, 1, false, false); if (plane_type == DRM_PLANE_TYPE_CURSOR) { vfio_display_update_cursor(dmabuf, &plane); diff --git a/hw/vfio/igd.c b/hw/vfio/igd.c index d7e4728fdc..e7952d15a0 100644 --- a/hw/vfio/igd.c +++ b/hw/vfio/igd.c @@ -103,6 +103,7 @@ static int igd_gen(VFIOPCIDevice *vdev) /* * Unfortunately, Intel changes it's specification quite often. This makes * it impossible to use a suitable default value for unknown devices. + * Return -1 for not applying any generation-specific quirks. */ return -1; } @@ -182,16 +183,13 @@ static bool vfio_pci_igd_opregion_init(VFIOPCIDevice *vdev, trace_vfio_pci_igd_opregion_enabled(vdev->vbasedev.name); - pci_set_long(vdev->pdev.config + IGD_ASLS, 0); - pci_set_long(vdev->pdev.wmask + IGD_ASLS, ~0); - pci_set_long(vdev->emulated_config_bits + IGD_ASLS, ~0); - return true; } -static bool vfio_pci_igd_setup_opregion(VFIOPCIDevice *vdev, Error **errp) +static bool vfio_pci_igd_opregion_detect(VFIOPCIDevice *vdev, + struct vfio_region_info **opregion, + Error **errp) { - g_autofree struct vfio_region_info *opregion = NULL; int ret; /* Hotplugging is not supported for opregion access */ @@ -202,17 +200,13 @@ static bool vfio_pci_igd_setup_opregion(VFIOPCIDevice *vdev, Error **errp) ret = vfio_device_get_region_info_type(&vdev->vbasedev, VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, - VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion); + VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, opregion); if (ret) { error_setg_errno(errp, -ret, "Device does not supports IGD OpRegion feature"); return false; } - if (!vfio_pci_igd_opregion_init(vdev, opregion, errp)) { - return false; - } - return true; } @@ -355,8 +349,8 @@ static int vfio_pci_igd_lpc_init(VFIOPCIDevice *vdev, static bool vfio_pci_igd_setup_lpc_bridge(VFIOPCIDevice *vdev, Error **errp) { - g_autofree struct vfio_region_info *host = NULL; - g_autofree struct vfio_region_info *lpc = NULL; + struct vfio_region_info *host = NULL; + struct vfio_region_info *lpc = NULL; PCIDevice *lpc_bridge; int ret; @@ -419,6 +413,44 @@ static bool vfio_pci_igd_setup_lpc_bridge(VFIOPCIDevice *vdev, Error **errp) return true; } +static bool vfio_pci_igd_override_gms(int gen, uint32_t gms, uint32_t *gmch) +{ + bool ret = false; + + if (gen == -1) { + error_report("x-igd-gms is not supported on this device"); + } else if (gen < 8) { + if (gms <= 0x10) { + *gmch &= ~(IGD_GMCH_GEN6_GMS_MASK << IGD_GMCH_GEN6_GMS_SHIFT); + *gmch |= gms << IGD_GMCH_GEN6_GMS_SHIFT; + ret = true; + } else { + error_report(QERR_INVALID_PARAMETER_VALUE, "x-igd-gms", "0~0x10"); + } + } else if (gen == 8) { + if (gms <= 0x40) { + *gmch &= ~(IGD_GMCH_GEN8_GMS_MASK << IGD_GMCH_GEN8_GMS_SHIFT); + *gmch |= gms << IGD_GMCH_GEN8_GMS_SHIFT; + ret = true; + } else { + error_report(QERR_INVALID_PARAMETER_VALUE, "x-igd-gms", "0~0x40"); + } + } else { + /* 0x0 to 0x40: 32MB increments starting at 0MB */ + /* 0xf0 to 0xfe: 4MB increments starting at 4MB */ + if ((gms <= 0x40) || (gms >= 0xf0 && gms <= 0xfe)) { + *gmch &= ~(IGD_GMCH_GEN8_GMS_MASK << IGD_GMCH_GEN8_GMS_SHIFT); + *gmch |= gms << IGD_GMCH_GEN8_GMS_SHIFT; + ret = true; + } else { + error_report(QERR_INVALID_PARAMETER_VALUE, + "x-igd-gms", "0~0x40 or 0xf0~0xfe"); + } + } + + return ret; +} + #define IGD_GGC_MMIO_OFFSET 0x108040 #define IGD_BDSM_MMIO_OFFSET 0x1080C0 @@ -428,41 +460,35 @@ void vfio_probe_igd_bar0_quirk(VFIOPCIDevice *vdev, int nr) VFIOConfigMirrorQuirk *ggc_mirror, *bdsm_mirror; int gen; - /* - * This must be an Intel VGA device at address 00:02.0 for us to even - * consider enabling legacy mode. Some driver have dependencies on the PCI - * bus address. - */ if (!vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, PCI_ANY_ID) || !vfio_is_vga(vdev) || nr != 0) { return; } - /* - * Only on IGD devices of gen 11 and above, the BDSM register is mirrored - * into MMIO space and read from MMIO space by the Windows driver. - */ + /* Only on IGD Gen6-12 device needs quirks in BAR 0 */ gen = igd_gen(vdev); if (gen < 6) { return; } - ggc_quirk = vfio_quirk_alloc(1); - ggc_mirror = ggc_quirk->data = g_malloc0(sizeof(*ggc_mirror)); - ggc_mirror->mem = ggc_quirk->mem; - ggc_mirror->vdev = vdev; - ggc_mirror->bar = nr; - ggc_mirror->offset = IGD_GGC_MMIO_OFFSET; - ggc_mirror->config_offset = IGD_GMCH; - - memory_region_init_io(ggc_mirror->mem, OBJECT(vdev), - &vfio_generic_mirror_quirk, ggc_mirror, - "vfio-igd-ggc-quirk", 2); - memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, - ggc_mirror->offset, ggc_mirror->mem, - 1); - - QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, ggc_quirk, next); + if (vdev->igd_gms) { + ggc_quirk = vfio_quirk_alloc(1); + ggc_mirror = ggc_quirk->data = g_malloc0(sizeof(*ggc_mirror)); + ggc_mirror->mem = ggc_quirk->mem; + ggc_mirror->vdev = vdev; + ggc_mirror->bar = nr; + ggc_mirror->offset = IGD_GGC_MMIO_OFFSET; + ggc_mirror->config_offset = IGD_GMCH; + + memory_region_init_io(ggc_mirror->mem, OBJECT(vdev), + &vfio_generic_mirror_quirk, ggc_mirror, + "vfio-igd-ggc-quirk", 2); + memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, + ggc_mirror->offset, ggc_mirror->mem, + 1); + + QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, ggc_quirk, next); + } bdsm_quirk = vfio_quirk_alloc(1); bdsm_mirror = bdsm_quirk->data = g_malloc0(sizeof(*bdsm_mirror)); @@ -484,44 +510,37 @@ void vfio_probe_igd_bar0_quirk(VFIOPCIDevice *vdev, int nr) static bool vfio_pci_igd_config_quirk(VFIOPCIDevice *vdev, Error **errp) { + struct vfio_region_info *opregion = NULL; int ret, gen; - uint64_t gms_size; + uint64_t gms_size = 0; uint64_t *bdsm_size; uint32_t gmch; bool legacy_mode_enabled = false; Error *err = NULL; - /* - * This must be an Intel VGA device at address 00:02.0 for us to even - * consider enabling legacy mode. The vBIOS has dependencies on the - * PCI bus address. - */ if (!vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, PCI_ANY_ID) || !vfio_is_vga(vdev)) { return true; } - /* - * IGD is not a standard, they like to change their specs often. We - * only attempt to support back to SandBridge and we hope that newer - * devices maintain compatibility with generation 8. - */ - gen = igd_gen(vdev); - if (gen == -1) { - error_report("IGD device %s is unsupported in legacy mode, " - "try SandyBridge or newer", vdev->vbasedev.name); + /* IGD device always comes with OpRegion */ + if (!vfio_pci_igd_opregion_detect(vdev, &opregion, errp)) { return true; } + info_report("OpRegion detected on Intel display %x.", vdev->device_id); + gen = igd_gen(vdev); gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, 4); /* * For backward compatibility, enable legacy mode when + * - Device geneation is 6 to 9 (including both) * - Machine type is i440fx (pc_piix) * - IGD device is at guest BDF 00:02.0 * - Not manually disabled by x-igd-legacy-mode=off */ if ((vdev->igd_legacy_mode != ON_OFF_AUTO_OFF) && + (gen >= 6 && gen <= 9) && !strcmp(MACHINE_GET_CLASS(qdev_get_machine())->family, "pc_piix") && (&vdev->pdev == pci_find_device(pci_device_root_bus(&vdev->pdev), 0, PCI_DEVFN(0x2, 0)))) { @@ -532,7 +551,7 @@ static bool vfio_pci_igd_config_quirk(VFIOPCIDevice *vdev, Error **errp) * - OpRegion * - Same LPC bridge and Host bridge VID/DID/SVID/SSID as host */ - g_autofree struct vfio_region_info *rom = NULL; + struct vfio_region_info *rom = NULL; legacy_mode_enabled = true; info_report("IGD legacy mode enabled, " @@ -566,13 +585,15 @@ static bool vfio_pci_igd_config_quirk(VFIOPCIDevice *vdev, Error **errp) vdev->features |= VFIO_FEATURE_ENABLE_IGD_LPC; } else if (vdev->igd_legacy_mode == ON_OFF_AUTO_ON) { error_setg(&err, - "Machine is not i440fx or assigned BDF is not 00:02.0"); + "Machine is not i440fx, assigned BDF is not 00:02.0, " + "or device %04x (gen %d) doesn't support legacy mode", + vdev->device_id, gen); goto error; } /* Setup OpRegion access */ if ((vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) && - !vfio_pci_igd_setup_opregion(vdev, errp)) { + !vfio_pci_igd_opregion_init(vdev, opregion, errp)) { goto error; } @@ -580,7 +601,15 @@ static bool vfio_pci_igd_config_quirk(VFIOPCIDevice *vdev, Error **errp) if ((vdev->features & VFIO_FEATURE_ENABLE_IGD_LPC) && !vfio_pci_igd_setup_lpc_bridge(vdev, errp)) { goto error; - } + } + + /* + * ASLS (OpRegion address) is read-only, emulated + * It contains HPA, guest firmware need to reprogram it with GPA. + */ + pci_set_long(vdev->pdev.config + IGD_ASLS, 0); + pci_set_long(vdev->pdev.wmask + IGD_ASLS, ~0); + pci_set_long(vdev->emulated_config_bits + IGD_ASLS, ~0); /* * Allow user to override dsm size using x-igd-gms option, in multiples of @@ -588,56 +617,44 @@ static bool vfio_pci_igd_config_quirk(VFIOPCIDevice *vdev, Error **errp) * set from DVMT Pre-Allocated option in host BIOS. */ if (vdev->igd_gms) { - if (gen < 8) { - if (vdev->igd_gms <= 0x10) { - gmch &= ~(IGD_GMCH_GEN6_GMS_MASK << IGD_GMCH_GEN6_GMS_SHIFT); - gmch |= vdev->igd_gms << IGD_GMCH_GEN6_GMS_SHIFT; - } else { - error_report(QERR_INVALID_PARAMETER_VALUE, - "x-igd-gms", "0~0x10"); - } - } else { - if (vdev->igd_gms <= 0x40) { - gmch &= ~(IGD_GMCH_GEN8_GMS_MASK << IGD_GMCH_GEN8_GMS_SHIFT); - gmch |= vdev->igd_gms << IGD_GMCH_GEN8_GMS_SHIFT; - } else { - error_report(QERR_INVALID_PARAMETER_VALUE, - "x-igd-gms", "0~0x40"); - } + if (!vfio_pci_igd_override_gms(gen, vdev->igd_gms, &gmch)) { + return false; } + + /* GMCH is read-only, emulated */ + pci_set_long(vdev->pdev.config + IGD_GMCH, gmch); + pci_set_long(vdev->pdev.wmask + IGD_GMCH, 0); + pci_set_long(vdev->emulated_config_bits + IGD_GMCH, ~0); } - gms_size = igd_stolen_memory_size(gen, gmch); + if (gen > 0) { + gms_size = igd_stolen_memory_size(gen, gmch); + + /* BDSM is read-write, emulated. BIOS needs to be able to write it */ + if (gen < 11) { + pci_set_long(vdev->pdev.config + IGD_BDSM, 0); + pci_set_long(vdev->pdev.wmask + IGD_BDSM, ~0); + pci_set_long(vdev->emulated_config_bits + IGD_BDSM, ~0); + } else { + pci_set_quad(vdev->pdev.config + IGD_BDSM_GEN11, 0); + pci_set_quad(vdev->pdev.wmask + IGD_BDSM_GEN11, ~0); + pci_set_quad(vdev->emulated_config_bits + IGD_BDSM_GEN11, ~0); + } + } /* * Request reserved memory for stolen memory via fw_cfg. VM firmware * must allocate a 1MB aligned reserved memory region below 4GB with - * the requested size (in bytes) for use by the Intel PCI class VGA - * device at VM address 00:02.0. The base address of this reserved - * memory region must be written to the device BDSM register at PCI - * config offset 0x5C. + * the requested size (in bytes) for use by the IGD device. The base + * address of this reserved memory region must be written to the + * device BDSM register. + * For newer device without BDSM register, this fw_cfg item is 0. */ bdsm_size = g_malloc(sizeof(*bdsm_size)); *bdsm_size = cpu_to_le64(gms_size); fw_cfg_add_file(fw_cfg_find(), "etc/igd-bdsm-size", bdsm_size, sizeof(*bdsm_size)); - /* GMCH is read-only, emulated */ - pci_set_long(vdev->pdev.config + IGD_GMCH, gmch); - pci_set_long(vdev->pdev.wmask + IGD_GMCH, 0); - pci_set_long(vdev->emulated_config_bits + IGD_GMCH, ~0); - - /* BDSM is read-write, emulated. The BIOS needs to be able to write it */ - if (gen < 11) { - pci_set_long(vdev->pdev.config + IGD_BDSM, 0); - pci_set_long(vdev->pdev.wmask + IGD_BDSM, ~0); - pci_set_long(vdev->emulated_config_bits + IGD_BDSM, ~0); - } else { - pci_set_quad(vdev->pdev.config + IGD_BDSM_GEN11, 0); - pci_set_quad(vdev->pdev.wmask + IGD_BDSM_GEN11, ~0); - pci_set_quad(vdev->emulated_config_bits + IGD_BDSM_GEN11, ~0); - } - trace_vfio_pci_igd_bdsm_enabled(vdev->vbasedev.name, (gms_size / MiB)); return true; @@ -664,8 +681,27 @@ error: */ static bool vfio_pci_kvmgt_config_quirk(VFIOPCIDevice *vdev, Error **errp) { + struct vfio_region_info *opregion = NULL; + int gen; + + if (!vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, PCI_ANY_ID) || + !vfio_is_vga(vdev)) { + return true; + } + + /* FIXME: Cherryview is Gen8, but don't support GVT-g */ + gen = igd_gen(vdev); + if (gen != 8 && gen != 9) { + return true; + } + + if (!vfio_pci_igd_opregion_detect(vdev, &opregion, errp)) { + /* Should never reach here, KVMGT always emulates OpRegion */ + return false; + } + if ((vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) && - !vfio_pci_igd_setup_opregion(vdev, errp)) { + !vfio_pci_igd_opregion_init(vdev, opregion, errp)) { return false; } diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c index 232c06dd15..af1c7ab10a 100644 --- a/hw/vfio/iommufd.c +++ b/hw/vfio/iommufd.c @@ -46,11 +46,28 @@ static int iommufd_cdev_map(const VFIOContainerBase *bcontainer, hwaddr iova, static int iommufd_cdev_unmap(const VFIOContainerBase *bcontainer, hwaddr iova, ram_addr_t size, - IOMMUTLBEntry *iotlb) + IOMMUTLBEntry *iotlb, bool unmap_all) { const VFIOIOMMUFDContainer *container = container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer); + /* unmap in halves */ + if (unmap_all) { + Int128 llsize = int128_rshift(int128_2_64(), 1); + int ret; + + ret = iommufd_backend_unmap_dma(container->be, container->ioas_id, + 0, int128_get64(llsize)); + + if (ret == 0) { + ret = iommufd_backend_unmap_dma(container->be, container->ioas_id, + int128_get64(llsize), + int128_get64(llsize)); + } + + return ret; + } + /* TODO: Handle dma_unmap_bitmap with iotlb args (migration) */ return iommufd_backend_unmap_dma(container->be, container->ioas_id, iova, size); @@ -588,14 +605,7 @@ found_container: iommufd_cdev_ram_block_discard_disable(false); } - vbasedev->group = 0; - vbasedev->num_irqs = dev_info.num_irqs; - vbasedev->num_regions = dev_info.num_regions; - vbasedev->flags = dev_info.flags; - vbasedev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET); - vbasedev->bcontainer = bcontainer; - QLIST_INSERT_HEAD(&bcontainer->device_list, vbasedev, container_next); - QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next); + vfio_device_prepare(vbasedev, bcontainer, &dev_info); trace_iommufd_cdev_device_info(vbasedev->name, devfd, vbasedev->num_irqs, vbasedev->num_regions, vbasedev->flags); @@ -622,9 +632,7 @@ static void iommufd_cdev_detach(VFIODevice *vbasedev) VFIOIOMMUFDContainer *container = container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer); - QLIST_REMOVE(vbasedev, global_next); - QLIST_REMOVE(vbasedev, container_next); - vbasedev->bcontainer = NULL; + vfio_device_unprepare(vbasedev); if (!vbasedev->ram_block_discard_allowed) { iommufd_cdev_ram_block_discard_disable(false); diff --git a/hw/vfio/listener.c b/hw/vfio/listener.c index 6f77e18a7a..bfacb3d8d9 100644 --- a/hw/vfio/listener.c +++ b/hw/vfio/listener.c @@ -172,7 +172,7 @@ static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) } } else { ret = vfio_container_dma_unmap(bcontainer, iova, - iotlb->addr_mask + 1, iotlb); + iotlb->addr_mask + 1, iotlb, false); if (ret) { error_setg(&local_err, "vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " @@ -201,7 +201,7 @@ static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, int ret; /* Unmap with a single call. */ - ret = vfio_container_dma_unmap(bcontainer, iova, size , NULL); + ret = vfio_container_dma_unmap(bcontainer, iova, size , NULL, false); if (ret) { error_report("%s: vfio_container_dma_unmap() failed: %s", __func__, strerror(-ret)); @@ -411,6 +411,32 @@ static bool vfio_get_section_iova_range(VFIOContainerBase *bcontainer, return true; } +static void vfio_listener_begin(MemoryListener *listener) +{ + VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, + listener); + void (*listener_begin)(VFIOContainerBase *bcontainer); + + listener_begin = VFIO_IOMMU_GET_CLASS(bcontainer)->listener_begin; + + if (listener_begin) { + listener_begin(bcontainer); + } +} + +static void vfio_listener_commit(MemoryListener *listener) +{ + VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, + listener); + void (*listener_commit)(VFIOContainerBase *bcontainer); + + listener_commit = VFIO_IOMMU_GET_CLASS(bcontainer)->listener_begin; + + if (listener_commit) { + listener_commit(bcontainer); + } +} + static void vfio_device_error_append(VFIODevice *vbasedev, Error **errp) { /* @@ -634,21 +660,14 @@ static void vfio_listener_region_del(MemoryListener *listener, } if (try_unmap) { + bool unmap_all = false; + if (int128_eq(llsize, int128_2_64())) { - /* The unmap ioctl doesn't accept a full 64-bit span. */ - llsize = int128_rshift(llsize, 1); - ret = vfio_container_dma_unmap(bcontainer, iova, - int128_get64(llsize), NULL); - if (ret) { - error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " - "0x%"HWADDR_PRIx") = %d (%s)", - bcontainer, iova, int128_get64(llsize), ret, - strerror(-ret)); - } - iova += int128_get64(llsize); + unmap_all = true; + llsize = int128_zero(); } - ret = vfio_container_dma_unmap(bcontainer, iova, - int128_get64(llsize), NULL); + ret = vfio_container_dma_unmap(bcontainer, iova, int128_get64(llsize), + NULL, unmap_all); if (ret) { error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " "0x%"HWADDR_PRIx") = %d (%s)", @@ -801,13 +820,17 @@ static void vfio_devices_dma_logging_stop(VFIOContainerBase *bcontainer) VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP; QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { + int ret; + if (!vbasedev->dirty_tracking) { continue; } - if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { + ret = vbasedev->io_ops->device_feature(vbasedev, feature); + + if (ret != 0) { warn_report("%s: Failed to stop DMA logging, err %d (%s)", - vbasedev->name, -errno, strerror(errno)); + vbasedev->name, -ret, strerror(-ret)); } vbasedev->dirty_tracking = false; } @@ -908,10 +931,9 @@ static bool vfio_devices_dma_logging_start(VFIOContainerBase *bcontainer, continue; } - ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); + ret = vbasedev->io_ops->device_feature(vbasedev, feature); if (ret) { - ret = -errno; - error_setg_errno(errp, errno, "%s: Failed to start DMA logging", + error_setg_errno(errp, -ret, "%s: Failed to start DMA logging", vbasedev->name); goto out; } @@ -1165,6 +1187,8 @@ static void vfio_listener_log_sync(MemoryListener *listener, static const MemoryListener vfio_memory_listener = { .name = "vfio", + .begin = vfio_listener_begin, + .commit = vfio_listener_commit, .region_add = vfio_listener_region_add, .region_del = vfio_listener_region_del, .log_global_start = vfio_listener_log_global_start, diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 6908bcc0d3..a1bfdfe375 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -241,7 +241,7 @@ static void vfio_intx_update(VFIOPCIDevice *vdev, PCIINTxRoute *route) static void vfio_intx_routing_notifier(PCIDevice *pdev) { - VFIOPCIDevice *vdev = VFIO_PCI(pdev); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(pdev); PCIINTxRoute route; if (vdev->interrupt != VFIO_INT_INTx) { @@ -381,7 +381,7 @@ static void vfio_msi_interrupt(void *opaque) static int vfio_enable_msix_no_vec(VFIOPCIDevice *vdev) { g_autofree struct vfio_irq_set *irq_set = NULL; - int ret = 0, argsz; + int argsz; int32_t *fd; argsz = sizeof(*irq_set) + sizeof(*fd); @@ -396,9 +396,7 @@ static int vfio_enable_msix_no_vec(VFIOPCIDevice *vdev) fd = (int32_t *)&irq_set->data; *fd = -1; - ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); - - return ret; + return vdev->vbasedev.io_ops->set_irqs(&vdev->vbasedev, irq_set); } static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix) @@ -455,7 +453,7 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix) fds[i] = fd; } - ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); + ret = vdev->vbasedev.io_ops->set_irqs(&vdev->vbasedev, irq_set); g_free(irq_set); @@ -516,7 +514,7 @@ static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg, static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr, MSIMessage *msg, IOHandler *handler) { - VFIOPCIDevice *vdev = VFIO_PCI(pdev); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(pdev); VFIOMSIVector *vector; int ret; bool resizing = !!(vdev->nr_vectors < nr + 1); @@ -581,7 +579,8 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr, vfio_device_irq_disable(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); ret = vfio_enable_vectors(vdev, true); if (ret) { - error_report("vfio: failed to enable vectors, %d", ret); + error_report("vfio: failed to enable vectors, %s", + strerror(-ret)); } } else { Error *err = NULL; @@ -621,7 +620,7 @@ static int vfio_msix_vector_use(PCIDevice *pdev, static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr) { - VFIOPCIDevice *vdev = VFIO_PCI(pdev); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(pdev); VFIOMSIVector *vector = &vdev->msi_vectors[nr]; trace_vfio_msix_vector_release(vdev->vbasedev.name, nr); @@ -695,7 +694,8 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) if (vdev->nr_vectors) { ret = vfio_enable_vectors(vdev, true); if (ret) { - error_report("vfio: failed to enable vectors, %d", ret); + error_report("vfio: failed to enable vectors, %s", + strerror(-ret)); } } else { /* @@ -712,7 +712,8 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) */ ret = vfio_enable_msix_no_vec(vdev); if (ret) { - error_report("vfio: failed to enable MSI-X, %d", ret); + error_report("vfio: failed to enable MSI-X, %s", + strerror(-ret)); } } @@ -765,7 +766,8 @@ retry: ret = vfio_enable_vectors(vdev, false); if (ret) { if (ret < 0) { - error_report("vfio: Error: Failed to setup MSI fds: %m"); + error_report("vfio: Error: Failed to setup MSI fds: %s", + strerror(-ret)); } else { error_report("vfio: Error: Failed to enable %d " "MSI vectors, retry with %d", vdev->nr_vectors, ret); @@ -881,18 +883,22 @@ static void vfio_update_msi(VFIOPCIDevice *vdev) static void vfio_pci_load_rom(VFIOPCIDevice *vdev) { - g_autofree struct vfio_region_info *reg_info = NULL; + VFIODevice *vbasedev = &vdev->vbasedev; + struct vfio_region_info *reg_info = NULL; uint64_t size; off_t off = 0; ssize_t bytes; + int ret; - if (vfio_device_get_region_info(&vdev->vbasedev, - VFIO_PCI_ROM_REGION_INDEX, ®_info)) { - error_report("vfio: Error getting ROM info: %m"); + ret = vfio_device_get_region_info(vbasedev, VFIO_PCI_ROM_REGION_INDEX, + ®_info); + + if (ret != 0) { + error_report("vfio: Error getting ROM info: %s", strerror(-ret)); return; } - trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size, + trace_vfio_pci_load_rom(vbasedev->name, (unsigned long)reg_info->size, (unsigned long)reg_info->offset, (unsigned long)reg_info->flags); @@ -901,8 +907,7 @@ static void vfio_pci_load_rom(VFIOPCIDevice *vdev) if (!vdev->rom_size) { vdev->rom_read_failed = true; - error_report("vfio-pci: Cannot read device rom at " - "%s", vdev->vbasedev.name); + error_report("vfio-pci: Cannot read device rom at %s", vbasedev->name); error_printf("Device option ROM contents are probably invalid " "(check dmesg).\nSkip option ROM probe with rombar=0, " "or load from file with romfile=\n"); @@ -913,18 +918,22 @@ static void vfio_pci_load_rom(VFIOPCIDevice *vdev) memset(vdev->rom, 0xff, size); while (size) { - bytes = pread(vdev->vbasedev.fd, vdev->rom + off, - size, vdev->rom_offset + off); + bytes = vbasedev->io_ops->region_read(vbasedev, + VFIO_PCI_ROM_REGION_INDEX, + off, size, vdev->rom + off); + if (bytes == 0) { break; } else if (bytes > 0) { off += bytes; size -= bytes; } else { - if (errno == EINTR || errno == EAGAIN) { + if (bytes == -EINTR || bytes == -EAGAIN) { continue; } - error_report("vfio: Error reading device ROM: %m"); + error_report("vfio: Error reading device ROM: %s", + strreaderror(bytes)); + break; } } @@ -960,6 +969,24 @@ static void vfio_pci_load_rom(VFIOPCIDevice *vdev) } } +/* "Raw" read of underlying config space. */ +static int vfio_pci_config_space_read(VFIOPCIDevice *vdev, off_t offset, + uint32_t size, void *data) +{ + return vdev->vbasedev.io_ops->region_read(&vdev->vbasedev, + VFIO_PCI_CONFIG_REGION_INDEX, + offset, size, data); +} + +/* "Raw" write of underlying config space. */ +static int vfio_pci_config_space_write(VFIOPCIDevice *vdev, off_t offset, + uint32_t size, void *data) +{ + return vdev->vbasedev.io_ops->region_write(&vdev->vbasedev, + VFIO_PCI_CONFIG_REGION_INDEX, + offset, size, data); +} + static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size) { VFIOPCIDevice *vdev = opaque; @@ -1012,10 +1039,9 @@ static const MemoryRegionOps vfio_rom_ops = { static void vfio_pci_size_rom(VFIOPCIDevice *vdev) { + VFIODevice *vbasedev = &vdev->vbasedev; uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK); - off_t offset = vdev->config_offset + PCI_ROM_ADDRESS; char *name; - int fd = vdev->vbasedev.fd; if (vdev->pdev.romfile || !vdev->pdev.rom_bar) { /* Since pci handles romfile, just print a message and return */ @@ -1032,11 +1058,12 @@ static void vfio_pci_size_rom(VFIOPCIDevice *vdev) * Use the same size ROM BAR as the physical device. The contents * will get filled in later when the guest tries to read it. */ - if (pread(fd, &orig, 4, offset) != 4 || - pwrite(fd, &size, 4, offset) != 4 || - pread(fd, &size, 4, offset) != 4 || - pwrite(fd, &orig, 4, offset) != 4) { - error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name); + if (vfio_pci_config_space_read(vdev, PCI_ROM_ADDRESS, 4, &orig) != 4 || + vfio_pci_config_space_write(vdev, PCI_ROM_ADDRESS, 4, &size) != 4 || + vfio_pci_config_space_read(vdev, PCI_ROM_ADDRESS, 4, &size) != 4 || + vfio_pci_config_space_write(vdev, PCI_ROM_ADDRESS, 4, &orig) != 4) { + + error_report("%s(%s) ROM access failed", __func__, vbasedev->name); return; } @@ -1169,7 +1196,7 @@ static const MemoryRegionOps vfio_vga_ops = { */ static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar) { - VFIOPCIDevice *vdev = VFIO_PCI(pdev); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(pdev); VFIORegion *region = &vdev->bars[bar].region; MemoryRegion *mmap_mr, *region_mr, *base_mr; PCIIORegion *r; @@ -1215,7 +1242,8 @@ static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar) */ uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len) { - VFIOPCIDevice *vdev = VFIO_PCI(pdev); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(pdev); + VFIODevice *vbasedev = &vdev->vbasedev; uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val; memcpy(&emu_bits, vdev->emulated_config_bits + addr, len); @@ -1228,12 +1256,12 @@ uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len) if (~emu_bits & (0xffffffffU >> (32 - len * 8))) { ssize_t ret; - ret = pread(vdev->vbasedev.fd, &phys_val, len, - vdev->config_offset + addr); + ret = vfio_pci_config_space_read(vdev, addr, len, &phys_val); if (ret != len) { - error_report("%s(%s, 0x%x, 0x%x) failed: %m", - __func__, vdev->vbasedev.name, addr, len); - return -errno; + error_report("%s(%s, 0x%x, 0x%x) failed: %s", + __func__, vbasedev->name, addr, len, + strreaderror(ret)); + return -1; } phys_val = le32_to_cpu(phys_val); } @@ -1248,16 +1276,19 @@ uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len) void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr, uint32_t val, int len) { - VFIOPCIDevice *vdev = VFIO_PCI(pdev); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(pdev); + VFIODevice *vbasedev = &vdev->vbasedev; uint32_t val_le = cpu_to_le32(val); + int ret; trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len); /* Write everything to VFIO, let it filter out what we can't write */ - if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr) - != len) { - error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m", - __func__, vdev->vbasedev.name, addr, val, len); + ret = vfio_pci_config_space_write(vdev, addr, len, &val_le); + if (ret != len) { + error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %s", + __func__, vbasedev->name, addr, val, len, + strwriteerror(ret)); } /* MSI/MSI-X Enabling/Disabling */ @@ -1345,9 +1376,11 @@ static bool vfio_msi_setup(VFIOPCIDevice *vdev, int pos, Error **errp) int ret, entries; Error *err = NULL; - if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl), - vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) { - error_setg_errno(errp, errno, "failed reading MSI PCI_CAP_FLAGS"); + ret = vfio_pci_config_space_read(vdev, pos + PCI_CAP_FLAGS, + sizeof(ctrl), &ctrl); + if (ret != sizeof(ctrl)) { + error_setg(errp, "failed reading MSI PCI_CAP_FLAGS: %s", + strreaderror(ret)); return false; } ctrl = le16_to_cpu(ctrl); @@ -1554,31 +1587,35 @@ static bool vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp) uint8_t pos; uint16_t ctrl; uint32_t table, pba; - int ret, fd = vdev->vbasedev.fd; - struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info), - .index = VFIO_PCI_MSIX_IRQ_INDEX }; + struct vfio_irq_info irq_info; VFIOMSIXInfo *msix; + int ret; pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX); if (!pos) { return true; } - if (pread(fd, &ctrl, sizeof(ctrl), - vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) { - error_setg_errno(errp, errno, "failed to read PCI MSIX FLAGS"); + ret = vfio_pci_config_space_read(vdev, pos + PCI_MSIX_FLAGS, + sizeof(ctrl), &ctrl); + if (ret != sizeof(ctrl)) { + error_setg(errp, "failed to read PCI MSIX FLAGS: %s", + strreaderror(ret)); return false; } - if (pread(fd, &table, sizeof(table), - vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) { - error_setg_errno(errp, errno, "failed to read PCI MSIX TABLE"); + ret = vfio_pci_config_space_read(vdev, pos + PCI_MSIX_TABLE, + sizeof(table), &table); + if (ret != sizeof(table)) { + error_setg(errp, "failed to read PCI MSIX TABLE: %s", + strreaderror(ret)); return false; } - if (pread(fd, &pba, sizeof(pba), - vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) { - error_setg_errno(errp, errno, "failed to read PCI MSIX PBA"); + ret = vfio_pci_config_space_read(vdev, pos + PCI_MSIX_PBA, + sizeof(pba), &pba); + if (ret != sizeof(pba)) { + error_setg(errp, "failed to read PCI MSIX PBA: %s", strreaderror(ret)); return false; } @@ -1593,7 +1630,8 @@ static bool vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp) msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK; msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; - ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info); + ret = vfio_device_get_irq_info(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX, + &irq_info); if (ret < 0) { error_setg_errno(errp, -ret, "failed to get MSI-X irq info"); g_free(msix); @@ -1737,10 +1775,10 @@ static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr) } /* Determine what type of BAR this is for registration */ - ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar), - vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr)); + ret = vfio_pci_config_space_read(vdev, PCI_BASE_ADDRESS_0 + (4 * nr), + sizeof(pci_bar), &pci_bar); if (ret != sizeof(pci_bar)) { - error_report("vfio: Failed to read BAR %d (%m)", nr); + error_report("vfio: Failed to read BAR %d: %s", nr, strreaderror(ret)); return; } @@ -2443,21 +2481,23 @@ void vfio_pci_pre_reset(VFIOPCIDevice *vdev) void vfio_pci_post_reset(VFIOPCIDevice *vdev) { + VFIODevice *vbasedev = &vdev->vbasedev; Error *err = NULL; - int nr; + int ret, nr; if (!vfio_intx_enable(vdev, &err)) { error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); } for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) { - off_t addr = vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr); + off_t addr = PCI_BASE_ADDRESS_0 + (4 * nr); uint32_t val = 0; uint32_t len = sizeof(val); - if (pwrite(vdev->vbasedev.fd, &val, len, addr) != len) { - error_report("%s(%s) reset bar %d failed: %m", __func__, - vdev->vbasedev.name, nr); + ret = vfio_pci_config_space_write(vdev, addr, len, &val); + if (ret != len) { + error_report("%s(%s) reset bar %d failed: %s", __func__, + vbasedev->name, nr, strwriteerror(ret)); } } @@ -2670,7 +2710,7 @@ static VFIODeviceOps vfio_pci_ops = { bool vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp) { VFIODevice *vbasedev = &vdev->vbasedev; - g_autofree struct vfio_region_info *reg_info = NULL; + struct vfio_region_info *reg_info = NULL; int ret; ret = vfio_device_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, ®_info); @@ -2735,8 +2775,8 @@ bool vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp) static bool vfio_populate_device(VFIOPCIDevice *vdev, Error **errp) { VFIODevice *vbasedev = &vdev->vbasedev; - g_autofree struct vfio_region_info *reg_info = NULL; - struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) }; + struct vfio_region_info *reg_info = NULL; + struct vfio_irq_info irq_info; int i, ret = -1; /* Sanity check device */ @@ -2797,12 +2837,10 @@ static bool vfio_populate_device(VFIOPCIDevice *vdev, Error **errp) } } - irq_info.index = VFIO_PCI_ERR_IRQ_INDEX; - - ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info); + ret = vfio_device_get_irq_info(vbasedev, VFIO_PCI_ERR_IRQ_INDEX, &irq_info); if (ret) { /* This can fail for an old kernel or legacy PCI dev */ - trace_vfio_populate_device_get_irq_info_failure(strerror(errno)); + trace_vfio_populate_device_get_irq_info_failure(strerror(-ret)); } else if (irq_info.count == 1) { vdev->pci_aer = true; } else { @@ -2911,17 +2949,18 @@ static void vfio_req_notifier_handler(void *opaque) static void vfio_register_req_notifier(VFIOPCIDevice *vdev) { - struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info), - .index = VFIO_PCI_REQ_IRQ_INDEX }; + struct vfio_irq_info irq_info; Error *err = NULL; int32_t fd; + int ret; if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) { return; } - if (ioctl(vdev->vbasedev.fd, - VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) { + ret = vfio_device_get_irq_info(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, + &irq_info); + if (ret < 0 || irq_info.count < 1) { return; } @@ -3090,11 +3129,12 @@ static bool vfio_interrupt_setup(VFIOPCIDevice *vdev, Error **errp) static void vfio_realize(PCIDevice *pdev, Error **errp) { ERRP_GUARD(); - VFIOPCIDevice *vdev = VFIO_PCI(pdev); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(pdev); VFIODevice *vbasedev = &vdev->vbasedev; int i, ret; char uuid[UUID_STR_LEN]; g_autofree char *name = NULL; + uint32_t config_space_size; if (vbasedev->fd < 0 && !vbasedev->sysfsdev) { if (!(~vdev->host.domain || ~vdev->host.bus || @@ -3149,13 +3189,14 @@ static void vfio_realize(PCIDevice *pdev, Error **errp) goto error; } + config_space_size = MIN(pci_config_size(&vdev->pdev), vdev->config_size); + /* Get a copy of config space */ - ret = pread(vbasedev->fd, vdev->pdev.config, - MIN(pci_config_size(&vdev->pdev), vdev->config_size), - vdev->config_offset); - if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) { - ret = ret < 0 ? -errno : -EFAULT; - error_setg_errno(errp, -ret, "failed to read device config space"); + ret = vfio_pci_config_space_read(vdev, 0, config_space_size, + vdev->pdev.config); + if (ret < (int)config_space_size) { + ret = ret < 0 ? -ret : EFAULT; + error_setg_errno(errp, ret, "failed to read device config space"); goto error; } @@ -3259,7 +3300,7 @@ error: static void vfio_instance_finalize(Object *obj) { - VFIOPCIDevice *vdev = VFIO_PCI(obj); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(obj); vfio_display_finalize(vdev); vfio_bars_finalize(vdev); @@ -3277,7 +3318,7 @@ static void vfio_instance_finalize(Object *obj) static void vfio_exitfn(PCIDevice *pdev) { - VFIOPCIDevice *vdev = VFIO_PCI(pdev); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(pdev); VFIODevice *vbasedev = &vdev->vbasedev; vfio_unregister_req_notifier(vdev); @@ -3301,7 +3342,7 @@ static void vfio_exitfn(PCIDevice *pdev) static void vfio_pci_reset(DeviceState *dev) { - VFIOPCIDevice *vdev = VFIO_PCI(dev); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(dev); trace_vfio_pci_reset(vdev->vbasedev.name); @@ -3341,7 +3382,7 @@ post_reset: static void vfio_instance_init(Object *obj) { PCIDevice *pci_dev = PCI_DEVICE(obj); - VFIOPCIDevice *vdev = VFIO_PCI(obj); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(obj); VFIODevice *vbasedev = &vdev->vbasedev; device_add_bootindex_property(obj, &vdev->bootindex, @@ -3362,6 +3403,31 @@ static void vfio_instance_init(Object *obj) pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS; } +static void vfio_pci_base_dev_class_init(ObjectClass *klass, const void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass); + + dc->desc = "VFIO PCI base device"; + set_bit(DEVICE_CATEGORY_MISC, dc->categories); + pdc->exit = vfio_exitfn; + pdc->config_read = vfio_pci_read_config; + pdc->config_write = vfio_pci_write_config; +} + +static const TypeInfo vfio_pci_base_dev_info = { + .name = TYPE_VFIO_PCI_BASE, + .parent = TYPE_PCI_DEVICE, + .instance_size = 0, + .abstract = true, + .class_init = vfio_pci_base_dev_class_init, + .interfaces = (const InterfaceInfo[]) { + { INTERFACE_PCIE_DEVICE }, + { INTERFACE_CONVENTIONAL_PCI_DEVICE }, + { } + }, +}; + static PropertyInfo vfio_pci_migration_multifd_transfer_prop; static const Property vfio_pci_dev_properties[] = { @@ -3385,7 +3451,7 @@ static const Property vfio_pci_dev_properties[] = { DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features, VFIO_FEATURE_ENABLE_REQ_BIT, true), DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features, - VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false), + VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, true), DEFINE_PROP_BIT("x-igd-lpc", VFIOPCIDevice, features, VFIO_FEATURE_ENABLE_IGD_LPC_BIT, false), DEFINE_PROP_ON_OFF_AUTO("x-igd-legacy-mode", VFIOPCIDevice, @@ -3432,7 +3498,8 @@ static const Property vfio_pci_dev_properties[] = { #ifdef CONFIG_IOMMUFD static void vfio_pci_set_fd(Object *obj, const char *str, Error **errp) { - vfio_device_set_fd(&VFIO_PCI(obj)->vbasedev, str, errp); + VFIOPCIDevice *vdev = VFIO_PCI_BASE(obj); + vfio_device_set_fd(&vdev->vbasedev, str, errp); } #endif @@ -3447,11 +3514,7 @@ static void vfio_pci_dev_class_init(ObjectClass *klass, const void *data) object_class_property_add_str(klass, "fd", NULL, vfio_pci_set_fd); #endif dc->desc = "VFIO-based PCI device assignment"; - set_bit(DEVICE_CATEGORY_MISC, dc->categories); pdc->realize = vfio_realize; - pdc->exit = vfio_exitfn; - pdc->config_read = vfio_pci_read_config; - pdc->config_write = vfio_pci_write_config; object_class_property_set_description(klass, /* 1.3 */ "host", @@ -3576,16 +3639,11 @@ static void vfio_pci_dev_class_init(ObjectClass *klass, const void *data) static const TypeInfo vfio_pci_dev_info = { .name = TYPE_VFIO_PCI, - .parent = TYPE_PCI_DEVICE, + .parent = TYPE_VFIO_PCI_BASE, .instance_size = sizeof(VFIOPCIDevice), .class_init = vfio_pci_dev_class_init, .instance_init = vfio_instance_init, .instance_finalize = vfio_instance_finalize, - .interfaces = (const InterfaceInfo[]) { - { INTERFACE_PCIE_DEVICE }, - { INTERFACE_CONVENTIONAL_PCI_DEVICE }, - { } - }, }; static const Property vfio_pci_dev_nohotplug_properties[] = { @@ -3632,6 +3690,7 @@ static void register_vfio_pci_dev_type(void) vfio_pci_migration_multifd_transfer_prop = qdev_prop_on_off_auto; vfio_pci_migration_multifd_transfer_prop.realized_set_allowed = true; + type_register_static(&vfio_pci_base_dev_info); type_register_static(&vfio_pci_dev_info); type_register_static(&vfio_pci_nohotplug_dev_info); } diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h index f835b1dbc2..5ce0fb916f 100644 --- a/hw/vfio/pci.h +++ b/hw/vfio/pci.h @@ -118,8 +118,16 @@ typedef struct VFIOMSIXInfo { bool noresize; } VFIOMSIXInfo; +/* + * TYPE_VFIO_PCI_BASE is an abstract type used to share code + * between VFIO implementations that use a kernel driver + * with those that use user sockets. + */ +#define TYPE_VFIO_PCI_BASE "vfio-pci-base" +OBJECT_DECLARE_SIMPLE_TYPE(VFIOPCIDevice, VFIO_PCI_BASE) + #define TYPE_VFIO_PCI "vfio-pci" -OBJECT_DECLARE_SIMPLE_TYPE(VFIOPCIDevice, VFIO_PCI) +/* TYPE_VFIO_PCI shares struct VFIOPCIDevice. */ struct VFIOPCIDevice { PCIDevice pdev; diff --git a/hw/vfio/platform.c b/hw/vfio/platform.c index ffb3681607..9a21f2e50a 100644 --- a/hw/vfio/platform.c +++ b/hw/vfio/platform.c @@ -474,10 +474,10 @@ static bool vfio_populate_device(VFIODevice *vbasedev, Error **errp) QSIMPLEQ_INIT(&vdev->pending_intp_queue); for (i = 0; i < vbasedev->num_irqs; i++) { - struct vfio_irq_info irq = { .argsz = sizeof(irq) }; + struct vfio_irq_info irq; + + ret = vfio_device_get_irq_info(vbasedev, i, &irq); - irq.index = i; - ret = ioctl(vbasedev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq); if (ret) { error_setg_errno(errp, -ret, "failed to get device irq info"); goto irq_err; diff --git a/hw/vfio/region.c b/hw/vfio/region.c index 04bf9eb098..34752c3f65 100644 --- a/hw/vfio/region.c +++ b/hw/vfio/region.c @@ -45,6 +45,7 @@ void vfio_region_write(void *opaque, hwaddr addr, uint32_t dword; uint64_t qword; } buf; + int ret; switch (size) { case 1: @@ -64,11 +65,13 @@ void vfio_region_write(void *opaque, hwaddr addr, break; } - if (pwrite(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { + ret = vbasedev->io_ops->region_write(vbasedev, region->nr, + addr, size, &buf); + if (ret != size) { error_report("%s(%s:region%d+0x%"HWADDR_PRIx", 0x%"PRIx64 - ",%d) failed: %m", + ",%d) failed: %s", __func__, vbasedev->name, region->nr, - addr, data, size); + addr, data, size, strwriteerror(ret)); } trace_vfio_region_write(vbasedev->name, region->nr, addr, data, size); @@ -96,11 +99,13 @@ uint64_t vfio_region_read(void *opaque, uint64_t qword; } buf; uint64_t data = 0; + int ret; - if (pread(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { - error_report("%s(%s:region%d+0x%"HWADDR_PRIx", %d) failed: %m", + ret = vbasedev->io_ops->region_read(vbasedev, region->nr, addr, size, &buf); + if (ret != size) { + error_report("%s(%s:region%d+0x%"HWADDR_PRIx", %d) failed: %s", __func__, vbasedev->name, region->nr, - addr, size); + addr, size, strreaderror(ret)); return (uint64_t)-1; } switch (size) { @@ -182,7 +187,7 @@ static int vfio_setup_region_sparse_mmaps(VFIORegion *region, int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region, int index, const char *name) { - g_autofree struct vfio_region_info *info = NULL; + struct vfio_region_info *info = NULL; int ret; ret = vfio_device_get_region_info(vbasedev, index, &info); diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c index 9a677e8ed7..78e0bc8f64 100644 --- a/hw/xen/xen-hvm-common.c +++ b/hw/xen/xen-hvm-common.c @@ -711,7 +711,7 @@ static int xen_map_ioreq_server(XenIOState *state) /* * If we fail to map the shared page with xenforeignmemory_map_resource() * or if we're using buffered ioreqs, we need xen_get_ioreq_server_info() - * to provide the the addresses to map the shared page and/or to get the + * to provide the addresses to map the shared page and/or to get the * event-channel port for buffered ioreqs. */ if (state->shared_page == NULL || state->has_bufioreq) { diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c index 698b5c53ed..e31d379702 100644 --- a/hw/xen/xen-mapcache.c +++ b/hw/xen/xen-mapcache.c @@ -75,7 +75,8 @@ typedef struct MapCache { } MapCache; static MapCache *mapcache; -static MapCache *mapcache_grants; +static MapCache *mapcache_grants_ro; +static MapCache *mapcache_grants_rw; static xengnttab_handle *xen_region_gnttabdev; static inline void mapcache_lock(MapCache *mc) @@ -176,9 +177,12 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void *opaque) * Grant mappings must use XC_PAGE_SIZE granularity since we can't * map anything beyond the number of pages granted to us. */ - mapcache_grants = xen_map_cache_init_single(f, opaque, - XC_PAGE_SHIFT, - max_mcache_size); + mapcache_grants_ro = xen_map_cache_init_single(f, opaque, + XC_PAGE_SHIFT, + max_mcache_size); + mapcache_grants_rw = xen_map_cache_init_single(f, opaque, + XC_PAGE_SHIFT, + max_mcache_size); setrlimit(RLIMIT_AS, &rlimit_as); } @@ -376,12 +380,12 @@ tryagain: entry = &mc->entry[address_index % mc->nr_buckets]; - while (entry && (lock || entry->lock) && entry->vaddr_base && - (entry->paddr_index != address_index || entry->size != cache_size || + while (entry && (!entry->vaddr_base || + entry->paddr_index != address_index || entry->size != cache_size || !test_bits(address_offset >> XC_PAGE_SHIFT, test_bit_size >> XC_PAGE_SHIFT, entry->valid_mapping))) { - if (!free_entry && !entry->lock) { + if (!free_entry && (!entry->lock || !entry->vaddr_base)) { free_entry = entry; free_pentry = pentry; } @@ -456,9 +460,13 @@ uint8_t *xen_map_cache(MemoryRegion *mr, bool is_write) { bool grant = xen_mr_is_grants(mr); - MapCache *mc = grant ? mapcache_grants : mapcache; + MapCache *mc = mapcache; uint8_t *p; + if (grant) { + mc = is_write ? mapcache_grants_rw : mapcache_grants_ro; + } + if (grant && !lock) { /* * Grants are only supported via address_space_map(). Anything @@ -523,7 +531,10 @@ ram_addr_t xen_ram_addr_from_mapcache(void *ptr) addr = xen_ram_addr_from_mapcache_single(mapcache, ptr); if (addr == RAM_ADDR_INVALID) { - addr = xen_ram_addr_from_mapcache_single(mapcache_grants, ptr); + addr = xen_ram_addr_from_mapcache_single(mapcache_grants_ro, ptr); + } + if (addr == RAM_ADDR_INVALID) { + addr = xen_ram_addr_from_mapcache_single(mapcache_grants_rw, ptr); } return addr; @@ -626,7 +637,8 @@ static void xen_invalidate_map_cache_entry_single(MapCache *mc, uint8_t *buffer) static void xen_invalidate_map_cache_entry_all(uint8_t *buffer) { xen_invalidate_map_cache_entry_single(mapcache, buffer); - xen_invalidate_map_cache_entry_single(mapcache_grants, buffer); + xen_invalidate_map_cache_entry_single(mapcache_grants_ro, buffer); + xen_invalidate_map_cache_entry_single(mapcache_grants_rw, buffer); } static void xen_invalidate_map_cache_entry_bh(void *opaque) diff --git a/include/accel/tcg/cpu-ldst.h b/include/accel/tcg/cpu-ldst.h index f97a730703..0de7f5eaa6 100644 --- a/include/accel/tcg/cpu-ldst.h +++ b/include/accel/tcg/cpu-ldst.h @@ -502,62 +502,4 @@ static inline uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr) return cpu_ldq_code_mmu(env, addr, oi, 0); } -/** - * tlb_vaddr_to_host: - * @env: CPUArchState - * @addr: guest virtual address to look up - * @access_type: 0 for read, 1 for write, 2 for execute - * @mmu_idx: MMU index to use for lookup - * - * Look up the specified guest virtual index in the TCG softmmu TLB. - * If we can translate a host virtual address suitable for direct RAM - * access, without causing a guest exception, then return it. - * Otherwise (TLB entry is for an I/O access, guest software - * TLB fill required, etc) return NULL. - */ -#ifdef CONFIG_USER_ONLY -static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, - MMUAccessType access_type, int mmu_idx) -{ - return g2h(env_cpu(env), addr); -} -#else -void *tlb_vaddr_to_host(CPUArchState *env, vaddr addr, - MMUAccessType access_type, int mmu_idx); -#endif - -/* - * For user-only, helpers that use guest to host address translation - * must protect the actual host memory access by recording 'retaddr' - * for the signal handler. This is required for a race condition in - * which another thread unmaps the page between a probe and the - * actual access. - */ -#ifdef CONFIG_USER_ONLY -extern __thread uintptr_t helper_retaddr; - -static inline void set_helper_retaddr(uintptr_t ra) -{ - helper_retaddr = ra; - /* - * Ensure that this write is visible to the SIGSEGV handler that - * may be invoked due to a subsequent invalid memory operation. - */ - signal_barrier(); -} - -static inline void clear_helper_retaddr(void) -{ - /* - * Ensure that previous memory operations have succeeded before - * removing the data visible to the signal handler. - */ - signal_barrier(); - helper_retaddr = 0; -} -#else -#define set_helper_retaddr(ra) do { } while (0) -#define clear_helper_retaddr() do { } while (0) -#endif - #endif /* ACCEL_TCG_CPU_LDST_H */ diff --git a/include/accel/tcg/cpu-ops.h b/include/accel/tcg/cpu-ops.h index 0e4352513d..cd22e5d5b9 100644 --- a/include/accel/tcg/cpu-ops.h +++ b/include/accel/tcg/cpu-ops.h @@ -16,6 +16,7 @@ #include "exec/memop.h" #include "exec/mmu-access-type.h" #include "exec/vaddr.h" +#include "accel/tcg/tb-cpu-state.h" #include "tcg/tcg-mo.h" struct TCGCPUOps { @@ -29,6 +30,13 @@ struct TCGCPUOps { bool mttcg_supported; /** + * @precise_smc: Stores which modify code within the current TB force + * the TB to exit; the next executed instruction will see + * the result of the store. + */ + bool precise_smc; + + /** * @guest_default_memory_order: default barrier that is required * for the guest memory ordering. */ @@ -54,6 +62,12 @@ struct TCGCPUOps { void (*translate_code)(CPUState *cpu, TranslationBlock *tb, int *max_insns, vaddr pc, void *host_pc); /** + * @get_tb_cpu_state: Extract CPU state for a TCG #TranslationBlock + * + * Fill in all data required to select or compile a TranslationBlock. + */ + TCGTBCPUState (*get_tb_cpu_state)(CPUState *cs); + /** * @synchronize_from_tb: Synchronize state from a TCG #TranslationBlock * * This is called when we abandon execution of a TB before starting it, @@ -143,11 +157,20 @@ struct TCGCPUOps { */ void (*record_sigbus)(CPUState *cpu, vaddr addr, MMUAccessType access_type, uintptr_t ra); + + /** + * untagged_addr: Remove an ignored tag from an address + * @cpu: cpu context + * @addr: tagged guest address + */ + vaddr (*untagged_addr)(CPUState *cs, vaddr addr); #else /** @do_interrupt: Callback for interrupt handling. */ void (*do_interrupt)(CPUState *cpu); /** @cpu_exec_interrupt: Callback for processing interrupts in cpu_exec */ bool (*cpu_exec_interrupt)(CPUState *cpu, int interrupt_request); + /** @cpu_exec_reset: Callback for reset in cpu_exec. */ + void (*cpu_exec_reset)(CPUState *cpu); /** * @cpu_exec_halt: Callback for handling halt in cpu_exec. * diff --git a/include/accel/tcg/getpc.h b/include/accel/tcg/getpc.h index 8a97ce34e7..0fc08addcf 100644 --- a/include/accel/tcg/getpc.h +++ b/include/accel/tcg/getpc.h @@ -8,10 +8,6 @@ #ifndef ACCEL_TCG_GETPC_H #define ACCEL_TCG_GETPC_H -#ifndef CONFIG_TCG -#error Can only include this header with TCG -#endif - /* GETPC is the true target of the return instruction that we'll execute. */ #ifdef CONFIG_TCG_INTERPRETER extern __thread uintptr_t tci_tb_ptr; diff --git a/include/accel/tcg/helper-retaddr.h b/include/accel/tcg/helper-retaddr.h new file mode 100644 index 0000000000..037fda2b83 --- /dev/null +++ b/include/accel/tcg/helper-retaddr.h @@ -0,0 +1,43 @@ +/* + * Get user helper pc for memory unwinding. + * SPDX-License-Identifier: LGPL-2.1-or-later + */ + +#ifndef ACCEL_TCG_HELPER_RETADDR_H +#define ACCEL_TCG_HELPER_RETADDR_H + +/* + * For user-only, helpers that use guest to host address translation + * must protect the actual host memory access by recording 'retaddr' + * for the signal handler. This is required for a race condition in + * which another thread unmaps the page between a probe and the + * actual access. + */ +#ifdef CONFIG_USER_ONLY +extern __thread uintptr_t helper_retaddr; + +static inline void set_helper_retaddr(uintptr_t ra) +{ + helper_retaddr = ra; + /* + * Ensure that this write is visible to the SIGSEGV handler that + * may be invoked due to a subsequent invalid memory operation. + */ + signal_barrier(); +} + +static inline void clear_helper_retaddr(void) +{ + /* + * Ensure that previous memory operations have succeeded before + * removing the data visible to the signal handler. + */ + signal_barrier(); + helper_retaddr = 0; +} +#else +#define set_helper_retaddr(ra) do { } while (0) +#define clear_helper_retaddr() do { } while (0) +#endif + +#endif /* ACCEL_TCG_HELPER_RETADDR_H */ diff --git a/include/accel/tcg/iommu.h b/include/accel/tcg/iommu.h new file mode 100644 index 0000000000..90cfd6c0ed --- /dev/null +++ b/include/accel/tcg/iommu.h @@ -0,0 +1,41 @@ +/* + * TCG IOMMU translations. + * + * Copyright (c) 2003 Fabrice Bellard + * SPDX-License-Identifier: LGPL-2.1-or-later + */ +#ifndef ACCEL_TCG_IOMMU_H +#define ACCEL_TCG_IOMMU_H + +#ifdef CONFIG_USER_ONLY +#error Cannot include accel/tcg/iommu.h from user emulation +#endif + +#include "exec/hwaddr.h" +#include "exec/memattrs.h" + +/** + * iotlb_to_section: + * @cpu: CPU performing the access + * @index: TCG CPU IOTLB entry + * + * Given a TCG CPU IOTLB entry, return the MemoryRegionSection that + * it refers to. @index will have been initially created and returned + * by memory_region_section_get_iotlb(). + */ +MemoryRegionSection *iotlb_to_section(CPUState *cpu, + hwaddr index, MemTxAttrs attrs); + +MemoryRegionSection *address_space_translate_for_iotlb(CPUState *cpu, + int asidx, + hwaddr addr, + hwaddr *xlat, + hwaddr *plen, + MemTxAttrs attrs, + int *prot); + +hwaddr memory_region_section_get_iotlb(CPUState *cpu, + MemoryRegionSection *section); + +#endif + diff --git a/include/exec/exec-all.h b/include/accel/tcg/probe.h index 944b579d91..dd9ecbbdf1 100644 --- a/include/exec/exec-all.h +++ b/include/accel/tcg/probe.h @@ -1,30 +1,14 @@ /* - * internal execution defines for qemu + * Probe guest virtual addresses for access permissions. * - * Copyright (c) 2003 Fabrice Bellard - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, see <http://www.gnu.org/licenses/>. + * Copyright (c) 2003 Fabrice Bellard + * SPDX-License-Identifier: LGPL-2.1-or-later */ - -#ifndef EXEC_ALL_H -#define EXEC_ALL_H +#ifndef ACCEL_TCG_PROBE_H +#define ACCEL_TCG_PROBE_H #include "exec/mmu-access-type.h" -#include "exec/translation-block.h" - -#if defined(CONFIG_TCG) -#include "accel/tcg/getpc.h" +#include "exec/vaddr.h" /** * probe_access: @@ -118,36 +102,21 @@ int probe_access_full_mmu(CPUArchState *env, vaddr addr, int size, void **phost, CPUTLBEntryFull **pfull); #endif /* !CONFIG_USER_ONLY */ -#endif /* CONFIG_TCG */ - -/* TranslationBlock invalidate API */ -void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr); -void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last); -void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr); - -#if !defined(CONFIG_USER_ONLY) /** - * iotlb_to_section: - * @cpu: CPU performing the access - * @index: TCG CPU IOTLB entry + * tlb_vaddr_to_host: + * @env: CPUArchState + * @addr: guest virtual address to look up + * @access_type: 0 for read, 1 for write, 2 for execute + * @mmu_idx: MMU index to use for lookup * - * Given a TCG CPU IOTLB entry, return the MemoryRegionSection that - * it refers to. @index will have been initially created and returned - * by memory_region_section_get_iotlb(). + * Look up the specified guest virtual index in the TCG softmmu TLB. + * If we can translate a host virtual address suitable for direct RAM + * access, without causing a guest exception, then return it. + * Otherwise (TLB entry is for an I/O access, guest software + * TLB fill required, etc) return NULL. */ -struct MemoryRegionSection *iotlb_to_section(CPUState *cpu, - hwaddr index, MemTxAttrs attrs); -#endif - -#if !defined(CONFIG_USER_ONLY) - -MemoryRegionSection * -address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr, - hwaddr *xlat, hwaddr *plen, - MemTxAttrs attrs, int *prot); -hwaddr memory_region_section_get_iotlb(CPUState *cpu, - MemoryRegionSection *section); -#endif +void *tlb_vaddr_to_host(CPUArchState *env, vaddr addr, + MMUAccessType access_type, int mmu_idx); #endif diff --git a/include/accel/tcg/tb-cpu-state.h b/include/accel/tcg/tb-cpu-state.h new file mode 100644 index 0000000000..8f912900ca --- /dev/null +++ b/include/accel/tcg/tb-cpu-state.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Definition of TCGTBCPUState. + */ + +#ifndef EXEC_TB_CPU_STATE_H +#define EXEC_TB_CPU_STATE_H + +#include "exec/vaddr.h" + +typedef struct TCGTBCPUState { + vaddr pc; + uint32_t flags; + uint32_t cflags; + uint64_t cs_base; +} TCGTBCPUState; + +#endif diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index dab1e7e580..a68485547d 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -182,7 +182,7 @@ void list_cpus(void); * @host_pc: the host pc within the translation * @data: output data * - * Attempt to load the the unwind state for a host pc occurring in + * Attempt to load the unwind state for a host pc occurring in * translated code. If @host_pc is not in translated code, the * function returns false; otherwise @data is loaded. * This is the same unwind info as given to restore_state_to_opc. diff --git a/include/exec/helper-proto-common.h b/include/exec/helper-proto-common.h index 16782ef46c..76e6c25bec 100644 --- a/include/exec/helper-proto-common.h +++ b/include/exec/helper-proto-common.h @@ -13,4 +13,6 @@ #include "exec/helper-proto.h.inc" #undef HELPER_H +#include "accel/tcg/getpc.h" + #endif /* HELPER_PROTO_COMMON_H */ diff --git a/include/exec/poison.h b/include/exec/poison.h index bc422719d8..a779adbb7a 100644 --- a/include/exec/poison.h +++ b/include/exec/poison.h @@ -37,7 +37,6 @@ #pragma GCC poison TARGET_NAME #pragma GCC poison TARGET_BIG_ENDIAN #pragma GCC poison TCG_GUEST_DEFAULT_MO -#pragma GCC poison TARGET_HAS_PRECISE_SMC #pragma GCC poison TARGET_LONG_BITS #pragma GCC poison TARGET_FMT_lx diff --git a/include/exec/translation-block.h b/include/exec/translation-block.h index 8b8e730561..cdce399eba 100644 --- a/include/exec/translation-block.h +++ b/include/exec/translation-block.h @@ -207,4 +207,8 @@ static inline void tb_set_page_addr1(TranslationBlock *tb, #endif } +/* TranslationBlock invalidate API */ +void tb_invalidate_phys_range(CPUState *cpu, tb_page_addr_t start, + tb_page_addr_t last); + #endif /* EXEC_TRANSLATION_BLOCK_H */ diff --git a/include/glib-compat.h b/include/glib-compat.h index 86be439ba0..2e32b90f05 100644 --- a/include/glib-compat.h +++ b/include/glib-compat.h @@ -37,6 +37,13 @@ #endif /* + * These functions perform function pointer casts which can cause function call + * failure on Emscripten. Use g_slist_sort_with_data and g_list_sort_with_data + * instead of these functions. + */ +#pragma GCC poison g_slist_sort g_list_sort + +/* * Note that because of the GLIB_VERSION_MAX_ALLOWED constant above, allowing * use of functions from newer GLib via this compat header needs a little * trickery to prevent warnings being emitted. diff --git a/include/hw/acpi/pcihp.h b/include/hw/acpi/pcihp.h index ac21a95913..a97904bada 100644 --- a/include/hw/acpi/pcihp.h +++ b/include/hw/acpi/pcihp.h @@ -58,7 +58,7 @@ typedef struct AcpiPciHpState { void acpi_pcihp_init(Object *owner, AcpiPciHpState *, PCIBus *root, MemoryRegion *io, uint16_t io_base); -bool acpi_pcihp_is_hotpluggbale_bus(AcpiPciHpState *s, BusState *bus); +bool acpi_pcihp_is_hotpluggable_bus(AcpiPciHpState *s, BusState *bus); void acpi_pcihp_device_pre_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp); void acpi_pcihp_device_plug_cb(HotplugHandler *hotplug_dev, AcpiPciHpState *s, diff --git a/include/hw/arm/aspeed.h b/include/hw/arm/aspeed.h index 9cae45a1c9..973277bea6 100644 --- a/include/hw/arm/aspeed.h +++ b/include/hw/arm/aspeed.h @@ -40,6 +40,7 @@ struct AspeedMachineClass { void (*i2c_init)(AspeedMachineState *bmc); uint32_t uart_default; bool sdhci_wp_inverted; + bool vbootrom; }; diff --git a/include/hw/arm/aspeed_soc.h b/include/hw/arm/aspeed_soc.h index f069d17d16..217ef0eafd 100644 --- a/include/hw/arm/aspeed_soc.h +++ b/include/hw/arm/aspeed_soc.h @@ -43,7 +43,7 @@ #include "hw/intc/arm_gicv3.h" #define ASPEED_SPIS_NUM 3 -#define ASPEED_EHCIS_NUM 2 +#define ASPEED_EHCIS_NUM 4 #define ASPEED_WDTS_NUM 8 #define ASPEED_CPUS_NUM 4 #define ASPEED_MACS_NUM 4 @@ -59,6 +59,7 @@ struct AspeedSoCState { MemoryRegion sram; MemoryRegion spi_boot_container; MemoryRegion spi_boot; + MemoryRegion vbootrom; AddressSpace dram_as; AspeedRtcState rtc; AspeedTimerCtrlState timerctrl; @@ -90,6 +91,8 @@ struct AspeedSoCState { SerialMM uart[ASPEED_UARTS_NUM]; Clock *sysclk; UnimplementedDeviceState iomem; + UnimplementedDeviceState iomem0; + UnimplementedDeviceState iomem1; UnimplementedDeviceState video; UnimplementedDeviceState emmc_boot_controller; UnimplementedDeviceState dpmcu; @@ -97,6 +100,7 @@ struct AspeedSoCState { UnimplementedDeviceState espi; UnimplementedDeviceState udc; UnimplementedDeviceState sgpiom; + UnimplementedDeviceState ltpi; UnimplementedDeviceState jtag[ASPEED_JTAG_NUM]; AspeedAPB2OPBState fsi[2]; }; @@ -142,6 +146,30 @@ struct Aspeed10x0SoCState { ARMv7MState armv7m; }; +struct Aspeed27x0SSPSoCState { + AspeedSoCState parent; + AspeedINTCState intc[2]; + UnimplementedDeviceState ipc[2]; + UnimplementedDeviceState scuio; + + ARMv7MState armv7m; +}; + +#define TYPE_ASPEED27X0SSP_SOC "aspeed27x0ssp-soc" +OBJECT_DECLARE_SIMPLE_TYPE(Aspeed27x0SSPSoCState, ASPEED27X0SSP_SOC) + +struct Aspeed27x0TSPSoCState { + AspeedSoCState parent; + AspeedINTCState intc[2]; + UnimplementedDeviceState ipc[2]; + UnimplementedDeviceState scuio; + + ARMv7MState armv7m; +}; + +#define TYPE_ASPEED27X0TSP_SOC "aspeed27x0tsp-soc" +OBJECT_DECLARE_SIMPLE_TYPE(Aspeed27x0TSPSoCState, ASPEED27X0TSP_SOC) + #define TYPE_ASPEED10X0_SOC "aspeed10x0-soc" OBJECT_DECLARE_SIMPLE_TYPE(Aspeed10x0SoCState, ASPEED10X0_SOC) @@ -169,8 +197,12 @@ struct AspeedSoCClass { const char *aspeed_soc_cpu_type(AspeedSoCClass *sc); enum { + ASPEED_DEV_VBOOTROM, ASPEED_DEV_SPI_BOOT, ASPEED_DEV_IOMEM, + ASPEED_DEV_IOMEM0, + ASPEED_DEV_IOMEM1, + ASPEED_DEV_LTPI, ASPEED_DEV_UART0, ASPEED_DEV_UART1, ASPEED_DEV_UART2, @@ -192,6 +224,8 @@ enum { ASPEED_DEV_SPI2, ASPEED_DEV_EHCI1, ASPEED_DEV_EHCI2, + ASPEED_DEV_EHCI3, + ASPEED_DEV_EHCI4, ASPEED_DEV_VIC, ASPEED_DEV_INTC, ASPEED_DEV_INTCIO, @@ -249,6 +283,8 @@ enum { ASPEED_DEV_SLIIO, ASPEED_GIC_DIST, ASPEED_GIC_REDIST, + ASPEED_DEV_IPC0, + ASPEED_DEV_IPC1, }; qemu_irq aspeed_soc_get_irq(AspeedSoCState *s, int dev); diff --git a/include/hw/arm/npcm8xx.h b/include/hw/arm/npcm8xx.h index 9812e6fa7e..3436abff99 100644 --- a/include/hw/arm/npcm8xx.h +++ b/include/hw/arm/npcm8xx.h @@ -36,6 +36,7 @@ #include "hw/usb/hcd-ehci.h" #include "hw/usb/hcd-ohci.h" #include "target/arm/cpu.h" +#include "hw/ssi/npcm_pspi.h" #define NPCM8XX_MAX_NUM_CPUS (4) @@ -99,6 +100,7 @@ struct NPCM8xxState { OHCISysBusState ohci[2]; NPCM7xxFIUState fiu[3]; NPCM7xxSDHCIState mmc; + NPCMPSPIState pspi; }; struct NPCM8xxClass { diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h index c8e94e6aed..9a1b0f53d2 100644 --- a/include/hw/arm/virt.h +++ b/include/hw/arm/virt.h @@ -117,14 +117,8 @@ typedef enum VirtGICType { struct VirtMachineClass { MachineClass parent; - bool disallow_affinity_adjustment; - bool no_its; bool no_tcg_its; - bool no_pmu; - bool claim_edge_triggered_timers; - bool smbios_old_sys_ver; bool no_highmem_compact; - bool no_highmem_ecam; bool no_ged; /* Machines < 4.2 have no support for ACPI GED device */ bool kvm_no_adjvtime; bool no_kvm_steal_time; diff --git a/include/hw/boards.h b/include/hw/boards.h index 765dc8dd35..a7b1fcffae 100644 --- a/include/hw/boards.h +++ b/include/hw/boards.h @@ -636,7 +636,11 @@ struct MachineState { /* * How many years/major releases for each phase * of the life cycle. Assumes use of versioning - * scheme where major is bumped each year + * scheme where major is bumped each year. + * + * These values must match the ver_machine_deprecation_version + * and ver_machine_deletion_version logic in docs/conf.py and + * the text in docs/about/deprecated.rst */ #define MACHINE_VER_DELETION_MAJOR 6 #define MACHINE_VER_DEPRECATION_MAJOR 3 @@ -650,11 +654,42 @@ struct MachineState { " years old are subject to deletion after " \ stringify(MACHINE_VER_DELETION_MAJOR) " years" -#define _MACHINE_VER_IS_EXPIRED_IMPL(cutoff, major, minor) \ +#define _MACHINE_VER_IS_CURRENT_EXPIRED(cutoff, major, minor) \ (((QEMU_VERSION_MAJOR - major) > cutoff) || \ (((QEMU_VERSION_MAJOR - major) == cutoff) && \ (QEMU_VERSION_MINOR - minor) >= 0)) +#define _MACHINE_VER_IS_NEXT_MINOR_EXPIRED(cutoff, major, minor) \ + (((QEMU_VERSION_MAJOR - major) > cutoff) || \ + (((QEMU_VERSION_MAJOR - major) == cutoff) && \ + ((QEMU_VERSION_MINOR + 1) - minor) >= 0)) + +#define _MACHINE_VER_IS_NEXT_MAJOR_EXPIRED(cutoff, major, minor) \ + ((((QEMU_VERSION_MAJOR + 1) - major) > cutoff) || \ + ((((QEMU_VERSION_MAJOR + 1) - major) == cutoff) && \ + (0 - minor) >= 0)) + +/* + * - The first check applies to formal releases + * - The second check applies to dev snapshots / release candidates + * where the next major version is the same. + * e.g. 9.0.50, 9.1.50, 9.0.90, 9.1.90 + * - The third check applies to dev snapshots / release candidates + * where the next major version will change. + * e.g. 9.2.50, 9.2.90 + * + * NB: this assumes we do 3 minor releases per year, before bumping major, + * and dev snapshots / release candidates are numbered with micro >= 50 + * If this ever changes the logic below will need modifying.... + */ +#define _MACHINE_VER_IS_EXPIRED_IMPL(cutoff, major, minor) \ + ((QEMU_VERSION_MICRO < 50 && \ + _MACHINE_VER_IS_CURRENT_EXPIRED(cutoff, major, minor)) || \ + (QEMU_VERSION_MICRO >= 50 && QEMU_VERSION_MINOR < 2 && \ + _MACHINE_VER_IS_NEXT_MINOR_EXPIRED(cutoff, major, minor)) || \ + (QEMU_VERSION_MICRO >= 50 && QEMU_VERSION_MINOR == 2 && \ + _MACHINE_VER_IS_NEXT_MAJOR_EXPIRED(cutoff, major, minor))) + #define _MACHINE_VER_IS_EXPIRED2(cutoff, major, minor) \ _MACHINE_VER_IS_EXPIRED_IMPL(cutoff, major, minor) #define _MACHINE_VER_IS_EXPIRED3(cutoff, major, minor, micro) \ @@ -719,28 +754,11 @@ struct MachineState { * suitable period of time has passed, it will cause * execution of the method to return, avoiding registration * of the machine - * - * The new deprecation and deletion policy for versioned - * machine types was introduced in QEMU 9.1.0. - * - * Under the new policy a number of old machine types (any - * prior to 2.12) would be liable for immediate deletion - * which would be a violation of our historical deprecation - * and removal policy - * - * Thus deletions are temporarily gated on existance of - * the env variable "QEMU_DELETE_MACHINES" / QEMU version - * number >= 10.1.0. This gate can be deleted in the 10.1.0 - * dev cycle */ #define MACHINE_VER_DELETION(...) \ do { \ if (MACHINE_VER_SHOULD_DELETE(__VA_ARGS__)) { \ - if (getenv("QEMU_DELETE_MACHINES") || \ - QEMU_VERSION_MAJOR > 10 || (QEMU_VERSION_MAJOR == 10 && \ - QEMU_VERSION_MINOR >= 1)) { \ - return; \ - } \ + return; \ } \ } while (0) diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index 12b2ff1f7d..1e87f7d393 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -1121,20 +1121,8 @@ bool cpu_exec_realizefn(CPUState *cpu, Error **errp); void cpu_exec_unrealizefn(CPUState *cpu); void cpu_exec_reset_hold(CPUState *cpu); -#ifdef COMPILING_PER_TARGET - extern const VMStateDescription vmstate_cpu_common; -#define VMSTATE_CPU() { \ - .name = "parent_obj", \ - .size = sizeof(CPUState), \ - .vmsd = &vmstate_cpu_common, \ - .flags = VMS_STRUCT, \ - .offset = 0, \ -} - -#endif /* COMPILING_PER_TARGET */ - #define UNASSIGNED_CPU_INDEX -1 #define UNASSIGNED_CLUSTER_INDEX -1 diff --git a/include/hw/hyperv/hyperv.h b/include/hw/hyperv/hyperv.h index d717b4e13d..63a8b65278 100644 --- a/include/hw/hyperv/hyperv.h +++ b/include/hw/hyperv/hyperv.h @@ -10,7 +10,8 @@ #ifndef HW_HYPERV_HYPERV_H #define HW_HYPERV_HYPERV_H -#include "cpu-qom.h" +#include "exec/hwaddr.h" +#include "hw/core/cpu.h" #include "hw/hyperv/hyperv-proto.h" typedef struct HvSintRoute HvSintRoute; diff --git a/include/hw/intc/aspeed_intc.h b/include/hw/intc/aspeed_intc.h index 3727ba24be..51288384a5 100644 --- a/include/hw/intc/aspeed_intc.h +++ b/include/hw/intc/aspeed_intc.h @@ -15,6 +15,11 @@ #define TYPE_ASPEED_INTC "aspeed.intc" #define TYPE_ASPEED_2700_INTC TYPE_ASPEED_INTC "-ast2700" #define TYPE_ASPEED_2700_INTCIO TYPE_ASPEED_INTC "io-ast2700" +#define TYPE_ASPEED_2700SSP_INTC TYPE_ASPEED_INTC "-ast2700ssp" +#define TYPE_ASPEED_2700SSP_INTCIO TYPE_ASPEED_INTC "io-ast2700ssp" +#define TYPE_ASPEED_2700TSP_INTC TYPE_ASPEED_INTC "-ast2700tsp" +#define TYPE_ASPEED_2700TSP_INTCIO TYPE_ASPEED_INTC "io-ast2700tsp" + OBJECT_DECLARE_TYPE(AspeedINTCState, AspeedINTCClass, ASPEED_INTC) #define ASPEED_INTC_MAX_INPINS 10 diff --git a/include/hw/intc/loongarch_extioi.h b/include/hw/intc/loongarch_extioi.h index 351f18afcf..4a6ae903e9 100644 --- a/include/hw/intc/loongarch_extioi.h +++ b/include/hw/intc/loongarch_extioi.h @@ -22,6 +22,7 @@ struct LoongArchExtIOIClass { DeviceRealize parent_realize; DeviceUnrealize parent_unrealize; + ResettablePhases parent_phases; }; #endif /* LOONGARCH_EXTIOI_H */ diff --git a/include/hw/intc/loongarch_extioi_common.h b/include/hw/intc/loongarch_extioi_common.h index 22d7880977..735bfee80a 100644 --- a/include/hw/intc/loongarch_extioi_common.h +++ b/include/hw/intc/loongarch_extioi_common.h @@ -94,6 +94,7 @@ struct LoongArchExtIOICommonClass { SysBusDeviceClass parent_class; DeviceRealize parent_realize; + ResettablePhases parent_phases; int (*pre_save)(void *s); int (*post_load)(void *s, int version_id); }; diff --git a/include/hw/intc/loongarch_ipi.h b/include/hw/intc/loongarch_ipi.h index 923bf21ecb..a7c6bf85d3 100644 --- a/include/hw/intc/loongarch_ipi.h +++ b/include/hw/intc/loongarch_ipi.h @@ -21,6 +21,7 @@ struct LoongarchIPIState { struct LoongarchIPIClass { LoongsonIPICommonClass parent_class; DeviceRealize parent_realize; + ResettablePhases parent_phases; }; #endif diff --git a/include/hw/intc/loongarch_pch_pic.h b/include/hw/intc/loongarch_pch_pic.h index 481cc58aed..839a59a43b 100644 --- a/include/hw/intc/loongarch_pch_pic.h +++ b/include/hw/intc/loongarch_pch_pic.h @@ -22,6 +22,7 @@ struct LoongarchPICClass { LoongArchPICCommonClass parent_class; DeviceRealize parent_realize; + ResettablePhases parent_phases; }; #endif /* HW_LOONGARCH_PCH_PIC_H */ diff --git a/include/hw/intc/loongarch_pic_common.h b/include/hw/intc/loongarch_pic_common.h index 43cce48978..d301377cd7 100644 --- a/include/hw/intc/loongarch_pic_common.h +++ b/include/hw/intc/loongarch_pic_common.h @@ -76,6 +76,7 @@ struct LoongArchPICCommonClass { SysBusDeviceClass parent_class; DeviceRealize parent_realize; + ResettablePhases parent_phases; int (*pre_save)(LoongArchPICCommonState *s); int (*post_load)(LoongArchPICCommonState *s, int version_id); }; diff --git a/include/hw/vfio/vfio-container-base.h b/include/hw/vfio/vfio-container-base.h index 5527e02722..3d392b0fd8 100644 --- a/include/hw/vfio/vfio-container-base.h +++ b/include/hw/vfio/vfio-container-base.h @@ -81,7 +81,7 @@ int vfio_container_dma_map(VFIOContainerBase *bcontainer, void *vaddr, bool readonly); int vfio_container_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova, ram_addr_t size, - IOMMUTLBEntry *iotlb); + IOMMUTLBEntry *iotlb, bool unmap_all); bool vfio_container_add_section_window(VFIOContainerBase *bcontainer, MemoryRegionSection *section, Error **errp); @@ -117,12 +117,25 @@ struct VFIOIOMMUClass { /* basic feature */ bool (*setup)(VFIOContainerBase *bcontainer, Error **errp); + void (*listener_begin)(VFIOContainerBase *bcontainer); + void (*listener_commit)(VFIOContainerBase *bcontainer); int (*dma_map)(const VFIOContainerBase *bcontainer, hwaddr iova, ram_addr_t size, void *vaddr, bool readonly); + /** + * @dma_unmap + * + * Unmap an address range from the container. + * + * @bcontainer: #VFIOContainerBase to use for unmap + * @iova: start address to unmap + * @size: size of the range to unmap + * @iotlb: The IOMMU TLB mapping entry (or NULL) + * @unmap_all: if set, unmap the entire address space + */ int (*dma_unmap)(const VFIOContainerBase *bcontainer, hwaddr iova, ram_addr_t size, - IOMMUTLBEntry *iotlb); + IOMMUTLBEntry *iotlb, bool unmap_all); bool (*attach_device)(const char *name, VFIODevice *vbasedev, AddressSpace *as, Error **errp); void (*detach_device)(VFIODevice *vbasedev); diff --git a/include/hw/vfio/vfio-device.h b/include/hw/vfio/vfio-device.h index 81c95bb51e..8bcb3c19f6 100644 --- a/include/hw/vfio/vfio-device.h +++ b/include/hw/vfio/vfio-device.h @@ -41,6 +41,7 @@ enum { }; typedef struct VFIODeviceOps VFIODeviceOps; +typedef struct VFIODeviceIOOps VFIODeviceIOOps; typedef struct VFIOMigration VFIOMigration; typedef struct IOMMUFDBackend IOMMUFDBackend; @@ -66,6 +67,7 @@ typedef struct VFIODevice { OnOffAuto migration_multifd_transfer; bool migration_events; VFIODeviceOps *ops; + VFIODeviceIOOps *io_ops; unsigned int num_irqs; unsigned int num_regions; unsigned int flags; @@ -81,6 +83,7 @@ typedef struct VFIODevice { IOMMUFDBackend *iommufd; VFIOIOASHwpt *hwpt; QLIST_ENTRY(VFIODevice) hwpt_next; + struct vfio_region_info **reginfo; } VFIODevice; struct VFIODeviceOps { @@ -115,6 +118,20 @@ struct VFIODeviceOps { int (*vfio_load_config)(VFIODevice *vdev, QEMUFile *f); }; +/* + * Given a return value of either a short number of bytes read or -errno, + * construct a meaningful error message. + */ +#define strreaderror(ret) \ + (ret < 0 ? strerror(-ret) : "short read") + +/* + * Given a return value of either a short number of bytes written or -errno, + * construct a meaningful error message. + */ +#define strwriteerror(ret) \ + (ret < 0 ? strerror(-ret) : "short write") + void vfio_device_irq_disable(VFIODevice *vbasedev, int index); void vfio_device_irq_unmask(VFIODevice *vbasedev, int index); void vfio_device_irq_mask(VFIODevice *vbasedev, int index); @@ -127,6 +144,9 @@ bool vfio_device_hiod_create_and_realize(VFIODevice *vbasedev, const char *typename, Error **errp); bool vfio_device_attach(char *name, VFIODevice *vbasedev, AddressSpace *as, Error **errp); +bool vfio_device_attach_by_iommu_type(const char *iommu_type, char *name, + VFIODevice *vbasedev, AddressSpace *as, + Error **errp); void vfio_device_detach(VFIODevice *vbasedev); VFIODevice *vfio_get_vfio_device(Object *obj); @@ -134,11 +154,73 @@ typedef QLIST_HEAD(VFIODeviceList, VFIODevice) VFIODeviceList; extern VFIODeviceList vfio_device_list; #ifdef CONFIG_LINUX +/* + * How devices communicate with the server. The default option is through + * ioctl() to the kernel VFIO driver, but vfio-user can use a socket to a remote + * process. + */ +struct VFIODeviceIOOps { + /** + * @device_feature + * + * Fill in feature info for the given device. + */ + int (*device_feature)(VFIODevice *vdev, struct vfio_device_feature *); + + /** + * @get_region_info + * + * Fill in @info with information on the region given by @info->index. + */ + int (*get_region_info)(VFIODevice *vdev, + struct vfio_region_info *info); + + /** + * @get_irq_info + * + * Fill in @irq with information on the IRQ given by @info->index. + */ + int (*get_irq_info)(VFIODevice *vdev, struct vfio_irq_info *irq); + + /** + * @set_irqs + * + * Configure IRQs as defined by @irqs. + */ + int (*set_irqs)(VFIODevice *vdev, struct vfio_irq_set *irqs); + + /** + * @region_read + * + * Read @size bytes from the region @nr at offset @off into the buffer + * @data. + */ + int (*region_read)(VFIODevice *vdev, uint8_t nr, off_t off, uint32_t size, + void *data); + + /** + * @region_write + * + * Write @size bytes to the region @nr at offset @off from the buffer + * @data. + */ + int (*region_write)(VFIODevice *vdev, uint8_t nr, off_t off, uint32_t size, + void *data); +}; + +void vfio_device_prepare(VFIODevice *vbasedev, VFIOContainerBase *bcontainer, + struct vfio_device_info *info); + +void vfio_device_unprepare(VFIODevice *vbasedev); + int vfio_device_get_region_info(VFIODevice *vbasedev, int index, struct vfio_region_info **info); int vfio_device_get_region_info_type(VFIODevice *vbasedev, uint32_t type, uint32_t subtype, struct vfio_region_info **info); bool vfio_device_has_region_cap(VFIODevice *vbasedev, int region, uint16_t cap_type); + +int vfio_device_get_irq_info(VFIODevice *vbasedev, int index, + struct vfio_irq_info *info); #endif /* Returns 0 on success, or a negative errno. */ diff --git a/include/hw/xen/interface/io/blkif.h b/include/hw/xen/interface/io/blkif.h index 22f1eef0c0..c5527999d1 100644 --- a/include/hw/xen/interface/io/blkif.h +++ b/include/hw/xen/interface/io/blkif.h @@ -324,7 +324,7 @@ * access (even when it should be read-only). If the frontend hits the * maximum number of allowed persistently mapped grants, it can fallback * to non persistent mode. This will cause a performance degradation, - * since the the backend driver will still try to map those grants + * since the backend driver will still try to map those grants * persistently. Since the persistent grants protocol is compatible with * the previous protocol, a frontend driver can choose to work in * persistent mode even when the backend doesn't support it. diff --git a/include/migration/register.h b/include/migration/register.h index c041ce32f2..b79dc81b8d 100644 --- a/include/migration/register.h +++ b/include/migration/register.h @@ -190,6 +190,21 @@ typedef struct SaveVMHandlers { /* This runs outside the BQL! */ /** + * @save_postcopy_prepare + * + * This hook will be invoked on the source side right before switching + * to postcopy (before VM stopped). + * + * @f: QEMUFile where to send the data + * @opaque: Data pointer passed to register_savevm_live() + * @errp: Error** used to report error message + * + * Returns: true if succeeded, false if error occured. When false is + * returned, @errp must be set. + */ + bool (*save_postcopy_prepare)(QEMUFile *f, void *opaque, Error **errp); + + /** * @state_pending_estimate * * This estimates the remaining data to transfer diff --git a/include/qemu/cacheflush.h b/include/qemu/cacheflush.h index ae20bcda73..76eb55d818 100644 --- a/include/qemu/cacheflush.h +++ b/include/qemu/cacheflush.h @@ -26,6 +26,13 @@ static inline void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) /* icache is coherent and does not require flushing. */ } +#elif defined(EMSCRIPTEN) + +static inline void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) +{ + /* Wasm doesn't have executable region of memory. */ +} + #else void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len); diff --git a/include/qemu/host-pci-mmio.h b/include/qemu/host-pci-mmio.h new file mode 100644 index 0000000000..a8ed9938ac --- /dev/null +++ b/include/qemu/host-pci-mmio.h @@ -0,0 +1,136 @@ +/* + * API for host PCI MMIO accesses (e.g. Linux VFIO BARs) + * + * Copyright 2025 IBM Corp. + * Author(s): Farhan Ali <alifm@linux.ibm.com> + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef HOST_PCI_MMIO_H +#define HOST_PCI_MMIO_H + +#include "qemu/bswap.h" +#include "qemu/s390x_pci_mmio.h" + +static inline uint8_t host_pci_ldub_p(const void *ioaddr) +{ + uint8_t ret = 0; +#ifdef __s390x__ + ret = s390x_pci_mmio_read_8(ioaddr); +#else + ret = ldub_p(ioaddr); +#endif + + return ret; +} + +static inline uint16_t host_pci_lduw_le_p(const void *ioaddr) +{ + uint16_t ret = 0; +#ifdef __s390x__ + ret = le16_to_cpu(s390x_pci_mmio_read_16(ioaddr)); +#else + ret = lduw_le_p(ioaddr); +#endif + + return ret; +} + +static inline uint32_t host_pci_ldl_le_p(const void *ioaddr) +{ + uint32_t ret = 0; +#ifdef __s390x__ + ret = le32_to_cpu(s390x_pci_mmio_read_32(ioaddr)); +#else + ret = ldl_le_p(ioaddr); +#endif + + return ret; +} + +static inline uint64_t host_pci_ldq_le_p(const void *ioaddr) +{ + uint64_t ret = 0; +#ifdef __s390x__ + ret = le64_to_cpu(s390x_pci_mmio_read_64(ioaddr)); +#else + ret = ldq_le_p(ioaddr); +#endif + + return ret; +} + +static inline void host_pci_stb_p(void *ioaddr, uint8_t val) +{ +#ifdef __s390x__ + s390x_pci_mmio_write_8(ioaddr, val); +#else + stb_p(ioaddr, val); +#endif +} + +static inline void host_pci_stw_le_p(void *ioaddr, uint16_t val) +{ +#ifdef __s390x__ + s390x_pci_mmio_write_16(ioaddr, cpu_to_le16(val)); +#else + stw_le_p(ioaddr, val); +#endif +} + +static inline void host_pci_stl_le_p(void *ioaddr, uint32_t val) +{ +#ifdef __s390x__ + s390x_pci_mmio_write_32(ioaddr, cpu_to_le32(val)); +#else + stl_le_p(ioaddr, val); +#endif +} + +static inline void host_pci_stq_le_p(void *ioaddr, uint64_t val) +{ +#ifdef __s390x__ + s390x_pci_mmio_write_64(ioaddr, cpu_to_le64(val)); +#else + stq_le_p(ioaddr, val); +#endif +} + +static inline uint64_t host_pci_ldn_le_p(const void *ioaddr, int sz) +{ + switch (sz) { + case 1: + return host_pci_ldub_p(ioaddr); + case 2: + return host_pci_lduw_le_p(ioaddr); + case 4: + return host_pci_ldl_le_p(ioaddr); + case 8: + return host_pci_ldq_le_p(ioaddr); + default: + g_assert_not_reached(); + } +} + +static inline void host_pci_stn_le_p(void *ioaddr, int sz, uint64_t v) +{ + switch (sz) { + case 1: + host_pci_stb_p(ioaddr, v); + break; + case 2: + host_pci_stw_le_p(ioaddr, v); + break; + case 4: + host_pci_stl_le_p(ioaddr, v); + break; + case 8: + host_pci_stq_le_p(ioaddr, v); + break; + default: + g_assert_not_reached(); + } +} + +#endif diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h index 4397a90680..96fe51bc39 100644 --- a/include/qemu/osdep.h +++ b/include/qemu/osdep.h @@ -8,7 +8,7 @@ * To avoid getting into possible circular include dependencies, this * file should not include any other QEMU headers, with the exceptions * of config-host.h, config-target.h, qemu/compiler.h, - * system/os-posix.h, system/os-win32.h, glib-compat.h and + * system/os-posix.h, system/os-win32.h, system/os-wasm.h, glib-compat.h and * qemu/typedefs.h, all of which are doing a similar job to this file * and are under similar constraints. * @@ -164,10 +164,14 @@ QEMU_EXTERN_C int daemon(int, int); #include "system/os-win32.h" #endif -#ifdef CONFIG_POSIX +#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) #include "system/os-posix.h" #endif +#if defined(EMSCRIPTEN) +#include "system/os-wasm.h" +#endif + #ifdef __cplusplus extern "C" { #endif diff --git a/include/qemu/s390x_pci_mmio.h b/include/qemu/s390x_pci_mmio.h new file mode 100644 index 0000000000..c5f63ecefa --- /dev/null +++ b/include/qemu/s390x_pci_mmio.h @@ -0,0 +1,24 @@ +/* + * s390x PCI MMIO definitions + * + * Copyright 2025 IBM Corp. + * Author(s): Farhan Ali <alifm@linux.ibm.com> + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ +#ifndef S390X_PCI_MMIO_H +#define S390X_PCI_MMIO_H + +#ifdef __s390x__ +uint8_t s390x_pci_mmio_read_8(const void *ioaddr); +uint16_t s390x_pci_mmio_read_16(const void *ioaddr); +uint32_t s390x_pci_mmio_read_32(const void *ioaddr); +uint64_t s390x_pci_mmio_read_64(const void *ioaddr); + +void s390x_pci_mmio_write_8(void *ioaddr, uint8_t val); +void s390x_pci_mmio_write_16(void *ioaddr, uint16_t val); +void s390x_pci_mmio_write_32(void *ioaddr, uint32_t val); +void s390x_pci_mmio_write_64(void *ioaddr, uint64_t val); +#endif /* __s390x__ */ + +#endif /* S390X_PCI_MMIO_H */ diff --git a/include/qemu/target-info-impl.h b/include/qemu/target-info-impl.h index d30805f7f2..1b51cbcfe1 100644 --- a/include/qemu/target-info-impl.h +++ b/include/qemu/target-info-impl.h @@ -14,6 +14,12 @@ typedef struct TargetInfo { /* runtime equivalent of TARGET_NAME definition */ const char *target_name; + /* runtime equivalent of TARGET_LONG_BITS definition */ + unsigned long_bits; + /* runtime equivalent of CPU_RESOLVING_TYPE definition */ + const char *cpu_type; + /* QOM typename machines for this binary must implement */ + const char *machine_typename; } TargetInfo; /** diff --git a/include/qemu/target-info.h b/include/qemu/target-info.h index 58d4136897..850a2958b9 100644 --- a/include/qemu/target-info.h +++ b/include/qemu/target-info.h @@ -17,6 +17,21 @@ const char *target_name(void); /** + * target_long_bits: + * + * Returns: number of bits in a long type for this target (i.e. 64). + */ +unsigned target_long_bits(void); + +/** + * target_machine_typename: + * + * Returns: Name of the QOM interface implemented by machines + * usable on this target binary. + */ +const char *target_machine_typename(void); + +/** * target_cpu_type: * * Returns: target CPU base QOM type name (i.e. TYPE_X86_CPU). diff --git a/include/standard-headers/asm-x86/setup_data.h b/include/standard-headers/asm-x86/setup_data.h index 09355f54c5..a483d72f42 100644 --- a/include/standard-headers/asm-x86/setup_data.h +++ b/include/standard-headers/asm-x86/setup_data.h @@ -18,7 +18,7 @@ #define SETUP_INDIRECT (1<<31) #define SETUP_TYPE_MAX (SETUP_ENUM_MAX | SETUP_INDIRECT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include "standard-headers/linux/types.h" @@ -78,6 +78,6 @@ struct ima_setup_data { uint64_t size; } QEMU_PACKED; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_SETUP_DATA_H */ diff --git a/include/standard-headers/drm/drm_fourcc.h b/include/standard-headers/drm/drm_fourcc.h index 708647776f..a8b759dcbc 100644 --- a/include/standard-headers/drm/drm_fourcc.h +++ b/include/standard-headers/drm/drm_fourcc.h @@ -420,6 +420,7 @@ extern "C" { #define DRM_FORMAT_MOD_VENDOR_ARM 0x08 #define DRM_FORMAT_MOD_VENDOR_ALLWINNER 0x09 #define DRM_FORMAT_MOD_VENDOR_AMLOGIC 0x0a +#define DRM_FORMAT_MOD_VENDOR_MTK 0x0b /* add more to the end as needed */ @@ -1452,6 +1453,46 @@ drm_fourcc_canonicalize_nvidia_format_mod(uint64_t modifier) */ #define AMLOGIC_FBC_OPTION_MEM_SAVING (1ULL << 0) +/* MediaTek modifiers + * Bits Parameter Notes + * ----- ------------------------ --------------------------------------------- + * 7: 0 TILE LAYOUT Values are MTK_FMT_MOD_TILE_* + * 15: 8 COMPRESSION Values are MTK_FMT_MOD_COMPRESS_* + * 23:16 10 BIT LAYOUT Values are MTK_FMT_MOD_10BIT_LAYOUT_* + * + */ + +#define DRM_FORMAT_MOD_MTK(__flags) fourcc_mod_code(MTK, __flags) + +/* + * MediaTek Tiled Modifier + * The lowest 8 bits of the modifier is used to specify the tiling + * layout. Only the 16L_32S tiling is used for now, but we define an + * "untiled" version and leave room for future expansion. + */ +#define MTK_FMT_MOD_TILE_MASK 0xf +#define MTK_FMT_MOD_TILE_NONE 0x0 +#define MTK_FMT_MOD_TILE_16L32S 0x1 + +/* + * Bits 8-15 specify compression options + */ +#define MTK_FMT_MOD_COMPRESS_MASK (0xf << 8) +#define MTK_FMT_MOD_COMPRESS_NONE (0x0 << 8) +#define MTK_FMT_MOD_COMPRESS_V1 (0x1 << 8) + +/* + * Bits 16-23 specify how the bits of 10 bit formats are + * stored out in memory + */ +#define MTK_FMT_MOD_10BIT_LAYOUT_MASK (0xf << 16) +#define MTK_FMT_MOD_10BIT_LAYOUT_PACKED (0x0 << 16) +#define MTK_FMT_MOD_10BIT_LAYOUT_LSBTILED (0x1 << 16) +#define MTK_FMT_MOD_10BIT_LAYOUT_LSBRASTER (0x2 << 16) + +/* alias for the most common tiling format */ +#define DRM_FORMAT_MOD_MTK_16L_32S_TILE DRM_FORMAT_MOD_MTK(MTK_FMT_MOD_TILE_16L32S) + /* * AMD modifiers * diff --git a/include/standard-headers/linux/const.h b/include/standard-headers/linux/const.h index 2122610de7..95ede23342 100644 --- a/include/standard-headers/linux/const.h +++ b/include/standard-headers/linux/const.h @@ -33,7 +33,7 @@ * Missing __asm__ support * * __BIT128() would not work in the __asm__ code, as it shifts an - * 'unsigned __init128' data type as direct representation of + * 'unsigned __int128' data type as direct representation of * 128 bit constants is not supported in the gcc compiler, as * they get silently truncated. * diff --git a/include/standard-headers/linux/ethtool.h b/include/standard-headers/linux/ethtool.h index e83382531c..5d1ad5fdea 100644 --- a/include/standard-headers/linux/ethtool.h +++ b/include/standard-headers/linux/ethtool.h @@ -2059,6 +2059,24 @@ enum ethtool_link_mode_bit_indices { ETHTOOL_LINK_MODE_10baseT1S_Half_BIT = 100, ETHTOOL_LINK_MODE_10baseT1S_P2MP_Half_BIT = 101, ETHTOOL_LINK_MODE_10baseT1BRR_Full_BIT = 102, + ETHTOOL_LINK_MODE_200000baseCR_Full_BIT = 103, + ETHTOOL_LINK_MODE_200000baseKR_Full_BIT = 104, + ETHTOOL_LINK_MODE_200000baseDR_Full_BIT = 105, + ETHTOOL_LINK_MODE_200000baseDR_2_Full_BIT = 106, + ETHTOOL_LINK_MODE_200000baseSR_Full_BIT = 107, + ETHTOOL_LINK_MODE_200000baseVR_Full_BIT = 108, + ETHTOOL_LINK_MODE_400000baseCR2_Full_BIT = 109, + ETHTOOL_LINK_MODE_400000baseKR2_Full_BIT = 110, + ETHTOOL_LINK_MODE_400000baseDR2_Full_BIT = 111, + ETHTOOL_LINK_MODE_400000baseDR2_2_Full_BIT = 112, + ETHTOOL_LINK_MODE_400000baseSR2_Full_BIT = 113, + ETHTOOL_LINK_MODE_400000baseVR2_Full_BIT = 114, + ETHTOOL_LINK_MODE_800000baseCR4_Full_BIT = 115, + ETHTOOL_LINK_MODE_800000baseKR4_Full_BIT = 116, + ETHTOOL_LINK_MODE_800000baseDR4_Full_BIT = 117, + ETHTOOL_LINK_MODE_800000baseDR4_2_Full_BIT = 118, + ETHTOOL_LINK_MODE_800000baseSR4_Full_BIT = 119, + ETHTOOL_LINK_MODE_800000baseVR4_Full_BIT = 120, /* must be last entry */ __ETHTOOL_LINK_MODE_MASK_NBITS @@ -2271,6 +2289,10 @@ static inline int ethtool_validate_duplex(uint8_t duplex) * be exploited to reduce the RSS queue spread. */ #define RXH_XFRM_SYM_XOR (1 << 0) +/* Similar to SYM_XOR, except that one copy of the XOR'ed fields is replaced by + * an OR of the same fields + */ +#define RXH_XFRM_SYM_OR_XOR (1 << 1) #define RXH_XFRM_NO_CHANGE 0xff /* L2-L4 network traffic flow types */ diff --git a/include/standard-headers/linux/fuse.h b/include/standard-headers/linux/fuse.h index d303effb2a..a2b5815d89 100644 --- a/include/standard-headers/linux/fuse.h +++ b/include/standard-headers/linux/fuse.h @@ -229,6 +229,9 @@ * - FUSE_URING_IN_OUT_HEADER_SZ * - FUSE_URING_OP_IN_OUT_SZ * - enum fuse_uring_cmd + * + * 7.43 + * - add FUSE_REQUEST_TIMEOUT */ #ifndef _LINUX_FUSE_H @@ -260,7 +263,7 @@ #define FUSE_KERNEL_VERSION 7 /** Minor version number of this interface */ -#define FUSE_KERNEL_MINOR_VERSION 42 +#define FUSE_KERNEL_MINOR_VERSION 43 /** The node ID of the root inode */ #define FUSE_ROOT_ID 1 @@ -431,6 +434,8 @@ struct fuse_file_lock { * of the request ID indicates resend requests * FUSE_ALLOW_IDMAP: allow creation of idmapped mounts * FUSE_OVER_IO_URING: Indicate that client supports io-uring + * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests. + * init_out.request_timeout contains the timeout (in secs) */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -473,11 +478,11 @@ struct fuse_file_lock { #define FUSE_PASSTHROUGH (1ULL << 37) #define FUSE_NO_EXPORT_SUPPORT (1ULL << 38) #define FUSE_HAS_RESEND (1ULL << 39) - /* Obsolete alias for FUSE_DIRECT_IO_ALLOW_MMAP */ #define FUSE_DIRECT_IO_RELAX FUSE_DIRECT_IO_ALLOW_MMAP #define FUSE_ALLOW_IDMAP (1ULL << 40) #define FUSE_OVER_IO_URING (1ULL << 41) +#define FUSE_REQUEST_TIMEOUT (1ULL << 42) /** * CUSE INIT request/reply flags @@ -905,7 +910,8 @@ struct fuse_init_out { uint16_t map_alignment; uint32_t flags2; uint32_t max_stack_depth; - uint32_t unused[6]; + uint16_t request_timeout; + uint16_t unused[11]; }; #define CUSE_INIT_INFO_MAX 4096 diff --git a/include/standard-headers/linux/pci_regs.h b/include/standard-headers/linux/pci_regs.h index 3445c4970e..ba326710f9 100644 --- a/include/standard-headers/linux/pci_regs.h +++ b/include/standard-headers/linux/pci_regs.h @@ -486,6 +486,7 @@ #define PCI_EXP_TYPE_RC_EC 0xa /* Root Complex Event Collector */ #define PCI_EXP_FLAGS_SLOT 0x0100 /* Slot implemented */ #define PCI_EXP_FLAGS_IRQ 0x3e00 /* Interrupt message number */ +#define PCI_EXP_FLAGS_FLIT 0x8000 /* Flit Mode Supported */ #define PCI_EXP_DEVCAP 0x04 /* Device capabilities */ #define PCI_EXP_DEVCAP_PAYLOAD 0x00000007 /* Max_Payload_Size */ #define PCI_EXP_DEVCAP_PHANTOM 0x00000018 /* Phantom functions */ @@ -795,6 +796,8 @@ #define PCI_ERR_CAP_ECRC_CHKC 0x00000080 /* ECRC Check Capable */ #define PCI_ERR_CAP_ECRC_CHKE 0x00000100 /* ECRC Check Enable */ #define PCI_ERR_CAP_PREFIX_LOG_PRESENT 0x00000800 /* TLP Prefix Log Present */ +#define PCI_ERR_CAP_TLP_LOG_FLIT 0x00040000 /* TLP was logged in Flit Mode */ +#define PCI_ERR_CAP_TLP_LOG_SIZE 0x00f80000 /* Logged TLP Size (only in Flit mode) */ #define PCI_ERR_HEADER_LOG 0x1c /* Header Log Register (16 bytes) */ #define PCI_ERR_ROOT_COMMAND 0x2c /* Root Error Command */ #define PCI_ERR_ROOT_CMD_COR_EN 0x00000001 /* Correctable Err Reporting Enable */ @@ -1013,7 +1016,7 @@ /* Resizable BARs */ #define PCI_REBAR_CAP 4 /* capability register */ -#define PCI_REBAR_CAP_SIZES 0x00FFFFF0 /* supported BAR sizes */ +#define PCI_REBAR_CAP_SIZES 0xFFFFFFF0 /* supported BAR sizes */ #define PCI_REBAR_CTRL 8 /* control register */ #define PCI_REBAR_CTRL_BAR_IDX 0x00000007 /* BAR index */ #define PCI_REBAR_CTRL_NBAR_MASK 0x000000E0 /* # of resizable BARs */ @@ -1061,8 +1064,9 @@ #define PCI_EXP_DPC_CAP_RP_EXT 0x0020 /* Root Port Extensions */ #define PCI_EXP_DPC_CAP_POISONED_TLP 0x0040 /* Poisoned TLP Egress Blocking Supported */ #define PCI_EXP_DPC_CAP_SW_TRIGGER 0x0080 /* Software Triggering Supported */ -#define PCI_EXP_DPC_RP_PIO_LOG_SIZE 0x0F00 /* RP PIO Log Size */ +#define PCI_EXP_DPC_RP_PIO_LOG_SIZE 0x0F00 /* RP PIO Log Size [3:0] */ #define PCI_EXP_DPC_CAP_DL_ACTIVE 0x1000 /* ERR_COR signal on DL_Active supported */ +#define PCI_EXP_DPC_RP_PIO_LOG_SIZE4 0x2000 /* RP PIO Log Size [4] */ #define PCI_EXP_DPC_CTL 0x06 /* DPC control */ #define PCI_EXP_DPC_CTL_EN_FATAL 0x0001 /* Enable trigger on ERR_FATAL message */ @@ -1205,9 +1209,12 @@ #define PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX 0x000000ff #define PCI_DOE_DATA_OBJECT_DISC_REQ_3_VER 0x0000ff00 #define PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID 0x0000ffff -#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL 0x00ff0000 +#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_TYPE 0x00ff0000 #define PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX 0xff000000 +/* Deprecated old name, replaced with PCI_DOE_DATA_OBJECT_DISC_RSP_3_TYPE */ +#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL PCI_DOE_DATA_OBJECT_DISC_RSP_3_TYPE + /* Compute Express Link (CXL r3.1, sec 8.1.5) */ #define PCI_DVSEC_CXL_PORT 3 #define PCI_DVSEC_CXL_PORT_CTL 0x0c diff --git a/include/standard-headers/linux/virtio_net.h b/include/standard-headers/linux/virtio_net.h index fc594fe5fc..982e854f14 100644 --- a/include/standard-headers/linux/virtio_net.h +++ b/include/standard-headers/linux/virtio_net.h @@ -327,6 +327,19 @@ struct virtio_net_rss_config { uint8_t hash_key_data[/* hash_key_length */]; }; +struct virtio_net_rss_config_hdr { + uint32_t hash_types; + uint16_t indirection_table_mask; + uint16_t unclassified_queue; + uint16_t indirection_table[/* 1 + indirection_table_mask */]; +}; + +struct virtio_net_rss_config_trailer { + uint16_t max_tx_vq; + uint8_t hash_key_length; + uint8_t hash_key_data[/* hash_key_length */]; +}; + #define VIRTIO_NET_CTRL_MQ_RSS_CONFIG 1 /* diff --git a/include/standard-headers/linux/virtio_snd.h b/include/standard-headers/linux/virtio_snd.h index 860f12e0a4..160d57899f 100644 --- a/include/standard-headers/linux/virtio_snd.h +++ b/include/standard-headers/linux/virtio_snd.h @@ -25,7 +25,7 @@ struct virtio_snd_config { uint32_t streams; /* # of available channel maps */ uint32_t chmaps; - /* # of available control elements */ + /* # of available control elements (if VIRTIO_SND_F_CTLS) */ uint32_t controls; }; diff --git a/include/system/hvf.h b/include/system/hvf.h index 730f927f03..7b45a2e198 100644 --- a/include/system/hvf.h +++ b/include/system/hvf.h @@ -14,19 +14,24 @@ #define HVF_H #include "qemu/accel.h" +#include "qemu/queue.h" +#include "exec/vaddr.h" #include "qom/object.h" #ifdef COMPILING_PER_TARGET -#include "cpu.h" +# ifdef CONFIG_HVF +# define CONFIG_HVF_IS_POSSIBLE +# endif /* !CONFIG_HVF */ +#else +# define CONFIG_HVF_IS_POSSIBLE +#endif /* COMPILING_PER_TARGET */ -#ifdef CONFIG_HVF +#ifdef CONFIG_HVF_IS_POSSIBLE extern bool hvf_allowed; #define hvf_enabled() (hvf_allowed) -#else /* !CONFIG_HVF */ +#else /* !CONFIG_HVF_IS_POSSIBLE */ #define hvf_enabled() 0 -#endif /* !CONFIG_HVF */ - -#endif /* COMPILING_PER_TARGET */ +#endif /* !CONFIG_HVF_IS_POSSIBLE */ #define TYPE_HVF_ACCEL ACCEL_CLASS_NAME("hvf") diff --git a/include/system/hvf_int.h b/include/system/hvf_int.h index 42ae18433f..8c8b84012d 100644 --- a/include/system/hvf_int.h +++ b/include/system/hvf_int.h @@ -11,6 +11,8 @@ #ifndef HVF_INT_H #define HVF_INT_H +#include "qemu/queue.h" + #ifdef __aarch64__ #include <Hypervisor/Hypervisor.h> typedef hv_vcpu_t hvf_vcpuid; diff --git a/include/system/os-wasm.h b/include/system/os-wasm.h new file mode 100644 index 0000000000..3abb3aaa03 --- /dev/null +++ b/include/system/os-wasm.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: MIT */ +/* + * posix specific declarations forked from os-posix.h, removing functions not + * working on Emscripten + * + * Copyright (c) 2003-2008 Fabrice Bellard + * Copyright (c) 2010 Jes Sorensen <Jes.Sorensen@redhat.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef QEMU_OS_WASM_H +#define QEMU_OS_WASM_H + +#include <sys/mman.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <arpa/inet.h> +#include <netdb.h> +#include <sys/un.h> + +#ifdef CONFIG_SYSMACROS +#include <sys/sysmacros.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +void os_set_line_buffering(void); +void os_setup_early_signal_handling(void); +void os_set_proc_name(const char *s); +void os_setup_signal_handling(void); +void os_setup_limits(void); +void os_setup_post(void); +int os_mlock(bool on_fault); +static inline int os_set_daemonize(bool d) +{ + return -1; +}; +bool is_daemonized(void); +static inline void os_daemonize(void) {} + +/** + * qemu_alloc_stack: + * @sz: pointer to a size_t holding the requested usable stack size + * + * Allocate memory that can be used as a stack, for instance for + * coroutines. If the memory cannot be allocated, this function + * will abort (like g_malloc()). This function also inserts an + * additional guard page to catch a potential stack overflow. + * Note that the memory required for the guard page and alignment + * and minimal stack size restrictions will increase the value of sz. + * + * The allocated stack must be freed with qemu_free_stack(). + * + * Returns: pointer to (the lowest address of) the stack memory. + */ +void *qemu_alloc_stack(size_t *sz); + +/** + * qemu_free_stack: + * @stack: stack to free + * @sz: size of stack in bytes + * + * Free a stack allocated via qemu_alloc_stack(). Note that sz must + * be exactly the adjusted stack size returned by qemu_alloc_stack. + */ +void qemu_free_stack(void *stack, size_t sz); + +/* POSIX and Mingw32 differ in the name of the stdio lock functions. */ + +static inline void qemu_flockfile(FILE *f) +{ + flockfile(f); +} + +static inline void qemu_funlockfile(FILE *f) +{ + funlockfile(f); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index b4e4425acb..15a1b1a4fa 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -24,7 +24,6 @@ #include "exec/cputlb.h" #include "exec/ramlist.h" #include "system/ramblock.h" -#include "exec/exec-all.h" #include "system/memory.h" #include "exec/target_page.h" #include "qemu/rcu.h" diff --git a/include/tcg/insn-start-words.h b/include/tcg/insn-start-words.h index d416d19bcf..c52aec50a7 100644 --- a/include/tcg/insn-start-words.h +++ b/include/tcg/insn-start-words.h @@ -1,13 +1,12 @@ /* SPDX-License-Identifier: MIT */ /* - * Define TARGET_INSN_START_WORDS + * Define INSN_START_WORDS * Copyright (c) 2008 Fabrice Bellard */ -#ifndef TARGET_INSN_START_WORDS +#ifndef TCG_INSN_START_WORDS +#define TCG_INSN_START_WORDS -#include "cpu-param.h" +#define INSN_START_WORDS 3 -# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS) - -#endif /* TARGET_INSN_START_WORDS */ +#endif /* TCG_INSN_START_WORDS */ diff --git a/include/tcg/tcg-op-common.h b/include/tcg/tcg-op-common.h index 009e2778c5..b439bdb385 100644 --- a/include/tcg/tcg-op-common.h +++ b/include/tcg/tcg-op-common.h @@ -135,6 +135,8 @@ void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh); void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh); +void tcg_gen_addcio_i32(TCGv_i32 r, TCGv_i32 co, + TCGv_i32 a, TCGv_i32 b, TCGv_i32 ci); void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2); void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2); void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2); @@ -238,6 +240,8 @@ void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh); void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh); +void tcg_gen_addcio_i64(TCGv_i64 r, TCGv_i64 co, + TCGv_i64 a, TCGv_i64 b, TCGv_i64 ci); void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2); void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2); void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2); diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h index cded92a447..c912578fdd 100644 --- a/include/tcg/tcg-op.h +++ b/include/tcg/tcg-op.h @@ -9,6 +9,7 @@ #define TCG_TCG_OP_H #include "tcg/tcg-op-common.h" +#include "tcg/insn-start-words.h" #include "exec/target_long.h" #ifndef TARGET_LONG_BITS @@ -23,24 +24,34 @@ # error #endif +#if INSN_START_WORDS != 3 +# error Mismatch with insn-start-words.h +#endif + #if TARGET_INSN_START_EXTRA_WORDS == 0 static inline void tcg_gen_insn_start(target_ulong pc) { - TCGOp *op = tcg_emit_op(INDEX_op_insn_start, 64 / TCG_TARGET_REG_BITS); + TCGOp *op = tcg_emit_op(INDEX_op_insn_start, + INSN_START_WORDS * 64 / TCG_TARGET_REG_BITS); tcg_set_insn_start_param(op, 0, pc); + tcg_set_insn_start_param(op, 1, 0); + tcg_set_insn_start_param(op, 2, 0); } #elif TARGET_INSN_START_EXTRA_WORDS == 1 static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1) { - TCGOp *op = tcg_emit_op(INDEX_op_insn_start, 2 * 64 / TCG_TARGET_REG_BITS); + TCGOp *op = tcg_emit_op(INDEX_op_insn_start, + INSN_START_WORDS * 64 / TCG_TARGET_REG_BITS); tcg_set_insn_start_param(op, 0, pc); tcg_set_insn_start_param(op, 1, a1); + tcg_set_insn_start_param(op, 2, 0); } #elif TARGET_INSN_START_EXTRA_WORDS == 2 static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1, target_ulong a2) { - TCGOp *op = tcg_emit_op(INDEX_op_insn_start, 3 * 64 / TCG_TARGET_REG_BITS); + TCGOp *op = tcg_emit_op(INDEX_op_insn_start, + INSN_START_WORDS * 64 / TCG_TARGET_REG_BITS); tcg_set_insn_start_param(op, 0, pc); tcg_set_insn_start_param(op, 1, a1); tcg_set_insn_start_param(op, 2, a2); @@ -253,6 +264,7 @@ DEF_ATOMIC2(tcg_gen_atomic_umax_fetch, i64) #define tcg_gen_movcond_tl tcg_gen_movcond_i64 #define tcg_gen_add2_tl tcg_gen_add2_i64 #define tcg_gen_sub2_tl tcg_gen_sub2_i64 +#define tcg_gen_addcio_tl tcg_gen_addcio_i64 #define tcg_gen_mulu2_tl tcg_gen_mulu2_i64 #define tcg_gen_muls2_tl tcg_gen_muls2_i64 #define tcg_gen_mulsu2_tl tcg_gen_mulsu2_i64 @@ -371,6 +383,7 @@ DEF_ATOMIC2(tcg_gen_atomic_umax_fetch, i64) #define tcg_gen_movcond_tl tcg_gen_movcond_i32 #define tcg_gen_add2_tl tcg_gen_add2_i32 #define tcg_gen_sub2_tl tcg_gen_sub2_i32 +#define tcg_gen_addcio_tl tcg_gen_addcio_i32 #define tcg_gen_mulu2_tl tcg_gen_mulu2_i32 #define tcg_gen_muls2_tl tcg_gen_muls2_i32 #define tcg_gen_mulsu2_tl tcg_gen_mulsu2_i32 diff --git a/include/tcg/tcg-opc.h b/include/tcg/tcg-opc.h index 5bf78b0764..e988edd93a 100644 --- a/include/tcg/tcg-opc.h +++ b/include/tcg/tcg-opc.h @@ -34,152 +34,87 @@ DEF(set_label, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_NOT_PRESENT) DEF(call, 0, 0, 3, TCG_OPF_CALL_CLOBBER | TCG_OPF_NOT_PRESENT) DEF(br, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_NOT_PRESENT) +DEF(brcond, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_COND_BRANCH | TCG_OPF_INT) DEF(mb, 0, 0, 1, TCG_OPF_NOT_PRESENT) -DEF(mov_i32, 1, 1, 0, TCG_OPF_NOT_PRESENT) -DEF(setcond_i32, 1, 2, 1, 0) -DEF(negsetcond_i32, 1, 2, 1, 0) -DEF(movcond_i32, 1, 4, 1, 0) -/* load/store */ -DEF(ld8u_i32, 1, 1, 1, 0) -DEF(ld8s_i32, 1, 1, 1, 0) -DEF(ld16u_i32, 1, 1, 1, 0) -DEF(ld16s_i32, 1, 1, 1, 0) -DEF(ld_i32, 1, 1, 1, 0) -DEF(st8_i32, 0, 2, 1, 0) -DEF(st16_i32, 0, 2, 1, 0) -DEF(st_i32, 0, 2, 1, 0) -/* arith */ -DEF(add_i32, 1, 2, 0, 0) -DEF(sub_i32, 1, 2, 0, 0) -DEF(mul_i32, 1, 2, 0, 0) -DEF(div_i32, 1, 2, 0, 0) -DEF(divu_i32, 1, 2, 0, 0) -DEF(rem_i32, 1, 2, 0, 0) -DEF(remu_i32, 1, 2, 0, 0) -DEF(div2_i32, 2, 3, 0, 0) -DEF(divu2_i32, 2, 3, 0, 0) -DEF(and_i32, 1, 2, 0, 0) -DEF(or_i32, 1, 2, 0, 0) -DEF(xor_i32, 1, 2, 0, 0) -/* shifts/rotates */ -DEF(shl_i32, 1, 2, 0, 0) -DEF(shr_i32, 1, 2, 0, 0) -DEF(sar_i32, 1, 2, 0, 0) -DEF(rotl_i32, 1, 2, 0, 0) -DEF(rotr_i32, 1, 2, 0, 0) -DEF(deposit_i32, 1, 2, 2, 0) -DEF(extract_i32, 1, 1, 2, 0) -DEF(sextract_i32, 1, 1, 2, 0) -DEF(extract2_i32, 1, 2, 1, 0) +DEF(mov, 1, 1, 0, TCG_OPF_INT | TCG_OPF_NOT_PRESENT) + +DEF(add, 1, 2, 0, TCG_OPF_INT) +DEF(and, 1, 2, 0, TCG_OPF_INT) +DEF(andc, 1, 2, 0, TCG_OPF_INT) +DEF(bswap16, 1, 1, 1, TCG_OPF_INT) +DEF(bswap32, 1, 1, 1, TCG_OPF_INT) +DEF(bswap64, 1, 1, 1, TCG_OPF_INT) +DEF(clz, 1, 2, 0, TCG_OPF_INT) +DEF(ctpop, 1, 1, 0, TCG_OPF_INT) +DEF(ctz, 1, 2, 0, TCG_OPF_INT) +DEF(deposit, 1, 2, 2, TCG_OPF_INT) +DEF(divs, 1, 2, 0, TCG_OPF_INT) +DEF(divs2, 2, 3, 0, TCG_OPF_INT) +DEF(divu, 1, 2, 0, TCG_OPF_INT) +DEF(divu2, 2, 3, 0, TCG_OPF_INT) +DEF(eqv, 1, 2, 0, TCG_OPF_INT) +DEF(extract, 1, 1, 2, TCG_OPF_INT) +DEF(extract2, 1, 2, 1, TCG_OPF_INT) +DEF(ld8u, 1, 1, 1, TCG_OPF_INT) +DEF(ld8s, 1, 1, 1, TCG_OPF_INT) +DEF(ld16u, 1, 1, 1, TCG_OPF_INT) +DEF(ld16s, 1, 1, 1, TCG_OPF_INT) +DEF(ld32u, 1, 1, 1, TCG_OPF_INT) +DEF(ld32s, 1, 1, 1, TCG_OPF_INT) +DEF(ld, 1, 1, 1, TCG_OPF_INT) +DEF(movcond, 1, 4, 1, TCG_OPF_INT) +DEF(mul, 1, 2, 0, TCG_OPF_INT) +DEF(muls2, 2, 2, 0, TCG_OPF_INT) +DEF(mulsh, 1, 2, 0, TCG_OPF_INT) +DEF(mulu2, 2, 2, 0, TCG_OPF_INT) +DEF(muluh, 1, 2, 0, TCG_OPF_INT) +DEF(nand, 1, 2, 0, TCG_OPF_INT) +DEF(neg, 1, 1, 0, TCG_OPF_INT) +DEF(negsetcond, 1, 2, 1, TCG_OPF_INT) +DEF(nor, 1, 2, 0, TCG_OPF_INT) +DEF(not, 1, 1, 0, TCG_OPF_INT) +DEF(or, 1, 2, 0, TCG_OPF_INT) +DEF(orc, 1, 2, 0, TCG_OPF_INT) +DEF(rems, 1, 2, 0, TCG_OPF_INT) +DEF(remu, 1, 2, 0, TCG_OPF_INT) +DEF(rotl, 1, 2, 0, TCG_OPF_INT) +DEF(rotr, 1, 2, 0, TCG_OPF_INT) +DEF(sar, 1, 2, 0, TCG_OPF_INT) +DEF(setcond, 1, 2, 1, TCG_OPF_INT) +DEF(sextract, 1, 1, 2, TCG_OPF_INT) +DEF(shl, 1, 2, 0, TCG_OPF_INT) +DEF(shr, 1, 2, 0, TCG_OPF_INT) +DEF(st8, 0, 2, 1, TCG_OPF_INT) +DEF(st16, 0, 2, 1, TCG_OPF_INT) +DEF(st32, 0, 2, 1, TCG_OPF_INT) +DEF(st, 0, 2, 1, TCG_OPF_INT) +DEF(sub, 1, 2, 0, TCG_OPF_INT) +DEF(xor, 1, 2, 0, TCG_OPF_INT) + +DEF(addco, 1, 2, 0, TCG_OPF_INT | TCG_OPF_CARRY_OUT) +DEF(addc1o, 1, 2, 0, TCG_OPF_INT | TCG_OPF_CARRY_OUT) +DEF(addci, 1, 2, 0, TCG_OPF_INT | TCG_OPF_CARRY_IN) +DEF(addcio, 1, 2, 0, TCG_OPF_INT | TCG_OPF_CARRY_IN | TCG_OPF_CARRY_OUT) + +DEF(subbo, 1, 2, 0, TCG_OPF_INT | TCG_OPF_CARRY_OUT) +DEF(subb1o, 1, 2, 0, TCG_OPF_INT | TCG_OPF_CARRY_OUT) +DEF(subbi, 1, 2, 0, TCG_OPF_INT | TCG_OPF_CARRY_IN) +DEF(subbio, 1, 2, 0, TCG_OPF_INT | TCG_OPF_CARRY_IN | TCG_OPF_CARRY_OUT) -DEF(brcond_i32, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_COND_BRANCH) - -DEF(add2_i32, 2, 4, 0, 0) -DEF(sub2_i32, 2, 4, 0, 0) -DEF(mulu2_i32, 2, 2, 0, 0) -DEF(muls2_i32, 2, 2, 0, 0) -DEF(muluh_i32, 1, 2, 0, 0) -DEF(mulsh_i32, 1, 2, 0, 0) DEF(brcond2_i32, 0, 4, 2, TCG_OPF_BB_END | TCG_OPF_COND_BRANCH) DEF(setcond2_i32, 1, 4, 1, 0) -DEF(ext8s_i32, 1, 1, 0, 0) -DEF(ext16s_i32, 1, 1, 0, 0) -DEF(ext8u_i32, 1, 1, 0, 0) -DEF(ext16u_i32, 1, 1, 0, 0) -DEF(bswap16_i32, 1, 1, 1, 0) -DEF(bswap32_i32, 1, 1, 1, 0) -DEF(not_i32, 1, 1, 0, 0) -DEF(neg_i32, 1, 1, 0, 0) -DEF(andc_i32, 1, 2, 0, 0) -DEF(orc_i32, 1, 2, 0, 0) -DEF(eqv_i32, 1, 2, 0, 0) -DEF(nand_i32, 1, 2, 0, 0) -DEF(nor_i32, 1, 2, 0, 0) -DEF(clz_i32, 1, 2, 0, 0) -DEF(ctz_i32, 1, 2, 0, 0) -DEF(ctpop_i32, 1, 1, 0, 0) - -DEF(mov_i64, 1, 1, 0, TCG_OPF_NOT_PRESENT) -DEF(setcond_i64, 1, 2, 1, 0) -DEF(negsetcond_i64, 1, 2, 1, 0) -DEF(movcond_i64, 1, 4, 1, 0) -/* load/store */ -DEF(ld8u_i64, 1, 1, 1, 0) -DEF(ld8s_i64, 1, 1, 1, 0) -DEF(ld16u_i64, 1, 1, 1, 0) -DEF(ld16s_i64, 1, 1, 1, 0) -DEF(ld32u_i64, 1, 1, 1, 0) -DEF(ld32s_i64, 1, 1, 1, 0) -DEF(ld_i64, 1, 1, 1, 0) -DEF(st8_i64, 0, 2, 1, 0) -DEF(st16_i64, 0, 2, 1, 0) -DEF(st32_i64, 0, 2, 1, 0) -DEF(st_i64, 0, 2, 1, 0) -/* arith */ -DEF(add_i64, 1, 2, 0, 0) -DEF(sub_i64, 1, 2, 0, 0) -DEF(mul_i64, 1, 2, 0, 0) -DEF(div_i64, 1, 2, 0, 0) -DEF(divu_i64, 1, 2, 0, 0) -DEF(rem_i64, 1, 2, 0, 0) -DEF(remu_i64, 1, 2, 0, 0) -DEF(div2_i64, 2, 3, 0, 0) -DEF(divu2_i64, 2, 3, 0, 0) -DEF(and_i64, 1, 2, 0, 0) -DEF(or_i64, 1, 2, 0, 0) -DEF(xor_i64, 1, 2, 0, 0) -/* shifts/rotates */ -DEF(shl_i64, 1, 2, 0, 0) -DEF(shr_i64, 1, 2, 0, 0) -DEF(sar_i64, 1, 2, 0, 0) -DEF(rotl_i64, 1, 2, 0, 0) -DEF(rotr_i64, 1, 2, 0, 0) -DEF(deposit_i64, 1, 2, 2, 0) -DEF(extract_i64, 1, 1, 2, 0) -DEF(sextract_i64, 1, 1, 2, 0) -DEF(extract2_i64, 1, 2, 1, 0) - /* size changing ops */ DEF(ext_i32_i64, 1, 1, 0, 0) DEF(extu_i32_i64, 1, 1, 0, 0) DEF(extrl_i64_i32, 1, 1, 0, 0) DEF(extrh_i64_i32, 1, 1, 0, 0) -DEF(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_COND_BRANCH) -DEF(ext8s_i64, 1, 1, 0, 0) -DEF(ext16s_i64, 1, 1, 0, 0) -DEF(ext32s_i64, 1, 1, 0, 0) -DEF(ext8u_i64, 1, 1, 0, 0) -DEF(ext16u_i64, 1, 1, 0, 0) -DEF(ext32u_i64, 1, 1, 0, 0) -DEF(bswap16_i64, 1, 1, 1, 0) -DEF(bswap32_i64, 1, 1, 1, 0) -DEF(bswap64_i64, 1, 1, 1, 0) -DEF(not_i64, 1, 1, 0, 0) -DEF(neg_i64, 1, 1, 0, 0) -DEF(andc_i64, 1, 2, 0, 0) -DEF(orc_i64, 1, 2, 0, 0) -DEF(eqv_i64, 1, 2, 0, 0) -DEF(nand_i64, 1, 2, 0, 0) -DEF(nor_i64, 1, 2, 0, 0) -DEF(clz_i64, 1, 2, 0, 0) -DEF(ctz_i64, 1, 2, 0, 0) -DEF(ctpop_i64, 1, 1, 0, 0) - -DEF(add2_i64, 2, 4, 0, 0) -DEF(sub2_i64, 2, 4, 0, 0) -DEF(mulu2_i64, 2, 2, 0, 0) -DEF(muls2_i64, 2, 2, 0, 0) -DEF(muluh_i64, 1, 2, 0, 0) -DEF(mulsh_i64, 1, 2, 0, 0) - #define DATA64_ARGS (TCG_TARGET_REG_BITS == 64 ? 1 : 2) -/* There are tcg_ctx->insn_start_words here, not just one. */ -DEF(insn_start, 0, 0, DATA64_ARGS, TCG_OPF_NOT_PRESENT) +DEF(insn_start, 0, 0, DATA64_ARGS * INSN_START_WORDS, TCG_OPF_NOT_PRESENT) DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END | TCG_OPF_NOT_PRESENT) DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END | TCG_OPF_NOT_PRESENT) @@ -188,22 +123,10 @@ DEF(goto_ptr, 0, 1, 0, TCG_OPF_BB_EXIT | TCG_OPF_BB_END) DEF(plugin_cb, 0, 0, 1, TCG_OPF_NOT_PRESENT) DEF(plugin_mem_cb, 0, 1, 1, TCG_OPF_NOT_PRESENT) -DEF(qemu_ld_i32, 1, 1, 1, - TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) -DEF(qemu_st_i32, 0, 1 + 1, 1, - TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) -DEF(qemu_ld_i64, DATA64_ARGS, 1, 1, - TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) -DEF(qemu_st_i64, 0, DATA64_ARGS + 1, 1, - TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) - -/* Only used by i386 to cope with stupid register constraints. */ -DEF(qemu_st8_i32, 0, 1 + 1, 1, - TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) - -/* Only for 64-bit hosts at the moment. */ -DEF(qemu_ld_i128, 2, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) -DEF(qemu_st_i128, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) +DEF(qemu_ld, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT) +DEF(qemu_st, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT) +DEF(qemu_ld2, 2, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT) +DEF(qemu_st2, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT) /* Host vector support. */ diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 84d99508b6..a8c00c72cc 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -34,6 +34,7 @@ #include "tcg-target-reg-bits.h" #include "tcg-target.h" #include "tcg/tcg-cond.h" +#include "tcg/insn-start-words.h" #include "tcg/debug-assert.h" /* XXX: make safe guess about sizes */ @@ -359,7 +360,6 @@ struct TCGContext { int page_mask; uint8_t page_bits; uint8_t tlb_dyn_max_bits; - uint8_t insn_start_words; TCGBar guest_mo; TCGRegSet reserved_regs; @@ -418,6 +418,11 @@ struct TCGContext { MemOp riscv_cur_vsew; TCGType riscv_cur_type; #endif + /* + * During the tcg_reg_alloc_op loop, we are within a sequence of + * carry-using opcodes like addco+addci. + */ + bool carry_live; GHashTable *const_table[TCG_TYPE_COUNT]; TCGTempSet free_temps[TCG_TYPE_COUNT]; @@ -577,18 +582,19 @@ static inline TCGv_vec temp_tcgv_vec(TCGTemp *t) return (TCGv_vec)temp_tcgv_i32(t); } -static inline TCGArg tcg_get_insn_param(TCGOp *op, int arg) +static inline TCGArg tcg_get_insn_param(TCGOp *op, unsigned arg) { return op->args[arg]; } -static inline void tcg_set_insn_param(TCGOp *op, int arg, TCGArg v) +static inline void tcg_set_insn_param(TCGOp *op, unsigned arg, TCGArg v) { op->args[arg] = v; } -static inline uint64_t tcg_get_insn_start_param(TCGOp *op, int arg) +static inline uint64_t tcg_get_insn_start_param(TCGOp *op, unsigned arg) { + tcg_debug_assert(arg < INSN_START_WORDS); if (TCG_TARGET_REG_BITS == 64) { return tcg_get_insn_param(op, arg); } else { @@ -597,8 +603,9 @@ static inline uint64_t tcg_get_insn_start_param(TCGOp *op, int arg) } } -static inline void tcg_set_insn_start_param(TCGOp *op, int arg, uint64_t v) +static inline void tcg_set_insn_start_param(TCGOp *op, unsigned arg, uint64_t v) { + tcg_debug_assert(arg < INSN_START_WORDS); if (TCG_TARGET_REG_BITS == 64) { tcg_set_insn_param(op, arg, v); } else { @@ -741,19 +748,25 @@ enum { /* Instruction has side effects: it cannot be removed if its outputs are not used, and might trigger exceptions. */ TCG_OPF_SIDE_EFFECTS = 0x08, + /* Instruction operands may be I32 or I64 */ + TCG_OPF_INT = 0x10, /* Instruction is optional and not implemented by the host, or insn is generic and should not be implemented by the host. */ TCG_OPF_NOT_PRESENT = 0x20, /* Instruction operands are vectors. */ TCG_OPF_VECTOR = 0x40, /* Instruction is a conditional branch. */ - TCG_OPF_COND_BRANCH = 0x80 + TCG_OPF_COND_BRANCH = 0x80, + /* Instruction produces carry out. */ + TCG_OPF_CARRY_OUT = 0x100, + /* Instruction consumes carry in. */ + TCG_OPF_CARRY_IN = 0x200, }; typedef struct TCGOpDef { const char *name; uint8_t nb_oargs, nb_iargs, nb_cargs, nb_args; - uint8_t flags; + uint16_t flags; } TCGOpDef; extern const TCGOpDef tcg_op_defs[]; diff --git a/include/ui/dmabuf.h b/include/ui/dmabuf.h index dc74ba895a..3decdca497 100644 --- a/include/ui/dmabuf.h +++ b/include/ui/dmabuf.h @@ -10,24 +10,29 @@ #ifndef DMABUF_H #define DMABUF_H +#define DMABUF_MAX_PLANES 4 + typedef struct QemuDmaBuf QemuDmaBuf; QemuDmaBuf *qemu_dmabuf_new(uint32_t width, uint32_t height, - uint32_t stride, uint32_t x, - uint32_t y, uint32_t backing_width, - uint32_t backing_height, uint32_t fourcc, - uint64_t modifier, int dmabuf_fd, + const uint32_t *offset, const uint32_t *stride, + uint32_t x, uint32_t y, + uint32_t backing_width, uint32_t backing_height, + uint32_t fourcc, uint64_t modifier, + const int32_t *dmabuf_fd, uint32_t num_planes, bool allow_fences, bool y0_top); void qemu_dmabuf_free(QemuDmaBuf *dmabuf); G_DEFINE_AUTOPTR_CLEANUP_FUNC(QemuDmaBuf, qemu_dmabuf_free); -int qemu_dmabuf_get_fd(QemuDmaBuf *dmabuf); -int qemu_dmabuf_dup_fd(QemuDmaBuf *dmabuf); +const int *qemu_dmabuf_get_fds(QemuDmaBuf *dmabuf, int *nfds); +void qemu_dmabuf_dup_fds(QemuDmaBuf *dmabuf, int *fds, int nfds); void qemu_dmabuf_close(QemuDmaBuf *dmabuf); uint32_t qemu_dmabuf_get_width(QemuDmaBuf *dmabuf); uint32_t qemu_dmabuf_get_height(QemuDmaBuf *dmabuf); -uint32_t qemu_dmabuf_get_stride(QemuDmaBuf *dmabuf); +const uint32_t *qemu_dmabuf_get_offsets(QemuDmaBuf *dmabuf, int *noffsets); +const uint32_t *qemu_dmabuf_get_strides(QemuDmaBuf *dmabuf, int *nstrides); +uint32_t qemu_dmabuf_get_num_planes(QemuDmaBuf *dmabuf); uint32_t qemu_dmabuf_get_fourcc(QemuDmaBuf *dmabuf); uint64_t qemu_dmabuf_get_modifier(QemuDmaBuf *dmabuf); uint32_t qemu_dmabuf_get_texture(QemuDmaBuf *dmabuf); @@ -44,6 +49,5 @@ void qemu_dmabuf_set_texture(QemuDmaBuf *dmabuf, uint32_t texture); void qemu_dmabuf_set_fence_fd(QemuDmaBuf *dmabuf, int32_t fence_fd); void qemu_dmabuf_set_sync(QemuDmaBuf *dmabuf, void *sync); void qemu_dmabuf_set_draw_submitted(QemuDmaBuf *dmabuf, bool draw_submitted); -void qemu_dmabuf_set_fd(QemuDmaBuf *dmabuf, int32_t fd); #endif diff --git a/include/ui/egl-helpers.h b/include/ui/egl-helpers.h index 4b8c0d2281..fb80e15142 100644 --- a/include/ui/egl-helpers.h +++ b/include/ui/egl-helpers.h @@ -46,8 +46,9 @@ extern int qemu_egl_rn_fd; extern struct gbm_device *qemu_egl_rn_gbm_dev; int egl_rendernode_init(const char *rendernode, DisplayGLMode mode); -int egl_get_fd_for_texture(uint32_t tex_id, EGLint *stride, EGLint *fourcc, - EGLuint64KHR *modifier); +bool egl_dmabuf_export_texture(uint32_t tex_id, int *fd, EGLint *offset, + EGLint *stride, EGLint *fourcc, int *num_planes, + EGLuint64KHR *modifier); void egl_dmabuf_import_texture(QemuDmaBuf *dmabuf); void egl_dmabuf_release_texture(QemuDmaBuf *dmabuf); diff --git a/include/user/cpu_loop.h b/include/user/cpu_loop.h index 589c66543f..ad8a1d711f 100644 --- a/include/user/cpu_loop.h +++ b/include/user/cpu_loop.h @@ -20,11 +20,9 @@ #ifndef USER_CPU_LOOP_H #define USER_CPU_LOOP_H -#include "exec/abi_ptr.h" +#include "exec/vaddr.h" #include "exec/mmu-access-type.h" -#include "exec/log.h" -#include "exec/target_long.h" -#include "special-errno.h" + /** * adjust_signal_pc: @@ -46,7 +44,7 @@ MMUAccessType adjust_signal_pc(uintptr_t *pc, bool is_write); * Return true if the write fault has been handled, and should be re-tried. */ bool handle_sigsegv_accerr_write(CPUState *cpu, sigset_t *old_set, - uintptr_t host_pc, abi_ptr guest_addr); + uintptr_t host_pc, vaddr guest_addr); /** * cpu_loop_exit_sigsegv: @@ -59,7 +57,7 @@ bool handle_sigsegv_accerr_write(CPUState *cpu, sigset_t *old_set, * Use the TCGCPUOps hook to record cpu state, do guest operating system * specific things to raise SIGSEGV, and jump to the main cpu loop. */ -G_NORETURN void cpu_loop_exit_sigsegv(CPUState *cpu, target_ulong addr, +G_NORETURN void cpu_loop_exit_sigsegv(CPUState *cpu, vaddr addr, MMUAccessType access_type, bool maperr, uintptr_t ra); @@ -73,7 +71,7 @@ G_NORETURN void cpu_loop_exit_sigsegv(CPUState *cpu, target_ulong addr, * Use the TCGCPUOps hook to record cpu state, do guest operating system * specific things to raise SIGBUS, and jump to the main cpu loop. */ -G_NORETURN void cpu_loop_exit_sigbus(CPUState *cpu, target_ulong addr, +G_NORETURN void cpu_loop_exit_sigbus(CPUState *cpu, vaddr addr, MMUAccessType access_type, uintptr_t ra); diff --git a/include/user/guest-host.h b/include/user/guest-host.h index 8d2079bbbb..8f7ef75896 100644 --- a/include/user/guest-host.h +++ b/include/user/guest-host.h @@ -8,9 +8,9 @@ #ifndef USER_GUEST_HOST_H #define USER_GUEST_HOST_H -#include "user/abitypes.h" +#include "exec/vaddr.h" #include "user/guest-base.h" -#include "cpu.h" +#include "accel/tcg/cpu-ops.h" /* * If non-zero, the guest virtual address space is a contiguous subset @@ -23,59 +23,48 @@ extern unsigned long reserved_va; /* - * Limit the guest addresses as best we can. - * - * When not using -R reserved_va, we cannot really limit the guest - * to less address space than the host. For 32-bit guests, this - * acts as a sanity check that we're not giving the guest an address - * that it cannot even represent. For 64-bit guests... the address - * might not be what the real kernel would give, but it is at least - * representable in the guest. - * - * TODO: Improve address allocation to avoid this problem, and to - * avoid setting bits at the top of guest addresses that might need - * to be used for tags. + * The last byte of the guest address space. + * If reserved_va is non-zero, guest_addr_max matches. + * If reserved_va is zero, guest_addr_max equals the full guest space. */ -#define GUEST_ADDR_MAX_ \ - ((MIN_CONST(TARGET_VIRT_ADDR_SPACE_BITS, TARGET_ABI_BITS) <= 32) ? \ - UINT32_MAX : ~0ul) -#define GUEST_ADDR_MAX (reserved_va ? : GUEST_ADDR_MAX_) +extern unsigned long guest_addr_max; -#ifndef TARGET_TAGGED_ADDRESSES -static inline abi_ptr cpu_untagged_addr(CPUState *cs, abi_ptr x) +static inline vaddr cpu_untagged_addr(CPUState *cs, vaddr x) { + const TCGCPUOps *tcg_ops = cs->cc->tcg_ops; + if (tcg_ops->untagged_addr) { + return tcg_ops->untagged_addr(cs, x); + } return x; } -#endif /* All direct uses of g2h and h2g need to go away for usermode softmmu. */ -static inline void *g2h_untagged(abi_ptr x) +static inline void *g2h_untagged(vaddr x) { return (void *)((uintptr_t)(x) + guest_base); } -static inline void *g2h(CPUState *cs, abi_ptr x) +static inline void *g2h(CPUState *cs, vaddr x) { return g2h_untagged(cpu_untagged_addr(cs, x)); } -static inline bool guest_addr_valid_untagged(abi_ulong x) +static inline bool guest_addr_valid_untagged(vaddr x) { - return x <= GUEST_ADDR_MAX; + return x <= guest_addr_max; } -static inline bool guest_range_valid_untagged(abi_ulong start, abi_ulong len) +static inline bool guest_range_valid_untagged(vaddr start, vaddr len) { - return len - 1 <= GUEST_ADDR_MAX && start <= GUEST_ADDR_MAX - len + 1; + return len - 1 <= guest_addr_max && start <= guest_addr_max - len + 1; } #define h2g_valid(x) \ - (HOST_LONG_BITS <= TARGET_VIRT_ADDR_SPACE_BITS || \ - (uintptr_t)(x) - guest_base <= GUEST_ADDR_MAX) + ((uintptr_t)(x) - guest_base <= guest_addr_max) #define h2g_nocheck(x) ({ \ uintptr_t __ret = (uintptr_t)(x) - guest_base; \ - (abi_ptr)__ret; \ + (vaddr)__ret; \ }) #define h2g(x) ({ \ diff --git a/include/user/page-protection.h b/include/user/page-protection.h index d5c8748d49..4bde664e4a 100644 --- a/include/user/page-protection.h +++ b/include/user/page-protection.h @@ -12,13 +12,12 @@ #error Cannot include this header from system emulation #endif -#include "cpu-param.h" -#include "exec/target_long.h" +#include "exec/vaddr.h" #include "exec/translation-block.h" -int page_unprotect(tb_page_addr_t address, uintptr_t pc); +int page_unprotect(CPUState *cpu, tb_page_addr_t address, uintptr_t pc); -int page_get_flags(target_ulong address); +int page_get_flags(vaddr address); /** * page_set_flags: @@ -31,9 +30,9 @@ int page_get_flags(target_ulong address); * The flag PAGE_WRITE_ORG is positioned automatically depending * on PAGE_WRITE. The mmap_lock should already be held. */ -void page_set_flags(target_ulong start, target_ulong last, int flags); +void page_set_flags(vaddr start, vaddr last, int flags); -void page_reset_target_data(target_ulong start, target_ulong last); +void page_reset_target_data(vaddr start, vaddr last); /** * page_check_range @@ -45,7 +44,7 @@ void page_reset_target_data(target_ulong start, target_ulong last); * Return false if any page is unmapped. Thus testing flags == 0 is * equivalent to testing for flags == PAGE_VALID. */ -bool page_check_range(target_ulong start, target_ulong last, int flags); +bool page_check_range(vaddr start, vaddr last, int flags); /** * page_check_range_empty: @@ -57,7 +56,7 @@ bool page_check_range(target_ulong start, target_ulong last, int flags); * The memory lock must be held so that the caller will can ensure * the result stays true until a new mapping can be installed. */ -bool page_check_range_empty(target_ulong start, target_ulong last); +bool page_check_range_empty(vaddr start, vaddr last); /** * page_find_range_empty @@ -71,26 +70,25 @@ bool page_check_range_empty(target_ulong start, target_ulong last); * The memory lock must be held, as the caller will want to ensure * the returned range stays empty until a new mapping can be installed. */ -target_ulong page_find_range_empty(target_ulong min, target_ulong max, - target_ulong len, target_ulong align); +vaddr page_find_range_empty(vaddr min, vaddr max, vaddr len, vaddr align); /** - * page_get_target_data(address) + * page_get_target_data * @address: guest virtual address + * @size: per-page size * - * Return TARGET_PAGE_DATA_SIZE bytes of out-of-band data to associate + * Return @size bytes of out-of-band data to associate * with the guest page at @address, allocating it if necessary. The * caller should already have verified that the address is valid. + * The value of @size must be the same for every call. * * The memory will be freed when the guest page is deallocated, * e.g. with the munmap system call. */ __attribute__((returns_nonnull)) -void *page_get_target_data(target_ulong address); - -typedef int (*walk_memory_regions_fn)(void *, target_ulong, - target_ulong, unsigned long); +void *page_get_target_data(vaddr address, size_t size); +typedef int (*walk_memory_regions_fn)(void *, vaddr, vaddr, int); int walk_memory_regions(void *, walk_memory_regions_fn); void page_dump(FILE *f); diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h index ec1e82bdc8..4e6aff08df 100644 --- a/linux-headers/asm-arm64/kvm.h +++ b/linux-headers/asm-arm64/kvm.h @@ -105,6 +105,7 @@ struct kvm_regs { #define KVM_ARM_VCPU_PTRAUTH_ADDRESS 5 /* VCPU uses address authentication */ #define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */ #define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */ +#define KVM_ARM_VCPU_HAS_EL2_E2H0 8 /* Limit NV support to E2H RES0 */ struct kvm_vcpu_init { __u32 target; @@ -365,6 +366,7 @@ enum { KVM_REG_ARM_STD_HYP_BIT_PV_TIME = 0, }; +/* Vendor hyper call function numbers 0-63 */ #define KVM_REG_ARM_VENDOR_HYP_BMAP KVM_REG_ARM_FW_FEAT_BMAP_REG(2) enum { @@ -372,6 +374,14 @@ enum { KVM_REG_ARM_VENDOR_HYP_BIT_PTP = 1, }; +/* Vendor hyper call function numbers 64-127 */ +#define KVM_REG_ARM_VENDOR_HYP_BMAP_2 KVM_REG_ARM_FW_FEAT_BMAP_REG(3) + +enum { + KVM_REG_ARM_VENDOR_HYP_BIT_DISCOVER_IMPL_VER = 0, + KVM_REG_ARM_VENDOR_HYP_BIT_DISCOVER_IMPL_CPUS = 1, +}; + /* Device Control API on vm fd */ #define KVM_ARM_VM_SMCCC_CTRL 0 #define KVM_ARM_VM_SMCCC_FILTER 0 @@ -394,6 +404,7 @@ enum { #define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6 #define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7 #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 +#define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT) diff --git a/linux-headers/asm-arm64/unistd_64.h b/linux-headers/asm-arm64/unistd_64.h index d4e90fff76..ee9aaebdf3 100644 --- a/linux-headers/asm-arm64/unistd_64.h +++ b/linux-headers/asm-arm64/unistd_64.h @@ -323,6 +323,7 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_UNISTD_64_H */ diff --git a/linux-headers/asm-generic/mman-common.h b/linux-headers/asm-generic/mman-common.h index 1ea2c4c33b..ef1c27fa3c 100644 --- a/linux-headers/asm-generic/mman-common.h +++ b/linux-headers/asm-generic/mman-common.h @@ -85,6 +85,7 @@ /* compatibility flags */ #define MAP_FILE 0 +#define PKEY_UNRESTRICTED 0x0 #define PKEY_DISABLE_ACCESS 0x1 #define PKEY_DISABLE_WRITE 0x2 #define PKEY_ACCESS_MASK (PKEY_DISABLE_ACCESS |\ diff --git a/linux-headers/asm-generic/unistd.h b/linux-headers/asm-generic/unistd.h index 88dc393c2b..2892a45023 100644 --- a/linux-headers/asm-generic/unistd.h +++ b/linux-headers/asm-generic/unistd.h @@ -849,9 +849,11 @@ __SYSCALL(__NR_getxattrat, sys_getxattrat) __SYSCALL(__NR_listxattrat, sys_listxattrat) #define __NR_removexattrat 466 __SYSCALL(__NR_removexattrat, sys_removexattrat) +#define __NR_open_tree_attr 467 +__SYSCALL(__NR_open_tree_attr, sys_open_tree_attr) #undef __NR_syscalls -#define __NR_syscalls 467 +#define __NR_syscalls 468 /* * 32 bit systems traditionally used different diff --git a/linux-headers/asm-loongarch/unistd_64.h b/linux-headers/asm-loongarch/unistd_64.h index 23fb96a8a7..50d22df8f7 100644 --- a/linux-headers/asm-loongarch/unistd_64.h +++ b/linux-headers/asm-loongarch/unistd_64.h @@ -319,6 +319,7 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_UNISTD_64_H */ diff --git a/linux-headers/asm-mips/unistd_n32.h b/linux-headers/asm-mips/unistd_n32.h index 9a75719644..bdcc2f460b 100644 --- a/linux-headers/asm-mips/unistd_n32.h +++ b/linux-headers/asm-mips/unistd_n32.h @@ -395,5 +395,6 @@ #define __NR_getxattrat (__NR_Linux + 464) #define __NR_listxattrat (__NR_Linux + 465) #define __NR_removexattrat (__NR_Linux + 466) +#define __NR_open_tree_attr (__NR_Linux + 467) #endif /* _ASM_UNISTD_N32_H */ diff --git a/linux-headers/asm-mips/unistd_n64.h b/linux-headers/asm-mips/unistd_n64.h index 7086783b0c..3b6b0193b6 100644 --- a/linux-headers/asm-mips/unistd_n64.h +++ b/linux-headers/asm-mips/unistd_n64.h @@ -371,5 +371,6 @@ #define __NR_getxattrat (__NR_Linux + 464) #define __NR_listxattrat (__NR_Linux + 465) #define __NR_removexattrat (__NR_Linux + 466) +#define __NR_open_tree_attr (__NR_Linux + 467) #endif /* _ASM_UNISTD_N64_H */ diff --git a/linux-headers/asm-mips/unistd_o32.h b/linux-headers/asm-mips/unistd_o32.h index b3825823e4..4609a4b4d3 100644 --- a/linux-headers/asm-mips/unistd_o32.h +++ b/linux-headers/asm-mips/unistd_o32.h @@ -441,5 +441,6 @@ #define __NR_getxattrat (__NR_Linux + 464) #define __NR_listxattrat (__NR_Linux + 465) #define __NR_removexattrat (__NR_Linux + 466) +#define __NR_open_tree_attr (__NR_Linux + 467) #endif /* _ASM_UNISTD_O32_H */ diff --git a/linux-headers/asm-powerpc/unistd_32.h b/linux-headers/asm-powerpc/unistd_32.h index 38ee4dc35d..5d38a427e0 100644 --- a/linux-headers/asm-powerpc/unistd_32.h +++ b/linux-headers/asm-powerpc/unistd_32.h @@ -448,6 +448,7 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_UNISTD_32_H */ diff --git a/linux-headers/asm-powerpc/unistd_64.h b/linux-headers/asm-powerpc/unistd_64.h index 5e5f156834..860a488e4d 100644 --- a/linux-headers/asm-powerpc/unistd_64.h +++ b/linux-headers/asm-powerpc/unistd_64.h @@ -420,6 +420,7 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_UNISTD_64_H */ diff --git a/linux-headers/asm-riscv/kvm.h b/linux-headers/asm-riscv/kvm.h index f06bc5efcd..5f59fd226c 100644 --- a/linux-headers/asm-riscv/kvm.h +++ b/linux-headers/asm-riscv/kvm.h @@ -182,6 +182,8 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_SVVPTC, KVM_RISCV_ISA_EXT_ZABHA, KVM_RISCV_ISA_EXT_ZICCRSE, + KVM_RISCV_ISA_EXT_ZAAMO, + KVM_RISCV_ISA_EXT_ZALRSC, KVM_RISCV_ISA_EXT_MAX, }; diff --git a/linux-headers/asm-riscv/unistd_32.h b/linux-headers/asm-riscv/unistd_32.h index 74f6127aed..a5e769f1d9 100644 --- a/linux-headers/asm-riscv/unistd_32.h +++ b/linux-headers/asm-riscv/unistd_32.h @@ -314,6 +314,7 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_UNISTD_32_H */ diff --git a/linux-headers/asm-riscv/unistd_64.h b/linux-headers/asm-riscv/unistd_64.h index bb6a15a2ec..8df4d64841 100644 --- a/linux-headers/asm-riscv/unistd_64.h +++ b/linux-headers/asm-riscv/unistd_64.h @@ -324,6 +324,7 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_UNISTD_64_H */ diff --git a/linux-headers/asm-s390/unistd_32.h b/linux-headers/asm-s390/unistd_32.h index 620201cb36..85eedbd18e 100644 --- a/linux-headers/asm-s390/unistd_32.h +++ b/linux-headers/asm-s390/unistd_32.h @@ -439,5 +439,6 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_S390_UNISTD_32_H */ diff --git a/linux-headers/asm-s390/unistd_64.h b/linux-headers/asm-s390/unistd_64.h index e7e4a10aaf..c03b1b9701 100644 --- a/linux-headers/asm-s390/unistd_64.h +++ b/linux-headers/asm-s390/unistd_64.h @@ -387,5 +387,6 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_S390_UNISTD_64_H */ diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h index 86f2c34e7a..dc591fb17e 100644 --- a/linux-headers/asm-x86/kvm.h +++ b/linux-headers/asm-x86/kvm.h @@ -557,6 +557,9 @@ struct kvm_x86_mce { #define KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE (1 << 7) #define KVM_XEN_HVM_CONFIG_SHARED_INFO_HVA (1 << 8) +#define KVM_XEN_MSR_MIN_INDEX 0x40000000u +#define KVM_XEN_MSR_MAX_INDEX 0x4fffffffu + struct kvm_xen_hvm_config { __u32 flags; __u32 msr; diff --git a/linux-headers/asm-x86/unistd_32.h b/linux-headers/asm-x86/unistd_32.h index a2eb492a75..491d6b4eb6 100644 --- a/linux-headers/asm-x86/unistd_32.h +++ b/linux-headers/asm-x86/unistd_32.h @@ -457,6 +457,7 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_UNISTD_32_H */ diff --git a/linux-headers/asm-x86/unistd_64.h b/linux-headers/asm-x86/unistd_64.h index 2f5fc400f5..7cf88bf9bd 100644 --- a/linux-headers/asm-x86/unistd_64.h +++ b/linux-headers/asm-x86/unistd_64.h @@ -380,6 +380,7 @@ #define __NR_getxattrat 464 #define __NR_listxattrat 465 #define __NR_removexattrat 466 +#define __NR_open_tree_attr 467 #endif /* _ASM_UNISTD_64_H */ diff --git a/linux-headers/asm-x86/unistd_x32.h b/linux-headers/asm-x86/unistd_x32.h index fecd832e7f..82959111e6 100644 --- a/linux-headers/asm-x86/unistd_x32.h +++ b/linux-headers/asm-x86/unistd_x32.h @@ -333,6 +333,7 @@ #define __NR_getxattrat (__X32_SYSCALL_BIT + 464) #define __NR_listxattrat (__X32_SYSCALL_BIT + 465) #define __NR_removexattrat (__X32_SYSCALL_BIT + 466) +#define __NR_open_tree_attr (__X32_SYSCALL_BIT + 467) #define __NR_rt_sigaction (__X32_SYSCALL_BIT + 512) #define __NR_rt_sigreturn (__X32_SYSCALL_BIT + 513) #define __NR_ioctl (__X32_SYSCALL_BIT + 514) diff --git a/linux-headers/linux/bits.h b/linux-headers/linux/bits.h index c0d00c0a98..58596d18f4 100644 --- a/linux-headers/linux/bits.h +++ b/linux-headers/linux/bits.h @@ -4,13 +4,9 @@ #ifndef _LINUX_BITS_H #define _LINUX_BITS_H -#define __GENMASK(h, l) \ - (((~_UL(0)) - (_UL(1) << (l)) + 1) & \ - (~_UL(0) >> (__BITS_PER_LONG - 1 - (h)))) +#define __GENMASK(h, l) (((~_UL(0)) << (l)) & (~_UL(0) >> (BITS_PER_LONG - 1 - (h)))) -#define __GENMASK_ULL(h, l) \ - (((~_ULL(0)) - (_ULL(1) << (l)) + 1) & \ - (~_ULL(0) >> (__BITS_PER_LONG_LONG - 1 - (h)))) +#define __GENMASK_ULL(h, l) (((~_ULL(0)) << (l)) & (~_ULL(0) >> (BITS_PER_LONG_LONG - 1 - (h)))) #define __GENMASK_U128(h, l) \ ((_BIT128((h)) << 1) - (_BIT128(l))) diff --git a/linux-headers/linux/const.h b/linux-headers/linux/const.h index 2122610de7..95ede23342 100644 --- a/linux-headers/linux/const.h +++ b/linux-headers/linux/const.h @@ -33,7 +33,7 @@ * Missing __asm__ support * * __BIT128() would not work in the __asm__ code, as it shifts an - * 'unsigned __init128' data type as direct representation of + * 'unsigned __int128' data type as direct representation of * 128 bit constants is not supported in the gcc compiler, as * they get silently truncated. * diff --git a/linux-headers/linux/iommufd.h b/linux-headers/linux/iommufd.h index ccbdca5e11..cb0f7d6b4d 100644 --- a/linux-headers/linux/iommufd.h +++ b/linux-headers/linux/iommufd.h @@ -55,6 +55,7 @@ enum { IOMMUFD_CMD_VIOMMU_ALLOC = 0x90, IOMMUFD_CMD_VDEVICE_ALLOC = 0x91, IOMMUFD_CMD_IOAS_CHANGE_PROCESS = 0x92, + IOMMUFD_CMD_VEVENTQ_ALLOC = 0x93, }; /** @@ -392,6 +393,9 @@ struct iommu_vfio_ioas { * Any domain attached to the non-PASID part of the * device must also be flagged, otherwise attaching a * PASID will blocked. + * For the user that wants to attach PASID, ioas is + * not recommended for both the non-PASID part + * and PASID part of the device. * If IOMMU does not support PASID it will return * error (-EOPNOTSUPP). */ @@ -608,9 +612,17 @@ enum iommu_hw_info_type { * IOMMU_HWPT_GET_DIRTY_BITMAP * IOMMU_HWPT_SET_DIRTY_TRACKING * + * @IOMMU_HW_CAP_PCI_PASID_EXEC: Execute Permission Supported, user ignores it + * when the struct + * iommu_hw_info::out_max_pasid_log2 is zero. + * @IOMMU_HW_CAP_PCI_PASID_PRIV: Privileged Mode Supported, user ignores it + * when the struct + * iommu_hw_info::out_max_pasid_log2 is zero. */ enum iommufd_hw_capabilities { IOMMU_HW_CAP_DIRTY_TRACKING = 1 << 0, + IOMMU_HW_CAP_PCI_PASID_EXEC = 1 << 1, + IOMMU_HW_CAP_PCI_PASID_PRIV = 1 << 2, }; /** @@ -626,6 +638,9 @@ enum iommufd_hw_capabilities { * iommu_hw_info_type. * @out_capabilities: Output the generic iommu capability info type as defined * in the enum iommu_hw_capabilities. + * @out_max_pasid_log2: Output the width of PASIDs. 0 means no PASID support. + * PCI devices turn to out_capabilities to check if the + * specific capabilities is supported or not. * @__reserved: Must be 0 * * Query an iommu type specific hardware information data from an iommu behind @@ -649,7 +664,8 @@ struct iommu_hw_info { __u32 data_len; __aligned_u64 data_uptr; __u32 out_data_type; - __u32 __reserved; + __u8 out_max_pasid_log2; + __u8 __reserved[3]; __aligned_u64 out_capabilities; }; #define IOMMU_GET_HW_INFO _IO(IOMMUFD_TYPE, IOMMUFD_CMD_GET_HW_INFO) @@ -1014,4 +1030,115 @@ struct iommu_ioas_change_process { #define IOMMU_IOAS_CHANGE_PROCESS \ _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_CHANGE_PROCESS) +/** + * enum iommu_veventq_flag - flag for struct iommufd_vevent_header + * @IOMMU_VEVENTQ_FLAG_LOST_EVENTS: vEVENTQ has lost vEVENTs + */ +enum iommu_veventq_flag { + IOMMU_VEVENTQ_FLAG_LOST_EVENTS = (1U << 0), +}; + +/** + * struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status + * @flags: Combination of enum iommu_veventq_flag + * @sequence: The sequence index of a vEVENT in the vEVENTQ, with a range of + * [0, INT_MAX] where the following index of INT_MAX is 0 + * + * Each iommufd_vevent_header reports a sequence index of the following vEVENT: + * + * +----------------------+-------+----------------------+-------+---+-------+ + * | header0 {sequence=0} | data0 | header1 {sequence=1} | data1 |...| dataN | + * +----------------------+-------+----------------------+-------+---+-------+ + * + * And this sequence index is expected to be monotonic to the sequence index of + * the previous vEVENT. If two adjacent sequence indexes has a delta larger than + * 1, it means that delta - 1 number of vEVENTs has lost, e.g. two lost vEVENTs: + * + * +-----+----------------------+-------+----------------------+-------+-----+ + * | ... | header3 {sequence=3} | data3 | header6 {sequence=6} | data6 | ... | + * +-----+----------------------+-------+----------------------+-------+-----+ + * + * If a vEVENT lost at the tail of the vEVENTQ and there is no following vEVENT + * providing the next sequence index, an IOMMU_VEVENTQ_FLAG_LOST_EVENTS header + * would be added to the tail, and no data would follow this header: + * + * +--+----------------------+-------+-----------------------------------------+ + * |..| header3 {sequence=3} | data3 | header4 {flags=LOST_EVENTS, sequence=4} | + * +--+----------------------+-------+-----------------------------------------+ + */ +struct iommufd_vevent_header { + __u32 flags; + __u32 sequence; +}; + +/** + * enum iommu_veventq_type - Virtual Event Queue Type + * @IOMMU_VEVENTQ_TYPE_DEFAULT: Reserved for future use + * @IOMMU_VEVENTQ_TYPE_ARM_SMMUV3: ARM SMMUv3 Virtual Event Queue + */ +enum iommu_veventq_type { + IOMMU_VEVENTQ_TYPE_DEFAULT = 0, + IOMMU_VEVENTQ_TYPE_ARM_SMMUV3 = 1, +}; + +/** + * struct iommu_vevent_arm_smmuv3 - ARM SMMUv3 Virtual Event + * (IOMMU_VEVENTQ_TYPE_ARM_SMMUV3) + * @evt: 256-bit ARM SMMUv3 Event record, little-endian. + * Reported event records: (Refer to "7.3 Event records" in SMMUv3 HW Spec) + * - 0x04 C_BAD_STE + * - 0x06 F_STREAM_DISABLED + * - 0x08 C_BAD_SUBSTREAMID + * - 0x0a C_BAD_CD + * - 0x10 F_TRANSLATION + * - 0x11 F_ADDR_SIZE + * - 0x12 F_ACCESS + * - 0x13 F_PERMISSION + * + * StreamID field reports a virtual device ID. To receive a virtual event for a + * device, a vDEVICE must be allocated via IOMMU_VDEVICE_ALLOC. + */ +struct iommu_vevent_arm_smmuv3 { + __aligned_le64 evt[4]; +}; + +/** + * struct iommu_veventq_alloc - ioctl(IOMMU_VEVENTQ_ALLOC) + * @size: sizeof(struct iommu_veventq_alloc) + * @flags: Must be 0 + * @viommu_id: virtual IOMMU ID to associate the vEVENTQ with + * @type: Type of the vEVENTQ. Must be defined in enum iommu_veventq_type + * @veventq_depth: Maximum number of events in the vEVENTQ + * @out_veventq_id: The ID of the new vEVENTQ + * @out_veventq_fd: The fd of the new vEVENTQ. User space must close the + * successfully returned fd after using it + * @__reserved: Must be 0 + * + * Explicitly allocate a virtual event queue interface for a vIOMMU. A vIOMMU + * can have multiple FDs for different types, but is confined to one per @type. + * User space should open the @out_veventq_fd to read vEVENTs out of a vEVENTQ, + * if there are vEVENTs available. A vEVENTQ will lose events due to overflow, + * if the number of the vEVENTs hits @veventq_depth. + * + * Each vEVENT in a vEVENTQ encloses a struct iommufd_vevent_header followed by + * a type-specific data structure, in a normal case: + * + * +-+---------+-------+---------+-------+-----+---------+-------+-+ + * | | header0 | data0 | header1 | data1 | ... | headerN | dataN | | + * +-+---------+-------+---------+-------+-----+---------+-------+-+ + * + * unless a tailing IOMMU_VEVENTQ_FLAG_LOST_EVENTS header is logged (refer to + * struct iommufd_vevent_header). + */ +struct iommu_veventq_alloc { + __u32 size; + __u32 flags; + __u32 viommu_id; + __u32 type; + __u32 veventq_depth; + __u32 out_veventq_id; + __u32 out_veventq_fd; + __u32 __reserved; +}; +#define IOMMU_VEVENTQ_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VEVENTQ_ALLOC) #endif diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h index 27181b3dd8..e5f3e8b5a0 100644 --- a/linux-headers/linux/kvm.h +++ b/linux-headers/linux/kvm.h @@ -921,6 +921,7 @@ struct kvm_enable_cap { #define KVM_CAP_PRE_FAULT_MEMORY 236 #define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237 #define KVM_CAP_X86_GUEST_MODE 238 +#define KVM_CAP_ARM_WRITABLE_IMP_ID_REGS 239 struct kvm_irq_routing_irqchip { __u32 irqchip; diff --git a/linux-headers/linux/psp-sev.h b/linux-headers/linux/psp-sev.h index 17bf191573..113c4ceb78 100644 --- a/linux-headers/linux/psp-sev.h +++ b/linux-headers/linux/psp-sev.h @@ -73,13 +73,20 @@ typedef enum { SEV_RET_INVALID_PARAM, SEV_RET_RESOURCE_LIMIT, SEV_RET_SECURE_DATA_INVALID, - SEV_RET_INVALID_KEY = 0x27, - SEV_RET_INVALID_PAGE_SIZE, - SEV_RET_INVALID_PAGE_STATE, - SEV_RET_INVALID_MDATA_ENTRY, - SEV_RET_INVALID_PAGE_OWNER, - SEV_RET_INVALID_PAGE_AEAD_OFLOW, - SEV_RET_RMP_INIT_REQUIRED, + SEV_RET_INVALID_PAGE_SIZE = 0x0019, + SEV_RET_INVALID_PAGE_STATE = 0x001A, + SEV_RET_INVALID_MDATA_ENTRY = 0x001B, + SEV_RET_INVALID_PAGE_OWNER = 0x001C, + SEV_RET_AEAD_OFLOW = 0x001D, + SEV_RET_EXIT_RING_BUFFER = 0x001F, + SEV_RET_RMP_INIT_REQUIRED = 0x0020, + SEV_RET_BAD_SVN = 0x0021, + SEV_RET_BAD_VERSION = 0x0022, + SEV_RET_SHUTDOWN_REQUIRED = 0x0023, + SEV_RET_UPDATE_FAILED = 0x0024, + SEV_RET_RESTORE_REQUIRED = 0x0025, + SEV_RET_RMP_INITIALIZATION_FAILED = 0x0026, + SEV_RET_INVALID_KEY = 0x0027, SEV_RET_MAX, } sev_ret_code; diff --git a/linux-headers/linux/stddef.h b/linux-headers/linux/stddef.h index e1416f7937..e1fcfcf3b3 100644 --- a/linux-headers/linux/stddef.h +++ b/linux-headers/linux/stddef.h @@ -70,4 +70,6 @@ #define __counted_by_be(m) #endif +#define __kernel_nonstring + #endif /* _LINUX_STDDEF_H */ diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h index 1b5e254d6a..79bf8c0cc5 100644 --- a/linux-headers/linux/vfio.h +++ b/linux-headers/linux/vfio.h @@ -671,6 +671,7 @@ enum { */ enum { VFIO_AP_REQ_IRQ_INDEX, + VFIO_AP_CFG_CHG_IRQ_INDEX, VFIO_AP_NUM_IRQS }; @@ -931,29 +932,34 @@ struct vfio_device_bind_iommufd { * VFIO_DEVICE_ATTACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 19, * struct vfio_device_attach_iommufd_pt) * @argsz: User filled size of this data. - * @flags: Must be 0. + * @flags: Flags for attach. * @pt_id: Input the target id which can represent an ioas or a hwpt * allocated via iommufd subsystem. * Output the input ioas id or the attached hwpt id which could * be the specified hwpt itself or a hwpt automatically created * for the specified ioas by kernel during the attachment. + * @pasid: The pasid to be attached, only meaningful when + * VFIO_DEVICE_ATTACH_PASID is set in @flags * * Associate the device with an address space within the bound iommufd. * Undo by VFIO_DEVICE_DETACH_IOMMUFD_PT or device fd close. This is only * allowed on cdev fds. * - * If a vfio device is currently attached to a valid hw_pagetable, without doing - * a VFIO_DEVICE_DETACH_IOMMUFD_PT, a second VFIO_DEVICE_ATTACH_IOMMUFD_PT ioctl - * passing in another hw_pagetable (hwpt) id is allowed. This action, also known - * as a hw_pagetable replacement, will replace the device's currently attached - * hw_pagetable with a new hw_pagetable corresponding to the given pt_id. + * If a vfio device or a pasid of this device is currently attached to a valid + * hw_pagetable (hwpt), without doing a VFIO_DEVICE_DETACH_IOMMUFD_PT, a second + * VFIO_DEVICE_ATTACH_IOMMUFD_PT ioctl passing in another hwpt id is allowed. + * This action, also known as a hw_pagetable replacement, will replace the + * currently attached hwpt of the device or the pasid of this device with a new + * hwpt corresponding to the given pt_id. * * Return: 0 on success, -errno on failure. */ struct vfio_device_attach_iommufd_pt { __u32 argsz; __u32 flags; +#define VFIO_DEVICE_ATTACH_PASID (1 << 0) __u32 pt_id; + __u32 pasid; }; #define VFIO_DEVICE_ATTACH_IOMMUFD_PT _IO(VFIO_TYPE, VFIO_BASE + 19) @@ -962,17 +968,21 @@ struct vfio_device_attach_iommufd_pt { * VFIO_DEVICE_DETACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 20, * struct vfio_device_detach_iommufd_pt) * @argsz: User filled size of this data. - * @flags: Must be 0. + * @flags: Flags for detach. + * @pasid: The pasid to be detached, only meaningful when + * VFIO_DEVICE_DETACH_PASID is set in @flags * - * Remove the association of the device and its current associated address - * space. After it, the device should be in a blocking DMA state. This is only - * allowed on cdev fds. + * Remove the association of the device or a pasid of the device and its current + * associated address space. After it, the device or the pasid should be in a + * blocking DMA state. This is only allowed on cdev fds. * * Return: 0 on success, -errno on failure. */ struct vfio_device_detach_iommufd_pt { __u32 argsz; __u32 flags; +#define VFIO_DEVICE_DETACH_PASID (1 << 0) + __u32 pasid; }; #define VFIO_DEVICE_DETACH_IOMMUFD_PT _IO(VFIO_TYPE, VFIO_BASE + 20) diff --git a/linux-user/elfload.c b/linux-user/elfload.c index fbfdec2f17..82ebf6a212 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -4059,8 +4059,7 @@ static void bswap_note(struct elf_note *en) /* * Calculate file (dump) size of given memory region. */ -static size_t vma_dump_size(target_ulong start, target_ulong end, - unsigned long flags) +static size_t vma_dump_size(vaddr start, vaddr end, int flags) { /* The area must be readable. */ if (!(flags & PAGE_READ)) { @@ -4253,14 +4252,14 @@ static int dump_write(int fd, const void *ptr, size_t size) return (0); } -static int wmr_page_unprotect_regions(void *opaque, target_ulong start, - target_ulong end, unsigned long flags) +static int wmr_page_unprotect_regions(void *opaque, vaddr start, + vaddr end, int flags) { if ((flags & (PAGE_WRITE | PAGE_WRITE_ORG)) == PAGE_WRITE_ORG) { size_t step = MAX(TARGET_PAGE_SIZE, qemu_real_host_page_size()); while (1) { - page_unprotect(start, 0); + page_unprotect(NULL, start, 0); if (end - start <= step) { break; } @@ -4275,8 +4274,8 @@ typedef struct { size_t size; } CountAndSizeRegions; -static int wmr_count_and_size_regions(void *opaque, target_ulong start, - target_ulong end, unsigned long flags) +static int wmr_count_and_size_regions(void *opaque, vaddr start, + vaddr end, int flags) { CountAndSizeRegions *css = opaque; @@ -4290,8 +4289,8 @@ typedef struct { off_t offset; } FillRegionPhdr; -static int wmr_fill_region_phdr(void *opaque, target_ulong start, - target_ulong end, unsigned long flags) +static int wmr_fill_region_phdr(void *opaque, vaddr start, + vaddr end, int flags) { FillRegionPhdr *d = opaque; struct elf_phdr *phdr = d->phdr; @@ -4313,8 +4312,8 @@ static int wmr_fill_region_phdr(void *opaque, target_ulong start, return 0; } -static int wmr_write_region(void *opaque, target_ulong start, - target_ulong end, unsigned long flags) +static int wmr_write_region(void *opaque, vaddr start, + vaddr end, int flags) { int fd = *(int *)opaque; size_t size = vma_dump_size(start, end, flags); diff --git a/linux-user/main.c b/linux-user/main.c index e2ec5970be..5ac5b55dc6 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -40,7 +40,6 @@ #include "qemu/plugin.h" #include "user/guest-base.h" #include "user/page-protection.h" -#include "exec/exec-all.h" #include "exec/gdbstub.h" #include "gdbstub/user.h" #include "tcg/startup.h" @@ -123,6 +122,7 @@ static const char *last_log_filename; #endif unsigned long reserved_va; +unsigned long guest_addr_max; static void usage(int exitcode); @@ -859,6 +859,13 @@ int main(int argc, char **argv, char **envp) /* MAX_RESERVED_VA + 1 is a large power of 2, so is aligned. */ reserved_va = max_reserved_va; } + if (reserved_va != 0) { + guest_addr_max = reserved_va; + } else if (MIN(TARGET_VIRT_ADDR_SPACE_BITS, TARGET_ABI_BITS) <= 32) { + guest_addr_max = UINT32_MAX; + } else { + guest_addr_max = ~0ul; + } /* * Temporarily disable diff --git a/linux-user/mmap.c b/linux-user/mmap.c index f88a80c31e..002e1e668e 100644 --- a/linux-user/mmap.c +++ b/linux-user/mmap.c @@ -645,7 +645,7 @@ static abi_long mmap_h_eq_g(abi_ulong start, abi_ulong len, * * However, this case is rather common with executable images, * so the workaround is important for even trivial tests, whereas - * the mmap of of a file being extended is less common. + * the mmap of a file being extended is less common. */ static abi_long mmap_h_lt_g(abi_ulong start, abi_ulong len, int host_prot, int mmap_flags, int page_flags, int fd, diff --git a/linux-user/signal.c b/linux-user/signal.c index 4dafc2c3a2..cd0e7398aa 100644 --- a/linux-user/signal.c +++ b/linux-user/signal.c @@ -750,7 +750,7 @@ void force_sigsegv(int oldsig) } #endif -void cpu_loop_exit_sigsegv(CPUState *cpu, target_ulong addr, +void cpu_loop_exit_sigsegv(CPUState *cpu, vaddr addr, MMUAccessType access_type, bool maperr, uintptr_t ra) { const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; @@ -766,7 +766,7 @@ void cpu_loop_exit_sigsegv(CPUState *cpu, target_ulong addr, cpu_loop_exit_restore(cpu, ra); } -void cpu_loop_exit_sigbus(CPUState *cpu, target_ulong addr, +void cpu_loop_exit_sigbus(CPUState *cpu, vaddr addr, MMUAccessType access_type, uintptr_t ra) { const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; diff --git a/linux-user/syscall.c b/linux-user/syscall.c index 5826ac3adb..23b901b713 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -8135,8 +8135,8 @@ static void open_self_maps_4(const struct open_self_maps_data *d, * Callback for walk_memory_regions, when read_self_maps() fails. * Proceed without the benefit of host /proc/self/maps cross-check. */ -static int open_self_maps_3(void *opaque, target_ulong guest_start, - target_ulong guest_end, unsigned long flags) +static int open_self_maps_3(void *opaque, vaddr guest_start, + vaddr guest_end, int flags) { static const MapInfo mi = { .is_priv = true }; @@ -8147,8 +8147,8 @@ static int open_self_maps_3(void *opaque, target_ulong guest_start, /* * Callback for walk_memory_regions, when read_self_maps() succeeds. */ -static int open_self_maps_2(void *opaque, target_ulong guest_start, - target_ulong guest_end, unsigned long flags) +static int open_self_maps_2(void *opaque, vaddr guest_start, + vaddr guest_end, int flags) { const struct open_self_maps_data *d = opaque; uintptr_t host_start = (uintptr_t)g2h_untagged(guest_start); diff --git a/linux-user/user-internals.h b/linux-user/user-internals.h index 4aa253b566..691b9a1775 100644 --- a/linux-user/user-internals.h +++ b/linux-user/user-internals.h @@ -19,7 +19,6 @@ #define LINUX_USER_USER_INTERNALS_H #include "user/thunk.h" -#include "exec/exec-all.h" #include "qemu/log.h" extern char *exec_path; diff --git a/meson.build b/meson.build index 8ae70dbe45..e819a7084c 100644 --- a/meson.build +++ b/meson.build @@ -50,9 +50,9 @@ genh = [] qapi_trace_events = [] bsd_oses = ['gnu/kfreebsd', 'freebsd', 'netbsd', 'openbsd', 'dragonfly', 'darwin'] -supported_oses = ['windows', 'freebsd', 'netbsd', 'openbsd', 'darwin', 'sunos', 'linux'] +supported_oses = ['windows', 'freebsd', 'netbsd', 'openbsd', 'darwin', 'sunos', 'linux', 'emscripten'] supported_cpus = ['ppc', 'ppc64', 's390x', 'riscv32', 'riscv64', 'x86', 'x86_64', - 'arm', 'aarch64', 'loongarch64', 'mips', 'mips64', 'sparc64'] + 'arm', 'aarch64', 'loongarch64', 'mips', 'mips64', 'sparc64', 'wasm32'] cpu = host_machine.cpu_family() @@ -94,12 +94,12 @@ have_rust = have_rust and add_languages('rust', native: true, required: get_option('rust').disable_auto_if(not have_system)) if have_rust rustc = meson.get_compiler('rust') - if rustc.version().version_compare('<1.63.0') + if rustc.version().version_compare('<1.77.0') if get_option('rust').enabled() - error('rustc version ' + rustc.version() + ' is unsupported. Please upgrade to at least 1.63.0') + error('rustc version ' + rustc.version() + ' is unsupported. Please upgrade to at least 1.77.0') else warning('rustc version ' + rustc.version() + ' is unsupported, disabling Rust compilation.') - message('Please upgrade to at least 1.63.0 to use Rust.') + message('Please upgrade to at least 1.77.0 to use Rust.') have_rust = false endif endif @@ -353,6 +353,8 @@ foreach lang : all_languages # endif #endif''') # ok + elif compiler.get_id() == 'emscripten' + # ok else error('You either need GCC v7.4 or Clang v10.0 (or XCode Clang v15.0) to compile QEMU') endif @@ -470,7 +472,10 @@ endif # instead, we can't add -no-pie because it overrides -shared: the linker then # tries to build an executable instead of a shared library and fails. So # don't add -no-pie anywhere and cross fingers. :( -if not get_option('b_pie') +# +# Emscripten doesn't support -no-pie but meson can't catch the compiler +# warning. So explicitly omit the flag for Emscripten. +if not get_option('b_pie') and host_os != 'emscripten' qemu_common_flags += cc.get_supported_arguments('-fno-pie', '-no-pie') endif @@ -514,6 +519,8 @@ ucontext_probe = ''' supported_backends = [] if host_os == 'windows' supported_backends += ['windows'] +elif host_os == 'emscripten' + supported_backends += ['wasm'] else if host_os != 'darwin' and cc.links(ucontext_probe) supported_backends += ['ucontext'] @@ -902,6 +909,10 @@ if get_option('tcg').allowed() if not get_option('tcg_interpreter') error('Unsupported CPU @0@, try --enable-tcg-interpreter'.format(cpu)) endif + elif host_arch == 'wasm32' + if not get_option('tcg_interpreter') + error('WebAssembly host requires --enable-tcg-interpreter') + endif elif get_option('tcg_interpreter') warning('Use of the TCG interpreter is not recommended on this host') warning('architecture. There is a native TCG execution backend available') @@ -2618,7 +2629,17 @@ config_host_data.set('CONFIG_FSTRIM', qga_fstrim) # has_header config_host_data.set('CONFIG_EPOLL', cc.has_header('sys/epoll.h')) config_host_data.set('CONFIG_LINUX_MAGIC_H', cc.has_header('linux/magic.h')) -config_host_data.set('CONFIG_VALGRIND_H', cc.has_header('valgrind/valgrind.h')) +valgrind = false +if get_option('valgrind').allowed() + if cc.has_header('valgrind/valgrind.h') + valgrind = true + else + if get_option('valgrind').enabled() + error('valgrind requested but valgrind.h not found') + endif + endif +endif +config_host_data.set('CONFIG_VALGRIND_H', valgrind) config_host_data.set('HAVE_BTRFS_H', cc.has_header('linux/btrfs.h')) config_host_data.set('HAVE_DRM_H', cc.has_header('libdrm/drm.h')) config_host_data.set('HAVE_OPENAT2_H', cc.has_header('linux/openat2.h')) @@ -2952,7 +2973,9 @@ config_host_data.set('CONFIG_ATOMIC64', cc.links(''' return 0; }''', args: qemu_isa_flags)) -has_int128_type = cc.compiles(''' +# has_int128_type is set to false on Emscripten to avoid errors by libffi +# during runtime. +has_int128_type = host_os != 'emscripten' and cc.compiles(''' __int128_t a; __uint128_t b; int main(void) { b = a; }''') @@ -3168,6 +3191,11 @@ if host_os == 'windows' }''', name: '_lock_file and _unlock_file')) endif +if spice.found() + config_host_data.set('HAVE_SPICE_QXL_GL_SCANOUT2', + cc.has_function('spice_qxl_gl_scanout2', dependencies: spice)) +endif + if host_os == 'windows' mingw_has_setjmp_longjmp = cc.links(''' #include <setjmp.h> @@ -3765,6 +3793,8 @@ if have_block # os-win32.c does not if host_os == 'windows' system_ss.add(files('os-win32.c')) + elif host_os == 'emscripten' + blockdev_ss.add(files('os-wasm.c')) else blockdev_ss.add(files('os-posix.c')) endif @@ -4506,7 +4536,11 @@ subdir('scripts') subdir('tools') subdir('pc-bios') subdir('docs') -subdir('tests') +# Tests are disabled on emscripten because they rely on host features that aren't +# supported by emscripten (e.g. fork and unix socket). +if host_os != 'emscripten' + subdir('tests') +endif if gtk.found() subdir('po') endif @@ -4609,7 +4643,6 @@ summary_info += {'Trace backends': ','.join(get_option('trace_backends'))} if 'simple' in get_option('trace_backends') summary_info += {'Trace output file': get_option('trace_file') + '-<pid>'} endif -summary_info += {'D-Bus display': dbus_display} summary_info += {'QOM debugging': get_option('qom_cast_debug')} summary_info += {'Relocatable install': get_option('relocatable')} summary_info += {'vhost-kernel support': have_vhost_kernel} @@ -4792,6 +4825,7 @@ summary_info = {} if host_os == 'darwin' summary_info += {'Cocoa support': cocoa} endif +summary_info += {'D-Bus display': dbus_display} summary_info += {'SDL support': sdl} summary_info += {'SDL image support': sdl_image} summary_info += {'GTK support': gtk} @@ -4905,6 +4939,7 @@ endif if host_os == 'darwin' summary_info += {'ParavirtualizedGraphics support': pvg} endif +summary_info += {'valgrind': valgrind} summary(summary_info, bool_yn: true, section: 'Dependencies') if host_arch == 'unknown' diff --git a/meson_options.txt b/meson_options.txt index 59d973bca0..cc66b46c63 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -34,7 +34,7 @@ option('fuzzing_engine', type : 'string', value : '', option('trace_file', type: 'string', value: 'trace', description: 'Trace file prefix for simple backend') option('coroutine_backend', type: 'combo', - choices: ['ucontext', 'sigaltstack', 'windows', 'auto'], + choices: ['ucontext', 'sigaltstack', 'windows', 'wasm', 'auto'], value: 'auto', description: 'coroutine backend to use') # Everything else can be set via --enable/--disable-* option @@ -115,6 +115,8 @@ option('dbus_display', type: 'feature', value: 'auto', description: '-display dbus support') option('tpm', type : 'feature', value : 'auto', description: 'TPM support') +option('valgrind', type : 'feature', value: 'auto', + description: 'valgrind debug support for coroutine stacks') # Do not enable it by default even for Mingw32, because it doesn't # work on Wine. diff --git a/migration/migration.c b/migration/migration.c index 55ec4bfab6..4697732bef 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -95,6 +95,9 @@ enum mig_rp_message_type { MIG_RP_MSG_MAX }; +/* Migration channel types */ +enum { CH_MAIN, CH_MULTIFD, CH_POSTCOPY }; + /* When we add fault tolerance, we could have several migrations at once. For now we don't need to add dynamic creation of migration */ @@ -259,6 +262,24 @@ migration_channels_and_transport_compatible(MigrationAddress *addr, return true; } +static bool +migration_capabilities_and_transport_compatible(MigrationAddress *addr, + Error **errp) +{ + if (addr->transport == MIGRATION_ADDRESS_TYPE_RDMA) { + return migrate_rdma_caps_check(migrate_get_current()->capabilities, + errp); + } + + return true; +} + +static bool migration_transport_compatible(MigrationAddress *addr, Error **errp) +{ + return migration_channels_and_transport_compatible(addr, errp) && + migration_capabilities_and_transport_compatible(addr, errp); +} + static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp) { uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp; @@ -750,7 +771,7 @@ static void qemu_start_incoming_migration(const char *uri, bool has_channels, } /* transport mechanism not suitable for migration? */ - if (!migration_channels_and_transport_compatible(addr, errp)) { + if (!migration_transport_compatible(addr, errp)) { return; } @@ -769,14 +790,6 @@ static void qemu_start_incoming_migration(const char *uri, bool has_channels, } #ifdef CONFIG_RDMA } else if (addr->transport == MIGRATION_ADDRESS_TYPE_RDMA) { - if (migrate_xbzrle()) { - error_setg(errp, "RDMA and XBZRLE can't be used together"); - return; - } - if (migrate_multifd()) { - error_setg(errp, "RDMA and multifd can't be used together"); - return; - } rdma_start_incoming_migration(&addr->u.rdma, errp); #endif } else if (addr->transport == MIGRATION_ADDRESS_TYPE_EXEC) { @@ -931,9 +944,8 @@ static void migration_incoming_setup(QEMUFile *f) { MigrationIncomingState *mis = migration_incoming_get_current(); - if (!mis->from_src_file) { - mis->from_src_file = f; - } + assert(!mis->from_src_file); + mis->from_src_file = f; qemu_file_set_blocking(f, false); } @@ -985,28 +997,19 @@ void migration_fd_process_incoming(QEMUFile *f) migration_incoming_process(); } -/* - * Returns true when we want to start a new incoming migration process, - * false otherwise. - */ -static bool migration_should_start_incoming(bool main_channel) +static bool migration_has_main_and_multifd_channels(void) { - /* Multifd doesn't start unless all channels are established */ - if (migrate_multifd()) { - return migration_has_all_channels(); + MigrationIncomingState *mis = migration_incoming_get_current(); + if (!mis->from_src_file) { + /* main channel not established */ + return false; } - /* Preempt channel only starts when the main channel is created */ - if (migrate_postcopy_preempt()) { - return main_channel; + if (migrate_multifd() && !multifd_recv_all_channels_created()) { + return false; } - /* - * For all the rest types of migration, we should only reach here when - * it's the main channel that's being created, and we should always - * proceed with this channel. - */ - assert(main_channel); + /* main and all multifd channels are established */ return true; } @@ -1015,59 +1018,81 @@ void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp) MigrationIncomingState *mis = migration_incoming_get_current(); Error *local_err = NULL; QEMUFile *f; - bool default_channel = true; + uint8_t channel; uint32_t channel_magic = 0; int ret = 0; - if (migrate_multifd() && !migrate_mapped_ram() && - !migrate_postcopy_ram() && - qio_channel_has_feature(ioc, QIO_CHANNEL_FEATURE_READ_MSG_PEEK)) { - /* - * With multiple channels, it is possible that we receive channels - * out of order on destination side, causing incorrect mapping of - * source channels on destination side. Check channel MAGIC to - * decide type of channel. Please note this is best effort, postcopy - * preempt channel does not send any magic number so avoid it for - * postcopy live migration. Also tls live migration already does - * tls handshake while initializing main channel so with tls this - * issue is not possible. - */ - ret = migration_channel_read_peek(ioc, (void *)&channel_magic, - sizeof(channel_magic), errp); + if (!migration_has_main_and_multifd_channels()) { + if (qio_channel_has_feature(ioc, QIO_CHANNEL_FEATURE_READ_MSG_PEEK)) { + /* + * With multiple channels, it is possible that we receive channels + * out of order on destination side, causing incorrect mapping of + * source channels on destination side. Check channel MAGIC to + * decide type of channel. Please note this is best effort, + * postcopy preempt channel does not send any magic number so + * avoid it for postcopy live migration. Also tls live migration + * already does tls handshake while initializing main channel so + * with tls this issue is not possible. + */ + ret = migration_channel_read_peek(ioc, (void *)&channel_magic, + sizeof(channel_magic), errp); + if (ret != 0) { + return; + } - if (ret != 0) { + channel_magic = be32_to_cpu(channel_magic); + if (channel_magic == QEMU_VM_FILE_MAGIC) { + channel = CH_MAIN; + } else if (channel_magic == MULTIFD_MAGIC) { + assert(migrate_multifd()); + channel = CH_MULTIFD; + } else if (!mis->from_src_file && + mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) { + /* reconnect main channel for postcopy recovery */ + channel = CH_MAIN; + } else { + error_setg(errp, "unknown channel magic: %u", channel_magic); + return; + } + } else if (mis->from_src_file && migrate_multifd()) { + /* + * Non-peekable channels like tls/file are processed as + * multifd channels when multifd is enabled. + */ + channel = CH_MULTIFD; + } else if (!mis->from_src_file) { + channel = CH_MAIN; + } else { + error_setg(errp, "non-peekable channel used without multifd"); return; } - - default_channel = (channel_magic == cpu_to_be32(QEMU_VM_FILE_MAGIC)); } else { - default_channel = !mis->from_src_file; + assert(migrate_postcopy_preempt()); + channel = CH_POSTCOPY; } if (multifd_recv_setup(errp) != 0) { return; } - if (default_channel) { + if (channel == CH_MAIN) { f = qemu_file_new_input(ioc); migration_incoming_setup(f); - } else { + } else if (channel == CH_MULTIFD) { /* Multiple connections */ - assert(migration_needs_multiple_sockets()); - if (migrate_multifd()) { - multifd_recv_new_channel(ioc, &local_err); - } else { - assert(migrate_postcopy_preempt()); - f = qemu_file_new_input(ioc); - postcopy_preempt_new_channel(mis, f); - } + multifd_recv_new_channel(ioc, &local_err); if (local_err) { error_propagate(errp, local_err); return; } + } else if (channel == CH_POSTCOPY) { + assert(!mis->postcopy_qemufile_dst); + f = qemu_file_new_input(ioc); + postcopy_preempt_new_channel(mis, f); + return; } - if (migration_should_start_incoming(default_channel)) { + if (migration_has_main_and_multifd_channels()) { /* If it's a recovery, we're done */ if (postcopy_try_recover()) { return; @@ -1084,18 +1109,13 @@ void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp) */ bool migration_has_all_channels(void) { - MigrationIncomingState *mis = migration_incoming_get_current(); - - if (!mis->from_src_file) { + if (!migration_has_main_and_multifd_channels()) { return false; } - if (migrate_multifd()) { - return multifd_recv_all_channels_created(); - } - - if (migrate_postcopy_preempt()) { - return mis->postcopy_qemufile_dst != NULL; + MigrationIncomingState *mis = migration_incoming_get_current(); + if (migrate_postcopy_preempt() && !mis->postcopy_qemufile_dst) { + return false; } return true; @@ -2208,7 +2228,7 @@ void qmp_migrate(const char *uri, bool has_channels, } /* transport mechanism not suitable for migration? */ - if (!migration_channels_and_transport_compatible(addr, errp)) { + if (!migration_transport_compatible(addr, errp)) { return; } @@ -2707,6 +2727,10 @@ static int postcopy_start(MigrationState *ms, Error **errp) } } + if (!qemu_savevm_state_postcopy_prepare(ms->to_dst_file, errp)) { + return -1; + } + trace_postcopy_start(); bql_lock(); trace_postcopy_start_set_run(); diff --git a/migration/multifd.c b/migration/multifd.c index 86c83e43c0..ec108af624 100644 --- a/migration/multifd.c +++ b/migration/multifd.c @@ -36,11 +36,6 @@ #include "io/channel-socket.h" #include "yank_functions.h" -/* Multiple fd's */ - -#define MULTIFD_MAGIC 0x11223344U -#define MULTIFD_VERSION 1 - typedef struct { uint32_t magic; uint32_t version; diff --git a/migration/multifd.h b/migration/multifd.h index 2d337e7b3b..9b6d81e7ed 100644 --- a/migration/multifd.h +++ b/migration/multifd.h @@ -49,6 +49,11 @@ bool multifd_queue_page(RAMBlock *block, ram_addr_t offset); bool multifd_recv(void); MultiFDRecvData *multifd_get_recv_data(void); +/* Multiple fd's */ + +#define MULTIFD_MAGIC 0x11223344U +#define MULTIFD_VERSION 1 + /* Multifd Compression flags */ #define MULTIFD_FLAG_SYNC (1 << 0) diff --git a/migration/options.c b/migration/options.c index b0ac2ea408..b6ae95358d 100644 --- a/migration/options.c +++ b/migration/options.c @@ -448,6 +448,24 @@ static bool migrate_incoming_started(void) return !!migration_incoming_get_current()->transport_data; } +bool migrate_rdma_caps_check(bool *caps, Error **errp) +{ + if (caps[MIGRATION_CAPABILITY_XBZRLE]) { + error_setg(errp, "RDMA and XBZRLE can't be used together"); + return false; + } + if (caps[MIGRATION_CAPABILITY_MULTIFD]) { + error_setg(errp, "RDMA and multifd can't be used together"); + return false; + } + if (caps[MIGRATION_CAPABILITY_POSTCOPY_RAM]) { + error_setg(errp, "RDMA and postcopy-ram can't be used together"); + return false; + } + + return true; +} + /** * @migration_caps_check - check capability compatibility * @@ -611,6 +629,13 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp) } } + /* + * On destination side, check the cases that capability is being set + * after incoming thread has started. + */ + if (migrate_rdma() && !migrate_rdma_caps_check(new_caps, errp)) { + return false; + } return true; } @@ -1193,6 +1218,11 @@ static void migrate_params_test_apply(MigrateSetParameters *params, dest->tls_hostname = params->tls_hostname->u.s; } + if (params->tls_authz) { + assert(params->tls_authz->type == QTYPE_QSTRING); + dest->tls_authz = params->tls_authz->u.s; + } + if (params->has_max_bandwidth) { dest->max_bandwidth = params->max_bandwidth; } diff --git a/migration/options.h b/migration/options.h index 762be4e641..82d839709e 100644 --- a/migration/options.h +++ b/migration/options.h @@ -57,6 +57,7 @@ bool migrate_tls(void); /* capabilities helpers */ +bool migrate_rdma_caps_check(bool *caps, Error **errp); bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp); /* parameters */ diff --git a/migration/ram.c b/migration/ram.c index cb8b2ed493..e12913b43e 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -91,6 +91,36 @@ XBZRLECacheStats xbzrle_counters; +/* + * This structure locates a specific location of a guest page. In QEMU, + * it's described in a tuple of (ramblock, offset). + */ +struct PageLocation { + RAMBlock *block; + unsigned long offset; +}; +typedef struct PageLocation PageLocation; + +/** + * PageLocationHint: describes a hint to a page location + * + * @valid set if the hint is vaild and to be consumed + * @location: the hint content + * + * In postcopy preempt mode, the urgent channel may provide hints to the + * background channel, so that QEMU source can try to migrate whatever is + * right after the requested urgent pages. + * + * This is based on the assumption that the VM (already running on the + * destination side) tends to access the memory with spatial locality. + * This is also the default behavior of vanilla postcopy (preempt off). + */ +struct PageLocationHint { + bool valid; + PageLocation location; +}; +typedef struct PageLocationHint PageLocationHint; + /* used by the search for pages to send */ struct PageSearchStatus { /* The migration channel used for a specific host page */ @@ -395,6 +425,13 @@ struct RAMState { * RAM migration. */ unsigned int postcopy_bmap_sync_requested; + /* + * Page hint during postcopy when preempt mode is on. Return path + * thread sets it, while background migration thread consumes it. + * + * Protected by @bitmap_mutex. + */ + PageLocationHint page_hint; }; typedef struct RAMState RAMState; @@ -1144,32 +1181,6 @@ static int save_zero_page(RAMState *rs, PageSearchStatus *pss, } /* - * @pages: the number of pages written by the control path, - * < 0 - error - * > 0 - number of pages written - * - * Return true if the pages has been saved, otherwise false is returned. - */ -static bool control_save_page(PageSearchStatus *pss, - ram_addr_t offset, int *pages) -{ - int ret; - - ret = rdma_control_save_page(pss->pss_channel, pss->block->offset, offset, - TARGET_PAGE_SIZE); - if (ret == RAM_SAVE_CONTROL_NOT_SUPP) { - return false; - } - - if (ret == RAM_SAVE_CONTROL_DELAYED) { - *pages = 1; - return true; - } - *pages = ret; - return true; -} - -/* * directly send the page to the stream * * Returns the number of pages written. @@ -1965,7 +1976,13 @@ static int ram_save_target_page(RAMState *rs, PageSearchStatus *pss) int res; /* Hand over to RDMA first */ - if (control_save_page(pss, offset, &res)) { + if (migrate_rdma()) { + res = rdma_control_save_page(pss->pss_channel, pss->block->offset, + offset, TARGET_PAGE_SIZE); + + if (res == RAM_SAVE_CONTROL_DELAYED) { + res = 1; + } return res; } @@ -2039,6 +2056,21 @@ static void pss_host_page_finish(PageSearchStatus *pss) pss->host_page_start = pss->host_page_end = 0; } +static void ram_page_hint_update(RAMState *rs, PageSearchStatus *pss) +{ + PageLocationHint *hint = &rs->page_hint; + + /* If there's a pending hint not consumed, don't bother */ + if (hint->valid) { + return; + } + + /* Provide a hint to the background stream otherwise */ + hint->location.block = pss->block; + hint->location.offset = pss->page; + hint->valid = true; +} + /* * Send an urgent host page specified by `pss'. Need to be called with * bitmap_mutex held. @@ -2084,6 +2116,7 @@ out: /* For urgent requests, flush immediately if sent */ if (sent) { qemu_fflush(pss->pss_channel); + ram_page_hint_update(rs, pss); } return ret; } @@ -2171,6 +2204,30 @@ static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss) return (res < 0 ? res : pages); } +static bool ram_page_hint_valid(RAMState *rs) +{ + /* There's only page hint during postcopy preempt mode */ + if (!postcopy_preempt_active()) { + return false; + } + + return rs->page_hint.valid; +} + +static void ram_page_hint_collect(RAMState *rs, RAMBlock **block, + unsigned long *page) +{ + PageLocationHint *hint = &rs->page_hint; + + assert(hint->valid); + + *block = hint->location.block; + *page = hint->location.offset; + + /* Mark the hint consumed */ + hint->valid = false; +} + /** * ram_find_and_save_block: finds a dirty page and sends it to f * @@ -2187,6 +2244,8 @@ static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss) static int ram_find_and_save_block(RAMState *rs) { PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY]; + unsigned long next_page; + RAMBlock *next_block; int pages = 0; /* No dirty page as there is zero RAM */ @@ -2206,7 +2265,14 @@ static int ram_find_and_save_block(RAMState *rs) rs->last_page = 0; } - pss_init(pss, rs->last_seen_block, rs->last_page); + if (ram_page_hint_valid(rs)) { + ram_page_hint_collect(rs, &next_block, &next_page); + } else { + next_block = rs->last_seen_block; + next_page = rs->last_page; + } + + pss_init(pss, next_block, next_page); while (true){ if (!get_queued_page(rs, pss)) { @@ -2339,6 +2405,13 @@ static void ram_save_cleanup(void *opaque) ram_state_cleanup(rsp); } +static void ram_page_hint_reset(PageLocationHint *hint) +{ + hint->location.block = NULL; + hint->location.offset = 0; + hint->valid = false; +} + static void ram_state_reset(RAMState *rs) { int i; @@ -2351,6 +2424,8 @@ static void ram_state_reset(RAMState *rs) rs->last_page = 0; rs->last_version = ram_list.version; rs->xbzrle_started = false; + + ram_page_hint_reset(&rs->page_hint); } #define MAX_WAIT 50 /* ms, half buffered_file limit */ @@ -4418,6 +4493,42 @@ static int ram_resume_prepare(MigrationState *s, void *opaque) return 0; } +static bool ram_save_postcopy_prepare(QEMUFile *f, void *opaque, Error **errp) +{ + int ret; + + if (migrate_multifd()) { + /* + * When multifd is enabled, source QEMU needs to make sure all the + * pages queued before postcopy starts have been flushed. + * + * The load of these pages must happen before switching to postcopy. + * It's because loading of guest pages (so far) in multifd recv + * threads is still non-atomic, so the load cannot happen with vCPUs + * running on the destination side. + * + * This flush and sync will guarantee that those pages are loaded + * _before_ postcopy starts on the destination. The rationale is, + * this happens before VM stops (and before source QEMU sends all + * the rest of the postcopy messages). So when the destination QEMU + * receives the postcopy messages, it must have received the sync + * message on the main channel (either RAM_SAVE_FLAG_MULTIFD_FLUSH, + * or RAM_SAVE_FLAG_EOS), and such message would guarantee that + * all previous guest pages queued in the multifd channels are + * completely loaded. + */ + ret = multifd_ram_flush_and_sync(f); + if (ret < 0) { + error_setg(errp, "%s: multifd flush and sync failed", __func__); + return false; + } + } + + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); + + return true; +} + void postcopy_preempt_shutdown_file(MigrationState *s) { qemu_put_be64(s->postcopy_qemufile_src, RAM_SAVE_FLAG_EOS); @@ -4437,6 +4548,7 @@ static SaveVMHandlers savevm_ram_handlers = { .load_setup = ram_load_setup, .load_cleanup = ram_load_cleanup, .resume_prepare = ram_resume_prepare, + .save_postcopy_prepare = ram_save_postcopy_prepare, }; static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host, diff --git a/migration/rdma.c b/migration/rdma.c index b31652baac..2d839fce6c 100644 --- a/migration/rdma.c +++ b/migration/rdma.c @@ -768,156 +768,12 @@ static void qemu_rdma_dump_gid(const char *who, struct rdma_cm_id *id) } /* - * As of now, IPv6 over RoCE / iWARP is not supported by linux. - * We will try the next addrinfo struct, and fail if there are - * no other valid addresses to bind against. - * - * If user is listening on '[::]', then we will not have a opened a device - * yet and have no way of verifying if the device is RoCE or not. - * - * In this case, the source VM will throw an error for ALL types of - * connections (both IPv4 and IPv6) if the destination machine does not have - * a regular infiniband network available for use. - * - * The only way to guarantee that an error is thrown for broken kernels is - * for the management software to choose a *specific* interface at bind time - * and validate what time of hardware it is. - * - * Unfortunately, this puts the user in a fix: - * - * If the source VM connects with an IPv4 address without knowing that the - * destination has bound to '[::]' the migration will unconditionally fail - * unless the management software is explicitly listening on the IPv4 - * address while using a RoCE-based device. - * - * If the source VM connects with an IPv6 address, then we're OK because we can - * throw an error on the source (and similarly on the destination). - * - * But in mixed environments, this will be broken for a while until it is fixed - * inside linux. - * - * We do provide a *tiny* bit of help in this function: We can list all of the - * devices in the system and check to see if all the devices are RoCE or - * Infiniband. - * - * If we detect that we have a *pure* RoCE environment, then we can safely - * thrown an error even if the management software has specified '[::]' as the - * bind address. - * - * However, if there is are multiple hetergeneous devices, then we cannot make - * this assumption and the user just has to be sure they know what they are - * doing. - * - * Patches are being reviewed on linux-rdma. - */ -static int qemu_rdma_broken_ipv6_kernel(struct ibv_context *verbs, Error **errp) -{ - /* This bug only exists in linux, to our knowledge. */ -#ifdef CONFIG_LINUX - struct ibv_port_attr port_attr; - - /* - * Verbs are only NULL if management has bound to '[::]'. - * - * Let's iterate through all the devices and see if there any pure IB - * devices (non-ethernet). - * - * If not, then we can safely proceed with the migration. - * Otherwise, there are no guarantees until the bug is fixed in linux. - */ - if (!verbs) { - int num_devices; - struct ibv_device **dev_list = ibv_get_device_list(&num_devices); - bool roce_found = false; - bool ib_found = false; - - for (int x = 0; x < num_devices; x++) { - verbs = ibv_open_device(dev_list[x]); - /* - * ibv_open_device() is not documented to set errno. If - * it does, it's somebody else's doc bug. If it doesn't, - * the use of errno below is wrong. - * TODO Find out whether ibv_open_device() sets errno. - */ - if (!verbs) { - if (errno == EPERM) { - continue; - } else { - error_setg_errno(errp, errno, - "could not open RDMA device context"); - return -1; - } - } - - if (ibv_query_port(verbs, 1, &port_attr)) { - ibv_close_device(verbs); - error_setg(errp, - "RDMA ERROR: Could not query initial IB port"); - return -1; - } - - if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { - ib_found = true; - } else if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { - roce_found = true; - } - - ibv_close_device(verbs); - - } - - if (roce_found) { - if (ib_found) { - warn_report("migrations may fail:" - " IPv6 over RoCE / iWARP in linux" - " is broken. But since you appear to have a" - " mixed RoCE / IB environment, be sure to only" - " migrate over the IB fabric until the kernel " - " fixes the bug."); - } else { - error_setg(errp, "RDMA ERROR: " - "You only have RoCE / iWARP devices in your systems" - " and your management software has specified '[::]'" - ", but IPv6 over RoCE / iWARP is not supported in Linux."); - return -1; - } - } - - return 0; - } - - /* - * If we have a verbs context, that means that some other than '[::]' was - * used by the management software for binding. In which case we can - * actually warn the user about a potentially broken kernel. - */ - - /* IB ports start with 1, not 0 */ - if (ibv_query_port(verbs, 1, &port_attr)) { - error_setg(errp, "RDMA ERROR: Could not query initial IB port"); - return -1; - } - - if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { - error_setg(errp, "RDMA ERROR: " - "Linux kernel's RoCE / iWARP does not support IPv6 " - "(but patches on linux-rdma in progress)"); - return -1; - } - -#endif - - return 0; -} - -/* * Figure out which RDMA device corresponds to the requested IP hostname * Also create the initial connection manager identifiers for opening * the connection. */ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp) { - Error *err = NULL; int ret; struct rdma_addrinfo *res; char port_str[16]; @@ -953,9 +809,8 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp) goto err_resolve_get_addr; } - /* Try all addresses, saving the first error in @err */ + /* Try all addresses, exit loop on first success of resolving address */ for (struct rdma_addrinfo *e = res; e != NULL; e = e->ai_next) { - Error **local_errp = err ? NULL : &err; inet_ntop(e->ai_family, &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip); @@ -964,25 +819,12 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp) ret = rdma_resolve_addr(rdma->cm_id, NULL, e->ai_dst_addr, RDMA_RESOLVE_TIMEOUT_MS); if (ret >= 0) { - if (e->ai_family == AF_INET6) { - ret = qemu_rdma_broken_ipv6_kernel(rdma->cm_id->verbs, - local_errp); - if (ret < 0) { - continue; - } - } - error_free(err); goto route; } } rdma_freeaddrinfo(res); - if (err) { - error_propagate(errp, err); - } else { - error_setg(errp, "RDMA ERROR: could not resolve address %s", - rdma->host); - } + error_setg(errp, "RDMA ERROR: could not resolve address %s", rdma->host); goto err_resolve_get_addr; route: @@ -2611,7 +2453,6 @@ err_rdma_source_connect: static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp) { - Error *err = NULL; int ret; struct rdma_cm_id *listen_id; char ip[40] = "unknown"; @@ -2661,9 +2502,8 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp) goto err_dest_init_bind_addr; } - /* Try all addresses, saving the first error in @err */ + /* Try all addresses */ for (e = res; e != NULL; e = e->ai_next) { - Error **local_errp = err ? NULL : &err; inet_ntop(e->ai_family, &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip); @@ -2672,24 +2512,12 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp) if (ret < 0) { continue; } - if (e->ai_family == AF_INET6) { - ret = qemu_rdma_broken_ipv6_kernel(listen_id->verbs, - local_errp); - if (ret < 0) { - continue; - } - } - error_free(err); break; } rdma_freeaddrinfo(res); if (!e) { - if (err) { - error_propagate(errp, err); - } else { - error_setg(errp, "RDMA ERROR: Error: could not rdma_bind_addr!"); - } + error_setg(errp, "RDMA ERROR: Error: could not rdma_bind_addr!"); goto err_dest_init_bind_addr; } @@ -3284,14 +3112,11 @@ err: int rdma_control_save_page(QEMUFile *f, ram_addr_t block_offset, ram_addr_t offset, size_t size) { - if (!migrate_rdma() || migration_in_postcopy()) { - return RAM_SAVE_CONTROL_NOT_SUPP; - } + assert(migrate_rdma()); int ret = qemu_rdma_save_page(f, block_offset, offset, size); - if (ret != RAM_SAVE_CONTROL_DELAYED && - ret != RAM_SAVE_CONTROL_NOT_SUPP) { + if (ret != RAM_SAVE_CONTROL_DELAYED) { if (ret < 0) { qemu_file_set_error(f, ret); } @@ -3829,7 +3654,7 @@ int rdma_block_notification_handle(QEMUFile *f, const char *name) int rdma_registration_start(QEMUFile *f, uint64_t flags) { - if (!migrate_rdma() || migration_in_postcopy()) { + if (!migrate_rdma()) { return 0; } @@ -3861,7 +3686,7 @@ int rdma_registration_stop(QEMUFile *f, uint64_t flags) RDMAControlHeader head = { .len = 0, .repeat = 1 }; int ret; - if (!migrate_rdma() || migration_in_postcopy()) { + if (!migrate_rdma()) { return 0; } diff --git a/migration/rdma.h b/migration/rdma.h index 4d3386b84a..f74f16a459 100644 --- a/migration/rdma.h +++ b/migration/rdma.h @@ -33,7 +33,6 @@ void rdma_start_incoming_migration(InetSocketAddress *host_port, Error **errp); #define RAM_CONTROL_ROUND 1 #define RAM_CONTROL_FINISH 3 -#define RAM_SAVE_CONTROL_NOT_SUPP -1000 #define RAM_SAVE_CONTROL_DELAYED -2000 #ifdef CONFIG_RDMA @@ -56,7 +55,7 @@ static inline int rdma_control_save_page(QEMUFile *f, ram_addr_t block_offset, ram_addr_t offset, size_t size) { - return RAM_SAVE_CONTROL_NOT_SUPP; + g_assert_not_reached(); } #endif #endif diff --git a/migration/savevm.c b/migration/savevm.c index 0c12e373b4..006514c3e3 100644 --- a/migration/savevm.c +++ b/migration/savevm.c @@ -1523,6 +1523,39 @@ void qemu_savevm_state_complete_postcopy(QEMUFile *f) qemu_fflush(f); } +bool qemu_savevm_state_postcopy_prepare(QEMUFile *f, Error **errp) +{ + SaveStateEntry *se; + bool ret; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->ops || !se->ops->save_postcopy_prepare) { + continue; + } + + if (se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } + + trace_savevm_section_start(se->idstr, se->section_id); + + save_section_header(f, se, QEMU_VM_SECTION_PART); + ret = se->ops->save_postcopy_prepare(f, se->opaque, errp); + save_section_footer(f, se); + + trace_savevm_section_end(se->idstr, se->section_id, ret); + + if (!ret) { + assert(*errp); + return false; + } + } + + return true; +} + int qemu_savevm_state_complete_precopy_iterable(QEMUFile *f, bool in_postcopy) { int64_t start_ts_each, end_ts_each; diff --git a/migration/savevm.h b/migration/savevm.h index 138c39a7f9..2d5e9c7166 100644 --- a/migration/savevm.h +++ b/migration/savevm.h @@ -45,6 +45,7 @@ void qemu_savevm_state_pending_exact(uint64_t *must_precopy, void qemu_savevm_state_pending_estimate(uint64_t *must_precopy, uint64_t *can_postcopy); int qemu_savevm_state_complete_precopy_iterable(QEMUFile *f, bool in_postcopy); +bool qemu_savevm_state_postcopy_prepare(QEMUFile *f, Error **errp); void qemu_savevm_send_ping(QEMUFile *f, uint32_t value); void qemu_savevm_send_open_return_path(QEMUFile *f); int qemu_savevm_send_packaged(QEMUFile *f, const uint8_t *buf, size_t len); diff --git a/monitor/qmp.c b/monitor/qmp.c index 2f46cf9e49..cb99a12d94 100644 --- a/monitor/qmp.c +++ b/monitor/qmp.c @@ -356,7 +356,8 @@ void qmp_dispatcher_co_wake(void) /* Write request before reading qmp_dispatcher_co_busy. */ smp_mb__before_rmw(); - if (!qatomic_xchg(&qmp_dispatcher_co_busy, true)) { + if (!qatomic_xchg(&qmp_dispatcher_co_busy, true) && + qatomic_read(&qmp_dispatcher_co)) { aio_co_wake(qmp_dispatcher_co); } } diff --git a/os-wasm.c b/os-wasm.c new file mode 100644 index 0000000000..d240c180c5 --- /dev/null +++ b/os-wasm.c @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: MIT */ +/* + * os-wasm.c + * Forked from os-posix.c, removing functions not working on Emscripten + * + * Copyright (c) 2003-2008 Fabrice Bellard + * Copyright (c) 2010 Red Hat, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "qemu/osdep.h" +#include <sys/resource.h> +#include <sys/wait.h> +#include <pwd.h> +#include <grp.h> +#include <libgen.h> + +#include "qemu/error-report.h" +#include "qemu/log.h" +#include "system/runstate.h" +#include "qemu/cutils.h" + +void os_setup_post(void){} +void os_set_line_buffering(void) +{ + setvbuf(stdout, NULL, _IOLBF, 0); +} +void os_setup_early_signal_handling(void) +{ + struct sigaction act; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &act, NULL); +} +void os_set_proc_name(const char *s) +{ + error_report("Change of process name not supported by your OS"); + exit(1); +} +static void termsig_handler(int signal, siginfo_t *info, void *c) +{ + qemu_system_killed(info->si_signo, info->si_pid); +} + +void os_setup_signal_handling(void) +{ + struct sigaction act; + + memset(&act, 0, sizeof(act)); + act.sa_sigaction = termsig_handler; + act.sa_flags = SA_SIGINFO; + sigaction(SIGINT, &act, NULL); + sigaction(SIGHUP, &act, NULL); + sigaction(SIGTERM, &act, NULL); +} +void os_setup_limits(void) +{ + struct rlimit nofile; + + if (getrlimit(RLIMIT_NOFILE, &nofile) < 0) { + warn_report("unable to query NOFILE limit: %s", strerror(errno)); + return; + } + + if (nofile.rlim_cur == nofile.rlim_max) { + return; + } + + nofile.rlim_cur = nofile.rlim_max; + + if (setrlimit(RLIMIT_NOFILE, &nofile) < 0) { + warn_report("unable to set NOFILE limit: %s", strerror(errno)); + return; + } +} +int os_mlock(bool on_fault) +{ +#ifdef HAVE_MLOCKALL + int ret = 0; + int flags = MCL_CURRENT | MCL_FUTURE; + + if (on_fault) { +#ifdef HAVE_MLOCK_ONFAULT + flags |= MCL_ONFAULT; +#else + error_report("mlockall: on_fault not supported"); + return -EINVAL; +#endif + } + + ret = mlockall(flags); + if (ret < 0) { + error_report("mlockall: %s", strerror(errno)); + } + + return ret; +#else + (void)on_fault; + return -ENOSYS; +#endif +} diff --git a/pc-bios/README b/pc-bios/README index f0f13e15f2..d009c37895 100644 --- a/pc-bios/README +++ b/pc-bios/README @@ -89,6 +89,12 @@ more features over time as needed. The source code is available at: https://github.com/google/vbootrom +- ast27x0_bootrom.bin is a simplified, free (Apache 2.0) boot ROM for + ASPEED AST27x0 BMC SOC. It currently implements the bare minimum to + load, parse, initialize and run boot images stored in SPI flash, but may grow + more features over time as needed. The source code is available at: + https://github.com/google/vbootrom + - hppa-firmware.img (32-bit) and hppa-firmware64.img (64-bit) are firmware files for the HP-PARISC (hppa) architecture. They are built form the SeaBIOS-hppa sources, which is a fork of SeaBIOS diff --git a/pc-bios/ast27x0_bootrom.bin b/pc-bios/ast27x0_bootrom.bin new file mode 100644 index 0000000000..0b9b3a2360 --- /dev/null +++ b/pc-bios/ast27x0_bootrom.bin Binary files differdiff --git a/pc-bios/meson.build b/pc-bios/meson.build index 34d8cc4f33..79bb2e1800 100644 --- a/pc-bios/meson.build +++ b/pc-bios/meson.build @@ -28,6 +28,7 @@ if unpack_edk2_blobs endif blobs = [ + 'ast27x0_bootrom.bin', 'bios.bin', 'bios-256k.bin', 'bios-microvm.bin', diff --git a/python/Makefile b/python/Makefile index 1fa4ba2498..764b79ccb2 100644 --- a/python/Makefile +++ b/python/Makefile @@ -9,13 +9,13 @@ help: @echo "make check-minreqs:" @echo " Run tests in the minreqs virtual environment." @echo " These tests use the oldest dependencies." - @echo " Requires: Python 3.8" - @echo " Hint (Fedora): 'sudo dnf install python3.8'" + @echo " Requires: Python 3.9" + @echo " Hint (Fedora): 'sudo dnf install python3.9'" @echo "" @echo "make check-tox:" @echo " Run tests against multiple python versions." @echo " These tests use the newest dependencies." - @echo " Requires: Python 3.8 - 3.11, and tox." + @echo " Requires: Python 3.9 - 3.11, and tox." @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.11'" @echo " The variable QEMU_TOX_EXTRA_ARGS can be use to pass extra" @echo " arguments to tox". @@ -59,7 +59,7 @@ PIP_INSTALL = pip install --disable-pip-version-check min-venv: $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate: setup.cfg tests/minreqs.txt @echo "VENV $(QEMU_MINVENV_DIR)" - @python3.8 -m venv $(QEMU_MINVENV_DIR) + @python3.9 -m venv $(QEMU_MINVENV_DIR) @( \ echo "ACTIVATE $(QEMU_MINVENV_DIR)"; \ . $(QEMU_MINVENV_DIR)/bin/activate; \ diff --git a/python/setup.cfg b/python/setup.cfg index cf5af7e664..c48dff280a 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -14,7 +14,6 @@ classifiers = Natural Language :: English Operating System :: OS Independent Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 @@ -23,7 +22,7 @@ classifiers = Typing :: Typed [options] -python_requires = >= 3.8 +python_requires = >= 3.9 packages = qemu.qmp qemu.machine @@ -78,7 +77,7 @@ exclude = __pycache__, [mypy] strict = True -python_version = 3.8 +python_version = 3.9 warn_unused_configs = True namespace_packages = True warn_unused_ignores = False @@ -186,7 +185,7 @@ multi_line_output=3 # of python available on your system to run this test. [tox:tox] -envlist = py38, py39, py310, py311, py312, py313 +envlist = py39, py310, py311, py312, py313 skip_missing_interpreters = true [testenv] diff --git a/python/tests/minreqs.txt b/python/tests/minreqs.txt index a3f423efd8..6445407ba8 100644 --- a/python/tests/minreqs.txt +++ b/python/tests/minreqs.txt @@ -1,5 +1,5 @@ # This file lists the ***oldest possible dependencies*** needed to run -# "make check" successfully under ***Python 3.8***. It is used primarily +# "make check" successfully under ***Python 3.9***. It is used primarily # by GitLab CI to ensure that our stated minimum versions in setup.cfg # are truthful and regularly validated. # diff --git a/qapi/machine-target.json b/qapi/machine-target.json index 541f93eeb7..426ce4ee82 100644 --- a/qapi/machine-target.json +++ b/qapi/machine-target.json @@ -350,7 +350,7 @@ # migration-safe in the future (since 4.1) # # @deprecated: If true, this CPU model is deprecated and may be -# removed in in some future version of QEMU according to the QEMU +# removed in some future version of QEMU according to the QEMU # deprecation policy. (since 5.2) # # @unavailable-features is a list of QOM property names that represent diff --git a/qapi/qom.json b/qapi/qom.json index 28ce24cd8d..04c118e4d6 100644 --- a/qapi/qom.json +++ b/qapi/qom.json @@ -871,7 +871,7 @@ # link characteristics read from PCIe Configuration space. # To get the full path latency from CPU to CXL attached DRAM # CXL device: Add the latency from CPU to Generic Port (from -# HMAT indexed via the the node ID in this SRAT structure) to +# HMAT indexed via the node ID in this SRAT structure) to # that for CXL bus links, the latency across intermediate switches # and from the EP port to the actual memory. Bandwidth is more # complex as there may be interleaving across multiple devices diff --git a/qemu-options.hx b/qemu-options.hx index dc694a99a3..aab53bcfe8 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -4862,7 +4862,7 @@ SRST Start right away with a saved state (``loadvm`` in monitor) ERST -#ifndef _WIN32 +#if !defined(_WIN32) && !defined(EMSCRIPTEN) DEF("daemonize", 0, QEMU_OPTION_daemonize, \ "-daemonize daemonize QEMU after initializing\n", QEMU_ARCH_ALL) #endif @@ -5249,7 +5249,7 @@ HXCOMM Internal use DEF("qtest", HAS_ARG, QEMU_OPTION_qtest, "", QEMU_ARCH_ALL) DEF("qtest-log", HAS_ARG, QEMU_OPTION_qtest_log, "", QEMU_ARCH_ALL) -#ifdef CONFIG_POSIX +#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) DEF("run-with", HAS_ARG, QEMU_OPTION_run_with, "-run-with [async-teardown=on|off][,chroot=dir][user=username|uid:gid]\n" " Set miscellaneous QEMU process lifecycle options:\n" diff --git a/qom/object.c b/qom/object.c index 664f0f24ae..7b013f40a0 100644 --- a/qom/object.c +++ b/qom/object.c @@ -485,7 +485,7 @@ bool object_apply_global_props(Object *obj, const GPtrArray *props, * Slot 0: accelerator's global property defaults * Slot 1: machine's global property defaults * Slot 2: global properties from legacy command line option - * Each is a GPtrArray of of GlobalProperty. + * Each is a GPtrArray of GlobalProperty. * Applied in order, later entries override earlier ones. */ static GPtrArray *object_compat_props[3]; diff --git a/qom/qom-qmp-cmds.c b/qom/qom-qmp-cmds.c index e866547618..293755f409 100644 --- a/qom/qom-qmp-cmds.c +++ b/qom/qom-qmp-cmds.c @@ -28,15 +28,11 @@ #include "qom/object_interfaces.h" #include "qom/qom-qobject.h" -ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp) +static Object *qom_resolve_path(const char *path, Error **errp) { - Object *obj; bool ambiguous = false; - ObjectPropertyInfoList *props = NULL; - ObjectProperty *prop; - ObjectPropertyIterator iter; + Object *obj = object_resolve_path(path, &ambiguous); - obj = object_resolve_path(path, &ambiguous); if (obj == NULL) { if (ambiguous) { error_setg(errp, "Path '%s' is ambiguous", path); @@ -44,6 +40,19 @@ ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp) error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND, "Device '%s' not found", path); } + } + return obj; +} + +ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp) +{ + Object *obj; + ObjectPropertyInfoList *props = NULL; + ObjectProperty *prop; + ObjectPropertyIterator iter; + + obj = qom_resolve_path(path, errp); + if (obj == NULL) { return NULL; } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 2ebf0a11ea..13d580c693 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -108,7 +108,6 @@ version = "0.1.0" dependencies = [ "libc", "qemu_api_macros", - "version_check", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index ab1185a814..d9faeecb10 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -7,10 +7,17 @@ members = [ "hw/timer/hpet", ] +[workspace.package] +edition = "2021" +homepage = "https://www.qemu.org" +license = "GPL-2.0-or-later" +repository = "https://gitlab.com/qemu-project/qemu/" +rust-version = "1.77.0" + [workspace.lints.rust] unexpected_cfgs = { level = "deny", check-cfg = [ 'cfg(MESON)', 'cfg(HAVE_GLIB_WITH_ALIGNED_ALLOC)', - 'cfg(has_offset_of)'] } +] } # Occasionally, we may need to silence warnings and clippy lints that # were only introduced in newer Rust compiler versions. Do not croak @@ -64,6 +71,7 @@ no_effect_underscore_binding = "deny" option_option = "deny" or_fun_call = "deny" ptr_as_ptr = "deny" +ptr_cast_constness = "deny" pub_underscore_fields = "deny" redundant_clone = "deny" redundant_closure_for_method_calls = "deny" @@ -81,11 +89,11 @@ suspicious_operation_groupings = "deny" transmute_ptr_to_ptr = "deny" transmute_undefined_repr = "deny" type_repetition_in_bounds = "deny" +uninlined_format_args = "deny" used_underscore_binding = "deny" # nice to have, but cannot be enabled yet #wildcard_imports = "deny" # still have many bindings::* imports -#ptr_cast_constness = "deny" # needs 1.65.0 for cast_mut()/cast_const() # these may have false positives #option_if_let_else = "deny" diff --git a/rust/clippy.toml b/rust/clippy.toml index 5d190f91de..58a62c0e63 100644 --- a/rust/clippy.toml +++ b/rust/clippy.toml @@ -1,2 +1,3 @@ doc-valid-idents = ["PrimeCell", ".."] -msrv = "1.63.0" +allow-mixed-uninlined-format-args = false +msrv = "1.77.0" diff --git a/rust/hw/char/pl011/Cargo.toml b/rust/hw/char/pl011/Cargo.toml index f2296cad58..a1f431ab4a 100644 --- a/rust/hw/char/pl011/Cargo.toml +++ b/rust/hw/char/pl011/Cargo.toml @@ -1,15 +1,16 @@ [package] name = "pl011" version = "0.1.0" -edition = "2021" authors = ["Manos Pitsidianakis <manos.pitsidianakis@linaro.org>"] -license = "GPL-2.0-or-later" description = "pl011 device model for QEMU" resolver = "2" publish = false -keywords = [] -categories = [] -rust-version = "1.63.0" + +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true [lib] crate-type = ["staticlib"] diff --git a/rust/hw/char/pl011/src/device.rs b/rust/hw/char/pl011/src/device.rs index bb2a0f207a..7c563ade9c 100644 --- a/rust/hw/char/pl011/src/device.rs +++ b/rust/hw/char/pl011/src/device.rs @@ -74,7 +74,7 @@ impl std::ops::Index<u32> for Fifo { } #[repr(C)] -#[derive(Debug, Default, qemu_api_macros::offsets)] +#[derive(Debug, Default)] pub struct PL011Registers { #[doc(alias = "fr")] pub flags: registers::Flags, @@ -98,7 +98,7 @@ pub struct PL011Registers { } #[repr(C)] -#[derive(qemu_api_macros::Object, qemu_api_macros::offsets)] +#[derive(qemu_api_macros::Object)] /// PL011 Device Model in QEMU pub struct PL011State { pub parent_obj: ParentField<SysBusDevice>, diff --git a/rust/hw/char/pl011/src/device_class.rs b/rust/hw/char/pl011/src/device_class.rs index b4d4a7eb43..d328d84632 100644 --- a/rust/hw/char/pl011/src/device_class.rs +++ b/rust/hw/char/pl011/src/device_class.rs @@ -3,13 +3,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later use std::{ - os::raw::{c_int, c_void}, + ffi::{c_int, c_void}, ptr::NonNull, }; use qemu_api::{ bindings::{qdev_prop_bool, qdev_prop_chr}, - c_str, prelude::*, vmstate::VMStateDescription, vmstate_clock, vmstate_fields, vmstate_of, vmstate_struct, vmstate_subsections, vmstate_unused, @@ -25,7 +24,7 @@ extern "C" fn pl011_clock_needed(opaque: *mut c_void) -> bool { /// Migration subsection for [`PL011State`] clock. static VMSTATE_PL011_CLOCK: VMStateDescription = VMStateDescription { - name: c_str!("pl011/clock").as_ptr(), + name: c"pl011/clock".as_ptr(), version_id: 1, minimum_version_id: 1, needed: Some(pl011_clock_needed), @@ -46,7 +45,7 @@ extern "C" fn pl011_post_load(opaque: *mut c_void, version_id: c_int) -> c_int { } static VMSTATE_PL011_REGS: VMStateDescription = VMStateDescription { - name: c_str!("pl011/regs").as_ptr(), + name: c"pl011/regs".as_ptr(), version_id: 2, minimum_version_id: 2, fields: vmstate_fields! { @@ -70,7 +69,7 @@ static VMSTATE_PL011_REGS: VMStateDescription = VMStateDescription { }; pub static VMSTATE_PL011: VMStateDescription = VMStateDescription { - name: c_str!("pl011").as_ptr(), + name: c"pl011".as_ptr(), version_id: 2, minimum_version_id: 2, post_load: Some(pl011_post_load), @@ -87,14 +86,14 @@ pub static VMSTATE_PL011: VMStateDescription = VMStateDescription { qemu_api::declare_properties! { PL011_PROPERTIES, qemu_api::define_property!( - c_str!("chardev"), + c"chardev", PL011State, char_backend, unsafe { &qdev_prop_chr }, CharBackend ), qemu_api::define_property!( - c_str!("migrate-clk"), + c"migrate-clk", PL011State, migrate_clock, unsafe { &qdev_prop_bool }, diff --git a/rust/hw/char/pl011/src/lib.rs b/rust/hw/char/pl011/src/lib.rs index dbae76991c..5c4fbc9d14 100644 --- a/rust/hw/char/pl011/src/lib.rs +++ b/rust/hw/char/pl011/src/lib.rs @@ -12,13 +12,11 @@ //! See [`PL011State`](crate::device::PL011State) for the device model type and //! the [`registers`] module for register types. -use qemu_api::c_str; - mod device; mod device_class; mod registers; pub use device::pl011_create; -pub const TYPE_PL011: &::std::ffi::CStr = c_str!("pl011"); -pub const TYPE_PL011_LUMINARY: &::std::ffi::CStr = c_str!("pl011_luminary"); +pub const TYPE_PL011: &::std::ffi::CStr = c"pl011"; +pub const TYPE_PL011_LUMINARY: &::std::ffi::CStr = c"pl011_luminary"; diff --git a/rust/hw/timer/hpet/Cargo.toml b/rust/hw/timer/hpet/Cargo.toml index 147f216e72..6f07502784 100644 --- a/rust/hw/timer/hpet/Cargo.toml +++ b/rust/hw/timer/hpet/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "hpet" version = "0.1.0" -edition = "2021" authors = ["Zhao Liu <zhao1.liu@intel.com>"] -license = "GPL-2.0-or-later" description = "IA-PC High Precision Event Timer emulation in Rust" -rust-version = "1.63.0" + +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true [lib] crate-type = ["staticlib"] diff --git a/rust/hw/timer/hpet/src/fw_cfg.rs b/rust/hw/timer/hpet/src/fw_cfg.rs index bef03727ea..aa08d28351 100644 --- a/rust/hw/timer/hpet/src/fw_cfg.rs +++ b/rust/hw/timer/hpet/src/fw_cfg.rs @@ -4,7 +4,7 @@ use std::ptr::addr_of_mut; -use qemu_api::{cell::bql_locked, impl_zeroable, zeroable::Zeroable}; +use qemu_api::{cell::bql_locked, zeroable::Zeroable}; /// Each `HPETState` represents a Event Timer Block. The v1 spec supports /// up to 8 blocks. QEMU only uses 1 block (in PC machine). @@ -18,7 +18,7 @@ pub struct HPETFwEntry { pub min_tick: u16, pub page_prot: u8, } -impl_zeroable!(HPETFwEntry); +unsafe impl Zeroable for HPETFwEntry {} #[repr(C, packed)] #[derive(Copy, Clone, Default)] @@ -26,7 +26,7 @@ pub struct HPETFwConfig { pub count: u8, pub hpet: [HPETFwEntry; HPET_MAX_NUM_EVENT_TIMER_BLOCK], } -impl_zeroable!(HPETFwConfig); +unsafe impl Zeroable for HPETFwConfig {} #[allow(non_upper_case_globals)] #[no_mangle] diff --git a/rust/hw/timer/hpet/src/hpet.rs b/rust/hw/timer/hpet/src/hpet.rs index cbd2ed4f6b..779681d650 100644 --- a/rust/hw/timer/hpet/src/hpet.rs +++ b/rust/hw/timer/hpet/src/hpet.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later use std::{ - ffi::CStr, + ffi::{c_int, c_void, CStr}, pin::Pin, ptr::{addr_of_mut, null_mut, NonNull}, slice::from_ref, @@ -14,7 +14,6 @@ use qemu_api::{ address_space_memory, address_space_stl_le, qdev_prop_bit, qdev_prop_bool, qdev_prop_uint32, qdev_prop_uint8, }, - c_str, cell::{BqlCell, BqlRefCell}, irq::InterruptSource, memory::{ @@ -25,7 +24,10 @@ use qemu_api::{ qom::{ObjectImpl, ObjectType, ParentField}, qom_isa, sysbus::{SysBusDevice, SysBusDeviceImpl}, - timer::{Timer, CLOCK_VIRTUAL}, + timer::{Timer, CLOCK_VIRTUAL, NANOSECONDS_PER_SECOND}, + vmstate::VMStateDescription, + vmstate_fields, vmstate_of, vmstate_struct, vmstate_subsections, vmstate_validate, + zeroable::Zeroable, }; use crate::fw_cfg::HPETFwConfig; @@ -180,7 +182,7 @@ fn timer_handler(timer_cell: &BqlRefCell<HPETTimer>) { /// HPET Timer Abstraction #[repr(C)] -#[derive(Debug, qemu_api_macros::offsets)] +#[derive(Debug)] pub struct HPETTimer { /// timer N index within the timer block (`HPETState`) #[doc(alias = "tn")] @@ -216,7 +218,7 @@ impl HPETTimer { // SAFETY: the HPETTimer will only be used after the timer // is initialized below. qemu_timer: unsafe { Timer::new() }, - state: NonNull::new(state as *const _ as *mut _).unwrap(), + state: NonNull::new((state as *const HPETState).cast_mut()).unwrap(), config: 0, cmp: 0, fsb: 0, @@ -520,7 +522,7 @@ impl HPETTimer { /// HPET Event Timer Block Abstraction #[repr(C)] -#[derive(qemu_api_macros::Object, qemu_api_macros::offsets)] +#[derive(qemu_api_macros::Object)] pub struct HPETState { parent_obj: ParentField<SysBusDevice>, iomem: MemoryRegion, @@ -561,6 +563,7 @@ pub struct HPETState { #[doc(alias = "timer")] timers: [BqlRefCell<HPETTimer>; HPET_MAX_TIMERS as usize], num_timers: BqlCell<u8>, + num_timers_save: BqlCell<u8>, /// Instance id (HPET timer block ID). hpet_id: BqlCell<usize>, @@ -839,6 +842,49 @@ impl HPETState { } } } + + fn pre_save(&self) -> i32 { + if self.is_hpet_enabled() { + self.counter.set(self.get_ticks()); + } + + /* + * The number of timers must match on source and destination, but it was + * also added to the migration stream. Check that it matches the value + * that was configured. + */ + self.num_timers_save.set(self.num_timers.get()); + 0 + } + + fn post_load(&self, _version_id: u8) -> i32 { + for timer in self.timers.iter().take(self.get_num_timers()) { + let mut t = timer.borrow_mut(); + + t.cmp64 = t.calculate_cmp64(t.get_state().counter.get(), t.cmp); + t.last = CLOCK_VIRTUAL.get_ns() - NANOSECONDS_PER_SECOND; + } + + // Recalculate the offset between the main counter and guest time + if !self.hpet_offset_saved { + self.hpet_offset + .set(ticks_to_ns(self.counter.get()) - CLOCK_VIRTUAL.get_ns()); + } + + 0 + } + + fn is_rtc_irq_level_needed(&self) -> bool { + self.rtc_irq_level.get() != 0 + } + + fn is_offset_needed(&self) -> bool { + self.is_hpet_enabled() && self.hpet_offset_saved + } + + fn validate_num_timers(&self, _version_id: u8) -> bool { + self.num_timers.get() == self.num_timers_save.get() + } } qom_isa!(HPETState: SysBusDevice, DeviceState, Object); @@ -861,7 +907,7 @@ impl ObjectImpl for HPETState { qemu_api::declare_properties! { HPET_PROPERTIES, qemu_api::define_property!( - c_str!("timers"), + c"timers", HPETState, num_timers, unsafe { &qdev_prop_uint8 }, @@ -869,7 +915,7 @@ qemu_api::declare_properties! { default = HPET_MIN_TIMERS ), qemu_api::define_property!( - c_str!("msi"), + c"msi", HPETState, flags, unsafe { &qdev_prop_bit }, @@ -878,7 +924,7 @@ qemu_api::declare_properties! { default = false, ), qemu_api::define_property!( - c_str!("hpet-intcap"), + c"hpet-intcap", HPETState, int_route_cap, unsafe { &qdev_prop_uint32 }, @@ -886,7 +932,7 @@ qemu_api::declare_properties! { default = 0 ), qemu_api::define_property!( - c_str!("hpet-offset-saved"), + c"hpet-offset-saved", HPETState, hpet_offset_saved, unsafe { &qdev_prop_bool }, @@ -895,11 +941,107 @@ qemu_api::declare_properties! { ), } +unsafe extern "C" fn hpet_rtc_irq_level_needed(opaque: *mut c_void) -> bool { + // SAFETY: + // the pointer is convertible to a reference + let state: &HPETState = unsafe { NonNull::new(opaque.cast::<HPETState>()).unwrap().as_ref() }; + state.is_rtc_irq_level_needed() +} + +unsafe extern "C" fn hpet_offset_needed(opaque: *mut c_void) -> bool { + // SAFETY: + // the pointer is convertible to a reference + let state: &HPETState = unsafe { NonNull::new(opaque.cast::<HPETState>()).unwrap().as_ref() }; + state.is_offset_needed() +} + +unsafe extern "C" fn hpet_pre_save(opaque: *mut c_void) -> c_int { + // SAFETY: + // the pointer is convertible to a reference + let state: &mut HPETState = + unsafe { NonNull::new(opaque.cast::<HPETState>()).unwrap().as_mut() }; + state.pre_save() as c_int +} + +unsafe extern "C" fn hpet_post_load(opaque: *mut c_void, version_id: c_int) -> c_int { + // SAFETY: + // the pointer is convertible to a reference + let state: &mut HPETState = + unsafe { NonNull::new(opaque.cast::<HPETState>()).unwrap().as_mut() }; + let version: u8 = version_id.try_into().unwrap(); + state.post_load(version) as c_int +} + +static VMSTATE_HPET_RTC_IRQ_LEVEL: VMStateDescription = VMStateDescription { + name: c"hpet/rtc_irq_level".as_ptr(), + version_id: 1, + minimum_version_id: 1, + needed: Some(hpet_rtc_irq_level_needed), + fields: vmstate_fields! { + vmstate_of!(HPETState, rtc_irq_level), + }, + ..Zeroable::ZERO +}; + +static VMSTATE_HPET_OFFSET: VMStateDescription = VMStateDescription { + name: c"hpet/offset".as_ptr(), + version_id: 1, + minimum_version_id: 1, + needed: Some(hpet_offset_needed), + fields: vmstate_fields! { + vmstate_of!(HPETState, hpet_offset), + }, + ..Zeroable::ZERO +}; + +static VMSTATE_HPET_TIMER: VMStateDescription = VMStateDescription { + name: c"hpet_timer".as_ptr(), + version_id: 1, + minimum_version_id: 1, + fields: vmstate_fields! { + vmstate_of!(HPETTimer, index), + vmstate_of!(HPETTimer, config), + vmstate_of!(HPETTimer, cmp), + vmstate_of!(HPETTimer, fsb), + vmstate_of!(HPETTimer, period), + vmstate_of!(HPETTimer, wrap_flag), + vmstate_of!(HPETTimer, qemu_timer), + }, + ..Zeroable::ZERO +}; + +const VALIDATE_TIMERS_NAME: &CStr = c"num_timers must match"; + +static VMSTATE_HPET: VMStateDescription = VMStateDescription { + name: c"hpet".as_ptr(), + version_id: 2, + minimum_version_id: 1, + pre_save: Some(hpet_pre_save), + post_load: Some(hpet_post_load), + fields: vmstate_fields! { + vmstate_of!(HPETState, config), + vmstate_of!(HPETState, int_status), + vmstate_of!(HPETState, counter), + vmstate_of!(HPETState, num_timers_save).with_version_id(2), + vmstate_validate!(HPETState, VALIDATE_TIMERS_NAME, HPETState::validate_num_timers), + vmstate_struct!(HPETState, timers[0 .. num_timers], &VMSTATE_HPET_TIMER, BqlRefCell<HPETTimer>, HPETState::validate_num_timers).with_version_id(0), + }, + subsections: vmstate_subsections! { + VMSTATE_HPET_RTC_IRQ_LEVEL, + VMSTATE_HPET_OFFSET, + }, + ..Zeroable::ZERO +}; + impl DeviceImpl for HPETState { fn properties() -> &'static [Property] { &HPET_PROPERTIES } + fn vmsd() -> Option<&'static VMStateDescription> { + Some(&VMSTATE_HPET) + } + const REALIZE: Option<fn(&Self)> = Some(Self::realize); } diff --git a/rust/hw/timer/hpet/src/lib.rs b/rust/hw/timer/hpet/src/lib.rs index 5e7c961c28..1954584a87 100644 --- a/rust/hw/timer/hpet/src/lib.rs +++ b/rust/hw/timer/hpet/src/lib.rs @@ -7,9 +7,7 @@ //! This library implements a device model for the IA-PC HPET (High //! Precision Event Timers) device in QEMU. -use qemu_api::c_str; - pub mod fw_cfg; pub mod hpet; -pub const TYPE_HPET: &::std::ffi::CStr = c_str!("hpet"); +pub const TYPE_HPET: &::std::ffi::CStr = c"hpet"; diff --git a/rust/qemu-api-macros/Cargo.toml b/rust/qemu-api-macros/Cargo.toml index 89dee1cfb3..0cd40c8e16 100644 --- a/rust/qemu-api-macros/Cargo.toml +++ b/rust/qemu-api-macros/Cargo.toml @@ -1,15 +1,16 @@ [package] name = "qemu_api_macros" version = "0.1.0" -edition = "2021" authors = ["Manos Pitsidianakis <manos.pitsidianakis@linaro.org>"] -license = "GPL-2.0-or-later" description = "Rust bindings for QEMU - Utility macros" resolver = "2" publish = false -keywords = [] -categories = [] -rust-version = "1.63.0" + +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true [lib] proc-macro = true diff --git a/rust/qemu-api-macros/src/lib.rs b/rust/qemu-api-macros/src/lib.rs index eda0d46d12..f97449bb30 100644 --- a/rust/qemu-api-macros/src/lib.rs +++ b/rust/qemu-api-macros/src/lib.rs @@ -6,7 +6,7 @@ use proc_macro::TokenStream; use quote::quote; use syn::{ parse_macro_input, parse_quote, punctuated::Punctuated, spanned::Spanned, token::Comma, Data, - DeriveInput, Field, Fields, FieldsUnnamed, Ident, Meta, Path, Token, Type, Variant, Visibility, + DeriveInput, Field, Fields, FieldsUnnamed, Ident, Meta, Path, Token, Variant, }; mod utils; @@ -16,50 +16,41 @@ fn get_fields<'a>( input: &'a DeriveInput, msg: &str, ) -> Result<&'a Punctuated<Field, Comma>, MacroError> { - if let Data::Struct(s) = &input.data { - if let Fields::Named(fs) = &s.fields { - Ok(&fs.named) - } else { - Err(MacroError::Message( - format!("Named fields required for {}", msg), - input.ident.span(), - )) - } - } else { - Err(MacroError::Message( - format!("Struct required for {}", msg), + let Data::Struct(ref s) = &input.data else { + return Err(MacroError::Message( + format!("Struct required for {msg}"), input.ident.span(), - )) - } + )); + }; + let Fields::Named(ref fs) = &s.fields else { + return Err(MacroError::Message( + format!("Named fields required for {msg}"), + input.ident.span(), + )); + }; + Ok(&fs.named) } fn get_unnamed_field<'a>(input: &'a DeriveInput, msg: &str) -> Result<&'a Field, MacroError> { - if let Data::Struct(s) = &input.data { - let unnamed = match &s.fields { - Fields::Unnamed(FieldsUnnamed { - unnamed: ref fields, - .. - }) => fields, - _ => { - return Err(MacroError::Message( - format!("Tuple struct required for {}", msg), - s.fields.span(), - )) - } - }; - if unnamed.len() != 1 { - return Err(MacroError::Message( - format!("A single field is required for {}", msg), - s.fields.span(), - )); - } - Ok(&unnamed[0]) - } else { - Err(MacroError::Message( - format!("Struct required for {}", msg), + let Data::Struct(ref s) = &input.data else { + return Err(MacroError::Message( + format!("Struct required for {msg}"), input.ident.span(), - )) + )); + }; + let Fields::Unnamed(FieldsUnnamed { ref unnamed, .. }) = &s.fields else { + return Err(MacroError::Message( + format!("Tuple struct required for {msg}"), + s.fields.span(), + )); + }; + if unnamed.len() != 1 { + return Err(MacroError::Message( + format!("A single field is required for {msg}"), + s.fields.span(), + )); } + Ok(&unnamed[0]) } fn is_c_repr(input: &DeriveInput, msg: &str) -> Result<(), MacroError> { @@ -69,7 +60,7 @@ fn is_c_repr(input: &DeriveInput, msg: &str) -> Result<(), MacroError> { Ok(()) } else { Err(MacroError::Message( - format!("#[repr(C)] required for {}", msg), + format!("#[repr(C)] required for {msg}"), input.ident.span(), )) } @@ -82,7 +73,7 @@ fn is_transparent_repr(input: &DeriveInput, msg: &str) -> Result<(), MacroError> Ok(()) } else { Err(MacroError::Message( - format!("#[repr(transparent)] required for {}", msg), + format!("#[repr(transparent)] required for {msg}"), input.ident.span(), )) } @@ -160,33 +151,6 @@ pub fn derive_opaque(input: TokenStream) -> TokenStream { TokenStream::from(expanded) } -#[rustfmt::skip::macros(quote)] -fn derive_offsets_or_error(input: DeriveInput) -> Result<proc_macro2::TokenStream, MacroError> { - is_c_repr(&input, "#[derive(offsets)]")?; - - let name = &input.ident; - let fields = get_fields(&input, "#[derive(offsets)]")?; - let field_names: Vec<&Ident> = fields.iter().map(|f| f.ident.as_ref().unwrap()).collect(); - let field_types: Vec<&Type> = fields.iter().map(|f| &f.ty).collect(); - let field_vis: Vec<&Visibility> = fields.iter().map(|f| &f.vis).collect(); - - Ok(quote! { - ::qemu_api::with_offsets! { - struct #name { - #(#field_vis #field_names: #field_types,)* - } - } - }) -} - -#[proc_macro_derive(offsets)] -pub fn derive_offsets(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input as DeriveInput); - let expanded = derive_offsets_or_error(input).unwrap_or_else(Into::into); - - TokenStream::from(expanded) -} - #[allow(non_snake_case)] fn get_repr_uN(input: &DeriveInput, msg: &str) -> Result<Path, MacroError> { let repr = input.attrs.iter().find(|attr| attr.path().is_ident("repr")); @@ -204,26 +168,25 @@ fn get_repr_uN(input: &DeriveInput, msg: &str) -> Result<Path, MacroError> { } Err(MacroError::Message( - format!("#[repr(u8/u16/u32/u64) required for {}", msg), + format!("#[repr(u8/u16/u32/u64) required for {msg}"), input.ident.span(), )) } fn get_variants(input: &DeriveInput) -> Result<&Punctuated<Variant, Comma>, MacroError> { - if let Data::Enum(e) = &input.data { - if let Some(v) = e.variants.iter().find(|v| v.fields != Fields::Unit) { - return Err(MacroError::Message( - "Cannot derive TryInto for enum with non-unit variants.".to_string(), - v.fields.span(), - )); - } - Ok(&e.variants) - } else { - Err(MacroError::Message( + let Data::Enum(ref e) = &input.data else { + return Err(MacroError::Message( "Cannot derive TryInto for union or struct.".to_string(), input.ident.span(), - )) + )); + }; + if let Some(v) = e.variants.iter().find(|v| v.fields != Fields::Unit) { + return Err(MacroError::Message( + "Cannot derive TryInto for enum with non-unit variants.".to_string(), + v.fields.span(), + )); } + Ok(&e.variants) } #[rustfmt::skip::macros(quote)] diff --git a/rust/qemu-api/Cargo.toml b/rust/qemu-api/Cargo.toml index 57747bc934..c96cf50e7a 100644 --- a/rust/qemu-api/Cargo.toml +++ b/rust/qemu-api/Cargo.toml @@ -1,26 +1,22 @@ [package] name = "qemu_api" version = "0.1.0" -edition = "2021" authors = ["Manos Pitsidianakis <manos.pitsidianakis@linaro.org>"] -license = "GPL-2.0-or-later" -readme = "README.md" -homepage = "https://www.qemu.org" description = "Rust bindings for QEMU" -repository = "https://gitlab.com/qemu-project/qemu/" +readme = "README.md" resolver = "2" publish = false -keywords = [] -categories = [] -rust-version = "1.63.0" + +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true [dependencies] qemu_api_macros = { path = "../qemu-api-macros" } libc = "0.2.162" -[build-dependencies] -version_check = "~0.9" - [features] default = ["debug_cell"] allocator = [] diff --git a/rust/qemu-api/build.rs b/rust/qemu-api/build.rs index 471e6c633d..1e720641d2 100644 --- a/rust/qemu-api/build.rs +++ b/rust/qemu-api/build.rs @@ -8,15 +8,13 @@ use std::os::unix::fs::symlink as symlink_file; use std::os::windows::fs::symlink_file; use std::{env, fs::remove_file, io::Result, path::Path}; -use version_check as rustc; - fn main() -> Result<()> { // Placing bindings.inc.rs in the source directory is supported // but not documented or encouraged. let path = env::var("MESON_BUILD_ROOT") .unwrap_or_else(|_| format!("{}/src", env!("CARGO_MANIFEST_DIR"))); - let file = format!("{}/bindings.inc.rs", path); + let file = format!("{path}/bindings.inc.rs"); let file = Path::new(&file); if !Path::new(&file).exists() { panic!(concat!( @@ -31,18 +29,13 @@ fn main() -> Result<()> { } let out_dir = env::var("OUT_DIR").unwrap(); - let dest_path = format!("{}/bindings.inc.rs", out_dir); + let dest_path = format!("{out_dir}/bindings.inc.rs"); let dest_path = Path::new(&dest_path); if dest_path.symlink_metadata().is_ok() { remove_file(dest_path)?; } symlink_file(file, dest_path)?; - // Check for available rustc features - if rustc::is_min_version("1.77.0").unwrap_or(false) { - println!("cargo:rustc-cfg=has_offset_of"); - } - println!("cargo:rerun-if-changed=build.rs"); Ok(()) } diff --git a/rust/qemu-api/meson.build b/rust/qemu-api/meson.build index 858685ddd4..1696df705b 100644 --- a/rust/qemu-api/meson.build +++ b/rust/qemu-api/meson.build @@ -5,9 +5,6 @@ _qemu_api_cfg = run_command(rustc_args, libc_dep = dependency('libc-0.2-rs') # _qemu_api_cfg += ['--cfg', 'feature="allocator"'] -if rustc.version().version_compare('>=1.77.0') - _qemu_api_cfg += ['--cfg', 'has_offset_of'] -endif if get_option('debug_mutex') _qemu_api_cfg += ['--cfg', 'feature="debug_cell"'] endif @@ -23,12 +20,10 @@ _qemu_api_rs = static_library( 'src/callbacks.rs', 'src/cell.rs', 'src/chardev.rs', - 'src/c_str.rs', 'src/errno.rs', 'src/irq.rs', 'src/memory.rs', 'src/module.rs', - 'src/offset_of.rs', 'src/prelude.rs', 'src/qdev.rs', 'src/qom.rs', diff --git a/rust/qemu-api/src/assertions.rs b/rust/qemu-api/src/assertions.rs index eb12e9499a..a2d38c877d 100644 --- a/rust/qemu-api/src/assertions.rs +++ b/rust/qemu-api/src/assertions.rs @@ -78,33 +78,26 @@ macro_rules! assert_same_type { /// ``` #[macro_export] macro_rules! assert_field_type { - ($t:ty, $i:tt, $ti:ty) => { + (@internal $param_name:ident, $ti:ty, $t:ty, $($field:tt)*) => { const _: () = { #[allow(unused)] - fn assert_field_type(v: $t) { - fn types_must_be_equal<T, U>(_: T) + fn assert_field_type($param_name: &$t) { + fn types_must_be_equal<T, U>(_: &T) where T: $crate::assertions::EqType<Itself = U>, { } - types_must_be_equal::<_, $ti>(v.$i); + types_must_be_equal::<_, $ti>(&$($field)*); } }; }; + ($t:ty, $i:tt, $ti:ty) => { + $crate::assert_field_type!(@internal v, $ti, $t, v.$i); + }; + ($t:ty, $i:tt, $ti:ty, num = $num:ident) => { - const _: () = { - #[allow(unused)] - fn assert_field_type(v: $t) { - fn types_must_be_equal<T, U>(_: T) - where - T: $crate::assertions::EqType<Itself = U>, - { - } - let index: usize = v.$num.try_into().unwrap(); - types_must_be_equal::<_, &$ti>(&v.$i[index]); - } - }; + $crate::assert_field_type!(@internal v, $ti, $t, v.$i[0]); }; } diff --git a/rust/qemu-api/src/c_str.rs b/rust/qemu-api/src/c_str.rs deleted file mode 100644 index 3fa61b59c7..0000000000 --- a/rust/qemu-api/src/c_str.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2024 Red Hat, Inc. -// Author(s): Paolo Bonzini <pbonzini@redhat.com> -// SPDX-License-Identifier: GPL-2.0-or-later - -#![doc(hidden)] -//! This module provides a macro to define a constant of type -//! [`CStr`](std::ffi::CStr), for compatibility with versions of -//! Rust that lack `c""` literals. -//! -//! Documentation is hidden because it only exposes macros, which -//! are exported directly from `qemu_api`. - -#[macro_export] -/// Given a string constant _without_ embedded or trailing NULs, return -/// a `CStr`. -/// -/// Needed for compatibility with Rust <1.77. -macro_rules! c_str { - ($str:expr) => {{ - const STRING: &str = concat!($str, "\0"); - const BYTES: &[u8] = STRING.as_bytes(); - - // "for" is not allowed in const context... oh well, - // everybody loves some lisp. This could be turned into - // a procedural macro if this is a problem; alternatively - // Rust 1.72 makes CStr::from_bytes_with_nul a const function. - const fn f(b: &[u8], i: usize) { - if i == b.len() - 1 { - } else if b[i] == 0 { - panic!("c_str argument contains NUL") - } else { - f(b, i + 1) - } - } - f(BYTES, 0); - - // SAFETY: absence of NULs apart from the final byte was checked above - unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(BYTES) } - }}; -} - -#[cfg(test)] -mod tests { - use std::ffi::CStr; - - use crate::c_str; - - #[test] - fn test_cstr_macro() { - let good = c_str!("🦀"); - let good_bytes = b"\xf0\x9f\xa6\x80\0"; - assert_eq!(good.to_bytes_with_nul(), good_bytes); - } - - #[test] - fn test_cstr_macro_const() { - const GOOD: &CStr = c_str!("🦀"); - const GOOD_BYTES: &[u8] = b"\xf0\x9f\xa6\x80\0"; - assert_eq!(GOOD.to_bytes_with_nul(), GOOD_BYTES); - } -} diff --git a/rust/qemu-api/src/cell.rs b/rust/qemu-api/src/cell.rs index ab0785a269..05ce09f6cb 100644 --- a/rust/qemu-api/src/cell.rs +++ b/rust/qemu-api/src/cell.rs @@ -77,13 +77,13 @@ //! //! ``` //! # use qemu_api::prelude::*; -//! # use qemu_api::{c_str, cell::BqlRefCell, irq::InterruptSource, irq::IRQState}; +//! # use qemu_api::{cell::BqlRefCell, irq::InterruptSource, irq::IRQState}; //! # use qemu_api::{sysbus::SysBusDevice, qom::Owned, qom::ParentField}; //! # const N_GPIOS: usize = 8; //! # struct PL061Registers { /* ... */ } //! # unsafe impl ObjectType for PL061State { //! # type Class = <SysBusDevice as ObjectType>::Class; -//! # const TYPE_NAME: &'static std::ffi::CStr = c_str!("pl061"); +//! # const TYPE_NAME: &'static std::ffi::CStr = c"pl061"; //! # } //! struct PL061State { //! parent_obj: ParentField<SysBusDevice>, @@ -1016,7 +1016,7 @@ impl<T> Opaque<T> { /// Returns a raw pointer to the opaque data. pub const fn as_ptr(&self) -> *const T { - self.as_mut_ptr() as *const _ + self.as_mut_ptr().cast_const() } /// Returns a raw pointer to the opaque data that can be passed to a diff --git a/rust/qemu-api/src/chardev.rs b/rust/qemu-api/src/chardev.rs index 11e6c45afa..6e0590d758 100644 --- a/rust/qemu-api/src/chardev.rs +++ b/rust/qemu-api/src/chardev.rs @@ -10,11 +10,10 @@ //! called. use std::{ - ffi::CStr, + ffi::{c_int, c_void, CStr}, fmt::{self, Debug}, io::{self, ErrorKind, Write}, marker::PhantomPinned, - os::raw::{c_int, c_void}, ptr::addr_of_mut, slice, }; @@ -161,7 +160,7 @@ impl CharBackend { receive_cb, event_cb, None, - (owner as *const T as *mut T).cast::<c_void>(), + (owner as *const T).cast_mut().cast::<c_void>(), core::ptr::null_mut(), true, ); diff --git a/rust/qemu-api/src/irq.rs b/rust/qemu-api/src/irq.rs index 1222d4fde3..1526e6f63a 100644 --- a/rust/qemu-api/src/irq.rs +++ b/rust/qemu-api/src/irq.rs @@ -4,7 +4,11 @@ //! Bindings for interrupt sources -use std::{ffi::CStr, marker::PhantomData, os::raw::c_int, ptr}; +use std::{ + ffi::{c_int, CStr}, + marker::PhantomData, + ptr, +}; use crate::{ bindings::{self, qemu_set_irq}, diff --git a/rust/qemu-api/src/lib.rs b/rust/qemu-api/src/lib.rs index 05f38b51d3..234a94e3c2 100644 --- a/rust/qemu-api/src/lib.rs +++ b/rust/qemu-api/src/lib.rs @@ -15,7 +15,6 @@ pub mod prelude; pub mod assertions; pub mod bitops; -pub mod c_str; pub mod callbacks; pub mod cell; pub mod chardev; @@ -23,7 +22,6 @@ pub mod errno; pub mod irq; pub mod memory; pub mod module; -pub mod offset_of; pub mod qdev; pub mod qom; pub mod sysbus; @@ -33,7 +31,7 @@ pub mod zeroable; use std::{ alloc::{GlobalAlloc, Layout}, - os::raw::c_void, + ffi::c_void, }; #[cfg(HAVE_GLIB_WITH_ALIGNED_ALLOC)] @@ -165,6 +163,3 @@ unsafe impl GlobalAlloc for QemuAllocator { } } } - -#[cfg(has_offset_of)] -pub use core::mem::offset_of; diff --git a/rust/qemu-api/src/memory.rs b/rust/qemu-api/src/memory.rs index fdb1ea11fc..9ef2694bd6 100644 --- a/rust/qemu-api/src/memory.rs +++ b/rust/qemu-api/src/memory.rs @@ -5,9 +5,8 @@ //! Bindings for `MemoryRegion`, `MemoryRegionOps` and `MemTxAttrs` use std::{ - ffi::{CStr, CString}, + ffi::{c_uint, c_void, CStr, CString}, marker::PhantomData, - os::raw::{c_uint, c_void}, }; pub use bindings::{hwaddr, MemTxAttrs}; diff --git a/rust/qemu-api/src/offset_of.rs b/rust/qemu-api/src/offset_of.rs deleted file mode 100644 index 373229bbde..0000000000 --- a/rust/qemu-api/src/offset_of.rs +++ /dev/null @@ -1,168 +0,0 @@ -// SPDX-License-Identifier: MIT - -#![doc(hidden)] -//! This module provides macros that emulate the functionality of -//! `core::mem::offset_of` on older versions of Rust. -//! -//! Documentation is hidden because it only exposes macros, which -//! are exported directly from `qemu_api`. - -/// This macro provides the same functionality as `core::mem::offset_of`, -/// except that only one level of field access is supported. The declaration -/// of the struct must be wrapped with `with_offsets! { }`. -/// -/// It is needed because `offset_of!` was only stabilized in Rust 1.77. -#[cfg(not(has_offset_of))] -#[macro_export] -macro_rules! offset_of { - ($Container:ty, $field:ident) => { - <$Container>::OFFSET_TO__.$field - }; -} - -/// A wrapper for struct declarations, that allows using `offset_of!` in -/// versions of Rust prior to 1.77 -#[macro_export] -macro_rules! with_offsets { - // This method to generate field offset constants comes from: - // - // https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=10a22a9b8393abd7b541d8fc844bc0df - // - // used under MIT license with permission of Yandros aka Daniel Henry-Mantilla - ( - $(#[$struct_meta:meta])* - $struct_vis:vis - struct $StructName:ident { - $( - $(#[$field_meta:meta])* - $field_vis:vis - $field_name:ident : $field_ty:ty - ),* - $(,)? - } - ) => ( - #[cfg(not(has_offset_of))] - const _: () = { - struct StructOffsetsHelper<T>(std::marker::PhantomData<T>); - const END_OF_PREV_FIELD: usize = 0; - - // populate StructOffsetsHelper<T> with associated consts, - // one for each field - $crate::with_offsets! { - @struct $StructName - @names [ $($field_name)* ] - @tys [ $($field_ty ,)*] - } - - // now turn StructOffsetsHelper<T>'s consts into a single struct, - // applying field visibility. This provides better error messages - // than if offset_of! used StructOffsetsHelper::<T> directly. - pub - struct StructOffsets { - $( - $field_vis - $field_name: usize, - )* - } - impl $StructName { - pub - const OFFSET_TO__: StructOffsets = StructOffsets { - $( - $field_name: StructOffsetsHelper::<$StructName>::$field_name, - )* - }; - } - }; - ); - - ( - @struct $StructName:ident - @names [] - @tys [] - ) => (); - - ( - @struct $StructName:ident - @names [$field_name:ident $($other_names:tt)*] - @tys [$field_ty:ty , $($other_tys:tt)*] - ) => ( - #[allow(non_local_definitions)] - #[allow(clippy::modulo_one)] - impl StructOffsetsHelper<$StructName> { - #[allow(nonstandard_style)] - const $field_name: usize = { - const ALIGN: usize = std::mem::align_of::<$field_ty>(); - const TRAIL: usize = END_OF_PREV_FIELD % ALIGN; - END_OF_PREV_FIELD + (if TRAIL == 0 { 0usize } else { ALIGN - TRAIL }) - }; - } - const _: () = { - const END_OF_PREV_FIELD: usize = - StructOffsetsHelper::<$StructName>::$field_name + - std::mem::size_of::<$field_ty>() - ; - $crate::with_offsets! { - @struct $StructName - @names [$($other_names)*] - @tys [$($other_tys)*] - } - }; - ); -} - -#[cfg(test)] -mod tests { - use crate::offset_of; - - #[repr(C)] - struct Foo { - a: u16, - b: u32, - c: u64, - d: u16, - } - - #[repr(C)] - struct Bar { - pub a: u16, - pub b: u64, - c: Foo, - d: u64, - } - - crate::with_offsets! { - #[repr(C)] - struct Bar { - pub a: u16, - pub b: u64, - c: Foo, - d: u64, - } - } - - #[repr(C)] - pub struct Baz { - b: u32, - a: u8, - } - crate::with_offsets! { - #[repr(C)] - pub struct Baz { - b: u32, - a: u8, - } - } - - #[test] - fn test_offset_of() { - const OFFSET_TO_C: usize = offset_of!(Bar, c); - - assert_eq!(offset_of!(Bar, a), 0); - assert_eq!(offset_of!(Bar, b), 8); - assert_eq!(OFFSET_TO_C, 16); - assert_eq!(offset_of!(Bar, d), 40); - - assert_eq!(offset_of!(Baz, b), 0); - assert_eq!(offset_of!(Baz, a), 4); - } -} diff --git a/rust/qemu-api/src/qdev.rs b/rust/qemu-api/src/qdev.rs index 18b4a9ba68..1279d7a58d 100644 --- a/rust/qemu-api/src/qdev.rs +++ b/rust/qemu-api/src/qdev.rs @@ -5,8 +5,7 @@ //! Bindings to create devices and access device functionality from Rust. use std::{ - ffi::{CStr, CString}, - os::raw::{c_int, c_void}, + ffi::{c_int, c_void, CStr, CString}, ptr::NonNull, }; @@ -191,7 +190,7 @@ macro_rules! define_property { // use associated function syntax for type checking name: ::std::ffi::CStr::as_ptr($name), info: $prop, - offset: $crate::offset_of!($state, $field) as isize, + offset: ::std::mem::offset_of!($state, $field) as isize, bitnr: $bitnr, set_default: true, defval: $crate::bindings::Property__bindgen_ty_1 { u: $defval as u64 }, @@ -203,7 +202,7 @@ macro_rules! define_property { // use associated function syntax for type checking name: ::std::ffi::CStr::as_ptr($name), info: $prop, - offset: $crate::offset_of!($state, $field) as isize, + offset: ::std::mem::offset_of!($state, $field) as isize, set_default: true, defval: $crate::bindings::Property__bindgen_ty_1 { u: $defval as u64 }, ..$crate::zeroable::Zeroable::ZERO @@ -214,7 +213,7 @@ macro_rules! define_property { // use associated function syntax for type checking name: ::std::ffi::CStr::as_ptr($name), info: $prop, - offset: $crate::offset_of!($state, $field) as isize, + offset: ::std::mem::offset_of!($state, $field) as isize, set_default: false, ..$crate::zeroable::Zeroable::ZERO } diff --git a/rust/qemu-api/src/qom.rs b/rust/qemu-api/src/qom.rs index f1b4022157..41e5a5e29a 100644 --- a/rust/qemu-api/src/qom.rs +++ b/rust/qemu-api/src/qom.rs @@ -93,11 +93,10 @@ //! without incurring into violations of orphan rules for traits. use std::{ - ffi::CStr, + ffi::{c_void, CStr}, fmt, mem::ManuallyDrop, ops::{Deref, DerefMut}, - os::raw::c_void, ptr::NonNull, }; @@ -389,7 +388,7 @@ where { #[allow(clippy::as_ptr_cast_mut)] { - self.as_ptr::<U>() as *mut _ + self.as_ptr::<U>().cast_mut() } } } @@ -535,9 +534,10 @@ pub trait ObjectImpl: ObjectType + IsA<Object> { /// While `klass`'s parent class is initialized on entry, the other fields /// are all zero; it is therefore assumed that all fields in `T` can be /// zeroed, otherwise it would not be possible to provide the class as a - /// `&mut T`. TODO: add a bound of [`Zeroable`](crate::zeroable::Zeroable) - /// to T; this is more easily done once Zeroable does not require a manual - /// implementation (Rust 1.75.0). + /// `&mut T`. TODO: it may be possible to add an unsafe trait that checks + /// that all fields *after the parent class* (but not the parent class + /// itself) are Zeroable. This unsafe trait can be added via a derive + /// macro. const CLASS_INIT: fn(&mut Self::Class); } @@ -638,7 +638,7 @@ impl<T: ObjectType> Owned<T> { // SAFETY NOTE: while NonNull requires a mutable pointer, only // Deref is implemented so the pointer passed to from_raw // remains const - Owned(NonNull::new(ptr as *mut T).unwrap()) + Owned(NonNull::new(ptr.cast_mut()).unwrap()) } /// Obtain a raw C pointer from a reference. `src` is consumed diff --git a/rust/qemu-api/src/timer.rs b/rust/qemu-api/src/timer.rs index f0b04ef95d..868bd88575 100644 --- a/rust/qemu-api/src/timer.rs +++ b/rust/qemu-api/src/timer.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later use std::{ - os::raw::{c_int, c_void}, + ffi::{c_int, c_void}, pin::Pin, }; @@ -81,7 +81,7 @@ impl Timer { scale as c_int, attributes as c_int, Some(timer_cb), - (opaque as *const T).cast::<c_void>() as *mut c_void, + (opaque as *const T).cast::<c_void>().cast_mut(), ) } } @@ -121,3 +121,5 @@ impl ClockType { pub const CLOCK_VIRTUAL: ClockType = ClockType { id: QEMUClockType::QEMU_CLOCK_VIRTUAL, }; + +pub const NANOSECONDS_PER_SECOND: u64 = 1000000000; diff --git a/rust/qemu-api/src/vmstate.rs b/rust/qemu-api/src/vmstate.rs index 1b2b12eadd..9c8b2398e9 100644 --- a/rust/qemu-api/src/vmstate.rs +++ b/rust/qemu-api/src/vmstate.rs @@ -25,7 +25,7 @@ //! functionality that is missing from `vmstate_of!`. use core::{marker::PhantomData, mem, ptr::NonNull}; -use std::os::raw::{c_int, c_void}; +use std::ffi::{c_int, c_void}; pub use crate::bindings::{VMStateDescription, VMStateField}; use crate::{ @@ -200,13 +200,14 @@ pub const fn vmstate_varray_flag<T: VMState>(_: PhantomData<T>) -> VMStateFlags /// and [`impl_vmstate_forward!`](crate::impl_vmstate_forward) help with this. #[macro_export] macro_rules! vmstate_of { - ($struct_name:ty, $field_name:ident $([0 .. $num:ident $(* $factor:expr)?])? $(,)?) => { + ($struct_name:ty, $field_name:ident $([0 .. $num:ident $(* $factor:expr)?])? $(, $test_fn:expr)? $(,)?) => { $crate::bindings::VMStateField { name: ::core::concat!(::core::stringify!($field_name), "\0") .as_bytes() .as_ptr() as *const ::std::os::raw::c_char, - offset: $crate::offset_of!($struct_name, $field_name), - $(num_offset: $crate::offset_of!($struct_name, $num),)? + offset: ::std::mem::offset_of!($struct_name, $field_name), + $(num_offset: ::std::mem::offset_of!($struct_name, $num),)? + $(field_exists: $crate::vmstate_exist_fn!($struct_name, $test_fn),)? // The calls to `call_func_with_field!` are the magic that // computes most of the VMStateField from the type of the field. info: $crate::info_enum_to_ref!($crate::call_func_with_field!( @@ -426,7 +427,7 @@ unsafe impl<T: VMState, const N: usize> VMState for [T; N] { macro_rules! vmstate_unused { ($size:expr) => {{ $crate::bindings::VMStateField { - name: $crate::c_str!("unused").as_ptr(), + name: c"unused".as_ptr(), size: $size, info: unsafe { ::core::ptr::addr_of!($crate::bindings::vmstate_info_unused_buffer) }, flags: $crate::bindings::VMStateFlags::VMS_BUFFER, @@ -435,6 +436,38 @@ macro_rules! vmstate_unused { }}; } +pub extern "C" fn rust_vms_test_field_exists<T, F: for<'a> FnCall<(&'a T, u8), bool>>( + opaque: *mut c_void, + version_id: c_int, +) -> bool { + // SAFETY: the opaque was passed as a reference to `T`. + let owner: &T = unsafe { &*(opaque.cast::<T>()) }; + let version: u8 = version_id.try_into().unwrap(); + F::call((owner, version)) +} + +pub type VMSFieldExistCb = unsafe extern "C" fn( + opaque: *mut std::os::raw::c_void, + version_id: std::os::raw::c_int, +) -> bool; + +#[macro_export] +macro_rules! vmstate_exist_fn { + ($struct_name:ty, $test_fn:expr) => {{ + const fn test_cb_builder__<T, F: for<'a> $crate::callbacks::FnCall<(&'a T, u8), bool>>( + _phantom: ::core::marker::PhantomData<F>, + ) -> $crate::vmstate::VMSFieldExistCb { + let _: () = F::ASSERT_IS_SOME; + $crate::vmstate::rust_vms_test_field_exists::<T, F> + } + + const fn phantom__<T>(_: &T) -> ::core::marker::PhantomData<T> { + ::core::marker::PhantomData + } + Some(test_cb_builder__::<$struct_name, _>(phantom__(&$test_fn))) + }}; +} + // FIXME: including the `vmsd` field in a `const` is not possible without // the const_refs_static feature (stabilized in Rust 1.83.0). Without it, // it is not possible to use VMS_STRUCT in a transparent manner using @@ -445,19 +478,20 @@ macro_rules! vmstate_unused { #[doc(alias = "VMSTATE_STRUCT")] #[macro_export] macro_rules! vmstate_struct { - ($struct_name:ty, $field_name:ident $([0 .. $num:ident $(* $factor:expr)?])?, $vmsd:expr, $type:ty $(,)?) => { + ($struct_name:ty, $field_name:ident $([0 .. $num:ident $(* $factor:expr)?])?, $vmsd:expr, $type:ty $(, $test_fn:expr)? $(,)?) => { $crate::bindings::VMStateField { name: ::core::concat!(::core::stringify!($field_name), "\0") .as_bytes() .as_ptr() as *const ::std::os::raw::c_char, - $(num_offset: $crate::offset_of!($struct_name, $num),)? + $(num_offset: ::std::mem::offset_of!($struct_name, $num),)? offset: { $crate::assert_field_type!($struct_name, $field_name, $type $(, num = $num)?); - $crate::offset_of!($struct_name, $field_name) + ::std::mem::offset_of!($struct_name, $field_name) }, size: ::core::mem::size_of::<$type>(), flags: $crate::bindings::VMStateFlags::VMS_STRUCT, vmsd: $vmsd, + $(field_exists: $crate::vmstate_exist_fn!($struct_name, $test_fn),)? ..$crate::zeroable::Zeroable::ZERO } $(.with_varray_flag_unchecked( $crate::call_func_with_field!( @@ -473,7 +507,7 @@ macro_rules! vmstate_struct { #[doc(alias = "VMSTATE_CLOCK")] #[macro_export] macro_rules! vmstate_clock { - ($struct_name:ty, $field_name:ident) => {{ + ($struct_name:ty, $field_name:ident $([0 .. $num:ident $(* $factor:expr)?])?) => {{ $crate::bindings::VMStateField { name: ::core::concat!(::core::stringify!($field_name), "\0") .as_bytes() @@ -482,9 +516,9 @@ macro_rules! vmstate_clock { $crate::assert_field_type!( $struct_name, $field_name, - $crate::qom::Owned<$crate::qdev::Clock> + $crate::qom::Owned<$crate::qdev::Clock> $(, num = $num)? ); - $crate::offset_of!($struct_name, $field_name) + ::std::mem::offset_of!($struct_name, $field_name) }, size: ::core::mem::size_of::<*const $crate::qdev::Clock>(), flags: $crate::bindings::VMStateFlags( @@ -493,7 +527,14 @@ macro_rules! vmstate_clock { ), vmsd: unsafe { ::core::ptr::addr_of!($crate::bindings::vmstate_clock) }, ..$crate::zeroable::Zeroable::ZERO - } + } $(.with_varray_flag_unchecked( + $crate::call_func_with_field!( + $crate::vmstate::vmstate_varray_flag, + $struct_name, + $num + ) + ) + $(.with_varray_multiply($factor))?)? }}; } @@ -514,43 +555,13 @@ macro_rules! vmstate_fields { }} } -pub extern "C" fn rust_vms_test_field_exists<T, F: for<'a> FnCall<(&'a T, u8), bool>>( - opaque: *mut c_void, - version_id: c_int, -) -> bool { - let owner: &T = unsafe { &*(opaque.cast::<T>()) }; - let version: u8 = version_id.try_into().unwrap(); - // SAFETY: the opaque was passed as a reference to `T`. - F::call((owner, version)) -} - -pub type VMSFieldExistCb = unsafe extern "C" fn( - opaque: *mut std::os::raw::c_void, - version_id: std::os::raw::c_int, -) -> bool; - #[doc(alias = "VMSTATE_VALIDATE")] #[macro_export] macro_rules! vmstate_validate { ($struct_name:ty, $test_name:expr, $test_fn:expr $(,)?) => { $crate::bindings::VMStateField { name: ::std::ffi::CStr::as_ptr($test_name), - field_exists: { - const fn test_cb_builder__< - T, - F: for<'a> $crate::callbacks::FnCall<(&'a T, u8), bool>, - >( - _phantom: ::core::marker::PhantomData<F>, - ) -> $crate::vmstate::VMSFieldExistCb { - let _: () = F::ASSERT_IS_SOME; - $crate::vmstate::rust_vms_test_field_exists::<T, F> - } - - const fn phantom__<T>(_: &T) -> ::core::marker::PhantomData<T> { - ::core::marker::PhantomData - } - Some(test_cb_builder__::<$struct_name, _>(phantom__(&$test_fn))) - }, + field_exists: $crate::vmstate_exist_fn!($struct_name, $test_fn), flags: $crate::bindings::VMStateFlags( $crate::bindings::VMStateFlags::VMS_MUST_EXIST.0 | $crate::bindings::VMStateFlags::VMS_ARRAY.0, diff --git a/rust/qemu-api/src/zeroable.rs b/rust/qemu-api/src/zeroable.rs index a3415a2ebc..d8239d0856 100644 --- a/rust/qemu-api/src/zeroable.rs +++ b/rust/qemu-api/src/zeroable.rs @@ -4,89 +4,17 @@ /// Encapsulates the requirement that /// `MaybeUninit::<Self>::zeroed().assume_init()` does not cause undefined -/// behavior. This trait in principle could be implemented as just: -/// -/// ``` -/// pub unsafe trait Zeroable: Default { -/// const ZERO: Self = unsafe { ::core::mem::MaybeUninit::<Self>::zeroed().assume_init() }; -/// } -/// ``` -/// -/// The need for a manual implementation is only because `zeroed()` cannot -/// be used as a `const fn` prior to Rust 1.75.0. Once we can assume a new -/// enough version of the compiler, we could provide a `#[derive(Zeroable)]` -/// macro to check at compile-time that all struct fields are Zeroable, and -/// use the above blanket implementation of the `ZERO` constant. +/// behavior. /// /// # Safety /// -/// Because the implementation of `ZERO` is manual, it does not make -/// any assumption on the safety of `zeroed()`. However, other users of the -/// trait could use it that way. Do not add this trait to a type unless -/// all-zeroes is a valid value for the type. In particular, remember that -/// raw pointers can be zero, but references and `NonNull<T>` cannot +/// Do not add this trait to a type unless all-zeroes is a valid value for the +/// type. In particular, raw pointers can be zero, but references and +/// `NonNull<T>` cannot. pub unsafe trait Zeroable: Default { - const ZERO: Self; -} - -/// A macro that acts similarly to [`core::mem::zeroed()`], only is const -/// -/// ## Safety -/// -/// Similar to `core::mem::zeroed()`, except this zeroes padding bits. Zeroed -/// padding usually isn't relevant to safety, but might be if a C union is used. -/// -/// Just like for `core::mem::zeroed()`, an all zero byte pattern might not -/// be a valid value for a type, as is the case for references `&T` and `&mut -/// T`. Reference types trigger a (denied by default) lint and cause immediate -/// undefined behavior if the lint is ignored -/// -/// ```rust compile_fail -/// use const_zero::const_zero; -/// // error: any use of this value will cause an error -/// // note: `#[deny(const_err)]` on by default -/// const STR: &str = unsafe{const_zero!(&'static str)}; -/// ``` -/// -/// `const_zero` does not work on unsized types: -/// -/// ```rust compile_fail -/// use const_zero::const_zero; -/// // error[E0277]: the size for values of type `[u8]` cannot be known at compilation time -/// const BYTES: [u8] = unsafe{const_zero!([u8])}; -/// ``` -/// ## Differences with `core::mem::zeroed` -/// -/// `const_zero` zeroes padding bits, while `core::mem::zeroed` doesn't -#[macro_export] -macro_rules! const_zero { - // This macro to produce a type-generic zero constant is taken from the - // const_zero crate (v0.1.1): - // - // https://docs.rs/const-zero/latest/src/const_zero/lib.rs.html - // - // and used under MIT license - ($type_:ty) => {{ - const TYPE_SIZE: ::core::primitive::usize = ::core::mem::size_of::<$type_>(); - union TypeAsBytes { - bytes: [::core::primitive::u8; TYPE_SIZE], - inner: ::core::mem::ManuallyDrop<$type_>, - } - const ZERO_BYTES: TypeAsBytes = TypeAsBytes { - bytes: [0; TYPE_SIZE], - }; - ::core::mem::ManuallyDrop::<$type_>::into_inner(ZERO_BYTES.inner) - }}; -} - -/// A wrapper to implement the `Zeroable` trait through the `const_zero` macro. -#[macro_export] -macro_rules! impl_zeroable { - ($type:ty) => { - unsafe impl $crate::zeroable::Zeroable for $type { - const ZERO: Self = unsafe { $crate::const_zero!($type) }; - } - }; + /// Return a value of Self whose memory representation consists of all + /// zeroes, with the possible exclusion of padding bytes. + const ZERO: Self = unsafe { ::core::mem::MaybeUninit::<Self>::zeroed().assume_init() }; } // bindgen does not derive Default here @@ -97,13 +25,13 @@ impl Default for crate::bindings::VMStateFlags { } } -impl_zeroable!(crate::bindings::Property__bindgen_ty_1); -impl_zeroable!(crate::bindings::Property); -impl_zeroable!(crate::bindings::VMStateFlags); -impl_zeroable!(crate::bindings::VMStateField); -impl_zeroable!(crate::bindings::VMStateDescription); -impl_zeroable!(crate::bindings::MemoryRegionOps__bindgen_ty_1); -impl_zeroable!(crate::bindings::MemoryRegionOps__bindgen_ty_2); -impl_zeroable!(crate::bindings::MemoryRegionOps); -impl_zeroable!(crate::bindings::MemTxAttrs); -impl_zeroable!(crate::bindings::CharBackend); +unsafe impl Zeroable for crate::bindings::Property__bindgen_ty_1 {} +unsafe impl Zeroable for crate::bindings::Property {} +unsafe impl Zeroable for crate::bindings::VMStateFlags {} +unsafe impl Zeroable for crate::bindings::VMStateField {} +unsafe impl Zeroable for crate::bindings::VMStateDescription {} +unsafe impl Zeroable for crate::bindings::MemoryRegionOps__bindgen_ty_1 {} +unsafe impl Zeroable for crate::bindings::MemoryRegionOps__bindgen_ty_2 {} +unsafe impl Zeroable for crate::bindings::MemoryRegionOps {} +unsafe impl Zeroable for crate::bindings::MemTxAttrs {} +unsafe impl Zeroable for crate::bindings::CharBackend {} diff --git a/rust/qemu-api/tests/tests.rs b/rust/qemu-api/tests/tests.rs index 99a7aab6fe..a658a49fcf 100644 --- a/rust/qemu-api/tests/tests.rs +++ b/rust/qemu-api/tests/tests.rs @@ -6,7 +6,6 @@ use std::{ffi::CStr, ptr::addr_of}; use qemu_api::{ bindings::{module_call_init, module_init_type, qdev_prop_bool}, - c_str, cell::{self, BqlCell}, declare_properties, define_property, prelude::*, @@ -21,12 +20,11 @@ mod vmstate_tests; // Test that macros can compile. pub static VMSTATE: VMStateDescription = VMStateDescription { - name: c_str!("name").as_ptr(), + name: c"name".as_ptr(), unmigratable: true, ..Zeroable::ZERO }; -#[derive(qemu_api_macros::offsets)] #[repr(C)] #[derive(qemu_api_macros::Object)] pub struct DummyState { @@ -49,7 +47,7 @@ impl DummyClass { declare_properties! { DUMMY_PROPERTIES, define_property!( - c_str!("migrate-clk"), + c"migrate-clk", DummyState, migrate_clock, unsafe { &qdev_prop_bool }, @@ -59,7 +57,7 @@ declare_properties! { unsafe impl ObjectType for DummyState { type Class = DummyClass; - const TYPE_NAME: &'static CStr = c_str!("dummy"); + const TYPE_NAME: &'static CStr = c"dummy"; } impl ObjectImpl for DummyState { @@ -79,7 +77,6 @@ impl DeviceImpl for DummyState { } } -#[derive(qemu_api_macros::offsets)] #[repr(C)] #[derive(qemu_api_macros::Object)] pub struct DummyChildState { @@ -94,7 +91,7 @@ pub struct DummyChildClass { unsafe impl ObjectType for DummyChildState { type Class = DummyChildClass; - const TYPE_NAME: &'static CStr = c_str!("dummy_child"); + const TYPE_NAME: &'static CStr = c"dummy_child"; } impl ObjectImpl for DummyChildState { diff --git a/rust/qemu-api/tests/vmstate_tests.rs b/rust/qemu-api/tests/vmstate_tests.rs index 8b93492a68..ad0fc5cd5d 100644 --- a/rust/qemu-api/tests/vmstate_tests.rs +++ b/rust/qemu-api/tests/vmstate_tests.rs @@ -2,14 +2,18 @@ // Author(s): Zhao Liu <zhai1.liu@intel.com> // SPDX-License-Identifier: GPL-2.0-or-later -use std::{ffi::CStr, mem::size_of, os::raw::c_void, ptr::NonNull, slice}; +use std::{ + ffi::{c_void, CStr}, + mem::size_of, + ptr::NonNull, + slice, +}; use qemu_api::{ bindings::{ vmstate_info_bool, vmstate_info_int32, vmstate_info_int64, vmstate_info_int8, vmstate_info_uint64, vmstate_info_uint8, vmstate_info_unused_buffer, VMStateFlags, }, - c_str, cell::{BqlCell, Opaque}, impl_vmstate_forward, vmstate::{VMStateDescription, VMStateField}, @@ -28,7 +32,7 @@ const FOO_ARRAY_MAX: usize = 3; // - VMSTATE_VARRAY_UINT16_UNSAFE // - VMSTATE_VARRAY_MULTIPLY #[repr(C)] -#[derive(qemu_api_macros::offsets)] +#[derive(Default)] struct FooA { arr: [u8; FOO_ARRAY_MAX], num: u16, @@ -38,7 +42,7 @@ struct FooA { } static VMSTATE_FOOA: VMStateDescription = VMStateDescription { - name: c_str!("foo_a").as_ptr(), + name: c"foo_a".as_ptr(), version_id: 1, minimum_version_id: 1, fields: vmstate_fields! { @@ -147,8 +151,9 @@ fn test_vmstate_varray_multiply() { // - VMSTATE_STRUCT_VARRAY_UINT8 // - (no C version) MULTIPLY variant of VMSTATE_STRUCT_VARRAY_UINT32 // - VMSTATE_ARRAY +// - VMSTATE_STRUCT_VARRAY_UINT8 with BqlCell wrapper & test_fn #[repr(C)] -#[derive(qemu_api_macros::offsets)] +#[derive(Default)] struct FooB { arr_a: [FooA; FOO_ARRAY_MAX], num_a: u8, @@ -158,10 +163,16 @@ struct FooB { val: bool, // FIXME: Use Timer array. Now we can't since it's hard to link savevm.c to test. arr_i64: [i64; FOO_ARRAY_MAX], + arr_a_wrap: [FooA; FOO_ARRAY_MAX], + num_a_wrap: BqlCell<u32>, +} + +fn validate_foob(_state: &FooB, _version_id: u8) -> bool { + true } static VMSTATE_FOOB: VMStateDescription = VMStateDescription { - name: c_str!("foo_b").as_ptr(), + name: c"foo_b".as_ptr(), version_id: 2, minimum_version_id: 1, fields: vmstate_fields! { @@ -170,13 +181,14 @@ static VMSTATE_FOOB: VMStateDescription = VMStateDescription { vmstate_struct!(FooB, arr_a[0 .. num_a], &VMSTATE_FOOA, FooA).with_version_id(1), vmstate_struct!(FooB, arr_a_mul[0 .. num_a_mul * 32], &VMSTATE_FOOA, FooA).with_version_id(2), vmstate_of!(FooB, arr_i64), + vmstate_struct!(FooB, arr_a_wrap[0 .. num_a_wrap], &VMSTATE_FOOA, FooA, validate_foob), }, ..Zeroable::ZERO }; #[test] fn test_vmstate_bool_v() { - let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 6) }; + let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 7) }; // 1st VMStateField ("val") in VMSTATE_FOOB (corresponding to VMSTATE_BOOL_V) assert_eq!( @@ -196,7 +208,7 @@ fn test_vmstate_bool_v() { #[test] fn test_vmstate_uint64() { - let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 6) }; + let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 7) }; // 2nd VMStateField ("wrap") in VMSTATE_FOOB (corresponding to VMSTATE_U64) assert_eq!( @@ -216,7 +228,7 @@ fn test_vmstate_uint64() { #[test] fn test_vmstate_struct_varray_uint8() { - let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 6) }; + let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 7) }; // 3rd VMStateField ("arr_a") in VMSTATE_FOOB (corresponding to // VMSTATE_STRUCT_VARRAY_UINT8) @@ -240,7 +252,7 @@ fn test_vmstate_struct_varray_uint8() { #[test] fn test_vmstate_struct_varray_uint32_multiply() { - let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 6) }; + let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 7) }; // 4th VMStateField ("arr_a_mul") in VMSTATE_FOOB (corresponding to // (no C version) MULTIPLY variant of VMSTATE_STRUCT_VARRAY_UINT32) @@ -266,7 +278,7 @@ fn test_vmstate_struct_varray_uint32_multiply() { #[test] fn test_vmstate_macro_array() { - let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 6) }; + let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 7) }; // 5th VMStateField ("arr_i64") in VMSTATE_FOOB (corresponding to // VMSTATE_ARRAY) @@ -283,9 +295,26 @@ fn test_vmstate_macro_array() { assert_eq!(foo_fields[4].flags, VMStateFlags::VMS_ARRAY); assert!(foo_fields[4].vmsd.is_null()); assert!(foo_fields[4].field_exists.is_none()); +} + +#[test] +fn test_vmstate_struct_varray_uint8_wrapper() { + let foo_fields: &[VMStateField] = unsafe { slice::from_raw_parts(VMSTATE_FOOB.fields, 7) }; + let mut foo_b: FooB = Default::default(); + let foo_b_p = std::ptr::addr_of_mut!(foo_b).cast::<c_void>(); + + // 6th VMStateField ("arr_a_wrap") in VMSTATE_FOOB (corresponding to + // VMSTATE_STRUCT_VARRAY_UINT8). Other fields are checked in + // test_vmstate_struct_varray_uint8. + assert_eq!( + unsafe { CStr::from_ptr(foo_fields[5].name) }.to_bytes_with_nul(), + b"arr_a_wrap\0" + ); + assert_eq!(foo_fields[5].num_offset, 228); + assert!(unsafe { foo_fields[5].field_exists.unwrap()(foo_b_p, 0) }); // The last VMStateField in VMSTATE_FOOB. - assert_eq!(foo_fields[5].flags, VMStateFlags::VMS_END); + assert_eq!(foo_fields[6].flags, VMStateFlags::VMS_END); } // =========================== Test VMSTATE_FOOC =========================== @@ -299,7 +328,6 @@ struct FooCWrapper([Opaque<*mut u8>; FOO_ARRAY_MAX]); // Though Opaque<> array i impl_vmstate_forward!(FooCWrapper); #[repr(C)] -#[derive(qemu_api_macros::offsets)] struct FooC { ptr: *const i32, ptr_a: NonNull<FooA>, @@ -308,7 +336,7 @@ struct FooC { } static VMSTATE_FOOC: VMStateDescription = VMStateDescription { - name: c_str!("foo_c").as_ptr(), + name: c"foo_c".as_ptr(), version_id: 3, minimum_version_id: 1, fields: vmstate_fields! { @@ -423,13 +451,13 @@ fn validate_food_2(_state: &FooD, _version_id: u8) -> bool { } static VMSTATE_FOOD: VMStateDescription = VMStateDescription { - name: c_str!("foo_d").as_ptr(), + name: c"foo_d".as_ptr(), version_id: 3, minimum_version_id: 1, fields: vmstate_fields! { - vmstate_validate!(FooD, c_str!("foo_d_0"), FooD::validate_food_0), - vmstate_validate!(FooD, c_str!("foo_d_1"), FooD::validate_food_1), - vmstate_validate!(FooD, c_str!("foo_d_2"), validate_food_2), + vmstate_validate!(FooD, c"foo_d_0", FooD::validate_food_0), + vmstate_validate!(FooD, c"foo_d_1", FooD::validate_food_1), + vmstate_validate!(FooD, c"foo_d_2", validate_food_2), }, ..Zeroable::ZERO }; diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml b/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml index dbcd2e076d..f11e980826 100644 --- a/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml +++ b/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml @@ -114,7 +114,7 @@ packages: - python3-venv - python3-yaml - rpm2cpio - - rustc + - rustc-1.77 - sed - socat - sparse diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml b/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml index 4b8ee3d885..6559cb2934 100644 --- a/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml +++ b/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml @@ -112,7 +112,7 @@ packages: - python3-venv - python3-yaml - rpm2cpio - - rustc + - rustc-1.77 - sed - socat - sparse diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 3e8e00852b..8a67a14e2e 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -80,7 +80,7 @@ meson_options_help() { printf "%s\n" ' --tls-priority=VALUE Default TLS protocol/cipher priority string' printf "%s\n" ' [NORMAL]' printf "%s\n" ' --with-coroutine=CHOICE coroutine backend to use (choices:' - printf "%s\n" ' auto/sigaltstack/ucontext/windows)' + printf "%s\n" ' auto/sigaltstack/ucontext/windows/wasm)' printf "%s\n" ' --with-pkgversion=VALUE use specified string as sub-version of the' printf "%s\n" ' package' printf "%s\n" ' --with-suffix=VALUE Suffix for QEMU data/modules/config directories' @@ -198,6 +198,7 @@ meson_options_help() { printf "%s\n" ' u2f U2F emulation support' printf "%s\n" ' uadk UADK Library support' printf "%s\n" ' usb-redir libusbredir support' + printf "%s\n" ' valgrind valgrind debug support for coroutine stacks' printf "%s\n" ' vde vde network backend support' printf "%s\n" ' vdi vdi image format support' printf "%s\n" ' vduse-blk-export' @@ -526,6 +527,8 @@ _meson_option_parse() { --disable-ubsan) printf "%s" -Dubsan=false ;; --enable-usb-redir) printf "%s" -Dusb_redir=enabled ;; --disable-usb-redir) printf "%s" -Dusb_redir=disabled ;; + --enable-valgrind) printf "%s" -Dvalgrind=enabled ;; + --disable-valgrind) printf "%s" -Dvalgrind=disabled ;; --enable-vde) printf "%s" -Dvde=enabled ;; --disable-vde) printf "%s" -Dvde=disabled ;; --enable-vdi) printf "%s" -Dvdi=enabled ;; diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini index 8109470a03..c9dbcec2db 100644 --- a/scripts/qapi/mypy.ini +++ b/scripts/qapi/mypy.ini @@ -1,4 +1,4 @@ [mypy] strict = True disallow_untyped_calls = False -python_version = 3.8 +python_version = 3.9 diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh new file mode 100755 index 0000000000..a39f2fb0e5 --- /dev/null +++ b/scripts/rdma-migration-helper.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Copied from blktests +get_ipv4_addr() +{ + ip -4 -o addr show dev "$1" | + sed -n 's/.*[[:blank:]]inet[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' | + head -1 | tr -d '\n' +} + +# existing rdma interfaces +rdma_interfaces() +{ + rdma link show | sed -nE 's/^link .* netdev ([^ ]+).*$/\1 /p' +} + +# existing valid ipv4 interfaces +ipv4_interfaces() +{ + ip -o addr show | awk '/inet / {print $2}' | grep -v -w lo +} + +rdma_rxe_detect() +{ + for r in $(rdma_interfaces) + do + ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return + done + + return 1 +} + +rdma_rxe_setup() +{ + for i in $(ipv4_interfaces) + do + rdma_interfaces | grep -qw $i && continue + rdma link add "${i}_rxe" type rxe netdev "$i" && { + echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)" + return + } + done + + echo "Failed to setup any new rdma/rxe link" >&2 + return 1 +} + +rdma_rxe_clean() +{ + modprobe -r rdma_rxe +} + +operation=${1:-detect} + +command -v rdma >/dev/null || { + echo "Command 'rdma' is not available, please install it first." >&2 + exit 1 +} + +if [ "$operation" == "setup" ] || [ "$operation" == "clean" ]; then + [ "$UID" == 0 ] || { + echo "Root privilege is required to setup/clean a rdma/rxe link" >&2 + exit 1 + } + rdma_rxe_"$operation" +elif [ "$operation" == "detect" ]; then + rdma_rxe_detect +else + echo "Usage: $0 [setup | detect | clean]" +fi diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh index 8913e4fb99..b43b8ef75a 100755 --- a/scripts/update-linux-headers.sh +++ b/scripts/update-linux-headers.sh @@ -177,7 +177,7 @@ EOF # Remove everything except the macros from bootparam.h avoiding the # unnecessary import of several video/ist/etc headers - sed -e '/__ASSEMBLY__/,/__ASSEMBLY__/d' \ + sed -e '/__ASSEMBLER__/,/__ASSEMBLER__/d' \ "$hdrdir/include/asm/bootparam.h" > "$hdrdir/bootparam.h" cp_portable "$hdrdir/bootparam.h" \ "$output/include/standard-headers/asm-$arch" diff --git a/scripts/vmstate-static-checker.py b/scripts/vmstate-static-checker.py index 9c0e6b81f2..25aca839a0 100755 --- a/scripts/vmstate-static-checker.py +++ b/scripts/vmstate-static-checker.py @@ -42,6 +42,7 @@ def check_fields_match(name, s_field, d_field): # Some fields changed names between qemu versions. This list # is used to allow such changes in each section / description. changed_names = { + 'acpi-ghes': ['ghes_addr_le', 'hw_error_le'], 'apic': ['timer', 'timer_expiry'], 'e1000': ['dev', 'parent_obj'], 'ehci': ['dev', 'pcidev'], diff --git a/semihosting/uaccess.c b/semihosting/uaccess.c index 81ffecaaba..4554844e15 100644 --- a/semihosting/uaccess.c +++ b/semihosting/uaccess.c @@ -9,7 +9,7 @@ #include "qemu/osdep.h" #include "accel/tcg/cpu-mmu-index.h" -#include "exec/exec-all.h" +#include "accel/tcg/probe.h" #include "exec/target_page.h" #include "exec/tlb-flags.h" #include "semihosting/uaccess.h" diff --git a/subprojects/bilge-impl-0.2-rs.wrap b/subprojects/bilge-impl-0.2-rs.wrap index d14c3dc769..4f84eca1cc 100644 --- a/subprojects/bilge-impl-0.2-rs.wrap +++ b/subprojects/bilge-impl-0.2-rs.wrap @@ -5,7 +5,6 @@ source_filename = bilge-impl-0.2.0.tar.gz source_hash = feb11e002038ad243af39c2068c8a72bcf147acf05025dcdb916fcc000adb2d8 #method = cargo patch_directory = bilge-impl-0.2-rs -diff_files = bilge-impl-1.63.0.patch # bump this version number on every change to meson.build or the patches: # v2 diff --git a/subprojects/packagefiles/bilge-impl-1.63.0.patch b/subprojects/packagefiles/bilge-impl-1.63.0.patch deleted file mode 100644 index 987428a6d6..0000000000 --- a/subprojects/packagefiles/bilge-impl-1.63.0.patch +++ /dev/null @@ -1,45 +0,0 @@ ---- a/src/shared/discriminant_assigner.rs -+++ b/src/shared/discriminant_assigner.rs -@@ -26,20 +26,20 @@ - let discriminant_expr = &discriminant.1; - let variant_name = &variant.ident; - -- let Expr::Lit(ExprLit { lit: Lit::Int(int), .. }) = discriminant_expr else { -+ if let Expr::Lit(ExprLit { lit: Lit::Int(int), .. }) = discriminant_expr { -+ let discriminant_value: u128 = int.base10_parse().unwrap_or_else(unreachable); -+ if discriminant_value > self.max_value() { -+ abort!(variant, "Value of variant exceeds the given number of bits") -+ } -+ -+ Some(discriminant_value) -+ } else { - abort!( - discriminant_expr, - "variant `{}` is not a number", variant_name; - help = "only literal integers currently supported" - ) -- }; -- -- let discriminant_value: u128 = int.base10_parse().unwrap_or_else(unreachable); -- if discriminant_value > self.max_value() { -- abort!(variant, "Value of variant exceeds the given number of bits") - } -- -- Some(discriminant_value) - } - - fn assign(&mut self, variant: &Variant) -> u128 { ---- a/src/shared/fallback.rs -+++ b/src/shared/fallback.rs -@@ -22,8 +22,9 @@ - } - Unnamed(fields) => { - let variant_fields = fields.unnamed.iter(); -- let Ok(fallback_value) = variant_fields.exactly_one() else { -- abort!(variant, "fallback variant must have exactly one field"; help = "use only one field or change to a unit variant") -+ let fallback_value = match variant_fields.exactly_one() { -+ Ok(ok) => ok, -+ _ => abort!(variant, "fallback variant must have exactly one field"; help = "use only one field or change to a unit variant") - }; - - if !is_last_variant { diff --git a/system/memory.c b/system/memory.c index 71434e7ad0..63b983efcd 100644 --- a/system/memory.c +++ b/system/memory.c @@ -1627,7 +1627,7 @@ bool memory_region_init_resizeable_ram(MemoryRegion *mr, return true; } -#ifdef CONFIG_POSIX +#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) bool memory_region_init_ram_from_file(MemoryRegion *mr, Object *owner, const char *name, diff --git a/system/physmem.c b/system/physmem.c index 16cf557d1a..a8a9ca309e 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -29,9 +29,9 @@ #ifdef CONFIG_TCG #include "accel/tcg/cpu-ops.h" +#include "accel/tcg/iommu.h" #endif /* CONFIG_TCG */ -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "exec/page-protection.h" #include "exec/target_page.h" @@ -587,6 +587,8 @@ MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat, return mr; } +#ifdef CONFIG_TCG + typedef struct TCGIOMMUNotifier { IOMMUNotifier n; MemoryRegion *mr; @@ -746,6 +748,33 @@ translate_fail: return &d->map.sections[PHYS_SECTION_UNASSIGNED]; } +MemoryRegionSection *iotlb_to_section(CPUState *cpu, + hwaddr index, MemTxAttrs attrs) +{ + int asidx = cpu_asidx_from_attrs(cpu, attrs); + CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx]; + AddressSpaceDispatch *d = cpuas->memory_dispatch; + int section_index = index & ~TARGET_PAGE_MASK; + MemoryRegionSection *ret; + + assert(section_index < d->map.sections_nb); + ret = d->map.sections + section_index; + assert(ret->mr); + assert(ret->mr->ops); + + return ret; +} + +/* Called from RCU critical section */ +hwaddr memory_region_section_get_iotlb(CPUState *cpu, + MemoryRegionSection *section) +{ + AddressSpaceDispatch *d = flatview_to_dispatch(section->fv); + return section - d->map.sections; +} + +#endif /* CONFIG_TCG */ + void cpu_address_space_init(CPUState *cpu, int asidx, const char *prefix, MemoryRegion *mr) { @@ -1002,14 +1031,6 @@ bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, return false; } -/* Called from RCU critical section */ -hwaddr memory_region_section_get_iotlb(CPUState *cpu, - MemoryRegionSection *section) -{ - AddressSpaceDispatch *d = flatview_to_dispatch(section->fv); - return section - d->map.sections; -} - static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end, uint16_t section); static subpage_t *subpage_init(FlatView *fv, hwaddr base); @@ -1243,7 +1264,7 @@ long qemu_maxrampagesize(void) return pagesize; } -#ifdef CONFIG_POSIX +#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) static int64_t get_file_size(int fd) { int64_t size; @@ -1978,7 +1999,7 @@ out_free: } } -#ifdef CONFIG_POSIX +#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, ram_addr_t max_size, qemu_ram_resize_cb resized, MemoryRegion *mr, uint32_t ram_flags, int fd, off_t offset, @@ -2158,7 +2179,8 @@ RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size, assert(!host ^ (ram_flags & RAM_PREALLOC)); assert(max_size >= size); -#ifdef CONFIG_POSIX /* ignore RAM_SHARED for Windows */ + /* ignore RAM_SHARED for Windows and emscripten*/ +#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) if (!host) { if (!share_flags && current_machine->aux_ram_share) { ram_flags |= RAM_SHARED; @@ -2255,7 +2277,7 @@ static void reclaim_ramblock(RAMBlock *block) ; } else if (xen_enabled()) { xen_invalidate_map_cache_entry(block->host); -#ifndef _WIN32 +#if !defined(_WIN32) && !defined(EMSCRIPTEN) } else if (block->fd >= 0) { qemu_ram_munmap(block->fd, block->host, block->max_length); close(block->fd); @@ -2669,23 +2691,6 @@ static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr) return phys_section_add(map, §ion); } -MemoryRegionSection *iotlb_to_section(CPUState *cpu, - hwaddr index, MemTxAttrs attrs) -{ - int asidx = cpu_asidx_from_attrs(cpu, attrs); - CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx]; - AddressSpaceDispatch *d = cpuas->memory_dispatch; - int section_index = index & ~TARGET_PAGE_MASK; - MemoryRegionSection *ret; - - assert(section_index < d->map.sections_nb); - ret = d->map.sections + section_index; - assert(ret->mr); - assert(ret->mr->ops); - - return ret; -} - static void io_mem_init(void) { memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL, @@ -2830,7 +2835,7 @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, } if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { assert(tcg_enabled()); - tb_invalidate_phys_range(addr, addr + length - 1); + tb_invalidate_phys_range(NULL, addr, addr + length - 1); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); diff --git a/system/vl.c b/system/vl.c index 520956f4a1..fd402b8ff8 100644 --- a/system/vl.c +++ b/system/vl.c @@ -27,6 +27,7 @@ #include "qemu/datadir.h" #include "qemu/units.h" #include "qemu/module.h" +#include "qemu/target-info.h" #include "exec/cpu-common.h" #include "exec/page-vary.h" #include "hw/qdev-properties.h" @@ -767,7 +768,7 @@ static QemuOptsList qemu_smp_opts = { }, }; -#if defined(CONFIG_POSIX) +#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) static QemuOptsList qemu_run_with_opts = { .name = "run-with", .head = QTAILQ_HEAD_INITIALIZER(qemu_run_with_opts.head), @@ -1564,7 +1565,7 @@ static void machine_help_func(const QDict *qdict) GSList *el; const char *type = qdict_get_try_str(qdict, "type"); - machines = object_class_get_list(TYPE_MACHINE, false); + machines = object_class_get_list(target_machine_typename(), false); if (type) { ObjectClass *machine_class = OBJECT_CLASS(find_machine(type, machines)); if (machine_class) { @@ -3678,7 +3679,7 @@ void qemu_init(int argc, char **argv) case QEMU_OPTION_nouserconfig: /* Nothing to be parsed here. Especially, do not error out below. */ break; -#if defined(CONFIG_POSIX) +#if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) case QEMU_OPTION_daemonize: os_set_daemonize(true); break; diff --git a/target-info-stub.c b/target-info-stub.c index 773a10188c..fecc0e7128 100644 --- a/target-info-stub.c +++ b/target-info-stub.c @@ -9,18 +9,17 @@ #include "qemu/osdep.h" #include "qemu/target-info.h" #include "qemu/target-info-impl.h" +#include "hw/boards.h" #include "cpu.h" static const TargetInfo target_info_stub = { .target_name = TARGET_NAME, + .long_bits = TARGET_LONG_BITS, + .cpu_type = CPU_RESOLVING_TYPE, + .machine_typename = TYPE_MACHINE, }; const TargetInfo *target_info(void) { return &target_info_stub; } - -const char *target_cpu_type(void) -{ - return CPU_RESOLVING_TYPE; -} diff --git a/target-info.c b/target-info.c index 84b18931e7..16fdca7aaa 100644 --- a/target-info.c +++ b/target-info.c @@ -14,3 +14,18 @@ const char *target_name(void) { return target_info()->target_name; } + +unsigned target_long_bits(void) +{ + return target_info()->long_bits; +} + +const char *target_cpu_type(void) +{ + return target_info()->cpu_type; +} + +const char *target_machine_typename(void) +{ + return target_info()->machine_typename; +} diff --git a/target/alpha/cpu.c b/target/alpha/cpu.c index 27e2008a4e..890b84c032 100644 --- a/target/alpha/cpu.c +++ b/target/alpha/cpu.c @@ -23,9 +23,9 @@ #include "qapi/error.h" #include "qemu/qemu-print.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "exec/target_page.h" +#include "accel/tcg/cpu-ops.h" #include "fpu/softfloat.h" @@ -41,6 +41,18 @@ static vaddr alpha_cpu_get_pc(CPUState *cs) return env->pc; } +static TCGTBCPUState alpha_get_tb_cpu_state(CPUState *cs) +{ + CPUAlphaState *env = cpu_env(cs); + uint32_t flags = env->flags & ENV_FLAG_TB_MASK; + +#ifdef CONFIG_USER_ONLY + flags |= TB_FLAG_UNALIGN * !cs->prctl_unalign_sigbus; +#endif + + return (TCGTBCPUState){ .pc = env->pc, .flags = flags }; +} + static void alpha_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -232,8 +244,6 @@ static const struct SysemuCPUOps alpha_sysemu_ops = { }; #endif -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps alpha_tcg_ops = { /* Alpha processors have a weak memory model */ .guest_default_memory_order = 0, @@ -241,6 +251,7 @@ static const TCGCPUOps alpha_tcg_ops = { .initialize = alpha_translate_init, .translate_code = alpha_translate_code, + .get_tb_cpu_state = alpha_get_tb_cpu_state, .synchronize_from_tb = alpha_cpu_synchronize_from_tb, .restore_state_to_opc = alpha_restore_state_to_opc, .mmu_index = alpha_cpu_mmu_index, @@ -252,6 +263,7 @@ static const TCGCPUOps alpha_tcg_ops = { .tlb_fill = alpha_cpu_tlb_fill, .cpu_exec_interrupt = alpha_cpu_exec_interrupt, .cpu_exec_halt = alpha_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = alpha_cpu_do_interrupt, .do_transaction_failed = alpha_cpu_do_transaction_failed, .do_unaligned_access = alpha_cpu_do_unaligned_access, diff --git a/target/alpha/cpu.h b/target/alpha/cpu.h index 849f673489..45944e46b5 100644 --- a/target/alpha/cpu.h +++ b/target/alpha/cpu.h @@ -464,17 +464,6 @@ void alpha_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr, MemTxResult response, uintptr_t retaddr); #endif -static inline void cpu_get_tb_cpu_state(CPUAlphaState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *pflags) -{ - *pc = env->pc; - *cs_base = 0; - *pflags = env->flags & ENV_FLAG_TB_MASK; -#ifdef CONFIG_USER_ONLY - *pflags |= TB_FLAG_UNALIGN * !env_cpu(env)->prctl_unalign_sigbus; -#endif -} - #ifdef CONFIG_USER_ONLY /* Copied from linux ieee_swcr_to_fpcr. */ static inline uint64_t alpha_ieee_swcr_to_fpcr(uint64_t swcr) diff --git a/target/alpha/fpu_helper.c b/target/alpha/fpu_helper.c index 6aefb9b851..30f3c7fd18 100644 --- a/target/alpha/fpu_helper.c +++ b/target/alpha/fpu_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" diff --git a/target/alpha/int_helper.c b/target/alpha/int_helper.c index 5672696f6f..6bfe63500e 100644 --- a/target/alpha/int_helper.c +++ b/target/alpha/int_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "qemu/host-utils.h" diff --git a/target/alpha/machine.c b/target/alpha/machine.c index f09834f635..5f302b166d 100644 --- a/target/alpha/machine.c +++ b/target/alpha/machine.c @@ -74,7 +74,7 @@ static const VMStateDescription vmstate_env = { }; static const VMStateField vmstate_cpu_fields[] = { - VMSTATE_CPU(), + VMSTATE_STRUCT(parent_obj, AlphaCPU, 0, vmstate_cpu_common, CPUState), VMSTATE_STRUCT(env, AlphaCPU, 1, vmstate_env, CPUAlphaState), VMSTATE_END_OF_LIST() }; diff --git a/target/alpha/mem_helper.c b/target/alpha/mem_helper.c index a4d5adb40c..2113fe33ae 100644 --- a/target/alpha/mem_helper.c +++ b/target/alpha/mem_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" static void do_unaligned_access(CPUAlphaState *env, vaddr addr, uintptr_t retaddr) diff --git a/target/alpha/translate.c b/target/alpha/translate.c index 7f3195a5dc..cebab0318c 100644 --- a/target/alpha/translate.c +++ b/target/alpha/translate.c @@ -21,7 +21,6 @@ #include "cpu.h" #include "system/cpus.h" #include "qemu/host-utils.h" -#include "exec/exec-all.h" #include "tcg/tcg-op.h" #include "exec/helper-proto.h" #include "exec/helper-gen.h" diff --git a/target/alpha/vax_helper.c b/target/alpha/vax_helper.c index f94fb519db..c1d201e7b4 100644 --- a/target/alpha/vax_helper.c +++ b/target/alpha/vax_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" diff --git a/target/arm/arm-qmp-cmds.c b/target/arm/arm-qmp-cmds.c index 883c0a0e8c..a1a944adb4 100644 --- a/target/arm/arm-qmp-cmds.c +++ b/target/arm/arm-qmp-cmds.c @@ -46,7 +46,7 @@ static inline void gic_cap_kvm_probe(GICCapability *v2, GICCapability *v3) #ifdef CONFIG_KVM int fdarray[3]; - if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, NULL)) { + if (!kvm_arm_create_scratch_host_vcpu(fdarray, NULL)) { return; } diff --git a/target/arm/cpu-param.h b/target/arm/cpu-param.h index 5c5bc8a009..8b46c7c570 100644 --- a/target/arm/cpu-param.h +++ b/target/arm/cpu-param.h @@ -17,14 +17,9 @@ #endif #ifdef CONFIG_USER_ONLY -# ifdef TARGET_AARCH64 -# define TARGET_TAGGED_ADDRESSES -# ifdef __FreeBSD__ -# define TARGET_PAGE_BITS 12 -# else +# if defined(TARGET_AARCH64) && defined(CONFIG_LINUX) /* Allow user-only to vary page size from 4k */ # define TARGET_PAGE_BITS_VARY -# endif # else # define TARGET_PAGE_BITS 12 # endif diff --git a/target/arm/cpu.c b/target/arm/cpu.c index 5e951675c6..45cb6fd7ee 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -33,7 +33,6 @@ #endif /* CONFIG_TCG */ #include "internals.h" #include "cpu-features.h" -#include "exec/exec-all.h" #include "exec/target_page.h" #include "hw/qdev-properties.h" #if !defined(CONFIG_USER_ONLY) @@ -2672,7 +2671,31 @@ static const char *arm_gdb_get_core_xml_file(CPUState *cs) return "arm-core.xml"; } -#ifndef CONFIG_USER_ONLY +#ifdef CONFIG_USER_ONLY +/** + * aarch64_untagged_addr: + * + * Remove any address tag from @x. This is explicitly related to the + * linux syscall TIF_TAGGED_ADDR setting, not TBI in general. + * + * There should be a better place to put this, but we need this in + * include/exec/cpu_ldst.h, and not some place linux-user specific. + * + * Note that arm-*-user will never set tagged_addr_enable. + */ +static vaddr aarch64_untagged_addr(CPUState *cs, vaddr x) +{ + CPUARMState *env = cpu_env(cs); + if (env->tagged_addr_enable) { + /* + * TBI is enabled for userspace but not kernelspace addresses. + * Only clear the tag if bit 55 is clear. + */ + x &= sextract64(x, 0, 56); + } + return x; +} +#else #include "hw/core/sysemu-cpu-ops.h" static const struct SysemuCPUOps arm_sysemu_ops = { @@ -2694,6 +2717,7 @@ static const TCGCPUOps arm_tcg_ops = { .initialize = arm_translate_init, .translate_code = arm_translate_code, + .get_tb_cpu_state = arm_get_tb_cpu_state, .synchronize_from_tb = arm_cpu_synchronize_from_tb, .debug_excp_handler = arm_debug_excp_handler, .restore_state_to_opc = arm_restore_state_to_opc, @@ -2702,10 +2726,12 @@ static const TCGCPUOps arm_tcg_ops = { #ifdef CONFIG_USER_ONLY .record_sigsegv = arm_cpu_record_sigsegv, .record_sigbus = arm_cpu_record_sigbus, + .untagged_addr = aarch64_untagged_addr, #else .tlb_fill_align = arm_cpu_tlb_fill_align, .cpu_exec_interrupt = arm_cpu_exec_interrupt, .cpu_exec_halt = arm_cpu_exec_halt, + .cpu_exec_reset = cpu_reset, .do_interrupt = arm_cpu_do_interrupt, .do_transaction_failed = arm_cpu_do_transaction_failed, .do_unaligned_access = arm_cpu_do_unaligned_access, diff --git a/target/arm/cpu.h b/target/arm/cpu.h index fdcf8cd1ae..6ed6409cb7 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -783,12 +783,9 @@ typedef struct CPUArchState { #else /* CONFIG_USER_ONLY */ /* For usermode syscall translation. */ bool eabi; -#endif /* CONFIG_USER_ONLY */ - -#ifdef TARGET_TAGGED_ADDRESSES /* Linux syscall tagged address support */ bool tagged_addr_enable; -#endif +#endif /* CONFIG_USER_ONLY */ } CPUARMState; static inline void set_feature(CPUARMState *env, int feature) @@ -3119,9 +3116,6 @@ static inline bool bswap_code(bool sctlr_b) #endif } -void cpu_get_tb_cpu_state(CPUARMState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags); - enum { QEMU_PSCI_CONDUIT_DISABLED = 0, QEMU_PSCI_CONDUIT_SMC = 1, @@ -3219,35 +3213,4 @@ extern const uint64_t pred_esz_masks[5]; #define LOG2_TAG_GRANULE 4 #define TAG_GRANULE (1 << LOG2_TAG_GRANULE) -#ifdef CONFIG_USER_ONLY - -#define TARGET_PAGE_DATA_SIZE (TARGET_PAGE_SIZE >> (LOG2_TAG_GRANULE + 1)) - -#ifdef TARGET_TAGGED_ADDRESSES -/** - * cpu_untagged_addr: - * @cs: CPU context - * @x: tagged address - * - * Remove any address tag from @x. This is explicitly related to the - * linux syscall TIF_TAGGED_ADDR setting, not TBI in general. - * - * There should be a better place to put this, but we need this in - * include/exec/cpu_ldst.h, and not some place linux-user specific. - */ -static inline target_ulong cpu_untagged_addr(CPUState *cs, target_ulong x) -{ - CPUARMState *env = cpu_env(cs); - if (env->tagged_addr_enable) { - /* - * TBI is enabled for userspace but not kernelspace addresses. - * Only clear the tag if bit 55 is clear. - */ - x &= sextract64(x, 0, 56); - } - return x; -} -#endif /* TARGET_TAGGED_ADDRESSES */ -#endif /* CONFIG_USER_ONLY */ - #endif diff --git a/target/arm/debug_helper.c b/target/arm/debug_helper.c index 473ee2af38..de7999f6a9 100644 --- a/target/arm/debug_helper.c +++ b/target/arm/debug_helper.c @@ -11,7 +11,6 @@ #include "internals.h" #include "cpu-features.h" #include "cpregs.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "exec/watchpoint.h" #include "system/tcg.h" diff --git a/target/arm/helper.c b/target/arm/helper.c index 7fb6e88630..4a2d1ecbfe 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -20,7 +20,6 @@ #include "qemu/bitops.h" #include "qemu/qemu-print.h" #include "exec/cputlb.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "hw/irq.h" #include "system/cpu-timers.h" @@ -30,6 +29,7 @@ #include "qapi/error.h" #include "qemu/guest-random.h" #ifdef CONFIG_TCG +#include "accel/tcg/probe.h" #include "semihosting/common-semi.h" #endif #include "cpregs.h" @@ -222,7 +222,7 @@ static void count_cpreg(gpointer key, gpointer opaque) } } -static gint cpreg_key_compare(gconstpointer a, gconstpointer b) +static gint cpreg_key_compare(gconstpointer a, gconstpointer b, gpointer d) { uint64_t aidx = cpreg_to_kvm_id((uintptr_t)a); uint64_t bidx = cpreg_to_kvm_id((uintptr_t)b); @@ -246,7 +246,7 @@ void init_cpreg_list(ARMCPU *cpu) int arraylen; keys = g_hash_table_get_keys(cpu->cp_regs); - keys = g_list_sort(keys, cpreg_key_compare); + keys = g_list_sort_with_data(keys, cpreg_key_compare, NULL); cpu->cpreg_array_len = 0; @@ -4987,7 +4987,7 @@ static void ic_ivau_write(CPUARMState *env, const ARMCPRegInfo *ri, mmap_lock(); - tb_invalidate_phys_range(start_address, end_address); + tb_invalidate_phys_range(env_cpu(env), start_address, end_address); mmap_unlock(); } @@ -11423,115 +11423,6 @@ ARMMMUIdx arm_mmu_idx(CPUARMState *env) return arm_mmu_idx_el(env, arm_current_el(env)); } -static bool mve_no_pred(CPUARMState *env) -{ - /* - * Return true if there is definitely no predication of MVE - * instructions by VPR or LTPSIZE. (Returning false even if there - * isn't any predication is OK; generated code will just be - * a little worse.) - * If the CPU does not implement MVE then this TB flag is always 0. - * - * NOTE: if you change this logic, the "recalculate s->mve_no_pred" - * logic in gen_update_fp_context() needs to be updated to match. - * - * We do not include the effect of the ECI bits here -- they are - * tracked in other TB flags. This simplifies the logic for - * "when did we emit code that changes the MVE_NO_PRED TB flag - * and thus need to end the TB?". - */ - if (cpu_isar_feature(aa32_mve, env_archcpu(env))) { - return false; - } - if (env->v7m.vpr) { - return false; - } - if (env->v7m.ltpsize < 4) { - return false; - } - return true; -} - -void cpu_get_tb_cpu_state(CPUARMState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *pflags) -{ - CPUARMTBFlags flags; - - assert_hflags_rebuild_correctly(env); - flags = env->hflags; - - if (EX_TBFLAG_ANY(flags, AARCH64_STATE)) { - *pc = env->pc; - if (cpu_isar_feature(aa64_bti, env_archcpu(env))) { - DP_TBFLAG_A64(flags, BTYPE, env->btype); - } - } else { - *pc = env->regs[15]; - - if (arm_feature(env, ARM_FEATURE_M)) { - if (arm_feature(env, ARM_FEATURE_M_SECURITY) && - FIELD_EX32(env->v7m.fpccr[M_REG_S], V7M_FPCCR, S) - != env->v7m.secure) { - DP_TBFLAG_M32(flags, FPCCR_S_WRONG, 1); - } - - if ((env->v7m.fpccr[env->v7m.secure] & R_V7M_FPCCR_ASPEN_MASK) && - (!(env->v7m.control[M_REG_S] & R_V7M_CONTROL_FPCA_MASK) || - (env->v7m.secure && - !(env->v7m.control[M_REG_S] & R_V7M_CONTROL_SFPA_MASK)))) { - /* - * ASPEN is set, but FPCA/SFPA indicate that there is no - * active FP context; we must create a new FP context before - * executing any FP insn. - */ - DP_TBFLAG_M32(flags, NEW_FP_CTXT_NEEDED, 1); - } - - bool is_secure = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK; - if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) { - DP_TBFLAG_M32(flags, LSPACT, 1); - } - - if (mve_no_pred(env)) { - DP_TBFLAG_M32(flags, MVE_NO_PRED, 1); - } - } else { - /* - * Note that XSCALE_CPAR shares bits with VECSTRIDE. - * Note that VECLEN+VECSTRIDE are RES0 for M-profile. - */ - if (arm_feature(env, ARM_FEATURE_XSCALE)) { - DP_TBFLAG_A32(flags, XSCALE_CPAR, env->cp15.c15_cpar); - } else { - DP_TBFLAG_A32(flags, VECLEN, env->vfp.vec_len); - DP_TBFLAG_A32(flags, VECSTRIDE, env->vfp.vec_stride); - } - if (env->vfp.xregs[ARM_VFP_FPEXC] & (1 << 30)) { - DP_TBFLAG_A32(flags, VFPEN, 1); - } - } - - DP_TBFLAG_AM32(flags, THUMB, env->thumb); - DP_TBFLAG_AM32(flags, CONDEXEC, env->condexec_bits); - } - - /* - * The SS_ACTIVE and PSTATE_SS bits correspond to the state machine - * states defined in the ARM ARM for software singlestep: - * SS_ACTIVE PSTATE.SS State - * 0 x Inactive (the TB flag for SS is always 0) - * 1 0 Active-pending - * 1 1 Active-not-pending - * SS_ACTIVE is set in hflags; PSTATE__SS is computed every TB. - */ - if (EX_TBFLAG_ANY(flags, SS_ACTIVE) && (env->pstate & PSTATE_SS)) { - DP_TBFLAG_ANY(flags, PSTATE__SS, 1); - } - - *pflags = flags.flags; - *cs_base = flags.flags2; -} - #ifdef TARGET_AARCH64 /* * The manual says that when SVE is enabled and VQ is widened the diff --git a/target/arm/hvf/hvf.c b/target/arm/hvf/hvf.c index 34ca36fab5..42258cc2d8 100644 --- a/target/arm/hvf/hvf.c +++ b/target/arm/hvf/hvf.c @@ -2278,28 +2278,23 @@ static inline bool hvf_arm_hw_debug_active(CPUState *cpu) return ((cur_hw_wps > 0) || (cur_hw_bps > 0)); } -static void hvf_arch_set_traps(void) +static void hvf_arch_set_traps(CPUState *cpu) { - CPUState *cpu; bool should_enable_traps = false; hv_return_t r = HV_SUCCESS; /* Check whether guest debugging is enabled for at least one vCPU; if it * is, enable exiting the guest on all vCPUs */ - CPU_FOREACH(cpu) { - should_enable_traps |= cpu->accel->guest_debug_enabled; - } - CPU_FOREACH(cpu) { - /* Set whether debug exceptions exit the guest */ - r = hv_vcpu_set_trap_debug_exceptions(cpu->accel->fd, - should_enable_traps); - assert_hvf_ok(r); + should_enable_traps |= cpu->accel->guest_debug_enabled; + /* Set whether debug exceptions exit the guest */ + r = hv_vcpu_set_trap_debug_exceptions(cpu->accel->fd, + should_enable_traps); + assert_hvf_ok(r); - /* Set whether accesses to debug registers exit the guest */ - r = hv_vcpu_set_trap_debug_reg_accesses(cpu->accel->fd, - should_enable_traps); - assert_hvf_ok(r); - } + /* Set whether accesses to debug registers exit the guest */ + r = hv_vcpu_set_trap_debug_reg_accesses(cpu->accel->fd, + should_enable_traps); + assert_hvf_ok(r); } void hvf_arch_update_guest_debug(CPUState *cpu) @@ -2340,7 +2335,7 @@ void hvf_arch_update_guest_debug(CPUState *cpu) deposit64(env->cp15.mdscr_el1, MDSCR_EL1_MDE_SHIFT, 1, 0); } - hvf_arch_set_traps(); + hvf_arch_set_traps(cpu); } bool hvf_arch_supports_guest_debug(void) diff --git a/target/arm/internals.h b/target/arm/internals.h index 4d3d84ffeb..660d3a88e0 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -28,6 +28,7 @@ #include "exec/hwaddr.h" #include "exec/vaddr.h" #include "exec/breakpoint.h" +#include "accel/tcg/tb-cpu-state.h" #include "hw/registerfields.h" #include "tcg/tcg-gvec-desc.h" #include "system/memory.h" @@ -372,6 +373,7 @@ void arm_restore_state_to_opc(CPUState *cs, const uint64_t *data); #ifdef CONFIG_TCG +TCGTBCPUState arm_get_tb_cpu_state(CPUState *cs); void arm_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb); /* Our implementation of TCGCPUOps::cpu_exec_halt */ @@ -1906,8 +1908,6 @@ static inline bool arm_fgt_active(CPUARMState *env, int el) (!arm_feature(env, ARM_FEATURE_EL3) || (env->cp15.scr_el3 & SCR_FGTEN)); } -void assert_hflags_rebuild_correctly(CPUARMState *env); - /* * Although the ARM implementation of hardware assisted debugging * allows for different breakpoints per-core, the current GDB diff --git a/target/arm/kvm.c b/target/arm/kvm.c index 97de8c7e93..9c62d12b23 100644 --- a/target/arm/kvm.c +++ b/target/arm/kvm.c @@ -100,8 +100,7 @@ static int kvm_arm_vcpu_finalize(ARMCPU *cpu, int feature) return kvm_vcpu_ioctl(CPU(cpu), KVM_ARM_VCPU_FINALIZE, &feature); } -bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try, - int *fdarray, +bool kvm_arm_create_scratch_host_vcpu(int *fdarray, struct kvm_vcpu_init *init) { int ret = 0, kvmfd = -1, vmfd = -1, cpufd = -1; @@ -150,40 +149,13 @@ bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try, struct kvm_vcpu_init preferred; ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, &preferred); - if (!ret) { - init->target = preferred.target; - } - } - if (ret >= 0) { - ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init); - if (ret < 0) { - goto err; - } - } else if (cpus_to_try) { - /* Old kernel which doesn't know about the - * PREFERRED_TARGET ioctl: we know it will only support - * creating one kind of guest CPU which is its preferred - * CPU type. - */ - struct kvm_vcpu_init try; - - while (*cpus_to_try != QEMU_KVM_ARM_TARGET_NONE) { - try.target = *cpus_to_try++; - memcpy(try.features, init->features, sizeof(init->features)); - ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, &try); - if (ret >= 0) { - break; - } - } if (ret < 0) { goto err; } - init->target = try.target; - } else { - /* Treat a NULL cpus_to_try argument the same as an empty - * list, which means we will fail the call since this must - * be an old kernel which doesn't support PREFERRED_TARGET. - */ + init->target = preferred.target; + } + ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init); + if (ret < 0) { goto err; } @@ -259,17 +231,6 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) uint64_t features = 0; int err; - /* Old kernels may not know about the PREFERRED_TARGET ioctl: however - * we know these will only support creating one kind of guest CPU, - * which is its preferred CPU type. Fortunately these old kernels - * support only a very limited number of CPUs. - */ - static const uint32_t cpus_to_try[] = { - KVM_ARM_TARGET_AEM_V8, - KVM_ARM_TARGET_FOUNDATION_V8, - KVM_ARM_TARGET_CORTEX_A57, - QEMU_KVM_ARM_TARGET_NONE - }; /* * target = -1 informs kvm_arm_create_scratch_host_vcpu() * to use the preferred target @@ -300,7 +261,7 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) features |= 1ULL << ARM_FEATURE_PMU; } - if (!kvm_arm_create_scratch_host_vcpu(cpus_to_try, fdarray, &init)) { + if (!kvm_arm_create_scratch_host_vcpu(fdarray, &init)) { return false; } @@ -1835,7 +1796,7 @@ uint32_t kvm_arm_sve_get_vls(ARMCPU *cpu) probed = true; - if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, &init)) { + if (!kvm_arm_create_scratch_host_vcpu(fdarray, &init)) { error_report("failed to create scratch VCPU with SVE enabled"); abort(); } diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h index 05c3de8cd4..5f17fc2f3d 100644 --- a/target/arm/kvm_arm.h +++ b/target/arm/kvm_arm.h @@ -97,10 +97,6 @@ void kvm_arm_reset_vcpu(ARMCPU *cpu); #ifdef CONFIG_KVM /** * kvm_arm_create_scratch_host_vcpu: - * @cpus_to_try: array of QEMU_KVM_ARM_TARGET_* values (terminated with - * QEMU_KVM_ARM_TARGET_NONE) to try as fallback if the kernel does not - * know the PREFERRED_TARGET ioctl. Passing NULL is the same as passing - * an empty array. * @fdarray: filled in with kvmfd, vmfd, cpufd file descriptors in that order * @init: filled in with the necessary values for creating a host * vcpu. If NULL is provided, will not init the vCPU (though the cpufd @@ -113,8 +109,7 @@ void kvm_arm_reset_vcpu(ARMCPU *cpu); * Returns: true on success (and fdarray and init are filled in), * false on failure (and fdarray and init are not valid). */ -bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try, - int *fdarray, +bool kvm_arm_create_scratch_host_vcpu(int *fdarray, struct kvm_vcpu_init *init); /** diff --git a/target/arm/ptw.c b/target/arm/ptw.c index e0e82ae507..89979c07e5 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -10,10 +10,10 @@ #include "qemu/log.h" #include "qemu/range.h" #include "qemu/main-loop.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "exec/tlb-flags.h" +#include "accel/tcg/probe.h" #include "cpu.h" #include "internals.h" #include "cpu-features.h" @@ -3551,13 +3551,9 @@ bool get_phys_addr_with_space_nogpc(CPUARMState *env, vaddr address, memop, result, fi); } -bool get_phys_addr(CPUARMState *env, vaddr address, - MMUAccessType access_type, MemOp memop, ARMMMUIdx mmu_idx, - GetPhysAddrResult *result, ARMMMUFaultInfo *fi) +static ARMSecuritySpace +arm_mmu_idx_to_security_space(CPUARMState *env, ARMMMUIdx mmu_idx) { - S1Translate ptw = { - .in_mmu_idx = mmu_idx, - }; ARMSecuritySpace ss; switch (mmu_idx) { @@ -3618,28 +3614,33 @@ bool get_phys_addr(CPUARMState *env, vaddr address, g_assert_not_reached(); } - ptw.in_space = ss; + return ss; +} + +bool get_phys_addr(CPUARMState *env, vaddr address, + MMUAccessType access_type, MemOp memop, ARMMMUIdx mmu_idx, + GetPhysAddrResult *result, ARMMMUFaultInfo *fi) +{ + S1Translate ptw = { + .in_mmu_idx = mmu_idx, + .in_space = arm_mmu_idx_to_security_space(env, mmu_idx), + }; + return get_phys_addr_gpc(env, &ptw, address, access_type, memop, result, fi); } -hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, - MemTxAttrs *attrs) +static hwaddr arm_cpu_get_phys_page(CPUARMState *env, vaddr addr, + MemTxAttrs *attrs, ARMMMUIdx mmu_idx) { - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; - ARMMMUIdx mmu_idx = arm_mmu_idx(env); - ARMSecuritySpace ss = arm_security_space(env); S1Translate ptw = { .in_mmu_idx = mmu_idx, - .in_space = ss, + .in_space = arm_mmu_idx_to_security_space(env, mmu_idx), .in_debug = true, }; GetPhysAddrResult res = {}; ARMMMUFaultInfo fi = {}; - bool ret; - - ret = get_phys_addr_gpc(env, &ptw, addr, MMU_DATA_LOAD, 0, &res, &fi); + bool ret = get_phys_addr_gpc(env, &ptw, addr, MMU_DATA_LOAD, 0, &res, &fi); *attrs = res.f.attrs; if (ret) { @@ -3647,3 +3648,33 @@ hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, } return res.f.phys_addr; } + +hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, + MemTxAttrs *attrs) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + ARMMMUIdx mmu_idx = arm_mmu_idx(env); + + hwaddr res = arm_cpu_get_phys_page(env, addr, attrs, mmu_idx); + + if (res != -1) { + return res; + } + + /* + * Memory may be accessible for an "unprivileged load/store" variant. + * In this case, get_a64_user_mem_index function generates an op using an + * unprivileged mmu idx, so we need to try with it. + */ + switch (mmu_idx) { + case ARMMMUIdx_E10_1: + case ARMMMUIdx_E10_1_PAN: + return arm_cpu_get_phys_page(env, addr, attrs, ARMMMUIdx_E10_0); + case ARMMMUIdx_E20_2: + case ARMMMUIdx_E20_2_PAN: + return arm_cpu_get_phys_page(env, addr, attrs, ARMMMUIdx_E20_0); + default: + return -1; + } +} diff --git a/target/arm/tcg-stubs.c b/target/arm/tcg-stubs.c index 93a15cad61..5e5166c049 100644 --- a/target/arm/tcg-stubs.c +++ b/target/arm/tcg-stubs.c @@ -21,10 +21,6 @@ void raise_exception_ra(CPUARMState *env, uint32_t excp, uint32_t syndrome, { g_assert_not_reached(); } -/* Temporarily while cpu_get_tb_cpu_state() is still in common code */ -void assert_hflags_rebuild_correctly(CPUARMState *env) -{ -} /* TLBI insns are only used by TCG, so we don't need to do anything for KVM */ void define_tlb_insn_regs(ARMCPU *cpu) diff --git a/target/arm/tcg/cpu-v7m.c b/target/arm/tcg/cpu-v7m.c index b34b657857..95b23d9b55 100644 --- a/target/arm/tcg/cpu-v7m.c +++ b/target/arm/tcg/cpu-v7m.c @@ -238,6 +238,7 @@ static const TCGCPUOps arm_v7m_tcg_ops = { .initialize = arm_translate_init, .translate_code = arm_translate_code, + .get_tb_cpu_state = arm_get_tb_cpu_state, .synchronize_from_tb = arm_cpu_synchronize_from_tb, .debug_excp_handler = arm_debug_excp_handler, .restore_state_to_opc = arm_restore_state_to_opc, @@ -250,6 +251,7 @@ static const TCGCPUOps arm_v7m_tcg_ops = { .tlb_fill_align = arm_cpu_tlb_fill_align, .cpu_exec_interrupt = arm_v7m_cpu_exec_interrupt, .cpu_exec_halt = arm_cpu_exec_halt, + .cpu_exec_reset = cpu_reset, .do_interrupt = arm_v7m_cpu_do_interrupt, .do_transaction_failed = arm_cpu_do_transaction_failed, .do_unaligned_access = arm_cpu_do_unaligned_access, diff --git a/target/arm/tcg/helper-a64.c b/target/arm/tcg/helper-a64.c index 842d9e6000..4f618ae390 100644 --- a/target/arm/tcg/helper-a64.c +++ b/target/arm/tcg/helper-a64.c @@ -29,8 +29,9 @@ #include "internals.h" #include "qemu/crc32c.h" #include "exec/cpu-common.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/helper-retaddr.h" +#include "accel/tcg/probe.h" #include "exec/target_page.h" #include "exec/tlb-flags.h" #include "qemu/int128.h" diff --git a/target/arm/tcg/hflags.c b/target/arm/tcg/hflags.c index e51d9f7b15..fd407a7b28 100644 --- a/target/arm/tcg/hflags.c +++ b/target/arm/tcg/hflags.c @@ -10,6 +10,8 @@ #include "internals.h" #include "cpu-features.h" #include "exec/helper-proto.h" +#include "exec/translation-block.h" +#include "accel/tcg/cpu-ops.h" #include "cpregs.h" static inline bool fgt_svc(CPUARMState *env, int el) @@ -498,7 +500,7 @@ void HELPER(rebuild_hflags_a64)(CPUARMState *env, int el) env->hflags = rebuild_hflags_a64(env, el, fp_el, mmu_idx); } -void assert_hflags_rebuild_correctly(CPUARMState *env) +static void assert_hflags_rebuild_correctly(CPUARMState *env) { #ifdef CONFIG_DEBUG_TCG CPUARMTBFlags c = env->hflags; @@ -513,3 +515,116 @@ void assert_hflags_rebuild_correctly(CPUARMState *env) } #endif } + +static bool mve_no_pred(CPUARMState *env) +{ + /* + * Return true if there is definitely no predication of MVE + * instructions by VPR or LTPSIZE. (Returning false even if there + * isn't any predication is OK; generated code will just be + * a little worse.) + * If the CPU does not implement MVE then this TB flag is always 0. + * + * NOTE: if you change this logic, the "recalculate s->mve_no_pred" + * logic in gen_update_fp_context() needs to be updated to match. + * + * We do not include the effect of the ECI bits here -- they are + * tracked in other TB flags. This simplifies the logic for + * "when did we emit code that changes the MVE_NO_PRED TB flag + * and thus need to end the TB?". + */ + if (cpu_isar_feature(aa32_mve, env_archcpu(env))) { + return false; + } + if (env->v7m.vpr) { + return false; + } + if (env->v7m.ltpsize < 4) { + return false; + } + return true; +} + +TCGTBCPUState arm_get_tb_cpu_state(CPUState *cs) +{ + CPUARMState *env = cpu_env(cs); + CPUARMTBFlags flags; + vaddr pc; + + assert_hflags_rebuild_correctly(env); + flags = env->hflags; + + if (EX_TBFLAG_ANY(flags, AARCH64_STATE)) { + pc = env->pc; + if (cpu_isar_feature(aa64_bti, env_archcpu(env))) { + DP_TBFLAG_A64(flags, BTYPE, env->btype); + } + } else { + pc = env->regs[15]; + + if (arm_feature(env, ARM_FEATURE_M)) { + if (arm_feature(env, ARM_FEATURE_M_SECURITY) && + FIELD_EX32(env->v7m.fpccr[M_REG_S], V7M_FPCCR, S) + != env->v7m.secure) { + DP_TBFLAG_M32(flags, FPCCR_S_WRONG, 1); + } + + if ((env->v7m.fpccr[env->v7m.secure] & R_V7M_FPCCR_ASPEN_MASK) && + (!(env->v7m.control[M_REG_S] & R_V7M_CONTROL_FPCA_MASK) || + (env->v7m.secure && + !(env->v7m.control[M_REG_S] & R_V7M_CONTROL_SFPA_MASK)))) { + /* + * ASPEN is set, but FPCA/SFPA indicate that there is no + * active FP context; we must create a new FP context before + * executing any FP insn. + */ + DP_TBFLAG_M32(flags, NEW_FP_CTXT_NEEDED, 1); + } + + bool is_secure = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK; + if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) { + DP_TBFLAG_M32(flags, LSPACT, 1); + } + + if (mve_no_pred(env)) { + DP_TBFLAG_M32(flags, MVE_NO_PRED, 1); + } + } else { + /* + * Note that XSCALE_CPAR shares bits with VECSTRIDE. + * Note that VECLEN+VECSTRIDE are RES0 for M-profile. + */ + if (arm_feature(env, ARM_FEATURE_XSCALE)) { + DP_TBFLAG_A32(flags, XSCALE_CPAR, env->cp15.c15_cpar); + } else { + DP_TBFLAG_A32(flags, VECLEN, env->vfp.vec_len); + DP_TBFLAG_A32(flags, VECSTRIDE, env->vfp.vec_stride); + } + if (env->vfp.xregs[ARM_VFP_FPEXC] & (1 << 30)) { + DP_TBFLAG_A32(flags, VFPEN, 1); + } + } + + DP_TBFLAG_AM32(flags, THUMB, env->thumb); + DP_TBFLAG_AM32(flags, CONDEXEC, env->condexec_bits); + } + + /* + * The SS_ACTIVE and PSTATE_SS bits correspond to the state machine + * states defined in the ARM ARM for software singlestep: + * SS_ACTIVE PSTATE.SS State + * 0 x Inactive (the TB flag for SS is always 0) + * 1 0 Active-pending + * 1 1 Active-not-pending + * SS_ACTIVE is set in hflags; PSTATE__SS is computed every TB. + */ + if (EX_TBFLAG_ANY(flags, SS_ACTIVE) && (env->pstate & PSTATE_SS)) { + DP_TBFLAG_ANY(flags, PSTATE__SS, 1); + } + + return (TCGTBCPUState){ + .pc = pc, + .flags = flags.flags, + .cs_base = flags.flags2, + }; +} diff --git a/target/arm/tcg/m_helper.c b/target/arm/tcg/m_helper.c index 37dc98dc35..6614719832 100644 --- a/target/arm/tcg/m_helper.c +++ b/target/arm/tcg/m_helper.c @@ -15,7 +15,6 @@ #include "qemu/main-loop.h" #include "qemu/bitops.h" #include "qemu/log.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #ifdef CONFIG_TCG #include "accel/tcg/cpu-ldst.h" diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c index 7dc5fb776b..0efc18a181 100644 --- a/target/arm/tcg/mte_helper.c +++ b/target/arm/tcg/mte_helper.c @@ -21,7 +21,6 @@ #include "qemu/log.h" #include "cpu.h" #include "internals.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #ifdef CONFIG_USER_ONLY #include "user/cpu_loop.h" @@ -30,6 +29,7 @@ #include "system/ram_addr.h" #endif #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "exec/helper-proto.h" #include "exec/tlb-flags.h" #include "accel/tcg/cpu-ops.h" @@ -37,7 +37,6 @@ #include "qemu/guest-random.h" #include "mte_helper.h" - static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude) { if (exclude == 0xffff) { @@ -63,6 +62,7 @@ uint8_t *allocation_tag_mem_probe(CPUARMState *env, int ptr_mmu_idx, bool probe, uintptr_t ra) { #ifdef CONFIG_USER_ONLY + const size_t page_data_size = TARGET_PAGE_SIZE >> (LOG2_TAG_GRANULE + 1); uint64_t clean_ptr = useronly_clean_ptr(ptr); int flags = page_get_flags(clean_ptr); uint8_t *tags; @@ -83,7 +83,7 @@ uint8_t *allocation_tag_mem_probe(CPUARMState *env, int ptr_mmu_idx, return NULL; } - tags = page_get_target_data(clean_ptr); + tags = page_get_target_data(clean_ptr, page_data_size); index = extract32(ptr, LOG2_TAG_GRANULE + 1, TARGET_PAGE_BITS - LOG2_TAG_GRANULE - 1); diff --git a/target/arm/tcg/mve_helper.c b/target/arm/tcg/mve_helper.c index f9f67d1f88..506d1c3475 100644 --- a/target/arm/tcg/mve_helper.c +++ b/target/arm/tcg/mve_helper.c @@ -23,7 +23,6 @@ #include "vec_internal.h" #include "exec/helper-proto.h" #include "accel/tcg/cpu-ldst.h" -#include "exec/exec-all.h" #include "tcg/tcg.h" #include "fpu/softfloat.h" #include "crypto/clmul.h" diff --git a/target/arm/tcg/op_helper.c b/target/arm/tcg/op_helper.c index 38d49cbb9d..dc3f83c37d 100644 --- a/target/arm/tcg/op_helper.c +++ b/target/arm/tcg/op_helper.c @@ -23,8 +23,8 @@ #include "exec/target_page.h" #include "internals.h" #include "cpu-features.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "cpregs.h" #define SIGNBIT (uint32_t)0x80000000 diff --git a/target/arm/tcg/pauth_helper.c b/target/arm/tcg/pauth_helper.c index 59bf27541d..c591c3052c 100644 --- a/target/arm/tcg/pauth_helper.c +++ b/target/arm/tcg/pauth_helper.c @@ -21,7 +21,6 @@ #include "cpu.h" #include "internals.h" #include "cpu-features.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "exec/helper-proto.h" #include "tcg/tcg-gvec-desc.h" diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c index 96b84c37a2..de0c6e54d4 100644 --- a/target/arm/tcg/sme_helper.c +++ b/target/arm/tcg/sme_helper.c @@ -23,7 +23,7 @@ #include "tcg/tcg-gvec-desc.h" #include "exec/helper-proto.h" #include "accel/tcg/cpu-ldst.h" -#include "exec/exec-all.h" +#include "accel/tcg/helper-retaddr.h" #include "qemu/int128.h" #include "fpu/softfloat.h" #include "vec_internal.h" diff --git a/target/arm/tcg/sve_helper.c b/target/arm/tcg/sve_helper.c index 87b6b4b3e6..a2c363a4e1 100644 --- a/target/arm/tcg/sve_helper.c +++ b/target/arm/tcg/sve_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "internals.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/helper-proto.h" #include "exec/target_page.h" @@ -31,7 +30,9 @@ #include "vec_internal.h" #include "sve_ldst_internal.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/helper-retaddr.h" #include "accel/tcg/cpu-ops.h" +#include "accel/tcg/probe.h" #ifdef CONFIG_USER_ONLY #include "user/page-protection.h" #endif diff --git a/target/arm/tcg/tlb_helper.c b/target/arm/tcg/tlb_helper.c index 8841f039bc..5ea4d6590f 100644 --- a/target/arm/tcg/tlb_helper.c +++ b/target/arm/tcg/tlb_helper.c @@ -9,7 +9,6 @@ #include "cpu.h" #include "internals.h" #include "cpu-features.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index 43408c71bb..52cf47e775 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -17,7 +17,6 @@ * License along with this library; if not, see <http://www.gnu.org/licenses/>. */ #include "qemu/osdep.h" -#include "exec/exec-all.h" #include "exec/target_page.h" #include "translate.h" #include "translate-a64.h" @@ -1076,11 +1075,9 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) TCGv_i64 cf_64 = tcg_temp_new_i64(); TCGv_i64 vf_64 = tcg_temp_new_i64(); TCGv_i64 tmp = tcg_temp_new_i64(); - TCGv_i64 zero = tcg_constant_i64(0); tcg_gen_extu_i32_i64(cf_64, cpu_CF); - tcg_gen_add2_i64(result, cf_64, t0, zero, cf_64, zero); - tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, zero); + tcg_gen_addcio_i64(result, cf_64, t0, t1, cf_64); tcg_gen_extrl_i64_i32(cpu_CF, cf_64); gen_set_NZ64(result); @@ -1094,12 +1091,10 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) TCGv_i32 t0_32 = tcg_temp_new_i32(); TCGv_i32 t1_32 = tcg_temp_new_i32(); TCGv_i32 tmp = tcg_temp_new_i32(); - TCGv_i32 zero = tcg_constant_i32(0); tcg_gen_extrl_i64_i32(t0_32, t0); tcg_gen_extrl_i64_i32(t1_32, t1); - tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, zero, cpu_CF, zero); - tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, zero); + tcg_gen_addcio_i32(cpu_NF, cpu_CF, t0_32, t1_32, cpu_CF); tcg_gen_mov_i32(cpu_ZF, cpu_NF); tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32); @@ -8600,7 +8595,7 @@ static bool trans_CCMP(DisasContext *s, arg_CCMP *a) tcg_gen_subi_i32(tcg_t2, tcg_t0, 1); nzcv = a->nzcv; - has_andc = tcg_op_supported(INDEX_op_andc_i32, TCG_TYPE_I32, 0); + has_andc = tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0); if (nzcv & 8) { /* N */ tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1); } else { diff --git a/target/arm/tcg/translate-sve.c b/target/arm/tcg/translate-sve.c index d23be477b4..f3cf028cb9 100644 --- a/target/arm/tcg/translate-sve.c +++ b/target/arm/tcg/translate-sve.c @@ -629,7 +629,7 @@ static void gen_bsl2n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) * = | ~(m | k) */ tcg_gen_and_i64(n, n, k); - if (tcg_op_supported(INDEX_op_orc_i64, TCG_TYPE_I64, 0)) { + if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) { tcg_gen_or_i64(m, m, k); tcg_gen_orc_i64(d, n, m); } else { diff --git a/target/arm/tcg/translate.c b/target/arm/tcg/translate.c index 273b860d57..e773ab7268 100644 --- a/target/arm/tcg/translate.c +++ b/target/arm/tcg/translate.c @@ -494,20 +494,9 @@ static void gen_add_CC(TCGv_i32 dest, TCGv_i32 t0, TCGv_i32 t1) static void gen_adc_CC(TCGv_i32 dest, TCGv_i32 t0, TCGv_i32 t1) { TCGv_i32 tmp = tcg_temp_new_i32(); - if (tcg_op_supported(INDEX_op_add2_i32, TCG_TYPE_I32, 0)) { - tcg_gen_movi_i32(tmp, 0); - tcg_gen_add2_i32(cpu_NF, cpu_CF, t0, tmp, cpu_CF, tmp); - tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1, tmp); - } else { - TCGv_i64 q0 = tcg_temp_new_i64(); - TCGv_i64 q1 = tcg_temp_new_i64(); - tcg_gen_extu_i32_i64(q0, t0); - tcg_gen_extu_i32_i64(q1, t1); - tcg_gen_add_i64(q0, q0, q1); - tcg_gen_extu_i32_i64(q1, cpu_CF); - tcg_gen_add_i64(q0, q0, q1); - tcg_gen_extr_i64_i32(cpu_NF, cpu_CF, q0); - } + + tcg_gen_addcio_i32(cpu_NF, cpu_CF, t0, t1, cpu_CF); + tcg_gen_mov_i32(cpu_ZF, cpu_NF); tcg_gen_xor_i32(cpu_VF, cpu_NF, t0); tcg_gen_xor_i32(tmp, t0, t1); @@ -7771,7 +7760,8 @@ static bool arm_check_ss_active(DisasContext *dc) static void arm_post_translate_insn(DisasContext *dc) { - if (dc->condjmp && dc->base.is_jmp == DISAS_NEXT) { + if (dc->condjmp && + (dc->base.is_jmp == DISAS_NEXT || dc->base.is_jmp == DISAS_TOO_MANY)) { if (dc->pc_save != dc->condlabel.pc_save) { gen_update_pc(dc, dc->condlabel.pc_save - dc->pc_save); } diff --git a/target/arm/tcg/translate.h b/target/arm/tcg/translate.h index 53e485d28a..1bfdb0fb9b 100644 --- a/target/arm/tcg/translate.h +++ b/target/arm/tcg/translate.h @@ -4,7 +4,6 @@ #include "cpu.h" #include "tcg/tcg-op.h" #include "tcg/tcg-op-gvec.h" -#include "exec/exec-all.h" #include "exec/translator.h" #include "exec/translation-block.h" #include "exec/helper-gen.h" diff --git a/target/avr/cpu.c b/target/avr/cpu.c index 3f261c6fec..250241541b 100644 --- a/target/avr/cpu.c +++ b/target/avr/cpu.c @@ -21,13 +21,13 @@ #include "qemu/osdep.h" #include "qapi/error.h" #include "qemu/qemu-print.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "system/address-spaces.h" #include "cpu.h" #include "disas/dis-asm.h" #include "tcg/debug-assert.h" #include "hw/qdev-properties.h" +#include "accel/tcg/cpu-ops.h" static void avr_cpu_set_pc(CPUState *cs, vaddr value) { @@ -54,6 +54,21 @@ static int avr_cpu_mmu_index(CPUState *cs, bool ifetch) return ifetch ? MMU_CODE_IDX : MMU_DATA_IDX; } +static TCGTBCPUState avr_get_tb_cpu_state(CPUState *cs) +{ + CPUAVRState *env = cpu_env(cs); + uint32_t flags = 0; + + if (env->fullacc) { + flags |= TB_FLAGS_FULL_ACCESS; + } + if (env->skip) { + flags |= TB_FLAGS_SKIP; + } + + return (TCGTBCPUState){ .pc = env->pc_w * 2, .flags = flags }; +} + static void avr_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -221,18 +236,18 @@ static const struct SysemuCPUOps avr_sysemu_ops = { .get_phys_page_debug = avr_cpu_get_phys_page_debug, }; -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps avr_tcg_ops = { .guest_default_memory_order = 0, .mttcg_supported = false, .initialize = avr_cpu_tcg_init, .translate_code = avr_cpu_translate_code, + .get_tb_cpu_state = avr_get_tb_cpu_state, .synchronize_from_tb = avr_cpu_synchronize_from_tb, .restore_state_to_opc = avr_restore_state_to_opc, .mmu_index = avr_cpu_mmu_index, .cpu_exec_interrupt = avr_cpu_exec_interrupt, .cpu_exec_halt = avr_cpu_has_work, + .cpu_exec_reset = cpu_reset, .tlb_fill = avr_cpu_tlb_fill, .do_interrupt = avr_cpu_do_interrupt, }; diff --git a/target/avr/cpu.h b/target/avr/cpu.h index d6666175a9..518e243d81 100644 --- a/target/avr/cpu.h +++ b/target/avr/cpu.h @@ -205,24 +205,6 @@ enum { TB_FLAGS_SKIP = 2, }; -static inline void cpu_get_tb_cpu_state(CPUAVRState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *pflags) -{ - uint32_t flags = 0; - - *pc = env->pc_w * 2; - *cs_base = 0; - - if (env->fullacc) { - flags |= TB_FLAGS_FULL_ACCESS; - } - if (env->skip) { - flags |= TB_FLAGS_SKIP; - } - - *pflags = flags; -} - static inline int cpu_interrupts_enabled(CPUAVRState *env) { return env->sregI != 0; diff --git a/target/avr/helper.c b/target/avr/helper.c index afa591470f..b9cd6d5ef2 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -23,7 +23,6 @@ #include "qemu/error-report.h" #include "cpu.h" #include "accel/tcg/cpu-ops.h" -#include "accel/tcg/getpc.h" #include "exec/cputlb.h" #include "exec/page-protection.h" #include "exec/target_page.h" diff --git a/target/avr/translate.c b/target/avr/translate.c index b9c592c899..804b0b21db 100644 --- a/target/avr/translate.c +++ b/target/avr/translate.c @@ -22,7 +22,6 @@ #include "qemu/qemu-print.h" #include "tcg/tcg.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "tcg/tcg-op.h" #include "exec/helper-proto.h" diff --git a/target/hexagon/cpu.c b/target/hexagon/cpu.c index a5d31c33bd..a5a04173ab 100644 --- a/target/hexagon/cpu.c +++ b/target/hexagon/cpu.c @@ -19,13 +19,13 @@ #include "qemu/qemu-print.h" #include "cpu.h" #include "internal.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "qapi/error.h" #include "hw/qdev-properties.h" #include "fpu/softfloat-helpers.h" #include "tcg/tcg.h" #include "exec/gdbstub.h" +#include "accel/tcg/cpu-ops.h" static void hexagon_v66_cpu_init(Object *obj) { } static void hexagon_v67_cpu_init(Object *obj) { } @@ -255,6 +255,22 @@ static vaddr hexagon_cpu_get_pc(CPUState *cs) return cpu_env(cs)->gpr[HEX_REG_PC]; } +static TCGTBCPUState hexagon_get_tb_cpu_state(CPUState *cs) +{ + CPUHexagonState *env = cpu_env(cs); + vaddr pc = env->gpr[HEX_REG_PC]; + uint32_t hex_flags = 0; + + if (pc == env->gpr[HEX_REG_SA0]) { + hex_flags = FIELD_DP32(hex_flags, TB_FLAGS, IS_TIGHT_LOOP, 1); + } + if (pc & PCALIGN_MASK) { + hexagon_raise_exception_err(env, HEX_CAUSE_PC_NOT_ALIGNED, 0); + } + + return (TCGTBCPUState){ .pc = pc, .flags = hex_flags }; +} + static void hexagon_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -322,14 +338,13 @@ static void hexagon_cpu_init(Object *obj) { } -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps hexagon_tcg_ops = { /* MTTCG not yet supported: require strict ordering */ .guest_default_memory_order = TCG_MO_ALL, .mttcg_supported = false, .initialize = hexagon_translate_init, .translate_code = hexagon_translate_code, + .get_tb_cpu_state = hexagon_get_tb_cpu_state, .synchronize_from_tb = hexagon_cpu_synchronize_from_tb, .restore_state_to_opc = hexagon_restore_state_to_opc, .mmu_index = hexagon_cpu_mmu_index, diff --git a/target/hexagon/cpu.h b/target/hexagon/cpu.h index c065fa8ddc..43a854f517 100644 --- a/target/hexagon/cpu.h +++ b/target/hexagon/cpu.h @@ -137,21 +137,6 @@ G_NORETURN void hexagon_raise_exception_err(CPUHexagonState *env, uint32_t exception, uintptr_t pc); -static inline void cpu_get_tb_cpu_state(CPUHexagonState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - uint32_t hex_flags = 0; - *pc = env->gpr[HEX_REG_PC]; - *cs_base = 0; - if (*pc == env->gpr[HEX_REG_SA0]) { - hex_flags = FIELD_DP32(hex_flags, TB_FLAGS, IS_TIGHT_LOOP, 1); - } - *flags = hex_flags; - if (*pc & PCALIGN_MASK) { - hexagon_raise_exception_err(env, HEX_CAUSE_PC_NOT_ALIGNED, 0); - } -} - typedef HexagonCPU ArchCPU; void hexagon_translate_init(void); diff --git a/target/hexagon/mmvec/macros.h b/target/hexagon/mmvec/macros.h index c1a88392c0..c7840fbf2e 100644 --- a/target/hexagon/mmvec/macros.h +++ b/target/hexagon/mmvec/macros.h @@ -22,6 +22,7 @@ #include "arch.h" #include "mmvec/system_ext_mmvec.h" #include "accel/tcg/getpc.h" +#include "accel/tcg/probe.h" #ifndef QEMU_GENERATE #define VdV (*(MMVector *restrict)(VdV_void)) diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index 3f3d86db2b..444799d3ad 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -17,8 +17,8 @@ #include "qemu/osdep.h" #include "qemu/log.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" #include "cpu.h" diff --git a/target/hppa/cpu.c b/target/hppa/cpu.c index b792cb247a..6465181543 100644 --- a/target/hppa/cpu.c +++ b/target/hppa/cpu.c @@ -24,12 +24,12 @@ #include "qemu/timer.h" #include "cpu.h" #include "qemu/module.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "exec/target_page.h" #include "fpu/softfloat.h" #include "tcg/tcg.h" #include "hw/hppa/hppa_hardware.h" +#include "accel/tcg/cpu-ops.h" static void hppa_cpu_set_pc(CPUState *cs, vaddr value) { @@ -51,11 +51,12 @@ static vaddr hppa_cpu_get_pc(CPUState *cs) env->iaoq_f & -4); } -void cpu_get_tb_cpu_state(CPUHPPAState *env, vaddr *pc, - uint64_t *pcsbase, uint32_t *pflags) +static TCGTBCPUState hppa_get_tb_cpu_state(CPUState *cs) { + CPUHPPAState *env = cpu_env(cs); uint32_t flags = 0; uint64_t cs_base = 0; + vaddr pc; /* * TB lookup assumes that PC contains the complete virtual address. @@ -63,7 +64,7 @@ void cpu_get_tb_cpu_state(CPUHPPAState *env, vaddr *pc, * incomplete virtual address. This also means that we must separate * out current cpu privilege from the low bits of IAOQ_F. */ - *pc = hppa_cpu_get_pc(env_cpu(env)); + pc = hppa_cpu_get_pc(env_cpu(env)); flags |= (env->iaoq_f & 3) << TB_FLAG_PRIV_SHIFT; /* @@ -99,8 +100,7 @@ void cpu_get_tb_cpu_state(CPUHPPAState *env, vaddr *pc, } #endif - *pcsbase = cs_base; - *pflags = flags; + return (TCGTBCPUState){ .pc = pc, .flags = flags, .cs_base = cs_base }; } static void hppa_cpu_synchronize_from_tb(CPUState *cs, @@ -250,8 +250,6 @@ static const struct SysemuCPUOps hppa_sysemu_ops = { }; #endif -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps hppa_tcg_ops = { /* PA-RISC 1.x processors have a strong memory model. */ /* @@ -264,6 +262,7 @@ static const TCGCPUOps hppa_tcg_ops = { .initialize = hppa_translate_init, .translate_code = hppa_translate_code, + .get_tb_cpu_state = hppa_get_tb_cpu_state, .synchronize_from_tb = hppa_cpu_synchronize_from_tb, .restore_state_to_opc = hppa_restore_state_to_opc, .mmu_index = hppa_cpu_mmu_index, @@ -272,6 +271,7 @@ static const TCGCPUOps hppa_tcg_ops = { .tlb_fill_align = hppa_cpu_tlb_fill_align, .cpu_exec_interrupt = hppa_cpu_exec_interrupt, .cpu_exec_halt = hppa_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = hppa_cpu_do_interrupt, .do_unaligned_access = hppa_cpu_do_unaligned_access, .do_transaction_failed = hppa_cpu_do_transaction_failed, diff --git a/target/hppa/cpu.h b/target/hppa/cpu.h index acc9937240..11d59d11ca 100644 --- a/target/hppa/cpu.h +++ b/target/hppa/cpu.h @@ -351,9 +351,6 @@ hwaddr hppa_abs_to_phys_pa2_w1(vaddr addr); #define CS_BASE_DIFFPAGE (1 << 12) #define CS_BASE_DIFFSPACE (1 << 13) -void cpu_get_tb_cpu_state(CPUHPPAState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *pflags); - target_ulong cpu_hppa_get_psw(CPUHPPAState *env); void cpu_hppa_put_psw(CPUHPPAState *env, target_ulong); void update_gva_offset_mask(CPUHPPAState *env); diff --git a/target/hppa/fpu_helper.c b/target/hppa/fpu_helper.c index a62d9d3083..ddd0a343d6 100644 --- a/target/hppa/fpu_helper.c +++ b/target/hppa/fpu_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" diff --git a/target/hppa/helper.c b/target/hppa/helper.c index ac7f58f0af..d7f8495d98 100644 --- a/target/hppa/helper.c +++ b/target/hppa/helper.c @@ -21,7 +21,6 @@ #include "qemu/log.h" #include "cpu.h" #include "fpu/softfloat.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "qemu/qemu-print.h" #include "hw/hppa/hppa_hardware.h" diff --git a/target/hppa/machine.c b/target/hppa/machine.c index bb47a2e689..13e555151a 100644 --- a/target/hppa/machine.c +++ b/target/hppa/machine.c @@ -216,7 +216,7 @@ static const VMStateDescription vmstate_env = { }; static const VMStateField vmstate_cpu_fields[] = { - VMSTATE_CPU(), + VMSTATE_STRUCT(parent_obj, HPPACPU, 0, vmstate_cpu_common, CPUState), VMSTATE_STRUCT(env, HPPACPU, 1, vmstate_env, CPUHPPAState), VMSTATE_END_OF_LIST() }; diff --git a/target/hppa/mem_helper.c b/target/hppa/mem_helper.c index 554d7bf4d1..9bdd0a6f23 100644 --- a/target/hppa/mem_helper.c +++ b/target/hppa/mem_helper.c @@ -20,9 +20,9 @@ #include "qemu/osdep.h" #include "qemu/log.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "accel/tcg/cpu-mmu-index.h" +#include "accel/tcg/probe.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "exec/helper-proto.h" diff --git a/target/hppa/op_helper.c b/target/hppa/op_helper.c index 2398ce2c64..0458378abb 100644 --- a/target/hppa/op_helper.c +++ b/target/hppa/op_helper.c @@ -20,9 +20,9 @@ #include "qemu/osdep.h" #include "qemu/log.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "qemu/timer.h" #include "trace.h" #ifdef CONFIG_USER_ONLY diff --git a/target/hppa/sys_helper.c b/target/hppa/sys_helper.c index 052a6a88a2..6e65fadcc7 100644 --- a/target/hppa/sys_helper.c +++ b/target/hppa/sys_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "qemu/log.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "qemu/timer.h" #include "system/runstate.h" diff --git a/target/hppa/translate.c b/target/hppa/translate.c index 14f3833322..7a81cfcb88 100644 --- a/target/hppa/translate.c +++ b/target/hppa/translate.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "qemu/host-utils.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "tcg/tcg-op.h" #include "tcg/tcg-op-gvec.h" @@ -1209,10 +1208,10 @@ static void do_add(DisasContext *ctx, unsigned rt, TCGv_i64 orig_in1, cb_msb = tcg_temp_new_i64(); cb = tcg_temp_new_i64(); - tcg_gen_add2_i64(dest, cb_msb, in1, ctx->zero, in2, ctx->zero); if (is_c) { - tcg_gen_add2_i64(dest, cb_msb, dest, cb_msb, - get_psw_carry(ctx, d), ctx->zero); + tcg_gen_addcio_i64(dest, cb_msb, in1, in2, get_psw_carry(ctx, d)); + } else { + tcg_gen_add2_i64(dest, cb_msb, in1, ctx->zero, in2, ctx->zero); } tcg_gen_xor_i64(cb, in1, in2); tcg_gen_xor_i64(cb, cb, dest); @@ -1308,9 +1307,7 @@ static void do_sub(DisasContext *ctx, unsigned rt, TCGv_i64 in1, if (is_b) { /* DEST,C = IN1 + ~IN2 + C. */ tcg_gen_not_i64(cb, in2); - tcg_gen_add2_i64(dest, cb_msb, in1, ctx->zero, - get_psw_carry(ctx, d), ctx->zero); - tcg_gen_add2_i64(dest, cb_msb, dest, cb_msb, cb, ctx->zero); + tcg_gen_addcio_i64(dest, cb_msb, in1, cb, get_psw_carry(ctx, d)); tcg_gen_xor_i64(cb, cb, in1); tcg_gen_xor_i64(cb, cb, dest); } else { @@ -3008,9 +3005,7 @@ static bool trans_ds(DisasContext *ctx, arg_rrr_cf *a) tcg_gen_xor_i64(add2, in2, addc); tcg_gen_andi_i64(addc, addc, 1); - tcg_gen_add2_i64(dest, cpu_psw_cb_msb, add1, ctx->zero, add2, ctx->zero); - tcg_gen_add2_i64(dest, cpu_psw_cb_msb, dest, cpu_psw_cb_msb, - addc, ctx->zero); + tcg_gen_addcio_i64(dest, cpu_psw_cb_msb, add1, add2, addc); /* Write back the result register. */ save_gpr(ctx, a->t, dest); @@ -3553,8 +3548,7 @@ static bool do_addb(DisasContext *ctx, unsigned r, TCGv_i64 in1, TCGv_i64 cb = tcg_temp_new_i64(); TCGv_i64 cb_msb = tcg_temp_new_i64(); - tcg_gen_movi_i64(cb_msb, 0); - tcg_gen_add2_i64(dest, cb_msb, in1, cb_msb, in2, cb_msb); + tcg_gen_add2_i64(dest, cb_msb, in1, ctx->zero, in2, ctx->zero); tcg_gen_xor_i64(cb, in1, in2); tcg_gen_xor_i64(cb, cb, dest); cb_cond = get_carry(ctx, d, cb, cb_msb); diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 6f21d5ed22..1ca6307c72 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -6240,7 +6240,7 @@ static void listflags(GList *features) } /* Sort alphabetically by type name, respecting X86CPUClass::ordering. */ -static gint x86_cpu_list_compare(gconstpointer a, gconstpointer b) +static gint x86_cpu_list_compare(gconstpointer a, gconstpointer b, gpointer d) { ObjectClass *class_a = (ObjectClass *)a; ObjectClass *class_b = (ObjectClass *)b; @@ -6261,7 +6261,7 @@ static gint x86_cpu_list_compare(gconstpointer a, gconstpointer b) static GSList *get_sorted_cpu_model_list(void) { GSList *list = object_class_get_list(TYPE_X86_CPU, false); - list = g_slist_sort(list, x86_cpu_list_compare); + list = g_slist_sort_with_data(list, x86_cpu_list_compare, NULL); return list; } @@ -6318,6 +6318,11 @@ static void x86_cpu_list_entry(gpointer data, gpointer user_data) qemu_printf(" %-20s %s\n", name, desc); } +static gint strcmp_wrap(gconstpointer a, gconstpointer b, gpointer d) +{ + return strcmp(a, b); +} + /* list available CPU models and flags */ static void x86_cpu_list(void) { @@ -6340,7 +6345,7 @@ static void x86_cpu_list(void) } } - names = g_list_sort(names, (GCompareFunc)strcmp); + names = g_list_sort_with_data(names, strcmp_wrap, NULL); qemu_printf("\nRecognized CPUID flags:\n"); listflags(names); diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 54bf9639f1..4f8ed8868e 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -35,10 +35,6 @@ #define XEN_NR_VIRQS 24 -/* support for self modifying code even if the modified instruction is - close to the modifying instruction */ -#define TARGET_HAS_PRECISE_SMC - #ifdef TARGET_X86_64 #define I386_ELF_MACHINE EM_X86_64 #define ELF_MACHINE_UNAME "x86_64" @@ -2603,20 +2599,6 @@ static inline bool is_mmu_index_32(int mmu_index) #include "hw/i386/apic.h" #endif -static inline void cpu_get_tb_cpu_state(CPUX86State *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *flags = env->hflags | - (env->eflags & (IOPL_MASK | TF_MASK | RF_MASK | VM_MASK | AC_MASK)); - if (env->hflags & HF_CS64_MASK) { - *cs_base = 0; - *pc = env->eip; - } else { - *cs_base = env->segs[R_CS].base; - *pc = (uint32_t)(*cs_base + env->eip); - } -} - void do_cpu_init(X86CPU *cpu); #define MCE_INJECT_BROADCAST 1 diff --git a/target/i386/emulate/x86_decode.c b/target/i386/emulate/x86_decode.c index 7fee219687..88be9479a8 100644 --- a/target/i386/emulate/x86_decode.c +++ b/target/i386/emulate/x86_decode.c @@ -26,7 +26,7 @@ static void decode_invalid(CPUX86State *env, struct x86_decode *decode) { - printf("%llx: failed to decode instruction ", env->eip); + printf(TARGET_FMT_lx ": failed to decode instruction ", env->eip); for (int i = 0; i < decode->opcode_len; i++) { printf("%x ", decode->opcode[i]); } @@ -1408,7 +1408,7 @@ struct decode_tbl _2op_inst[] = { }; struct decode_x87_tbl invl_inst_x87 = {0x0, 0, 0, 0, 0, false, false, NULL, - NULL, decode_invalid, 0}; + NULL, decode_invalid}; struct decode_x87_tbl _x87_inst[] = { {0xd8, 0, 3, X86_DECODE_CMD_FADD, 10, false, false, @@ -1456,8 +1456,7 @@ struct decode_x87_tbl _x87_inst[] = { decode_x87_modrm_st0, NULL, decode_d9_4}, {0xd9, 4, 0, X86_DECODE_CMD_INVL, 4, false, false, decode_x87_modrm_bytep, NULL, NULL}, - {0xd9, 5, 3, X86_DECODE_CMD_FLDxx, 10, false, false, NULL, NULL, NULL, - RFLAGS_MASK_NONE}, + {0xd9, 5, 3, X86_DECODE_CMD_FLDxx, 10, false, false, NULL, NULL, NULL}, {0xd9, 5, 0, X86_DECODE_CMD_FLDCW, 2, false, false, decode_x87_modrm_bytep, NULL, NULL}, @@ -1478,20 +1477,17 @@ struct decode_x87_tbl _x87_inst[] = { decode_x87_modrm_st0, NULL}, {0xda, 3, 3, X86_DECODE_CMD_FCMOV, 10, false, false, decode_x87_modrm_st0, decode_x87_modrm_st0, NULL}, - {0xda, 4, 3, X86_DECODE_CMD_INVL, 10, false, false, NULL, NULL, NULL, - RFLAGS_MASK_NONE}, + {0xda, 4, 3, X86_DECODE_CMD_INVL, 10, false, false, NULL, NULL, NULL}, {0xda, 4, 0, X86_DECODE_CMD_FSUB, 4, false, false, decode_x87_modrm_st0, decode_x87_modrm_intp, NULL}, {0xda, 5, 3, X86_DECODE_CMD_FUCOM, 10, false, true, decode_x87_modrm_st0, decode_decode_x87_modrm_st0, NULL}, {0xda, 5, 0, X86_DECODE_CMD_FSUB, 4, true, false, decode_x87_modrm_st0, decode_x87_modrm_intp, NULL}, - {0xda, 6, 3, X86_DECODE_CMD_INVL, 10, false, false, NULL, NULL, NULL, - RFLAGS_MASK_NONE}, + {0xda, 6, 3, X86_DECODE_CMD_INVL, 10, false, false, NULL, NULL, NULL}, {0xda, 6, 0, X86_DECODE_CMD_FDIV, 4, false, false, decode_x87_modrm_st0, decode_x87_modrm_intp, NULL}, - {0xda, 7, 3, X86_DECODE_CMD_INVL, 10, false, false, NULL, NULL, NULL, - RFLAGS_MASK_NONE}, + {0xda, 7, 3, X86_DECODE_CMD_INVL, 10, false, false, NULL, NULL, NULL}, {0xda, 7, 0, X86_DECODE_CMD_FDIV, 4, true, false, decode_x87_modrm_st0, decode_x87_modrm_intp, NULL}, @@ -1511,8 +1507,7 @@ struct decode_x87_tbl _x87_inst[] = { decode_x87_modrm_intp, NULL, NULL}, {0xdb, 4, 3, X86_DECODE_CMD_INVL, 10, false, false, NULL, NULL, decode_db_4}, - {0xdb, 4, 0, X86_DECODE_CMD_INVL, 10, false, false, NULL, NULL, NULL, - RFLAGS_MASK_NONE}, + {0xdb, 4, 0, X86_DECODE_CMD_INVL, 10, false, false, NULL, NULL, NULL}, {0xdb, 5, 3, X86_DECODE_CMD_FUCOMI, 10, false, false, decode_x87_modrm_st0, decode_x87_modrm_st0, NULL}, {0xdb, 5, 0, X86_DECODE_CMD_FLD, 10, false, false, diff --git a/target/i386/emulate/x86_emu.c b/target/i386/emulate/x86_emu.c index 26a4876aac..7773b51b95 100644 --- a/target/i386/emulate/x86_emu.c +++ b/target/i386/emulate/x86_emu.c @@ -1241,7 +1241,7 @@ static void init_cmd_handler(void) bool exec_instruction(CPUX86State *env, struct x86_decode *ins) { if (!_cmd_handler[ins->cmd].handler) { - printf("Unimplemented handler (%llx) for %d (%x %x) \n", env->eip, + printf("Unimplemented handler (" TARGET_FMT_lx ") for %d (%x %x) \n", env->eip, ins->cmd, ins->opcode[0], ins->opcode_len > 1 ? ins->opcode[1] : 0); env->eip += ins->len; diff --git a/target/i386/helper.c b/target/i386/helper.c index 197fdac7dd..e0aaed3c4c 100644 --- a/target/i386/helper.c +++ b/target/i386/helper.c @@ -526,7 +526,7 @@ void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank, static inline target_ulong get_memio_eip(CPUX86State *env) { #ifdef CONFIG_TCG - uint64_t data[TARGET_INSN_START_WORDS]; + uint64_t data[INSN_START_WORDS]; CPUState *cs = env_cpu(env); if (!cpu_unwind_state_data(cs, cs->mem_io_pc, data)) { diff --git a/target/i386/hvf/hvf.c b/target/i386/hvf/hvf.c index 23ebf2550a..99e37a33e5 100644 --- a/target/i386/hvf/hvf.c +++ b/target/i386/hvf/hvf.c @@ -76,6 +76,7 @@ #include "qemu/main-loop.h" #include "qemu/accel.h" #include "target/i386/cpu.h" +#include "exec/target_page.h" static Error *invtsc_mig_blocker; diff --git a/target/i386/tcg/access.c b/target/i386/tcg/access.c index 0fdd587edd..97e3f0e756 100644 --- a/target/i386/tcg/access.c +++ b/target/i386/tcg/access.c @@ -4,7 +4,7 @@ #include "qemu/osdep.h" #include "cpu.h" #include "accel/tcg/cpu-ldst.h" -#include "exec/exec-all.h" +#include "accel/tcg/probe.h" #include "exec/target_page.h" #include "access.h" diff --git a/target/i386/tcg/emit.c.inc b/target/i386/tcg/emit.c.inc index ca6bc2ea82..1a7fab9333 100644 --- a/target/i386/tcg/emit.c.inc +++ b/target/i386/tcg/emit.c.inc @@ -19,16 +19,6 @@ * License along with this library; if not, see <http://www.gnu.org/licenses/>. */ -/* - * Sometimes, knowing what the backend has can produce better code. - * The exact opcode to check depends on 32- vs. 64-bit. - */ -#ifdef TARGET_X86_64 -#define INDEX_op_extract2_tl INDEX_op_extract2_i64 -#else -#define INDEX_op_extract2_tl INDEX_op_extract2_i32 -#endif - #define MMX_OFFSET(reg) \ ({ assert((reg) >= 0 && (reg) <= 7); \ offsetof(CPUX86State, fpregs[reg].mmx); }) @@ -352,7 +342,7 @@ static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv break; case X86_OP_SEG: /* Note that gen_movl_seg takes care of interrupt shadow and TF. */ - gen_movl_seg(s, op->n, s->T0); + gen_movl_seg(s, op->n, v, op->n == R_SS); break; case X86_OP_INT: if (op->has_ea) { @@ -2392,7 +2382,7 @@ static void gen_lxx_seg(DisasContext *s, X86DecodedInsn *decode, int seg) gen_op_ld_v(s, MO_16, s->T1, s->A0); /* load the segment here to handle exceptions properly */ - gen_movl_seg(s, seg, s->T1); + gen_movl_seg(s, seg, s->T1, false); } static void gen_LDS(DisasContext *s, X86DecodedInsn *decode) @@ -3023,7 +3013,7 @@ static void gen_PMOVMSKB(DisasContext *s, X86DecodedInsn *decode) tcg_gen_ld8u_tl(s->T0, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); while (vec_len > 8) { vec_len -= 8; - if (tcg_op_supported(INDEX_op_extract2_tl, TCG_TYPE_TL, 0)) { + if (tcg_op_supported(INDEX_op_extract2, TCG_TYPE_TL, 0)) { /* * Load the next byte of the result into the high byte of T. * TCG does a similar expansion of deposit to shl+extract2; by diff --git a/target/i386/tcg/excp_helper.c b/target/i386/tcg/excp_helper.c index de71e68f98..6fb8036d98 100644 --- a/target/i386/tcg/excp_helper.c +++ b/target/i386/tcg/excp_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "qemu/log.h" #include "system/runstate.h" #include "exec/helper-proto.h" diff --git a/target/i386/tcg/helper-tcg.h b/target/i386/tcg/helper-tcg.h index 54d845379c..6b3f19855f 100644 --- a/target/i386/tcg/helper-tcg.h +++ b/target/i386/tcg/helper-tcg.h @@ -20,7 +20,6 @@ #ifndef I386_HELPER_TCG_H #define I386_HELPER_TCG_H -#include "exec/exec-all.h" #include "qemu/host-utils.h" /* Maximum instruction code size */ diff --git a/target/i386/tcg/int_helper.c b/target/i386/tcg/int_helper.c index 1a02e9d843..46741d9f68 100644 --- a/target/i386/tcg/int_helper.c +++ b/target/i386/tcg/int_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "qemu/log.h" #include "cpu.h" -#include "exec/exec-all.h" #include "qemu/host-utils.h" #include "exec/helper-proto.h" #include "qapi/error.h" diff --git a/target/i386/tcg/mem_helper.c b/target/i386/tcg/mem_helper.c index 84a0815217..9e7c2d8029 100644 --- a/target/i386/tcg/mem_helper.c +++ b/target/i386/tcg/mem_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "qemu/int128.h" #include "qemu/atomic128.h" diff --git a/target/i386/tcg/mpx_helper.c b/target/i386/tcg/mpx_helper.c index a0f816dfae..fa8abcc482 100644 --- a/target/i386/tcg/mpx_helper.c +++ b/target/i386/tcg/mpx_helper.c @@ -21,7 +21,6 @@ #include "cpu.h" #include "exec/helper-proto.h" #include "accel/tcg/cpu-ldst.h" -#include "exec/exec-all.h" #include "exec/target_page.h" #include "helper-tcg.h" diff --git a/target/i386/tcg/seg_helper.c b/target/i386/tcg/seg_helper.c index 3af902e0ec..0ca081b286 100644 --- a/target/i386/tcg/seg_helper.c +++ b/target/i386/tcg/seg_helper.c @@ -22,8 +22,8 @@ #include "cpu.h" #include "qemu/log.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "exec/log.h" #include "helper-tcg.h" #include "seg_helper.h" diff --git a/target/i386/tcg/system/bpt_helper.c b/target/i386/tcg/system/bpt_helper.c index 08ccd3f5e6..aebb5caac3 100644 --- a/target/i386/tcg/system/bpt_helper.c +++ b/target/i386/tcg/system/bpt_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "exec/watchpoint.h" #include "tcg/helper-tcg.h" diff --git a/target/i386/tcg/system/excp_helper.c b/target/i386/tcg/system/excp_helper.c index 93614aa3e5..c162621587 100644 --- a/target/i386/tcg/system/excp_helper.c +++ b/target/i386/tcg/system/excp_helper.c @@ -20,6 +20,7 @@ #include "qemu/osdep.h" #include "cpu.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "exec/cputlb.h" #include "exec/page-protection.h" #include "exec/target_page.h" diff --git a/target/i386/tcg/tcg-cpu.c b/target/i386/tcg/tcg-cpu.c index e53aaa31bf..179dfdf064 100644 --- a/target/i386/tcg/tcg-cpu.c +++ b/target/i386/tcg/tcg-cpu.c @@ -24,6 +24,7 @@ #include "accel/accel-cpu-target.h" #include "exec/translation-block.h" #include "exec/target_page.h" +#include "accel/tcg/cpu-ops.h" #include "tcg-cpu.h" /* Frob eflags into and out of the CPU temporary format. */ @@ -47,6 +48,25 @@ static void x86_cpu_exec_exit(CPUState *cs) env->eflags = cpu_compute_eflags(env); } +static TCGTBCPUState x86_get_tb_cpu_state(CPUState *cs) +{ + CPUX86State *env = cpu_env(cs); + uint32_t flags, cs_base; + vaddr pc; + + flags = env->hflags | + (env->eflags & (IOPL_MASK | TF_MASK | RF_MASK | VM_MASK | AC_MASK)); + if (env->hflags & HF_CS64_MASK) { + cs_base = 0; + pc = env->eip; + } else { + cs_base = env->segs[R_CS].base; + pc = (uint32_t)(cs_base + env->eip); + } + + return (TCGTBCPUState){ .pc = pc, .flags = flags, .cs_base = cs_base }; +} + static void x86_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -120,18 +140,27 @@ static bool x86_debug_check_breakpoint(CPUState *cs) /* RF disables all architectural breakpoints. */ return !(env->eflags & RF_MASK); } -#endif -#include "accel/tcg/cpu-ops.h" +static void x86_cpu_exec_reset(CPUState *cs) +{ + CPUArchState *env = cpu_env(cs); + + cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0, 0); + do_cpu_init(env_archcpu(env)); + cs->exception_index = EXCP_HALTED; +} +#endif const TCGCPUOps x86_tcg_ops = { .mttcg_supported = true, + .precise_smc = true, /* * The x86 has a strong memory model with some store-after-load re-ordering */ .guest_default_memory_order = TCG_MO_ALL & ~TCG_MO_ST_LD, .initialize = tcg_x86_init, .translate_code = x86_translate_code, + .get_tb_cpu_state = x86_get_tb_cpu_state, .synchronize_from_tb = x86_cpu_synchronize_from_tb, .restore_state_to_opc = x86_restore_state_to_opc, .mmu_index = x86_cpu_mmu_index, @@ -146,6 +175,7 @@ const TCGCPUOps x86_tcg_ops = { .do_interrupt = x86_cpu_do_interrupt, .cpu_exec_halt = x86_cpu_exec_halt, .cpu_exec_interrupt = x86_cpu_exec_interrupt, + .cpu_exec_reset = x86_cpu_exec_reset, .do_unaligned_access = x86_cpu_do_unaligned_access, .debug_excp_handler = breakpoint_handler, .debug_check_breakpoint = x86_debug_check_breakpoint, diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 8a641951cd..0fcddc2ec0 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -21,7 +21,6 @@ #include "qemu/host-utils.h" #include "cpu.h" #include "accel/tcg/cpu-mmu-index.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "tcg/tcg-op.h" #include "tcg/tcg-op-gvec.h" @@ -2026,27 +2025,36 @@ static void gen_op_movl_seg_real(DisasContext *s, X86Seg seg_reg, TCGv seg) /* move SRC to seg_reg and compute if the CPU state may change. Never call this function with seg_reg == R_CS */ -static void gen_movl_seg(DisasContext *s, X86Seg seg_reg, TCGv src) +static void gen_movl_seg(DisasContext *s, X86Seg seg_reg, TCGv src, bool inhibit_irq) { if (PE(s) && !VM86(s)) { TCGv_i32 sel = tcg_temp_new_i32(); tcg_gen_trunc_tl_i32(sel, src); gen_helper_load_seg(tcg_env, tcg_constant_i32(seg_reg), sel); - /* abort translation because the addseg value may change or - because ss32 may change. For R_SS, translation must always - stop as a special handling must be done to disable hardware - interrupts for the next instruction */ - if (seg_reg == R_SS) { - s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; - } else if (CODE32(s) && seg_reg < R_FS) { + + /* For move to DS/ES/SS, the addseg or ss32 flags may change. */ + if (CODE32(s) && seg_reg < R_FS) { s->base.is_jmp = DISAS_EOB_NEXT; } } else { gen_op_movl_seg_real(s, seg_reg, src); - if (seg_reg == R_SS) { - s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; - } + } + + /* + * For MOV or POP to SS (but not LSS) translation must always + * stop as a special handling must be done to disable hardware + * interrupts for the next instruction. + * + * This is the last instruction, so it's okay to overwrite + * HF_TF_MASK; the next TB will start with the flag set. + * + * DISAS_EOB_INHIBIT_IRQ is a superset of DISAS_EOB_NEXT which + * might have been set above. + */ + if (inhibit_irq) { + s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; + s->flags &= ~HF_TF_MASK; } } @@ -2297,7 +2305,7 @@ gen_eob(DisasContext *s, int mode) if (mode == DISAS_EOB_RECHECK_TF) { gen_helper_rechecking_single_step(tcg_env); tcg_gen_exit_tb(NULL, 0); - } else if ((s->flags & HF_TF_MASK) && mode != DISAS_EOB_INHIBIT_IRQ) { + } else if (s->flags & HF_TF_MASK) { gen_helper_single_step(tcg_env); } else if (mode == DISAS_JUMP && /* give irqs a chance to happen */ diff --git a/target/i386/tcg/user/excp_helper.c b/target/i386/tcg/user/excp_helper.c index b3bdb7831a..98fab4cbc3 100644 --- a/target/i386/tcg/user/excp_helper.c +++ b/target/i386/tcg/user/excp_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "tcg/helper-tcg.h" void x86_cpu_record_sigsegv(CPUState *cs, vaddr addr, diff --git a/target/i386/tcg/user/seg_helper.c b/target/i386/tcg/user/seg_helper.c index 5692dd5195..263f59937f 100644 --- a/target/i386/tcg/user/seg_helper.c +++ b/target/i386/tcg/user/seg_helper.c @@ -21,7 +21,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "tcg/helper-tcg.h" #include "tcg/seg_helper.h" diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c index 8ad45b453d..f7535d1be7 100644 --- a/target/loongarch/cpu.c +++ b/target/loongarch/cpu.c @@ -15,7 +15,6 @@ #include "system/kvm.h" #include "kvm/kvm_loongarch.h" #include "hw/qdev-properties.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "cpu.h" #include "internals.h" @@ -30,6 +29,7 @@ #endif #ifdef CONFIG_TCG #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/cpu-ops.h" #include "tcg/tcg.h" #endif #include "tcg/tcg_loongarch.h" @@ -336,6 +336,20 @@ static bool loongarch_cpu_exec_interrupt(CPUState *cs, int interrupt_request) } #endif +static TCGTBCPUState loongarch_get_tb_cpu_state(CPUState *cs) +{ + CPULoongArchState *env = cpu_env(cs); + uint32_t flags; + + flags = env->CSR_CRMD & (R_CSR_CRMD_PLV_MASK | R_CSR_CRMD_PG_MASK); + flags |= FIELD_EX64(env->CSR_EUEN, CSR_EUEN, FPE) * HW_FLAGS_EUEN_FPE; + flags |= FIELD_EX64(env->CSR_EUEN, CSR_EUEN, SXE) * HW_FLAGS_EUEN_SXE; + flags |= FIELD_EX64(env->CSR_EUEN, CSR_EUEN, ASXE) * HW_FLAGS_EUEN_ASXE; + flags |= is_va32(env) * HW_FLAGS_VA32; + + return (TCGTBCPUState){ .pc = env->pc, .flags = flags }; +} + static void loongarch_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -862,14 +876,13 @@ static void loongarch_cpu_dump_state(CPUState *cs, FILE *f, int flags) } #ifdef CONFIG_TCG -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps loongarch_tcg_ops = { .guest_default_memory_order = 0, .mttcg_supported = true, .initialize = loongarch_translate_init, .translate_code = loongarch_translate_code, + .get_tb_cpu_state = loongarch_get_tb_cpu_state, .synchronize_from_tb = loongarch_cpu_synchronize_from_tb, .restore_state_to_opc = loongarch_restore_state_to_opc, .mmu_index = loongarch_cpu_mmu_index, @@ -878,6 +891,7 @@ static const TCGCPUOps loongarch_tcg_ops = { .tlb_fill = loongarch_cpu_tlb_fill, .cpu_exec_interrupt = loongarch_cpu_exec_interrupt, .cpu_exec_halt = loongarch_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = loongarch_cpu_do_interrupt, .do_transaction_failed = loongarch_cpu_do_transaction_failed, #endif diff --git a/target/loongarch/cpu.h b/target/loongarch/cpu.h index 70ff56e60c..262bf87f7b 100644 --- a/target/loongarch/cpu.h +++ b/target/loongarch/cpu.h @@ -492,18 +492,6 @@ static inline void set_pc(CPULoongArchState *env, uint64_t value) #define HW_FLAGS_VA32 0x20 #define HW_FLAGS_EUEN_ASXE 0x40 -static inline void cpu_get_tb_cpu_state(CPULoongArchState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *pc = env->pc; - *cs_base = 0; - *flags = env->CSR_CRMD & (R_CSR_CRMD_PLV_MASK | R_CSR_CRMD_PG_MASK); - *flags |= FIELD_EX64(env->CSR_EUEN, CSR_EUEN, FPE) * HW_FLAGS_EUEN_FPE; - *flags |= FIELD_EX64(env->CSR_EUEN, CSR_EUEN, SXE) * HW_FLAGS_EUEN_SXE; - *flags |= FIELD_EX64(env->CSR_EUEN, CSR_EUEN, ASXE) * HW_FLAGS_EUEN_ASXE; - *flags |= is_va32(env) * HW_FLAGS_VA32; -} - #define CPU_RESOLVING_TYPE TYPE_LOONGARCH_CPU void loongarch_cpu_post_init(Object *obj); diff --git a/target/loongarch/tcg/fpu_helper.c b/target/loongarch/tcg/fpu_helper.c index fc3fd0561e..fc9c64c20a 100644 --- a/target/loongarch/tcg/fpu_helper.c +++ b/target/loongarch/tcg/fpu_helper.c @@ -8,7 +8,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "fpu/softfloat.h" #include "internals.h" diff --git a/target/loongarch/tcg/iocsr_helper.c b/target/loongarch/tcg/iocsr_helper.c index e62170de3c..c155f48564 100644 --- a/target/loongarch/tcg/iocsr_helper.c +++ b/target/loongarch/tcg/iocsr_helper.c @@ -9,7 +9,6 @@ #include "cpu.h" #include "qemu/host-utils.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #define GET_MEMTXATTRS(cas) \ diff --git a/target/loongarch/tcg/op_helper.c b/target/loongarch/tcg/op_helper.c index 94e3b28016..16ac0d43bc 100644 --- a/target/loongarch/tcg/op_helper.c +++ b/target/loongarch/tcg/op_helper.c @@ -10,7 +10,6 @@ #include "cpu.h" #include "qemu/host-utils.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "internals.h" #include "qemu/crc32c.h" diff --git a/target/loongarch/tcg/tlb_helper.c b/target/loongarch/tcg/tlb_helper.c index af208d75ae..dc48b0f4d2 100644 --- a/target/loongarch/tcg/tlb_helper.c +++ b/target/loongarch/tcg/tlb_helper.c @@ -13,7 +13,6 @@ #include "internals.h" #include "exec/helper-proto.h" #include "exec/cputlb.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "accel/tcg/cpu-ldst.h" diff --git a/target/loongarch/tcg/vec_helper.c b/target/loongarch/tcg/vec_helper.c index 3faf52cbc4..a270998e63 100644 --- a/target/loongarch/tcg/vec_helper.c +++ b/target/loongarch/tcg/vec_helper.c @@ -7,7 +7,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" #include "internals.h" diff --git a/target/m68k/cpu.c b/target/m68k/cpu.c index 6f33b86c7d..c5196a612e 100644 --- a/target/m68k/cpu.c +++ b/target/m68k/cpu.c @@ -23,6 +23,8 @@ #include "cpu.h" #include "migration/vmstate.h" #include "fpu/softfloat.h" +#include "exec/translation-block.h" +#include "accel/tcg/cpu-ops.h" static void m68k_cpu_set_pc(CPUState *cs, vaddr value) { @@ -38,6 +40,24 @@ static vaddr m68k_cpu_get_pc(CPUState *cs) return cpu->env.pc; } +static TCGTBCPUState m68k_get_tb_cpu_state(CPUState *cs) +{ + CPUM68KState *env = cpu_env(cs); + uint32_t flags; + + flags = (env->macsr >> 4) & TB_FLAGS_MACSR; + if (env->sr & SR_S) { + flags |= TB_FLAGS_MSR_S; + flags |= (env->sfc << (TB_FLAGS_SFC_S_BIT - 2)) & TB_FLAGS_SFC_S; + flags |= (env->dfc << (TB_FLAGS_DFC_S_BIT - 2)) & TB_FLAGS_DFC_S; + } + if (M68K_SR_TRACE(env->sr) == M68K_SR_TRACE_ANY_INS) { + flags |= TB_FLAGS_TRACE; + } + + return (TCGTBCPUState){ .pc = env->pc, .flags = flags }; +} + static void m68k_restore_state_to_opc(CPUState *cs, const TranslationBlock *tb, const uint64_t *data) @@ -586,8 +606,6 @@ static const struct SysemuCPUOps m68k_sysemu_ops = { }; #endif /* !CONFIG_USER_ONLY */ -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps m68k_tcg_ops = { /* MTTCG not yet supported: require strict ordering */ .guest_default_memory_order = TCG_MO_ALL, @@ -595,6 +613,7 @@ static const TCGCPUOps m68k_tcg_ops = { .initialize = m68k_tcg_init, .translate_code = m68k_translate_code, + .get_tb_cpu_state = m68k_get_tb_cpu_state, .restore_state_to_opc = m68k_restore_state_to_opc, .mmu_index = m68k_cpu_mmu_index, @@ -602,6 +621,7 @@ static const TCGCPUOps m68k_tcg_ops = { .tlb_fill = m68k_cpu_tlb_fill, .cpu_exec_interrupt = m68k_cpu_exec_interrupt, .cpu_exec_halt = m68k_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = m68k_cpu_do_interrupt, .do_transaction_failed = m68k_cpu_transaction_failed, #endif /* !CONFIG_USER_ONLY */ diff --git a/target/m68k/cpu.h b/target/m68k/cpu.h index 39d0b9d6d7..d9db6a486a 100644 --- a/target/m68k/cpu.h +++ b/target/m68k/cpu.h @@ -605,22 +605,6 @@ void m68k_cpu_transaction_failed(CPUState *cs, hwaddr physaddr, vaddr addr, #define TB_FLAGS_TRACE 16 #define TB_FLAGS_TRACE_BIT (1 << TB_FLAGS_TRACE) -static inline void cpu_get_tb_cpu_state(CPUM68KState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *pc = env->pc; - *cs_base = 0; - *flags = (env->macsr >> 4) & TB_FLAGS_MACSR; - if (env->sr & SR_S) { - *flags |= TB_FLAGS_MSR_S; - *flags |= (env->sfc << (TB_FLAGS_SFC_S_BIT - 2)) & TB_FLAGS_SFC_S; - *flags |= (env->dfc << (TB_FLAGS_DFC_S_BIT - 2)) & TB_FLAGS_DFC_S; - } - if (M68K_SR_TRACE(env->sr) == M68K_SR_TRACE_ANY_INS) { - *flags |= TB_FLAGS_TRACE; - } -} - void dump_mmu(CPUM68KState *env); #endif diff --git a/target/m68k/fpu_helper.c b/target/m68k/fpu_helper.c index ac4a0d85be..56012863c8 100644 --- a/target/m68k/fpu_helper.c +++ b/target/m68k/fpu_helper.c @@ -21,7 +21,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "softfloat.h" diff --git a/target/m68k/helper.c b/target/m68k/helper.c index 3b880dd4d9..15f110fa7a 100644 --- a/target/m68k/helper.c +++ b/target/m68k/helper.c @@ -21,7 +21,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "exec/cputlb.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "exec/gdbstub.h" diff --git a/target/m68k/op_helper.c b/target/m68k/op_helper.c index 242aecccbb..f29ae12af8 100644 --- a/target/m68k/op_helper.c +++ b/target/m68k/op_helper.c @@ -20,7 +20,6 @@ #include "qemu/log.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "semihosting/semihost.h" diff --git a/target/m68k/translate.c b/target/m68k/translate.c index b1266a7875..97afceb129 100644 --- a/target/m68k/translate.c +++ b/target/m68k/translate.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "exec/target_page.h" #include "tcg/tcg-op.h" diff --git a/target/microblaze/cpu.c b/target/microblaze/cpu.c index 00a2730de4..d069e40e70 100644 --- a/target/microblaze/cpu.c +++ b/target/microblaze/cpu.c @@ -27,11 +27,11 @@ #include "cpu.h" #include "qemu/module.h" #include "hw/qdev-properties.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "exec/gdbstub.h" #include "exec/translation-block.h" #include "fpu/softfloat-helpers.h" +#include "accel/tcg/cpu-ops.h" #include "tcg/tcg.h" static const struct { @@ -95,6 +95,17 @@ static vaddr mb_cpu_get_pc(CPUState *cs) return cpu->env.pc; } +static TCGTBCPUState mb_get_tb_cpu_state(CPUState *cs) +{ + CPUMBState *env = cpu_env(cs); + + return (TCGTBCPUState){ + .pc = env->pc, + .flags = (env->iflags & IFLAGS_TB_MASK) | (env->msr & MSR_TB_MASK), + .cs_base = (env->iflags & IMM_FLAG ? env->imm : 0), + }; +} + static void mb_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -424,8 +435,6 @@ static const struct SysemuCPUOps mb_sysemu_ops = { }; #endif -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps mb_tcg_ops = { /* MicroBlaze is always in-order. */ .guest_default_memory_order = TCG_MO_ALL, @@ -433,6 +442,7 @@ static const TCGCPUOps mb_tcg_ops = { .initialize = mb_tcg_init, .translate_code = mb_translate_code, + .get_tb_cpu_state = mb_get_tb_cpu_state, .synchronize_from_tb = mb_cpu_synchronize_from_tb, .restore_state_to_opc = mb_restore_state_to_opc, .mmu_index = mb_cpu_mmu_index, @@ -441,6 +451,7 @@ static const TCGCPUOps mb_tcg_ops = { .tlb_fill = mb_cpu_tlb_fill, .cpu_exec_interrupt = mb_cpu_exec_interrupt, .cpu_exec_halt = mb_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = mb_cpu_do_interrupt, .do_transaction_failed = mb_cpu_transaction_failed, .do_unaligned_access = mb_cpu_do_unaligned_access, diff --git a/target/microblaze/cpu.h b/target/microblaze/cpu.h index d511f22a55..6ad8643f2e 100644 --- a/target/microblaze/cpu.h +++ b/target/microblaze/cpu.h @@ -419,14 +419,6 @@ static inline bool mb_cpu_is_big_endian(CPUState *cs) return !cpu->cfg.endi; } -static inline void cpu_get_tb_cpu_state(CPUMBState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *pc = env->pc; - *flags = (env->iflags & IFLAGS_TB_MASK) | (env->msr & MSR_TB_MASK); - *cs_base = (*flags & IMM_FLAG ? env->imm : 0); -} - #if !defined(CONFIG_USER_ONLY) bool mb_cpu_tlb_fill(CPUState *cs, vaddr address, int size, MMUAccessType access_type, int mmu_idx, diff --git a/target/microblaze/machine.c b/target/microblaze/machine.c index 51705e4f5c..a4cf38dc89 100644 --- a/target/microblaze/machine.c +++ b/target/microblaze/machine.c @@ -93,7 +93,7 @@ static const VMStateDescription vmstate_env = { }; static const VMStateField vmstate_cpu_fields[] = { - VMSTATE_CPU(), + VMSTATE_STRUCT(parent_obj, MicroBlazeCPU, 0, vmstate_cpu_common, CPUState), VMSTATE_STRUCT(env, MicroBlazeCPU, 1, vmstate_env, CPUMBState), VMSTATE_END_OF_LIST() }; diff --git a/target/microblaze/op_helper.c b/target/microblaze/op_helper.c index 4624ce5b67..9e838dfa15 100644 --- a/target/microblaze/op_helper.c +++ b/target/microblaze/op_helper.c @@ -23,7 +23,6 @@ #include "cpu.h" #include "exec/helper-proto.h" #include "qemu/host-utils.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "fpu/softfloat.h" diff --git a/target/microblaze/translate.c b/target/microblaze/translate.c index 7dcad6cf0d..671b1ae4db 100644 --- a/target/microblaze/translate.c +++ b/target/microblaze/translate.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "tcg/tcg-op.h" #include "exec/helper-proto.h" @@ -311,11 +310,7 @@ static void gen_add(TCGv_i32 out, TCGv_i32 ina, TCGv_i32 inb) /* Input and output carry. */ static void gen_addc(TCGv_i32 out, TCGv_i32 ina, TCGv_i32 inb) { - TCGv_i32 zero = tcg_constant_i32(0); - TCGv_i32 tmp = tcg_temp_new_i32(); - - tcg_gen_add2_i32(tmp, cpu_msr_c, ina, zero, cpu_msr_c, zero); - tcg_gen_add2_i32(out, cpu_msr_c, tmp, cpu_msr_c, inb, zero); + tcg_gen_addcio_i32(out, cpu_msr_c, ina, inb, cpu_msr_c); } /* Input carry, but no output carry. */ @@ -544,12 +539,10 @@ static void gen_rsub(TCGv_i32 out, TCGv_i32 ina, TCGv_i32 inb) /* Input and output carry. */ static void gen_rsubc(TCGv_i32 out, TCGv_i32 ina, TCGv_i32 inb) { - TCGv_i32 zero = tcg_constant_i32(0); TCGv_i32 tmp = tcg_temp_new_i32(); tcg_gen_not_i32(tmp, ina); - tcg_gen_add2_i32(tmp, cpu_msr_c, tmp, zero, cpu_msr_c, zero); - tcg_gen_add2_i32(out, cpu_msr_c, tmp, cpu_msr_c, inb, zero); + tcg_gen_addcio_i32(out, cpu_msr_c, tmp, inb, cpu_msr_c); } /* No input or output carry. */ diff --git a/target/mips/cpu.c b/target/mips/cpu.c index d13361a150..4cbfb9435a 100644 --- a/target/mips/cpu.c +++ b/target/mips/cpu.c @@ -29,7 +29,6 @@ #include "qemu/module.h" #include "system/kvm.h" #include "system/qtest.h" -#include "exec/exec-all.h" #include "hw/qdev-properties.h" #include "hw/qdev-clock.h" #include "fpu_helper.h" @@ -550,12 +549,24 @@ static int mips_cpu_mmu_index(CPUState *cs, bool ifunc) return mips_env_mmu_index(cpu_env(cs)); } +static TCGTBCPUState mips_get_tb_cpu_state(CPUState *cs) +{ + CPUMIPSState *env = cpu_env(cs); + + return (TCGTBCPUState){ + .pc = env->active_tc.PC, + .flags = env->hflags & (MIPS_HFLAG_TMASK | MIPS_HFLAG_BMASK | + MIPS_HFLAG_HWRENA_ULR), + }; +} + static const TCGCPUOps mips_tcg_ops = { .mttcg_supported = TARGET_LONG_BITS == 32, .guest_default_memory_order = 0, .initialize = mips_tcg_init, .translate_code = mips_translate_code, + .get_tb_cpu_state = mips_get_tb_cpu_state, .synchronize_from_tb = mips_cpu_synchronize_from_tb, .restore_state_to_opc = mips_restore_state_to_opc, .mmu_index = mips_cpu_mmu_index, @@ -564,6 +575,7 @@ static const TCGCPUOps mips_tcg_ops = { .tlb_fill = mips_cpu_tlb_fill, .cpu_exec_interrupt = mips_cpu_exec_interrupt, .cpu_exec_halt = mips_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = mips_cpu_do_interrupt, .do_transaction_failed = mips_cpu_do_transaction_failed, .do_unaligned_access = mips_cpu_do_unaligned_access, diff --git a/target/mips/cpu.h b/target/mips/cpu.h index d16f9a7220..5cd4c6c818 100644 --- a/target/mips/cpu.h +++ b/target/mips/cpu.h @@ -1366,15 +1366,6 @@ void cpu_mips_clock_init(MIPSCPU *cpu); /* helper.c */ target_ulong exception_resume_pc(CPUMIPSState *env); -static inline void cpu_get_tb_cpu_state(CPUMIPSState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *pc = env->active_tc.PC; - *cs_base = 0; - *flags = env->hflags & (MIPS_HFLAG_TMASK | MIPS_HFLAG_BMASK | - MIPS_HFLAG_HWRENA_ULR); -} - /** * mips_cpu_create_with_clock: * @typename: a MIPS CPU type. diff --git a/target/mips/system/physaddr.c b/target/mips/system/physaddr.c index 505781d84c..b8e1a5ac98 100644 --- a/target/mips/system/physaddr.c +++ b/target/mips/system/physaddr.c @@ -18,7 +18,6 @@ */ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "../internal.h" diff --git a/target/mips/tcg/exception.c b/target/mips/tcg/exception.c index 1a8902ea1b..d32bcebf46 100644 --- a/target/mips/tcg/exception.c +++ b/target/mips/tcg/exception.c @@ -23,7 +23,6 @@ #include "cpu.h" #include "internal.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" target_ulong exception_resume_pc(CPUMIPSState *env) diff --git a/target/mips/tcg/fpu_helper.c b/target/mips/tcg/fpu_helper.c index 45d593de48..36af980802 100644 --- a/target/mips/tcg/fpu_helper.c +++ b/target/mips/tcg/fpu_helper.c @@ -24,7 +24,6 @@ #include "cpu.h" #include "internal.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "fpu/softfloat.h" #include "fpu_helper.h" diff --git a/target/mips/tcg/ldst_helper.c b/target/mips/tcg/ldst_helper.c index 2fb879fcbc..10319bf03a 100644 --- a/target/mips/tcg/ldst_helper.c +++ b/target/mips/tcg/ldst_helper.c @@ -23,7 +23,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "exec/memop.h" #include "internal.h" diff --git a/target/mips/tcg/msa_helper.c b/target/mips/tcg/msa_helper.c index e349344647..f554b3d10e 100644 --- a/target/mips/tcg/msa_helper.c +++ b/target/mips/tcg/msa_helper.c @@ -21,8 +21,8 @@ #include "cpu.h" #include "internal.h" #include "tcg/tcg.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "exec/helper-proto.h" #include "exec/memop.h" #include "exec/target_page.h" diff --git a/target/mips/tcg/op_helper.c b/target/mips/tcg/op_helper.c index 65403f1a87..b906d10204 100644 --- a/target/mips/tcg/op_helper.c +++ b/target/mips/tcg/op_helper.c @@ -22,7 +22,6 @@ #include "cpu.h" #include "internal.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "exec/memop.h" #include "fpu_helper.h" diff --git a/target/mips/tcg/system/special_helper.c b/target/mips/tcg/system/special_helper.c index 3ce3ae1e12..b54cbe88a3 100644 --- a/target/mips/tcg/system/special_helper.c +++ b/target/mips/tcg/system/special_helper.c @@ -22,7 +22,6 @@ #include "qemu/log.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "internal.h" diff --git a/target/mips/tcg/system/tlb_helper.c b/target/mips/tcg/system/tlb_helper.c index e477ef812a..eccaf3624c 100644 --- a/target/mips/tcg/system/tlb_helper.c +++ b/target/mips/tcg/system/tlb_helper.c @@ -22,7 +22,6 @@ #include "cpu.h" #include "internal.h" #include "exec/cputlb.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "accel/tcg/cpu-ldst.h" diff --git a/target/openrisc/cpu.c b/target/openrisc/cpu.c index 2ec267efec..054ad33360 100644 --- a/target/openrisc/cpu.c +++ b/target/openrisc/cpu.c @@ -21,9 +21,9 @@ #include "qapi/error.h" #include "qemu/qemu-print.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "fpu/softfloat-helpers.h" +#include "accel/tcg/cpu-ops.h" #include "tcg/tcg.h" static void openrisc_cpu_set_pc(CPUState *cs, vaddr value) @@ -41,6 +41,18 @@ static vaddr openrisc_cpu_get_pc(CPUState *cs) return cpu->env.pc; } +static TCGTBCPUState openrisc_get_tb_cpu_state(CPUState *cs) +{ + CPUOpenRISCState *env = cpu_env(cs); + + return (TCGTBCPUState){ + .pc = env->pc, + .flags = ((env->dflag ? TB_FLAGS_DFLAG : 0) + | (cpu_get_gpr(env, 0) ? 0 : TB_FLAGS_R0_0) + | (env->sr & (SR_SM | SR_DME | SR_IME | SR_OVE))), + }; +} + static void openrisc_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -240,14 +252,13 @@ static const struct SysemuCPUOps openrisc_sysemu_ops = { }; #endif -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps openrisc_tcg_ops = { .guest_default_memory_order = 0, .mttcg_supported = true, .initialize = openrisc_translate_init, .translate_code = openrisc_translate_code, + .get_tb_cpu_state = openrisc_get_tb_cpu_state, .synchronize_from_tb = openrisc_cpu_synchronize_from_tb, .restore_state_to_opc = openrisc_restore_state_to_opc, .mmu_index = openrisc_cpu_mmu_index, @@ -256,6 +267,7 @@ static const TCGCPUOps openrisc_tcg_ops = { .tlb_fill = openrisc_cpu_tlb_fill, .cpu_exec_interrupt = openrisc_cpu_exec_interrupt, .cpu_exec_halt = openrisc_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = openrisc_cpu_do_interrupt, #endif /* !CONFIG_USER_ONLY */ }; diff --git a/target/openrisc/cpu.h b/target/openrisc/cpu.h index 569819bfb0..f4bcf00b07 100644 --- a/target/openrisc/cpu.h +++ b/target/openrisc/cpu.h @@ -349,16 +349,6 @@ static inline void cpu_set_gpr(CPUOpenRISCState *env, int i, uint32_t val) env->shadow_gpr[0][i] = val; } -static inline void cpu_get_tb_cpu_state(CPUOpenRISCState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *pc = env->pc; - *cs_base = 0; - *flags = (env->dflag ? TB_FLAGS_DFLAG : 0) - | (cpu_get_gpr(env, 0) ? 0 : TB_FLAGS_R0_0) - | (env->sr & (SR_SM | SR_DME | SR_IME | SR_OVE)); -} - static inline uint32_t cpu_get_sr(const CPUOpenRISCState *env) { return (env->sr diff --git a/target/openrisc/exception.c b/target/openrisc/exception.c index 8699c3dcea..e213be36b6 100644 --- a/target/openrisc/exception.c +++ b/target/openrisc/exception.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exception.h" G_NORETURN void raise_exception(OpenRISCCPU *cpu, uint32_t excp) diff --git a/target/openrisc/exception_helper.c b/target/openrisc/exception_helper.c index 1f5be4bed9..c2c9d13652 100644 --- a/target/openrisc/exception_helper.c +++ b/target/openrisc/exception_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "exception.h" diff --git a/target/openrisc/fpu_helper.c b/target/openrisc/fpu_helper.c index 8b81d2f62f..dba997255c 100644 --- a/target/openrisc/fpu_helper.c +++ b/target/openrisc/fpu_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" diff --git a/target/openrisc/interrupt.c b/target/openrisc/interrupt.c index b3b5b40577..486823094c 100644 --- a/target/openrisc/interrupt.c +++ b/target/openrisc/interrupt.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "qemu/log.h" #include "cpu.h" -#include "exec/exec-all.h" #include "gdbstub/helpers.h" #include "qemu/host-utils.h" #ifndef CONFIG_USER_ONLY diff --git a/target/openrisc/interrupt_helper.c b/target/openrisc/interrupt_helper.c index ab4ea88b69..1553ebc71b 100644 --- a/target/openrisc/interrupt_helper.c +++ b/target/openrisc/interrupt_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" void HELPER(rfe)(CPUOpenRISCState *env) diff --git a/target/openrisc/machine.c b/target/openrisc/machine.c index 3574e571cb..081c706d02 100644 --- a/target/openrisc/machine.c +++ b/target/openrisc/machine.c @@ -136,7 +136,7 @@ const VMStateDescription vmstate_openrisc_cpu = { .minimum_version_id = 1, .post_load = cpu_post_load, .fields = (const VMStateField[]) { - VMSTATE_CPU(), + VMSTATE_STRUCT(parent_obj, OpenRISCCPU, 0, vmstate_cpu_common, CPUState), VMSTATE_STRUCT(env, OpenRISCCPU, 1, vmstate_env, CPUOpenRISCState), VMSTATE_END_OF_LIST() } diff --git a/target/openrisc/sys_helper.c b/target/openrisc/sys_helper.c index 92badf017f..d96b41a01c 100644 --- a/target/openrisc/sys_helper.c +++ b/target/openrisc/sys_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "exec/target_page.h" #include "exec/helper-proto.h" @@ -219,7 +218,7 @@ target_ulong HELPER(mfspr)(CPUOpenRISCState *env, target_ulong rd, { OpenRISCCPU *cpu = env_archcpu(env); #ifndef CONFIG_USER_ONLY - uint64_t data[TARGET_INSN_START_WORDS]; + uint64_t data[INSN_START_WORDS]; MachineState *ms = MACHINE(qdev_get_machine()); CPUState *cs = env_cpu(env); int idx; diff --git a/target/openrisc/translate.c b/target/openrisc/translate.c index d4ce60188b..5ab3bc7021 100644 --- a/target/openrisc/translate.c +++ b/target/openrisc/translate.c @@ -21,7 +21,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "accel/tcg/cpu-mmu-index.h" -#include "exec/exec-all.h" #include "tcg/tcg-op.h" #include "qemu/log.h" #include "qemu/bitops.h" @@ -221,8 +220,7 @@ static void gen_addc(DisasContext *dc, TCGv dest, TCGv srca, TCGv srcb) TCGv t0 = tcg_temp_new(); TCGv res = tcg_temp_new(); - tcg_gen_add2_tl(res, cpu_sr_cy, srca, dc->zero, cpu_sr_cy, dc->zero); - tcg_gen_add2_tl(res, cpu_sr_cy, res, cpu_sr_cy, srcb, dc->zero); + tcg_gen_addcio_tl(res, cpu_sr_cy, srca, srcb, cpu_sr_cy); tcg_gen_xor_tl(cpu_sr_ov, srca, srcb); tcg_gen_xor_tl(t0, res, srcb); tcg_gen_andc_tl(cpu_sr_ov, t0, cpu_sr_ov); diff --git a/target/ppc/cpu.h b/target/ppc/cpu.h index 13115a89ff..6b90543811 100644 --- a/target/ppc/cpu.h +++ b/target/ppc/cpu.h @@ -2751,19 +2751,6 @@ void cpu_write_xer(CPUPPCState *env, target_ulong xer); */ #define is_book3s_arch2x(ctx) (!!((ctx)->insns_flags & PPC_SEGMENT_64B)) -#ifdef CONFIG_DEBUG_TCG -void cpu_get_tb_cpu_state(CPUPPCState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags); -#else -static inline void cpu_get_tb_cpu_state(CPUPPCState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *pc = env->nip; - *cs_base = 0; - *flags = env->hflags; -} -#endif - G_NORETURN void raise_exception_err_ra(CPUPPCState *env, uint32_t exception, uint32_t error_code, uintptr_t raddr); diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c index b0973b6df9..9642812a71 100644 --- a/target/ppc/cpu_init.c +++ b/target/ppc/cpu_init.c @@ -45,7 +45,6 @@ #include "internal.h" #include "spr_common.h" #include "power8-pmu.h" - #ifndef CONFIG_USER_ONLY #include "hw/boards.h" #include "hw/intc/intc.h" @@ -7115,7 +7114,7 @@ PowerPCCPUClass *ppc_cpu_get_family_class(PowerPCCPUClass *pcc) } /* Sort by PVR, ordering special case "host" last. */ -static gint ppc_cpu_list_compare(gconstpointer a, gconstpointer b) +static gint ppc_cpu_list_compare(gconstpointer a, gconstpointer b, gpointer d) { ObjectClass *oc_a = (ObjectClass *)a; ObjectClass *oc_b = (ObjectClass *)b; @@ -7183,7 +7182,7 @@ static void ppc_cpu_list(void) qemu_printf("Available CPUs:\n"); list = object_class_get_list(TYPE_POWERPC_CPU, false); - list = g_slist_sort(list, ppc_cpu_list_compare); + list = g_slist_sort_with_data(list, ppc_cpu_list_compare, NULL); g_slist_foreach(list, ppc_cpu_list_entry, NULL); g_slist_free(list); @@ -7483,6 +7482,7 @@ static const TCGCPUOps ppc_tcg_ops = { .guest_default_memory_order = 0, .initialize = ppc_translate_init, .translate_code = ppc_translate_code, + .get_tb_cpu_state = ppc_get_tb_cpu_state, .restore_state_to_opc = ppc_restore_state_to_opc, .mmu_index = ppc_cpu_mmu_index, @@ -7492,6 +7492,7 @@ static const TCGCPUOps ppc_tcg_ops = { .tlb_fill = ppc_cpu_tlb_fill, .cpu_exec_interrupt = ppc_cpu_exec_interrupt, .cpu_exec_halt = ppc_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = ppc_cpu_do_interrupt, .cpu_exec_enter = ppc_cpu_exec_enter, .cpu_exec_exit = ppc_cpu_exec_exit, diff --git a/target/ppc/excp_helper.c b/target/ppc/excp_helper.c index da8b525a41..1efdc4066e 100644 --- a/target/ppc/excp_helper.c +++ b/target/ppc/excp_helper.c @@ -24,7 +24,6 @@ #include "system/system.h" #include "system/runstate.h" #include "cpu.h" -#include "exec/exec-all.h" #include "internal.h" #include "helper_regs.h" #include "hw/ppc/ppc.h" diff --git a/target/ppc/fpu_helper.c b/target/ppc/fpu_helper.c index d93cfed17b..07b782f971 100644 --- a/target/ppc/fpu_helper.c +++ b/target/ppc/fpu_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "internal.h" #include "fpu/softfloat.h" diff --git a/target/ppc/helper_regs.c b/target/ppc/helper_regs.c index f211bc9830..7e5726871e 100644 --- a/target/ppc/helper_regs.c +++ b/target/ppc/helper_regs.c @@ -27,6 +27,8 @@ #include "power8-pmu.h" #include "cpu-models.h" #include "spr_common.h" +#include "accel/tcg/cpu-ops.h" +#include "internal.h" /* Swap temporary saved registers with GPRs */ void hreg_swap_gpr_tgpr(CPUPPCState *env) @@ -255,26 +257,23 @@ void hreg_update_pmu_hflags(CPUPPCState *env) env->hflags |= hreg_compute_pmu_hflags_value(env); } -#ifdef CONFIG_DEBUG_TCG -void cpu_get_tb_cpu_state(CPUPPCState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) +TCGTBCPUState ppc_get_tb_cpu_state(CPUState *cs) { + CPUPPCState *env = cpu_env(cs); uint32_t hflags_current = env->hflags; - uint32_t hflags_rebuilt; - - *pc = env->nip; - *cs_base = 0; - *flags = hflags_current; - hflags_rebuilt = hreg_compute_hflags_value(env); +#ifdef CONFIG_DEBUG_TCG + uint32_t hflags_rebuilt = hreg_compute_hflags_value(env); if (unlikely(hflags_current != hflags_rebuilt)) { cpu_abort(env_cpu(env), "TCG hflags mismatch (current:0x%08x rebuilt:0x%08x)\n", hflags_current, hflags_rebuilt); } -} #endif + return (TCGTBCPUState){ .pc = env->nip, .flags = hflags_current }; +} + void cpu_interrupt_exittb(CPUState *cs) { /* diff --git a/target/ppc/internal.h b/target/ppc/internal.h index 9012d3809c..7723350227 100644 --- a/target/ppc/internal.h +++ b/target/ppc/internal.h @@ -21,6 +21,7 @@ #include "exec/breakpoint.h" #include "hw/registerfields.h" #include "exec/page-protection.h" +#include "accel/tcg/tb-cpu-state.h" /* PM instructions */ typedef enum { @@ -308,4 +309,6 @@ static inline int ger_pack_masks(int pmsk, int ymsk, int xmsk) return msk; } +TCGTBCPUState ppc_get_tb_cpu_state(CPUState *cs); + #endif /* PPC_INTERNAL_H */ diff --git a/target/ppc/machine.c b/target/ppc/machine.c index 98df5b4a3a..d72e5ecb94 100644 --- a/target/ppc/machine.c +++ b/target/ppc/machine.c @@ -1,6 +1,5 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "system/kvm.h" #include "system/tcg.h" #include "helper_regs.h" diff --git a/target/ppc/mem_helper.c b/target/ppc/mem_helper.c index d7e8d678f4..6ab71a6fcb 100644 --- a/target/ppc/mem_helper.c +++ b/target/ppc/mem_helper.c @@ -19,12 +19,13 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/target_page.h" #include "qemu/host-utils.h" #include "exec/helper-proto.h" #include "helper_regs.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/helper-retaddr.h" +#include "accel/tcg/probe.h" #include "internal.h" #include "qemu/atomic128.h" diff --git a/target/ppc/misc_helper.c b/target/ppc/misc_helper.c index 46ae454afd..e7d9462518 100644 --- a/target/ppc/misc_helper.c +++ b/target/ppc/misc_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "qemu/log.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "exec/helper-proto.h" #include "qemu/error-report.h" diff --git a/target/ppc/mmu-hash32.c b/target/ppc/mmu-hash32.c index 5bd3efe70e..8b980a5aa9 100644 --- a/target/ppc/mmu-hash32.c +++ b/target/ppc/mmu-hash32.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "system/kvm.h" diff --git a/target/ppc/mmu-hash64.c b/target/ppc/mmu-hash64.c index 3ba4810497..dd337558aa 100644 --- a/target/ppc/mmu-hash64.c +++ b/target/ppc/mmu-hash64.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "qemu/units.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "qemu/error-report.h" #include "qemu/qemu-print.h" diff --git a/target/ppc/mmu-radix64.c b/target/ppc/mmu-radix64.c index 4ab5f3bb92..33ac341290 100644 --- a/target/ppc/mmu-radix64.c +++ b/target/ppc/mmu-radix64.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "qemu/error-report.h" #include "system/kvm.h" diff --git a/target/ppc/mmu_common.c b/target/ppc/mmu_common.c index 394a0c9bb6..52d48615ac 100644 --- a/target/ppc/mmu_common.c +++ b/target/ppc/mmu_common.c @@ -24,7 +24,6 @@ #include "kvm_ppc.h" #include "mmu-hash64.h" #include "mmu-hash32.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "exec/log.h" diff --git a/target/ppc/mmu_helper.c b/target/ppc/mmu_helper.c index 2138666122..ac60705402 100644 --- a/target/ppc/mmu_helper.c +++ b/target/ppc/mmu_helper.c @@ -25,7 +25,6 @@ #include "mmu-hash64.h" #include "mmu-hash32.h" #include "exec/cputlb.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "exec/log.h" diff --git a/target/ppc/power8-pmu.c b/target/ppc/power8-pmu.c index db9ee8e96b..2a7a5b493a 100644 --- a/target/ppc/power8-pmu.c +++ b/target/ppc/power8-pmu.c @@ -13,7 +13,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "helper_regs.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "qemu/error-report.h" #include "qemu/timer.h" diff --git a/target/ppc/tcg-excp_helper.c b/target/ppc/tcg-excp_helper.c index 2b15e5f2f0..f835be5156 100644 --- a/target/ppc/tcg-excp_helper.c +++ b/target/ppc/tcg-excp_helper.c @@ -21,7 +21,6 @@ #include "qemu/log.h" #include "target/ppc/cpu.h" #include "accel/tcg/cpu-ldst.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "system/runstate.h" diff --git a/target/ppc/timebase_helper.c b/target/ppc/timebase_helper.c index 73120323b4..7209b418fb 100644 --- a/target/ppc/timebase_helper.c +++ b/target/ppc/timebase_helper.c @@ -20,7 +20,6 @@ #include "cpu.h" #include "hw/ppc/ppc.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "qemu/log.h" #include "qemu/main-loop.h" diff --git a/target/ppc/translate.c b/target/ppc/translate.c index fea2f2ce23..27f90c3cc5 100644 --- a/target/ppc/translate.c +++ b/target/ppc/translate.c @@ -21,7 +21,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "internal.h" -#include "exec/exec-all.h" #include "exec/target_page.h" #include "tcg/tcg-op.h" #include "tcg/tcg-op-gvec.h" @@ -1746,11 +1745,10 @@ static inline void gen_op_arith_add(DisasContext *ctx, TCGv ret, TCGv arg1, tcg_gen_mov_tl(ca32, ca); } } else { - TCGv zero = tcg_constant_tl(0); if (add_ca) { - tcg_gen_add2_tl(t0, ca, arg1, zero, ca, zero); - tcg_gen_add2_tl(t0, ca, t0, ca, arg2, zero); + tcg_gen_addcio_tl(t0, ca, arg1, arg2, ca); } else { + TCGv zero = tcg_constant_tl(0); tcg_gen_add2_tl(t0, ca, arg1, zero, arg2, zero); } gen_op_arith_compute_ca32(ctx, t0, arg1, arg2, ca32, 0); @@ -1949,11 +1947,9 @@ static inline void gen_op_arith_subf(DisasContext *ctx, TCGv ret, TCGv arg1, tcg_gen_mov_tl(cpu_ca32, cpu_ca); } } else if (add_ca) { - TCGv zero, inv1 = tcg_temp_new(); + TCGv inv1 = tcg_temp_new(); tcg_gen_not_tl(inv1, arg1); - zero = tcg_constant_tl(0); - tcg_gen_add2_tl(t0, cpu_ca, arg2, zero, cpu_ca, zero); - tcg_gen_add2_tl(t0, cpu_ca, t0, cpu_ca, inv1, zero); + tcg_gen_addcio_tl(t0, cpu_ca, arg2, inv1, cpu_ca); gen_op_arith_compute_ca32(ctx, t0, inv1, arg2, cpu_ca32, 0); } else { tcg_gen_setcond_tl(TCG_COND_GEU, cpu_ca, arg2, arg1); diff --git a/target/ppc/user_only_helper.c b/target/ppc/user_only_helper.c index a4d07a0d0d..ae210eb847 100644 --- a/target/ppc/user_only_helper.c +++ b/target/ppc/user_only_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "internal.h" void ppc_cpu_record_sigsegv(CPUState *cs, vaddr address, diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index e0604f4c78..d92874baa0 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -24,7 +24,6 @@ #include "cpu.h" #include "cpu_vendorid.h" #include "internals.h" -#include "exec/exec-all.h" #include "qapi/error.h" #include "qapi/visitor.h" #include "qemu/error-report.h" diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index 167909c89b..c66ac3bc27 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -802,9 +802,6 @@ static inline uint32_t vext_get_vlmax(uint32_t vlenb, uint32_t vsew, return vlen >> (vsew + 3 - lmul); } -void cpu_get_tb_cpu_state(CPURISCVState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *pflags); - bool riscv_cpu_is_32bit(RISCVCPU *cpu); bool riscv_cpu_virt_mem_enabled(CPURISCVState *env); diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c index 619c76cc00..d5039f69a9 100644 --- a/target/riscv/cpu_helper.c +++ b/target/riscv/cpu_helper.c @@ -24,7 +24,6 @@ #include "internals.h" #include "pmu.h" #include "exec/cputlb.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "system/memory.h" @@ -136,103 +135,6 @@ bool riscv_env_smode_dbltrp_enabled(CPURISCVState *env, bool virt) #endif } -void cpu_get_tb_cpu_state(CPURISCVState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *pflags) -{ - RISCVCPU *cpu = env_archcpu(env); - RISCVExtStatus fs, vs; - uint32_t flags = 0; - bool pm_signext = riscv_cpu_virt_mem_enabled(env); - - *pc = env->xl == MXL_RV32 ? env->pc & UINT32_MAX : env->pc; - *cs_base = 0; - - if (cpu->cfg.ext_zve32x) { - /* - * If env->vl equals to VLMAX, we can use generic vector operation - * expanders (GVEC) to accerlate the vector operations. - * However, as LMUL could be a fractional number. The maximum - * vector size can be operated might be less than 8 bytes, - * which is not supported by GVEC. So we set vl_eq_vlmax flag to true - * only when maxsz >= 8 bytes. - */ - - /* lmul encoded as in DisasContext::lmul */ - int8_t lmul = sextract32(FIELD_EX64(env->vtype, VTYPE, VLMUL), 0, 3); - uint32_t vsew = FIELD_EX64(env->vtype, VTYPE, VSEW); - uint32_t vlmax = vext_get_vlmax(cpu->cfg.vlenb, vsew, lmul); - uint32_t maxsz = vlmax << vsew; - bool vl_eq_vlmax = (env->vstart == 0) && (vlmax == env->vl) && - (maxsz >= 8); - flags = FIELD_DP32(flags, TB_FLAGS, VILL, env->vill); - flags = FIELD_DP32(flags, TB_FLAGS, SEW, vsew); - flags = FIELD_DP32(flags, TB_FLAGS, LMUL, - FIELD_EX64(env->vtype, VTYPE, VLMUL)); - flags = FIELD_DP32(flags, TB_FLAGS, VL_EQ_VLMAX, vl_eq_vlmax); - flags = FIELD_DP32(flags, TB_FLAGS, VTA, - FIELD_EX64(env->vtype, VTYPE, VTA)); - flags = FIELD_DP32(flags, TB_FLAGS, VMA, - FIELD_EX64(env->vtype, VTYPE, VMA)); - flags = FIELD_DP32(flags, TB_FLAGS, VSTART_EQ_ZERO, env->vstart == 0); - } else { - flags = FIELD_DP32(flags, TB_FLAGS, VILL, 1); - } - - if (cpu_get_fcfien(env)) { - /* - * For Forward CFI, only the expectation of a lpad at - * the start of the block is tracked via env->elp. env->elp - * is turned on during jalr translation. - */ - flags = FIELD_DP32(flags, TB_FLAGS, FCFI_LP_EXPECTED, env->elp); - flags = FIELD_DP32(flags, TB_FLAGS, FCFI_ENABLED, 1); - } - - if (cpu_get_bcfien(env)) { - flags = FIELD_DP32(flags, TB_FLAGS, BCFI_ENABLED, 1); - } - -#ifdef CONFIG_USER_ONLY - fs = EXT_STATUS_DIRTY; - vs = EXT_STATUS_DIRTY; -#else - flags = FIELD_DP32(flags, TB_FLAGS, PRIV, env->priv); - - flags |= riscv_env_mmu_index(env, 0); - fs = get_field(env->mstatus, MSTATUS_FS); - vs = get_field(env->mstatus, MSTATUS_VS); - - if (env->virt_enabled) { - flags = FIELD_DP32(flags, TB_FLAGS, VIRT_ENABLED, 1); - /* - * Merge DISABLED and !DIRTY states using MIN. - * We will set both fields when dirtying. - */ - fs = MIN(fs, get_field(env->mstatus_hs, MSTATUS_FS)); - vs = MIN(vs, get_field(env->mstatus_hs, MSTATUS_VS)); - } - - /* With Zfinx, floating point is enabled/disabled by Smstateen. */ - if (!riscv_has_ext(env, RVF)) { - fs = (smstateen_acc_ok(env, 0, SMSTATEEN0_FCSR) == RISCV_EXCP_NONE) - ? EXT_STATUS_DIRTY : EXT_STATUS_DISABLED; - } - - if (cpu->cfg.debug && !icount_enabled()) { - flags = FIELD_DP32(flags, TB_FLAGS, ITRIGGER, env->itrigger_enabled); - } -#endif - - flags = FIELD_DP32(flags, TB_FLAGS, FS, fs); - flags = FIELD_DP32(flags, TB_FLAGS, VS, vs); - flags = FIELD_DP32(flags, TB_FLAGS, XL, env->xl); - flags = FIELD_DP32(flags, TB_FLAGS, AXL, cpu_address_xl(env)); - flags = FIELD_DP32(flags, TB_FLAGS, PM_PMM, riscv_pm_get_pmm(env)); - flags = FIELD_DP32(flags, TB_FLAGS, PM_SIGNEXTEND, pm_signext); - - *pflags = flags; -} - RISCVPmPmm riscv_pm_get_pmm(CPURISCVState *env) { #ifndef CONFIG_USER_ONLY diff --git a/target/riscv/crypto_helper.c b/target/riscv/crypto_helper.c index bb084e00ef..a0fb54bc50 100644 --- a/target/riscv/crypto_helper.c +++ b/target/riscv/crypto_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "crypto/aes.h" #include "crypto/aes-round.h" diff --git a/target/riscv/csr.c b/target/riscv/csr.c index c52c87faae..a32e1455c9 100644 --- a/target/riscv/csr.c +++ b/target/riscv/csr.c @@ -24,10 +24,10 @@ #include "tcg/tcg-cpu.h" #include "pmu.h" #include "time_helper.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "exec/tb-flush.h" #include "exec/icount.h" +#include "accel/tcg/getpc.h" #include "qemu/guest-random.h" #include "qapi/error.h" #include <stdbool.h> diff --git a/target/riscv/debug.c b/target/riscv/debug.c index 8564f0b371..5664466749 100644 --- a/target/riscv/debug.c +++ b/target/riscv/debug.c @@ -28,7 +28,6 @@ #include "qapi/error.h" #include "cpu.h" #include "trace.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "exec/watchpoint.h" #include "system/cpu-timers.h" diff --git a/target/riscv/fpu_helper.c b/target/riscv/fpu_helper.c index 91b1a56d10..706bdfa61d 100644 --- a/target/riscv/fpu_helper.c +++ b/target/riscv/fpu_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "qemu/host-utils.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" #include "internals.h" diff --git a/target/riscv/m128_helper.c b/target/riscv/m128_helper.c index ec14aaa901..7d9b83b1b2 100644 --- a/target/riscv/m128_helper.c +++ b/target/riscv/m128_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" target_ulong HELPER(divu_i128)(CPURISCVState *env, diff --git a/target/riscv/op_helper.c b/target/riscv/op_helper.c index 5b0db2c45a..05316f2088 100644 --- a/target/riscv/op_helper.c +++ b/target/riscv/op_helper.c @@ -21,9 +21,9 @@ #include "qemu/osdep.h" #include "cpu.h" #include "internals.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "exec/helper-proto.h" #include "exec/tlb-flags.h" #include "trace.h" diff --git a/target/riscv/tcg/tcg-cpu.c b/target/riscv/tcg/tcg-cpu.c index 54ac54f2e1..55e00972b7 100644 --- a/target/riscv/tcg/tcg-cpu.c +++ b/target/riscv/tcg/tcg-cpu.c @@ -18,7 +18,6 @@ */ #include "qemu/osdep.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "tcg-cpu.h" #include "cpu.h" @@ -37,6 +36,7 @@ #ifndef CONFIG_USER_ONLY #include "hw/boards.h" #include "system/tcg.h" +#include "exec/icount.h" #endif /* Hash that stores user set extensions */ @@ -98,6 +98,103 @@ static int riscv_cpu_mmu_index(CPUState *cs, bool ifetch) return riscv_env_mmu_index(cpu_env(cs), ifetch); } +static TCGTBCPUState riscv_get_tb_cpu_state(CPUState *cs) +{ + CPURISCVState *env = cpu_env(cs); + RISCVCPU *cpu = env_archcpu(env); + RISCVExtStatus fs, vs; + uint32_t flags = 0; + bool pm_signext = riscv_cpu_virt_mem_enabled(env); + + if (cpu->cfg.ext_zve32x) { + /* + * If env->vl equals to VLMAX, we can use generic vector operation + * expanders (GVEC) to accerlate the vector operations. + * However, as LMUL could be a fractional number. The maximum + * vector size can be operated might be less than 8 bytes, + * which is not supported by GVEC. So we set vl_eq_vlmax flag to true + * only when maxsz >= 8 bytes. + */ + + /* lmul encoded as in DisasContext::lmul */ + int8_t lmul = sextract32(FIELD_EX64(env->vtype, VTYPE, VLMUL), 0, 3); + uint32_t vsew = FIELD_EX64(env->vtype, VTYPE, VSEW); + uint32_t vlmax = vext_get_vlmax(cpu->cfg.vlenb, vsew, lmul); + uint32_t maxsz = vlmax << vsew; + bool vl_eq_vlmax = (env->vstart == 0) && (vlmax == env->vl) && + (maxsz >= 8); + flags = FIELD_DP32(flags, TB_FLAGS, VILL, env->vill); + flags = FIELD_DP32(flags, TB_FLAGS, SEW, vsew); + flags = FIELD_DP32(flags, TB_FLAGS, LMUL, + FIELD_EX64(env->vtype, VTYPE, VLMUL)); + flags = FIELD_DP32(flags, TB_FLAGS, VL_EQ_VLMAX, vl_eq_vlmax); + flags = FIELD_DP32(flags, TB_FLAGS, VTA, + FIELD_EX64(env->vtype, VTYPE, VTA)); + flags = FIELD_DP32(flags, TB_FLAGS, VMA, + FIELD_EX64(env->vtype, VTYPE, VMA)); + flags = FIELD_DP32(flags, TB_FLAGS, VSTART_EQ_ZERO, env->vstart == 0); + } else { + flags = FIELD_DP32(flags, TB_FLAGS, VILL, 1); + } + + if (cpu_get_fcfien(env)) { + /* + * For Forward CFI, only the expectation of a lpad at + * the start of the block is tracked via env->elp. env->elp + * is turned on during jalr translation. + */ + flags = FIELD_DP32(flags, TB_FLAGS, FCFI_LP_EXPECTED, env->elp); + flags = FIELD_DP32(flags, TB_FLAGS, FCFI_ENABLED, 1); + } + + if (cpu_get_bcfien(env)) { + flags = FIELD_DP32(flags, TB_FLAGS, BCFI_ENABLED, 1); + } + +#ifdef CONFIG_USER_ONLY + fs = EXT_STATUS_DIRTY; + vs = EXT_STATUS_DIRTY; +#else + flags = FIELD_DP32(flags, TB_FLAGS, PRIV, env->priv); + + flags |= riscv_env_mmu_index(env, 0); + fs = get_field(env->mstatus, MSTATUS_FS); + vs = get_field(env->mstatus, MSTATUS_VS); + + if (env->virt_enabled) { + flags = FIELD_DP32(flags, TB_FLAGS, VIRT_ENABLED, 1); + /* + * Merge DISABLED and !DIRTY states using MIN. + * We will set both fields when dirtying. + */ + fs = MIN(fs, get_field(env->mstatus_hs, MSTATUS_FS)); + vs = MIN(vs, get_field(env->mstatus_hs, MSTATUS_VS)); + } + + /* With Zfinx, floating point is enabled/disabled by Smstateen. */ + if (!riscv_has_ext(env, RVF)) { + fs = (smstateen_acc_ok(env, 0, SMSTATEEN0_FCSR) == RISCV_EXCP_NONE) + ? EXT_STATUS_DIRTY : EXT_STATUS_DISABLED; + } + + if (cpu->cfg.debug && !icount_enabled()) { + flags = FIELD_DP32(flags, TB_FLAGS, ITRIGGER, env->itrigger_enabled); + } +#endif + + flags = FIELD_DP32(flags, TB_FLAGS, FS, fs); + flags = FIELD_DP32(flags, TB_FLAGS, VS, vs); + flags = FIELD_DP32(flags, TB_FLAGS, XL, env->xl); + flags = FIELD_DP32(flags, TB_FLAGS, AXL, cpu_address_xl(env)); + flags = FIELD_DP32(flags, TB_FLAGS, PM_PMM, riscv_pm_get_pmm(env)); + flags = FIELD_DP32(flags, TB_FLAGS, PM_SIGNEXTEND, pm_signext); + + return (TCGTBCPUState){ + .pc = env->xl == MXL_RV32 ? env->pc & UINT32_MAX : env->pc, + .flags = flags + }; +} + static void riscv_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -146,6 +243,7 @@ const TCGCPUOps riscv_tcg_ops = { .initialize = riscv_translate_init, .translate_code = riscv_translate_code, + .get_tb_cpu_state = riscv_get_tb_cpu_state, .synchronize_from_tb = riscv_cpu_synchronize_from_tb, .restore_state_to_opc = riscv_restore_state_to_opc, .mmu_index = riscv_cpu_mmu_index, @@ -154,6 +252,7 @@ const TCGCPUOps riscv_tcg_ops = { .tlb_fill = riscv_cpu_tlb_fill, .cpu_exec_interrupt = riscv_cpu_exec_interrupt, .cpu_exec_halt = riscv_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = riscv_cpu_do_interrupt, .do_transaction_failed = riscv_cpu_do_transaction_failed, .do_unaligned_access = riscv_cpu_do_unaligned_access, diff --git a/target/riscv/translate.c b/target/riscv/translate.c index cef61b5b29..85128f997b 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -20,7 +20,6 @@ #include "qemu/log.h" #include "cpu.h" #include "tcg/tcg-op.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "exec/helper-gen.h" #include "exec/target_page.h" diff --git a/target/riscv/vcrypto_helper.c b/target/riscv/vcrypto_helper.c index 1526de96f5..9a0d9b4f53 100644 --- a/target/riscv/vcrypto_helper.c +++ b/target/riscv/vcrypto_helper.c @@ -26,7 +26,6 @@ #include "crypto/aes-round.h" #include "crypto/sm4.h" #include "exec/memop.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "internals.h" #include "vector_internals.h" diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index b8ae704457..8eea3e6df0 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -21,8 +21,8 @@ #include "qemu/bitops.h" #include "cpu.h" #include "exec/memop.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "exec/page-protection.h" #include "exec/helper-proto.h" #include "exec/tlb-flags.h" diff --git a/target/riscv/zce_helper.c b/target/riscv/zce_helper.c index 50d65f386c..55221f5f37 100644 --- a/target/riscv/zce_helper.c +++ b/target/riscv/zce_helper.c @@ -18,7 +18,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "accel/tcg/cpu-ldst.h" diff --git a/target/rx/cpu.c b/target/rx/cpu.c index a51b543028..36eba75545 100644 --- a/target/rx/cpu.c +++ b/target/rx/cpu.c @@ -28,6 +28,7 @@ #include "hw/loader.h" #include "fpu/softfloat.h" #include "tcg/debug-assert.h" +#include "accel/tcg/cpu-ops.h" static void rx_cpu_set_pc(CPUState *cs, vaddr value) { @@ -43,6 +44,17 @@ static vaddr rx_cpu_get_pc(CPUState *cs) return cpu->env.pc; } +static TCGTBCPUState rx_get_tb_cpu_state(CPUState *cs) +{ + CPURXState *env = cpu_env(cs); + uint32_t flags = 0; + + flags = FIELD_DP32(flags, PSW, PM, env->psw_pm); + flags = FIELD_DP32(flags, PSW, U, env->psw_u); + + return (TCGTBCPUState){ .pc = env->pc, .flags = flags }; +} + static void rx_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -201,8 +213,6 @@ static const struct SysemuCPUOps rx_sysemu_ops = { .get_phys_page_debug = rx_cpu_get_phys_page_debug, }; -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps rx_tcg_ops = { /* MTTCG not yet supported: require strict ordering */ .guest_default_memory_order = TCG_MO_ALL, @@ -210,6 +220,7 @@ static const TCGCPUOps rx_tcg_ops = { .initialize = rx_translate_init, .translate_code = rx_translate_code, + .get_tb_cpu_state = rx_get_tb_cpu_state, .synchronize_from_tb = rx_cpu_synchronize_from_tb, .restore_state_to_opc = rx_restore_state_to_opc, .mmu_index = rx_cpu_mmu_index, @@ -217,6 +228,7 @@ static const TCGCPUOps rx_tcg_ops = { .cpu_exec_interrupt = rx_cpu_exec_interrupt, .cpu_exec_halt = rx_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = rx_cpu_do_interrupt, }; diff --git a/target/rx/cpu.h b/target/rx/cpu.h index 5c19c83219..ba5761b647 100644 --- a/target/rx/cpu.h +++ b/target/rx/cpu.h @@ -153,15 +153,6 @@ void rx_cpu_unpack_psw(CPURXState *env, uint32_t psw, int rte); #define RX_CPU_IRQ 0 #define RX_CPU_FIR 1 -static inline void cpu_get_tb_cpu_state(CPURXState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *pc = env->pc; - *cs_base = 0; - *flags = FIELD_DP32(0, PSW, PM, env->psw_pm); - *flags = FIELD_DP32(*flags, PSW, U, env->psw_u); -} - static inline uint32_t rx_cpu_pack_psw(CPURXState *env) { uint32_t psw = 0; diff --git a/target/rx/op_helper.c b/target/rx/op_helper.c index a2f1f3824d..2b190a4b2c 100644 --- a/target/rx/op_helper.c +++ b/target/rx/op_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "qemu/bitops.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "accel/tcg/cpu-ldst.h" #include "fpu/softfloat.h" diff --git a/target/rx/translate.c b/target/rx/translate.c index bbda703be8..19a9584a82 100644 --- a/target/rx/translate.c +++ b/target/rx/translate.c @@ -20,7 +20,6 @@ #include "qemu/bswap.h" #include "qemu/qemu-print.h" #include "cpu.h" -#include "exec/exec-all.h" #include "tcg/tcg-op.h" #include "exec/helper-proto.h" #include "exec/helper-gen.h" diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c index 3d644f5e23..9c1158ebcc 100644 --- a/target/s390x/cpu.c +++ b/target/s390x/cpu.c @@ -302,15 +302,16 @@ static const Property s390x_cpu_properties[] = { #ifdef CONFIG_TCG #include "accel/tcg/cpu-ops.h" +#include "tcg/tcg_s390x.h" static int s390x_cpu_mmu_index(CPUState *cs, bool ifetch) { return s390x_env_mmu_index(cpu_env(cs), ifetch); } -void cpu_get_tb_cpu_state(CPUS390XState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *pflags) +static TCGTBCPUState s390x_get_tb_cpu_state(CPUState *cs) { + CPUS390XState *env = cpu_env(cs); uint32_t flags; if (env->psw.addr & 1) { @@ -322,9 +323,6 @@ void cpu_get_tb_cpu_state(CPUS390XState *env, vaddr *pc, tcg_s390_program_interrupt(env, PGM_SPECIFICATION, 0); } - *pc = env->psw.addr; - *cs_base = env->ex_value; - flags = (env->psw.mask >> FLAG_MASK_PSW_SHIFT) & FLAG_MASK_PSW; if (env->psw.mask & PSW_MASK_PER) { flags |= env->cregs[9] & (FLAG_MASK_PER_BRANCH | @@ -341,11 +339,17 @@ void cpu_get_tb_cpu_state(CPUS390XState *env, vaddr *pc, if (env->cregs[0] & CR0_VECTOR) { flags |= FLAG_MASK_VECTOR; } - *pflags = flags; + + return (TCGTBCPUState){ + .pc = env->psw.addr, + .flags = flags, + .cs_base = env->ex_value, + }; } static const TCGCPUOps s390_tcg_ops = { .mttcg_supported = true, + .precise_smc = true, /* * The z/Architecture has a strong memory model with some * store-after-load re-ordering. @@ -354,6 +358,7 @@ static const TCGCPUOps s390_tcg_ops = { .initialize = s390x_translate_init, .translate_code = s390x_translate_code, + .get_tb_cpu_state = s390x_get_tb_cpu_state, .restore_state_to_opc = s390x_restore_state_to_opc, .mmu_index = s390x_cpu_mmu_index, @@ -364,6 +369,7 @@ static const TCGCPUOps s390_tcg_ops = { .tlb_fill = s390_cpu_tlb_fill, .cpu_exec_interrupt = s390_cpu_exec_interrupt, .cpu_exec_halt = s390_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = s390_cpu_do_interrupt, .debug_excp_handler = s390x_cpu_debug_excp_handler, .do_unaligned_access = s390x_cpu_do_unaligned_access, diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h index d9ca2506e2..aa931cb674 100644 --- a/target/s390x/cpu.h +++ b/target/s390x/cpu.h @@ -35,8 +35,6 @@ #define ELF_MACHINE_UNAME "S390X" -#define TARGET_HAS_PRECISE_SMC - #define MMU_USER_IDX 0 #define S390_MAX_CPUS 248 @@ -413,15 +411,6 @@ static inline int s390x_env_mmu_index(CPUS390XState *env, bool ifetch) #endif } -#ifdef CONFIG_TCG - -#include "tcg/tcg_s390x.h" - -void cpu_get_tb_cpu_state(CPUS390XState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags); - -#endif /* CONFIG_TCG */ - /* PER bits from control register 9 */ #define PER_CR9_EVENT_BRANCH 0x80000000 #define PER_CR9_EVENT_IFETCH 0x40000000 diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c index 8951f1b36f..954a7a99a9 100644 --- a/target/s390x/cpu_models.c +++ b/target/s390x/cpu_models.c @@ -373,7 +373,7 @@ static void s390_print_cpu_model_list_entry(gpointer data, gpointer user_data) g_free(name); } -static gint s390_cpu_list_compare(gconstpointer a, gconstpointer b) +static gint s390_cpu_list_compare(gconstpointer a, gconstpointer b, gpointer d) { const S390CPUClass *cc_a = S390_CPU_CLASS((ObjectClass *)a); const S390CPUClass *cc_b = S390_CPU_CLASS((ObjectClass *)b); @@ -415,7 +415,7 @@ void s390_cpu_list(void) qemu_printf("Available CPUs:\n"); list = object_class_get_list(TYPE_S390_CPU, false); - list = g_slist_sort(list, s390_cpu_list_compare); + list = g_slist_sort_with_data(list, s390_cpu_list_compare, NULL); g_slist_foreach(list, s390_print_cpu_model_list_entry, NULL); g_slist_free(list); diff --git a/target/s390x/interrupt.c b/target/s390x/interrupt.c index 4ae6e2ddea..1dca835c5d 100644 --- a/target/s390x/interrupt.c +++ b/target/s390x/interrupt.c @@ -11,7 +11,6 @@ #include "cpu.h" #include "kvm/kvm_s390x.h" #include "s390x-internal.h" -#include "exec/exec-all.h" #include "system/kvm.h" #include "system/tcg.h" #include "hw/s390x/ioinst.h" diff --git a/target/s390x/kvm/pv.c b/target/s390x/kvm/pv.c index fe0a72c416..2bc916a545 100644 --- a/target/s390x/kvm/pv.c +++ b/target/s390x/kvm/pv.c @@ -30,7 +30,7 @@ static struct kvm_s390_pv_info_vm info_vm; static struct kvm_s390_pv_info_dump info_dump; static int __s390_pv_cmd(uint32_t cmd, const char *cmdname, void *data, - int *pvrc) + struct S390PVResponse *pv_resp) { struct kvm_pv_cmd pv_cmd = { .cmd = cmd, @@ -47,8 +47,10 @@ static int __s390_pv_cmd(uint32_t cmd, const char *cmdname, void *data, "IOCTL rc: %d", cmd, cmdname, pv_cmd.rc, pv_cmd.rrc, rc); } - if (pvrc) { - *pvrc = pv_cmd.rc; + if (pv_resp) { + pv_resp->cmd = cmd; + pv_resp->rc = pv_cmd.rc; + pv_resp->rrc = pv_cmd.rrc; } return rc; } @@ -57,16 +59,15 @@ static int __s390_pv_cmd(uint32_t cmd, const char *cmdname, void *data, * This macro lets us pass the command as a string to the function so * we can print it on an error. */ -#define s390_pv_cmd(cmd, data) __s390_pv_cmd(cmd, #cmd, data, NULL) -#define s390_pv_cmd_pvrc(cmd, data, pvrc) __s390_pv_cmd(cmd, #cmd, data, pvrc) -#define s390_pv_cmd_exit(cmd, data) \ -{ \ - int rc; \ - \ - rc = __s390_pv_cmd(cmd, #cmd, data, NULL); \ - if (rc) { \ - exit(1); \ - } \ +#define s390_pv_cmd(cmd, data) __s390_pv_cmd(cmd, #cmd, data, NULL) +#define s390_pv_cmd_pv_resp(cmd, data, pv_resp) \ + __s390_pv_cmd(cmd, #cmd, data, pv_resp) + +static void s390_pv_cmd_exit(uint32_t cmd, void *data) +{ + if (s390_pv_cmd(cmd, data)) { + exit(1); + } } int s390_pv_query_info(void) @@ -147,18 +148,20 @@ bool s390_pv_vm_try_disable_async(S390CcwMachineState *ms) return true; } -int s390_pv_set_sec_parms(uint64_t origin, uint64_t length, Error **errp) +#define UV_RC_SSC_INVAL_HOSTKEY 0x0108 +int s390_pv_set_sec_parms(uint64_t origin, uint64_t length, + struct S390PVResponse *pv_resp, Error **errp) { - int ret, pvrc; + int ret; struct kvm_s390_pv_sec_parm args = { .origin = origin, .length = length, }; - ret = s390_pv_cmd_pvrc(KVM_PV_SET_SEC_PARMS, &args, &pvrc); + ret = s390_pv_cmd_pv_resp(KVM_PV_SET_SEC_PARMS, &args, pv_resp); if (ret) { error_setg(errp, "Failed to set secure execution parameters"); - if (pvrc == 0x108) { + if (pv_resp->rc == UV_RC_SSC_INVAL_HOSTKEY) { error_append_hint(errp, "Please check whether the image is " "correctly encrypted for this host\n"); } @@ -170,7 +173,8 @@ int s390_pv_set_sec_parms(uint64_t origin, uint64_t length, Error **errp) /* * Called for each component in the SE type IPL parameter block 0. */ -int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak) +int s390_pv_unpack(uint64_t addr, uint64_t size, + uint64_t tweak, struct S390PVResponse *pv_resp) { struct kvm_s390_pv_unp args = { .addr = addr, @@ -178,7 +182,7 @@ int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak) .tweak = tweak, }; - return s390_pv_cmd(KVM_PV_UNPACK, &args); + return s390_pv_cmd_pv_resp(KVM_PV_UNPACK, &args, pv_resp); } void s390_pv_prep_reset(void) @@ -186,9 +190,9 @@ void s390_pv_prep_reset(void) s390_pv_cmd_exit(KVM_PV_PREP_RESET, NULL); } -int s390_pv_verify(void) +int s390_pv_verify(struct S390PVResponse *pv_resp) { - return s390_pv_cmd(KVM_PV_VERIFY, NULL); + return s390_pv_cmd_pv_resp(KVM_PV_VERIFY, NULL, pv_resp); } void s390_pv_unshare(void) @@ -196,13 +200,29 @@ void s390_pv_unshare(void) s390_pv_cmd_exit(KVM_PV_UNSHARE_ALL, NULL); } -void s390_pv_inject_reset_error(CPUState *cs) +void s390_pv_inject_reset_error(CPUState *cs, + struct S390PVResponse pv_resp) { int r1 = (cs->kvm_run->s390_sieic.ipa & 0x00f0) >> 4; CPUS390XState *env = &S390_CPU(cs)->env; + union { + struct { + uint16_t pv_cmd; + uint16_t pv_rrc; + uint16_t pv_rc; + uint16_t diag_rc; + }; + uint64_t regs; + } resp = { + .pv_cmd = pv_resp.cmd, + .pv_rrc = pv_resp.rrc, + .pv_rc = pv_resp.rc, + .diag_rc = DIAG_308_RC_INVAL_FOR_PV + }; + /* Report that we are unable to enter protected mode */ - env->regs[r1 + 1] = DIAG_308_RC_INVAL_FOR_PV; + env->regs[r1 + 1] = resp.regs; } uint64_t kvm_s390_pv_dmp_get_size_cpu(void) diff --git a/target/s390x/kvm/pv.h b/target/s390x/kvm/pv.h index 5e9c8bd351..94e885e933 100644 --- a/target/s390x/kvm/pv.h +++ b/target/s390x/kvm/pv.h @@ -16,6 +16,12 @@ #include "system/kvm.h" #include "hw/s390x/s390-virtio-ccw.h" +struct S390PVResponse { + uint16_t cmd; + uint16_t rrc; + uint16_t rc; +}; + #ifdef CONFIG_KVM #include "cpu.h" @@ -42,12 +48,15 @@ int s390_pv_query_info(void); int s390_pv_vm_enable(void); void s390_pv_vm_disable(void); bool s390_pv_vm_try_disable_async(S390CcwMachineState *ms); -int s390_pv_set_sec_parms(uint64_t origin, uint64_t length, Error **errp); -int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak); +int s390_pv_set_sec_parms(uint64_t origin, uint64_t length, + struct S390PVResponse *pv_resp, Error **errp); +int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak, + struct S390PVResponse *pv_resp); void s390_pv_prep_reset(void); -int s390_pv_verify(void); +int s390_pv_verify(struct S390PVResponse *pv_resp); void s390_pv_unshare(void); -void s390_pv_inject_reset_error(CPUState *cs); +void s390_pv_inject_reset_error(CPUState *cs, + struct S390PVResponse pv_resp); uint64_t kvm_s390_pv_dmp_get_size_cpu(void); uint64_t kvm_s390_pv_dmp_get_size_mem_state(void); uint64_t kvm_s390_pv_dmp_get_size_completion_data(void); @@ -63,12 +72,15 @@ static inline int s390_pv_vm_enable(void) { return 0; } static inline void s390_pv_vm_disable(void) {} static inline bool s390_pv_vm_try_disable_async(S390CcwMachineState *ms) { return false; } static inline int s390_pv_set_sec_parms(uint64_t origin, uint64_t length, + struct S390PVResponse *pv_resp, Error **errp) { return 0; } -static inline int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak) { return 0; } +static inline int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak, + struct S390PVResponse *pv_resp) { return 0; } static inline void s390_pv_prep_reset(void) {} -static inline int s390_pv_verify(void) { return 0; } +static inline int s390_pv_verify(struct S390PVResponse *pv_resp) { return 0; } static inline void s390_pv_unshare(void) {} -static inline void s390_pv_inject_reset_error(CPUState *cs) {}; +static inline void s390_pv_inject_reset_error(CPUState *cs, + struct S390PVResponse pv_resp) {}; static inline uint64_t kvm_s390_pv_dmp_get_size_cpu(void) { return 0; } static inline uint64_t kvm_s390_pv_dmp_get_size_mem_state(void) { return 0; } static inline uint64_t kvm_s390_pv_dmp_get_size_completion_data(void) { return 0; } diff --git a/target/s390x/mmu_helper.c b/target/s390x/mmu_helper.c index 0e133cb9a5..00946e9c0f 100644 --- a/target/s390x/mmu_helper.c +++ b/target/s390x/mmu_helper.c @@ -23,7 +23,6 @@ #include "kvm/kvm_s390x.h" #include "system/kvm.h" #include "system/tcg.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "hw/hw.h" diff --git a/target/s390x/sigp.c b/target/s390x/sigp.c index a3347f1236..5e95c4978f 100644 --- a/target/s390x/sigp.c +++ b/target/s390x/sigp.c @@ -16,7 +16,6 @@ #include "system/runstate.h" #include "system/address-spaces.h" #include "exec/cputlb.h" -#include "exec/exec-all.h" #include "system/tcg.h" #include "trace.h" #include "qapi/qapi-types-machine.h" diff --git a/target/s390x/tcg/cc_helper.c b/target/s390x/tcg/cc_helper.c index b36f8cdc8b..6595ac763c 100644 --- a/target/s390x/tcg/cc_helper.c +++ b/target/s390x/tcg/cc_helper.c @@ -22,7 +22,6 @@ #include "cpu.h" #include "s390x-internal.h" #include "tcg_s390x.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "qemu/host-utils.h" diff --git a/target/s390x/tcg/crypto_helper.c b/target/s390x/tcg/crypto_helper.c index 642c1b18c4..4447bb66ee 100644 --- a/target/s390x/tcg/crypto_helper.c +++ b/target/s390x/tcg/crypto_helper.c @@ -17,7 +17,6 @@ #include "s390x-internal.h" #include "tcg_s390x.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" static uint64_t R(uint64_t x, int c) diff --git a/target/s390x/tcg/excp_helper.c b/target/s390x/tcg/excp_helper.c index 6cd813e1ab..e4c75d0ce0 100644 --- a/target/s390x/tcg/excp_helper.c +++ b/target/s390x/tcg/excp_helper.c @@ -23,7 +23,6 @@ #include "cpu.h" #include "exec/helper-proto.h" #include "exec/cputlb.h" -#include "exec/exec-all.h" #include "exec/target_page.h" #include "exec/watchpoint.h" #include "s390x-internal.h" diff --git a/target/s390x/tcg/fpu_helper.c b/target/s390x/tcg/fpu_helper.c index 5041c13962..1ba43715ac 100644 --- a/target/s390x/tcg/fpu_helper.c +++ b/target/s390x/tcg/fpu_helper.c @@ -22,7 +22,6 @@ #include "cpu.h" #include "s390x-internal.h" #include "tcg_s390x.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" diff --git a/target/s390x/tcg/int_helper.c b/target/s390x/tcg/int_helper.c index 253c036415..fbda396f5b 100644 --- a/target/s390x/tcg/int_helper.c +++ b/target/s390x/tcg/int_helper.c @@ -22,7 +22,6 @@ #include "cpu.h" #include "s390x-internal.h" #include "tcg_s390x.h" -#include "exec/exec-all.h" #include "qemu/host-utils.h" #include "exec/helper-proto.h" #include "accel/tcg/cpu-ldst.h" diff --git a/target/s390x/tcg/mem_helper.c b/target/s390x/tcg/mem_helper.c index 0cdfd380ce..a03609a140 100644 --- a/target/s390x/tcg/mem_helper.c +++ b/target/s390x/tcg/mem_helper.c @@ -25,13 +25,14 @@ #include "tcg_s390x.h" #include "exec/helper-proto.h" #include "exec/cpu-common.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "exec/page-protection.h" #include "accel/tcg/cpu-ldst.h" +#include "accel/tcg/probe.h" #include "exec/target_page.h" #include "exec/tlb-flags.h" #include "accel/tcg/cpu-ops.h" +#include "accel/tcg/helper-retaddr.h" #include "qemu/int128.h" #include "qemu/atomic128.h" diff --git a/target/s390x/tcg/misc_helper.c b/target/s390x/tcg/misc_helper.c index d5088493ea..f7101be574 100644 --- a/target/s390x/tcg/misc_helper.c +++ b/target/s390x/tcg/misc_helper.c @@ -26,7 +26,6 @@ #include "qemu/host-utils.h" #include "exec/helper-proto.h" #include "qemu/timer.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "accel/tcg/cpu-ldst.h" #include "exec/target_page.h" diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index 00073c5560..c7e8574438 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -31,7 +31,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "s390x-internal.h" -#include "exec/exec-all.h" #include "tcg/tcg-op.h" #include "tcg/tcg-op-gvec.h" #include "qemu/log.h" @@ -1250,11 +1249,7 @@ static DisasJumpType op_addc32(DisasContext *s, DisasOps *o) static DisasJumpType op_addc64(DisasContext *s, DisasOps *o) { compute_carry(s); - - TCGv_i64 zero = tcg_constant_i64(0); - tcg_gen_add2_i64(o->out, cc_src, o->in1, zero, cc_src, zero); - tcg_gen_add2_i64(o->out, cc_src, o->out, cc_src, o->in2, zero); - + tcg_gen_addcio_i64(o->out, cc_src, o->in1, o->in2, cc_src); return DISAS_NEXT; } diff --git a/target/s390x/tcg/vec_fpu_helper.c b/target/s390x/tcg/vec_fpu_helper.c index 1bbaa82fe8..744f800fb6 100644 --- a/target/s390x/tcg/vec_fpu_helper.c +++ b/target/s390x/tcg/vec_fpu_helper.c @@ -15,7 +15,6 @@ #include "vec.h" #include "tcg_s390x.h" #include "tcg/tcg-gvec-desc.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" diff --git a/target/s390x/tcg/vec_helper.c b/target/s390x/tcg/vec_helper.c index 781ccc565b..46ec4a947d 100644 --- a/target/s390x/tcg/vec_helper.c +++ b/target/s390x/tcg/vec_helper.c @@ -17,7 +17,6 @@ #include "tcg/tcg-gvec-desc.h" #include "exec/helper-proto.h" #include "accel/tcg/cpu-ldst.h" -#include "exec/exec-all.h" void HELPER(gvec_vbperm)(void *v1, const void *v2, const void *v3, uint32_t desc) diff --git a/target/sh4/cpu.c b/target/sh4/cpu.c index 57d7b5fbc8..b35f18e250 100644 --- a/target/sh4/cpu.c +++ b/target/sh4/cpu.c @@ -24,9 +24,9 @@ #include "qemu/qemu-print.h" #include "cpu.h" #include "migration/vmstate.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "fpu/softfloat-helpers.h" +#include "accel/tcg/cpu-ops.h" #include "tcg/tcg.h" static void superh_cpu_set_pc(CPUState *cs, vaddr value) @@ -43,6 +43,29 @@ static vaddr superh_cpu_get_pc(CPUState *cs) return cpu->env.pc; } +static TCGTBCPUState superh_get_tb_cpu_state(CPUState *cs) +{ + CPUSH4State *env = cpu_env(cs); + uint32_t flags; + + flags = env->flags + | (env->fpscr & TB_FLAG_FPSCR_MASK) + | (env->sr & TB_FLAG_SR_MASK) + | (env->movcal_backup ? TB_FLAG_PENDING_MOVCA : 0); /* Bit 3 */ +#ifdef CONFIG_USER_ONLY + flags |= TB_FLAG_UNALIGN * !cs->prctl_unalign_sigbus; +#endif + + return (TCGTBCPUState){ + .pc = env->pc, + .flags = flags, +#ifdef CONFIG_USER_ONLY + /* For a gUSA region, notice the end of the region. */ + .cs_base = flags & TB_FLAG_GUSA_MASK ? env->gregs[0] : 0, +#endif + }; +} + static void superh_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -259,8 +282,6 @@ static const struct SysemuCPUOps sh4_sysemu_ops = { }; #endif -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps superh_tcg_ops = { /* MTTCG not yet supported: require strict ordering */ .guest_default_memory_order = TCG_MO_ALL, @@ -268,6 +289,7 @@ static const TCGCPUOps superh_tcg_ops = { .initialize = sh4_translate_init, .translate_code = sh4_translate_code, + .get_tb_cpu_state = superh_get_tb_cpu_state, .synchronize_from_tb = superh_cpu_synchronize_from_tb, .restore_state_to_opc = superh_restore_state_to_opc, .mmu_index = sh4_cpu_mmu_index, @@ -276,6 +298,7 @@ static const TCGCPUOps superh_tcg_ops = { .tlb_fill = superh_cpu_tlb_fill, .cpu_exec_interrupt = superh_cpu_exec_interrupt, .cpu_exec_halt = superh_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = superh_cpu_do_interrupt, .do_unaligned_access = superh_cpu_do_unaligned_access, .io_recompile_replay_branch = superh_io_recompile_replay_branch, diff --git a/target/sh4/cpu.h b/target/sh4/cpu.h index 906f99ddf0..c41ab70dd7 100644 --- a/target/sh4/cpu.h +++ b/target/sh4/cpu.h @@ -380,19 +380,4 @@ static inline void cpu_write_sr(CPUSH4State *env, target_ulong sr) env->sr = sr & ~((1u << SR_M) | (1u << SR_Q) | (1u << SR_T)); } -static inline void cpu_get_tb_cpu_state(CPUSH4State *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *pc = env->pc; - /* For a gUSA region, notice the end of the region. */ - *cs_base = env->flags & TB_FLAG_GUSA_MASK ? env->gregs[0] : 0; - *flags = env->flags - | (env->fpscr & TB_FLAG_FPSCR_MASK) - | (env->sr & TB_FLAG_SR_MASK) - | (env->movcal_backup ? TB_FLAG_PENDING_MOVCA : 0); /* Bit 3 */ -#ifdef CONFIG_USER_ONLY - *flags |= TB_FLAG_UNALIGN * !env_cpu(env)->prctl_unalign_sigbus; -#endif -} - #endif /* SH4_CPU_H */ diff --git a/target/sh4/helper.c b/target/sh4/helper.c index b41d14d5d7..fb7642bda1 100644 --- a/target/sh4/helper.c +++ b/target/sh4/helper.c @@ -21,7 +21,6 @@ #include "cpu.h" #include "exec/cputlb.h" -#include "exec/exec-all.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "exec/log.h" diff --git a/target/sh4/op_helper.c b/target/sh4/op_helper.c index e7fcad3c1b..557b1bf497 100644 --- a/target/sh4/op_helper.c +++ b/target/sh4/op_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include "fpu/softfloat.h" diff --git a/target/sh4/translate.c b/target/sh4/translate.c index d796ad52c4..bf8828fce8 100644 --- a/target/sh4/translate.c +++ b/target/sh4/translate.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "tcg/tcg-op.h" #include "exec/helper-proto.h" #include "exec/helper-gen.h" @@ -695,14 +694,8 @@ static void _decode_opc(DisasContext * ctx) tcg_gen_add_i32(REG(B11_8), REG(B11_8), REG(B7_4)); return; case 0x300e: /* addc Rm,Rn */ - { - TCGv t0, t1; - t0 = tcg_constant_tl(0); - t1 = tcg_temp_new(); - tcg_gen_add2_i32(t1, cpu_sr_t, cpu_sr_t, t0, REG(B7_4), t0); - tcg_gen_add2_i32(REG(B11_8), cpu_sr_t, - REG(B11_8), t0, t1, cpu_sr_t); - } + tcg_gen_addcio_i32(REG(B11_8), cpu_sr_t, + REG(B11_8), REG(B7_4), cpu_sr_t); return; case 0x300f: /* addv Rm,Rn */ { @@ -1940,16 +1933,16 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) NEXT_INSN; switch (ctx->opcode & 0xf00f) { case 0x300c: /* add Rm,Rn */ - op_opc = INDEX_op_add_i32; + op_opc = INDEX_op_add; goto do_reg_op; case 0x2009: /* and Rm,Rn */ - op_opc = INDEX_op_and_i32; + op_opc = INDEX_op_and; goto do_reg_op; case 0x200a: /* xor Rm,Rn */ - op_opc = INDEX_op_xor_i32; + op_opc = INDEX_op_xor; goto do_reg_op; case 0x200b: /* or Rm,Rn */ - op_opc = INDEX_op_or_i32; + op_opc = INDEX_op_or; do_reg_op: /* The operation register should be as expected, and the other input cannot depend on the load. */ @@ -1976,7 +1969,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) goto fail; } op_dst = B11_8; - op_opc = INDEX_op_xor_i32; + op_opc = INDEX_op_xor; op_arg = tcg_constant_i32(-1); break; @@ -1984,7 +1977,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) if (op_dst != B11_8 || mv_src >= 0) { goto fail; } - op_opc = INDEX_op_add_i32; + op_opc = INDEX_op_add; op_arg = tcg_constant_i32(B7_0s); break; @@ -1995,7 +1988,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) if ((ld_dst == B11_8) + (ld_dst == B7_4) != 1 || mv_src >= 0) { goto fail; } - op_opc = INDEX_op_setcond_i32; /* placeholder */ + op_opc = INDEX_op_setcond; /* placeholder */ op_src = (ld_dst == B11_8 ? B7_4 : B11_8); op_arg = REG(op_src); @@ -2030,7 +2023,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) if (ld_dst != B11_8 || ld_dst != B7_4 || mv_src >= 0) { goto fail; } - op_opc = INDEX_op_setcond_i32; + op_opc = INDEX_op_setcond; op_arg = tcg_constant_i32(0); NEXT_INSN; @@ -2087,7 +2080,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) ctx->memidx, ld_mop); break; - case INDEX_op_add_i32: + case INDEX_op_add: if (op_dst != st_src) { goto fail; } @@ -2105,7 +2098,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) } break; - case INDEX_op_and_i32: + case INDEX_op_and: if (op_dst != st_src) { goto fail; } @@ -2119,7 +2112,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) } break; - case INDEX_op_or_i32: + case INDEX_op_or: if (op_dst != st_src) { goto fail; } @@ -2133,7 +2126,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) } break; - case INDEX_op_xor_i32: + case INDEX_op_xor: if (op_dst != st_src) { goto fail; } @@ -2147,7 +2140,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) } break; - case INDEX_op_setcond_i32: + case INDEX_op_setcond: if (st_src == ld_dst) { goto fail; } diff --git a/target/sparc/cpu.c b/target/sparc/cpu.c index bc753d5f62..2a3e408923 100644 --- a/target/sparc/cpu.c +++ b/target/sparc/cpu.c @@ -23,7 +23,6 @@ #include "qemu/module.h" #include "qemu/qemu-print.h" #include "accel/tcg/cpu-mmu-index.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "hw/qdev-properties.h" #include "qapi/visitor.h" @@ -717,13 +716,11 @@ static void sparc_cpu_synchronize_from_tb(CPUState *cs, cpu->env.npc = tb->cs_base; } -void cpu_get_tb_cpu_state(CPUSPARCState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *pflags) +static TCGTBCPUState sparc_get_tb_cpu_state(CPUState *cs) { - uint32_t flags; - *pc = env->pc; - *cs_base = env->npc; - flags = cpu_mmu_index(env_cpu(env), false); + CPUSPARCState *env = cpu_env(cs); + uint32_t flags = cpu_mmu_index(cs, false); + #ifndef CONFIG_USER_ONLY if (cpu_supervisor_mode(env)) { flags |= TB_FLAG_SUPER; @@ -752,7 +749,12 @@ void cpu_get_tb_cpu_state(CPUSPARCState *env, vaddr *pc, } #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_SPARC64 */ - *pflags = flags; + + return (TCGTBCPUState){ + .pc = env->pc, + .flags = flags, + .cs_base = env->npc, + }; } static void sparc_restore_state_to_opc(CPUState *cs, @@ -1027,6 +1029,7 @@ static const TCGCPUOps sparc_tcg_ops = { .initialize = sparc_tcg_init, .translate_code = sparc_translate_code, + .get_tb_cpu_state = sparc_get_tb_cpu_state, .synchronize_from_tb = sparc_cpu_synchronize_from_tb, .restore_state_to_opc = sparc_restore_state_to_opc, .mmu_index = sparc_cpu_mmu_index, @@ -1035,6 +1038,7 @@ static const TCGCPUOps sparc_tcg_ops = { .tlb_fill = sparc_cpu_tlb_fill, .cpu_exec_interrupt = sparc_cpu_exec_interrupt, .cpu_exec_halt = sparc_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = sparc_cpu_do_interrupt, .do_transaction_failed = sparc_cpu_do_transaction_failed, .do_unaligned_access = sparc_cpu_do_unaligned_access, diff --git a/target/sparc/cpu.h b/target/sparc/cpu.h index 37fd1e066e..31cb3d97eb 100644 --- a/target/sparc/cpu.h +++ b/target/sparc/cpu.h @@ -741,9 +741,6 @@ trap_state* cpu_tsptr(CPUSPARCState* env); #define TB_FLAG_FSR_QNE (1 << 8) #define TB_FLAG_ASI_SHIFT 24 -void cpu_get_tb_cpu_state(CPUSPARCState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *pflags); - static inline bool tb_fpu_enabled(int tb_flags) { #if defined(CONFIG_USER_ONLY) diff --git a/target/sparc/fop_helper.c b/target/sparc/fop_helper.c index c25097d07f..a49334150d 100644 --- a/target/sparc/fop_helper.c +++ b/target/sparc/fop_helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" diff --git a/target/sparc/helper.c b/target/sparc/helper.c index 7846ddd6f6..9163b9d46a 100644 --- a/target/sparc/helper.c +++ b/target/sparc/helper.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "qemu/timer.h" #include "qemu/host-utils.h" #include "exec/helper-proto.h" diff --git a/target/sparc/ldst_helper.c b/target/sparc/ldst_helper.c index 4c5dba19d1..2c63eb9e03 100644 --- a/target/sparc/ldst_helper.c +++ b/target/sparc/ldst_helper.c @@ -23,7 +23,6 @@ #include "cpu.h" #include "tcg/tcg.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "exec/cputlb.h" #include "exec/page-protection.h" #include "exec/target_page.h" diff --git a/target/sparc/machine.c b/target/sparc/machine.c index 222e5709c5..4dd75aff74 100644 --- a/target/sparc/machine.c +++ b/target/sparc/machine.c @@ -1,6 +1,5 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "qemu/timer.h" #include "migration/cpu.h" diff --git a/target/sparc/translate.c b/target/sparc/translate.c index adebddf27b..b922e53bf1 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -22,7 +22,6 @@ #include "cpu.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "exec/target_page.h" #include "tcg/tcg-op.h" #include "tcg/tcg-op-gvec.h" @@ -396,8 +395,7 @@ static void gen_op_addcc_int(TCGv dst, TCGv src1, TCGv src2, TCGv cin) TCGv z = tcg_constant_tl(0); if (cin) { - tcg_gen_add2_tl(cpu_cc_N, cpu_cc_C, src1, z, cin, z); - tcg_gen_add2_tl(cpu_cc_N, cpu_cc_C, cpu_cc_N, cpu_cc_C, src2, z); + tcg_gen_addcio_tl(cpu_cc_N, cpu_cc_C, src1, src2, cin); } else { tcg_gen_add2_tl(cpu_cc_N, cpu_cc_C, src1, z, src2, z); } diff --git a/target/sparc/win_helper.c b/target/sparc/win_helper.c index 0c4b09f2c1..9ad9d01e8b 100644 --- a/target/sparc/win_helper.c +++ b/target/sparc/win_helper.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "qemu/main-loop.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "trace.h" diff --git a/target/tricore/cpu.c b/target/tricore/cpu.c index 098cd06c54..e56f90fde9 100644 --- a/target/tricore/cpu.c +++ b/target/tricore/cpu.c @@ -20,10 +20,10 @@ #include "qemu/osdep.h" #include "qapi/error.h" #include "cpu.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" #include "qemu/error-report.h" #include "tcg/debug-assert.h" +#include "accel/tcg/cpu-ops.h" static inline void set_feature(CPUTriCoreState *env, int feature) { @@ -45,6 +45,16 @@ static vaddr tricore_cpu_get_pc(CPUState *cs) return cpu_env(cs)->PC; } +static TCGTBCPUState tricore_get_tb_cpu_state(CPUState *cs) +{ + CPUTriCoreState *env = cpu_env(cs); + + return (TCGTBCPUState){ + .pc = env->PC, + .flags = FIELD_DP32(0, TB_FLAGS, PRIV, extract32(env->PSW, 10, 2)), + }; +} + static void tricore_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { @@ -169,20 +179,20 @@ static const struct SysemuCPUOps tricore_sysemu_ops = { .get_phys_page_debug = tricore_cpu_get_phys_page_debug, }; -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps tricore_tcg_ops = { /* MTTCG not yet supported: require strict ordering */ .guest_default_memory_order = TCG_MO_ALL, .mttcg_supported = false, .initialize = tricore_tcg_init, .translate_code = tricore_translate_code, + .get_tb_cpu_state = tricore_get_tb_cpu_state, .synchronize_from_tb = tricore_cpu_synchronize_from_tb, .restore_state_to_opc = tricore_restore_state_to_opc, .mmu_index = tricore_cpu_mmu_index, .tlb_fill = tricore_cpu_tlb_fill, .cpu_exec_interrupt = tricore_cpu_exec_interrupt, .cpu_exec_halt = tricore_cpu_has_work, + .cpu_exec_reset = cpu_reset, }; static void tricore_cpu_class_init(ObjectClass *c, const void *data) diff --git a/target/tricore/cpu.h b/target/tricore/cpu.h index c76e65f818..82085fbc32 100644 --- a/target/tricore/cpu.h +++ b/target/tricore/cpu.h @@ -258,18 +258,6 @@ void tricore_tcg_init(void); void tricore_translate_code(CPUState *cs, TranslationBlock *tb, int *max_insns, vaddr pc, void *host_pc); -static inline void cpu_get_tb_cpu_state(CPUTriCoreState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - uint32_t new_flags = 0; - *pc = env->PC; - *cs_base = 0; - - new_flags |= FIELD_DP32(new_flags, TB_FLAGS, PRIV, - extract32(env->PSW, 10, 2)); - *flags = new_flags; -} - #define CPU_RESOLVING_TYPE TYPE_TRICORE_CPU /* helpers.c */ diff --git a/target/tricore/op_helper.c b/target/tricore/op_helper.c index ae559b6922..9910c13f4b 100644 --- a/target/tricore/op_helper.c +++ b/target/tricore/op_helper.c @@ -18,7 +18,6 @@ #include "cpu.h" #include "qemu/host-utils.h" #include "exec/helper-proto.h" -#include "exec/exec-all.h" #include "accel/tcg/cpu-ldst.h" #include <zlib.h> /* for crc32 */ diff --git a/target/tricore/translate.c b/target/tricore/translate.c index 7cd26d8eab..3d0e7a10bd 100644 --- a/target/tricore/translate.c +++ b/target/tricore/translate.c @@ -20,7 +20,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "tcg/tcg-op.h" #include "accel/tcg/cpu-ldst.h" #include "qemu/qemu-print.h" @@ -1346,15 +1345,11 @@ static inline void gen_addi_CC(TCGv ret, TCGv r1, int32_t con) static inline void gen_addc_CC(TCGv ret, TCGv r1, TCGv r2) { - TCGv carry = tcg_temp_new_i32(); - TCGv t0 = tcg_temp_new_i32(); + TCGv t0 = tcg_temp_new_i32(); TCGv result = tcg_temp_new_i32(); - tcg_gen_movi_tl(t0, 0); - tcg_gen_setcondi_tl(TCG_COND_NE, carry, cpu_PSW_C, 0); /* Addition, carry and set C/V/SV bits */ - tcg_gen_add2_i32(result, cpu_PSW_C, r1, t0, carry, t0); - tcg_gen_add2_i32(result, cpu_PSW_C, result, cpu_PSW_C, r2, t0); + tcg_gen_addcio_i32(result, cpu_PSW_C, r1, r2, cpu_PSW_C); /* calc V bit */ tcg_gen_xor_tl(cpu_PSW_V, result, r1); tcg_gen_xor_tl(t0, r1, r2); @@ -3981,7 +3976,7 @@ static void decode_bit_andacc(DisasContext *ctx) pos1, pos2, &tcg_gen_andc_tl, &tcg_gen_and_tl); break; case OPC2_32_BIT_AND_NOR_T: - if (tcg_op_supported(INDEX_op_andc_i32, TCG_TYPE_I32, 0)) { + if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0)) { gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2], pos1, pos2, &tcg_gen_or_tl, &tcg_gen_andc_tl); } else { @@ -4114,7 +4109,7 @@ static void decode_bit_orand(DisasContext *ctx) pos1, pos2, &tcg_gen_andc_tl, &tcg_gen_or_tl); break; case OPC2_32_BIT_OR_NOR_T: - if (tcg_op_supported(INDEX_op_orc_i32, TCG_TYPE_I32, 0)) { + if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I32, 0)) { gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2], pos1, pos2, &tcg_gen_or_tl, &tcg_gen_orc_tl); } else { diff --git a/target/xtensa/cpu.c b/target/xtensa/cpu.c index 27d6e40195..91b71b6caa 100644 --- a/target/xtensa/cpu.c +++ b/target/xtensa/cpu.c @@ -35,6 +35,7 @@ #include "qemu/module.h" #include "migration/vmstate.h" #include "hw/qdev-clock.h" +#include "accel/tcg/cpu-ops.h" #ifndef CONFIG_USER_ONLY #include "system/memory.h" #endif @@ -54,6 +55,80 @@ static vaddr xtensa_cpu_get_pc(CPUState *cs) return cpu->env.pc; } +static TCGTBCPUState xtensa_get_tb_cpu_state(CPUState *cs) +{ + CPUXtensaState *env = cpu_env(cs); + uint32_t flags = 0; + target_ulong cs_base = 0; + + flags |= xtensa_get_ring(env); + if (env->sregs[PS] & PS_EXCM) { + flags |= XTENSA_TBFLAG_EXCM; + } else if (xtensa_option_enabled(env->config, XTENSA_OPTION_LOOP)) { + target_ulong lend_dist = + env->sregs[LEND] - (env->pc & -(1u << TARGET_PAGE_BITS)); + + /* + * 0 in the csbase_lend field means that there may not be a loopback + * for any instruction that starts inside this page. Any other value + * means that an instruction that ends at this offset from the page + * start may loop back and will need loopback code to be generated. + * + * lend_dist is 0 when LEND points to the start of the page, but + * no instruction that starts inside this page may end at offset 0, + * so it's still correct. + * + * When an instruction ends at a page boundary it may only start in + * the previous page. lend_dist will be encoded as TARGET_PAGE_SIZE + * for the TB that contains this instruction. + */ + if (lend_dist < (1u << TARGET_PAGE_BITS) + env->config->max_insn_size) { + target_ulong lbeg_off = env->sregs[LEND] - env->sregs[LBEG]; + + cs_base = lend_dist; + if (lbeg_off < 256) { + cs_base |= lbeg_off << XTENSA_CSBASE_LBEG_OFF_SHIFT; + } + } + } + if (xtensa_option_enabled(env->config, XTENSA_OPTION_EXTENDED_L32R) && + (env->sregs[LITBASE] & 1)) { + flags |= XTENSA_TBFLAG_LITBASE; + } + if (xtensa_option_enabled(env->config, XTENSA_OPTION_DEBUG)) { + if (xtensa_get_cintlevel(env) < env->config->debug_level) { + flags |= XTENSA_TBFLAG_DEBUG; + } + if (xtensa_get_cintlevel(env) < env->sregs[ICOUNTLEVEL]) { + flags |= XTENSA_TBFLAG_ICOUNT; + } + } + if (xtensa_option_enabled(env->config, XTENSA_OPTION_COPROCESSOR)) { + flags |= env->sregs[CPENABLE] << XTENSA_TBFLAG_CPENABLE_SHIFT; + } + if (xtensa_option_enabled(env->config, XTENSA_OPTION_WINDOWED_REGISTER) && + (env->sregs[PS] & (PS_WOE | PS_EXCM)) == PS_WOE) { + uint32_t windowstart = xtensa_replicate_windowstart(env) >> + (env->sregs[WINDOW_BASE] + 1); + uint32_t w = ctz32(windowstart | 0x8); + + flags |= (w << XTENSA_TBFLAG_WINDOW_SHIFT) | XTENSA_TBFLAG_CWOE; + flags |= extract32(env->sregs[PS], PS_CALLINC_SHIFT, + PS_CALLINC_LEN) << XTENSA_TBFLAG_CALLINC_SHIFT; + } else { + flags |= 3 << XTENSA_TBFLAG_WINDOW_SHIFT; + } + if (env->yield_needed) { + flags |= XTENSA_TBFLAG_YIELD; + } + + return (TCGTBCPUState){ + .pc = env->pc, + .flags = flags, + .cs_base = cs_base, + }; +} + static void xtensa_restore_state_to_opc(CPUState *cs, const TranslationBlock *tb, const uint64_t *data) @@ -229,8 +304,6 @@ static const struct SysemuCPUOps xtensa_sysemu_ops = { }; #endif -#include "accel/tcg/cpu-ops.h" - static const TCGCPUOps xtensa_tcg_ops = { /* Xtensa processors have a weak memory model */ .guest_default_memory_order = 0, @@ -239,6 +312,7 @@ static const TCGCPUOps xtensa_tcg_ops = { .initialize = xtensa_translate_init, .translate_code = xtensa_translate_code, .debug_excp_handler = xtensa_breakpoint_handler, + .get_tb_cpu_state = xtensa_get_tb_cpu_state, .restore_state_to_opc = xtensa_restore_state_to_opc, .mmu_index = xtensa_cpu_mmu_index, @@ -246,6 +320,7 @@ static const TCGCPUOps xtensa_tcg_ops = { .tlb_fill = xtensa_cpu_tlb_fill, .cpu_exec_interrupt = xtensa_cpu_exec_interrupt, .cpu_exec_halt = xtensa_cpu_has_work, + .cpu_exec_reset = cpu_reset, .do_interrupt = xtensa_cpu_do_interrupt, .do_transaction_failed = xtensa_cpu_do_transaction_failed, .do_unaligned_access = xtensa_cpu_do_unaligned_access, diff --git a/target/xtensa/cpu.h b/target/xtensa/cpu.h index c03ed71c94..74122ebe15 100644 --- a/target/xtensa/cpu.h +++ b/target/xtensa/cpu.h @@ -733,74 +733,6 @@ static inline uint32_t xtensa_replicate_windowstart(CPUXtensaState *env) #define XTENSA_CSBASE_LBEG_OFF_MASK 0x00ff0000 #define XTENSA_CSBASE_LBEG_OFF_SHIFT 16 -static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, vaddr *pc, - uint64_t *cs_base, uint32_t *flags) -{ - *pc = env->pc; - *cs_base = 0; - *flags = 0; - *flags |= xtensa_get_ring(env); - if (env->sregs[PS] & PS_EXCM) { - *flags |= XTENSA_TBFLAG_EXCM; - } else if (xtensa_option_enabled(env->config, XTENSA_OPTION_LOOP)) { - target_ulong lend_dist = - env->sregs[LEND] - (env->pc & -(1u << TARGET_PAGE_BITS)); - - /* - * 0 in the csbase_lend field means that there may not be a loopback - * for any instruction that starts inside this page. Any other value - * means that an instruction that ends at this offset from the page - * start may loop back and will need loopback code to be generated. - * - * lend_dist is 0 when LEND points to the start of the page, but - * no instruction that starts inside this page may end at offset 0, - * so it's still correct. - * - * When an instruction ends at a page boundary it may only start in - * the previous page. lend_dist will be encoded as TARGET_PAGE_SIZE - * for the TB that contains this instruction. - */ - if (lend_dist < (1u << TARGET_PAGE_BITS) + env->config->max_insn_size) { - target_ulong lbeg_off = env->sregs[LEND] - env->sregs[LBEG]; - - *cs_base = lend_dist; - if (lbeg_off < 256) { - *cs_base |= lbeg_off << XTENSA_CSBASE_LBEG_OFF_SHIFT; - } - } - } - if (xtensa_option_enabled(env->config, XTENSA_OPTION_EXTENDED_L32R) && - (env->sregs[LITBASE] & 1)) { - *flags |= XTENSA_TBFLAG_LITBASE; - } - if (xtensa_option_enabled(env->config, XTENSA_OPTION_DEBUG)) { - if (xtensa_get_cintlevel(env) < env->config->debug_level) { - *flags |= XTENSA_TBFLAG_DEBUG; - } - if (xtensa_get_cintlevel(env) < env->sregs[ICOUNTLEVEL]) { - *flags |= XTENSA_TBFLAG_ICOUNT; - } - } - if (xtensa_option_enabled(env->config, XTENSA_OPTION_COPROCESSOR)) { - *flags |= env->sregs[CPENABLE] << XTENSA_TBFLAG_CPENABLE_SHIFT; - } - if (xtensa_option_enabled(env->config, XTENSA_OPTION_WINDOWED_REGISTER) && - (env->sregs[PS] & (PS_WOE | PS_EXCM)) == PS_WOE) { - uint32_t windowstart = xtensa_replicate_windowstart(env) >> - (env->sregs[WINDOW_BASE] + 1); - uint32_t w = ctz32(windowstart | 0x8); - - *flags |= (w << XTENSA_TBFLAG_WINDOW_SHIFT) | XTENSA_TBFLAG_CWOE; - *flags |= extract32(env->sregs[PS], PS_CALLINC_SHIFT, - PS_CALLINC_LEN) << XTENSA_TBFLAG_CALLINC_SHIFT; - } else { - *flags |= 3 << XTENSA_TBFLAG_WINDOW_SHIFT; - } - if (env->yield_needed) { - *flags |= XTENSA_TBFLAG_YIELD; - } -} - XtensaCPU *xtensa_cpu_create_with_clock(const char *cpu_type, Clock *cpu_refclk); diff --git a/target/xtensa/dbg_helper.c b/target/xtensa/dbg_helper.c index c4f4298a50..3b91f7c38a 100644 --- a/target/xtensa/dbg_helper.c +++ b/target/xtensa/dbg_helper.c @@ -30,7 +30,6 @@ #include "cpu.h" #include "exec/helper-proto.h" #include "qemu/host-utils.h" -#include "exec/exec-all.h" #include "exec/watchpoint.h" #include "system/address-spaces.h" diff --git a/target/xtensa/exc_helper.c b/target/xtensa/exc_helper.c index ca629f071d..b611c9bf97 100644 --- a/target/xtensa/exc_helper.c +++ b/target/xtensa/exc_helper.c @@ -32,7 +32,6 @@ #include "exec/helper-proto.h" #include "qemu/host-utils.h" #include "qemu/atomic.h" -#include "exec/exec-all.h" void HELPER(exception)(CPUXtensaState *env, uint32_t excp) { diff --git a/target/xtensa/fpu_helper.c b/target/xtensa/fpu_helper.c index 53fc7cfd2a..5358060c50 100644 --- a/target/xtensa/fpu_helper.c +++ b/target/xtensa/fpu_helper.c @@ -30,7 +30,6 @@ #include "cpu.h" #include "exec/helper-proto.h" #include "qemu/host-utils.h" -#include "exec/exec-all.h" #include "fpu/softfloat.h" enum { diff --git a/target/xtensa/mmu_helper.c b/target/xtensa/mmu_helper.c index a7dd810055..71330fc84b 100644 --- a/target/xtensa/mmu_helper.c +++ b/target/xtensa/mmu_helper.c @@ -34,7 +34,7 @@ #include "qemu/host-utils.h" #include "exec/cputlb.h" #include "accel/tcg/cpu-mmu-index.h" -#include "exec/exec-all.h" +#include "accel/tcg/probe.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "system/memory.h" diff --git a/target/xtensa/op_helper.c b/target/xtensa/op_helper.c index c125fa4946..fc47ebaaf5 100644 --- a/target/xtensa/op_helper.c +++ b/target/xtensa/op_helper.c @@ -30,7 +30,6 @@ #include "exec/helper-proto.h" #include "exec/page-protection.h" #include "qemu/host-utils.h" -#include "exec/exec-all.h" #include "system/memory.h" #include "qemu/atomic.h" #include "qemu/timer.h" diff --git a/target/xtensa/translate.c b/target/xtensa/translate.c index 2af83c07c2..34ae2f4e16 100644 --- a/target/xtensa/translate.c +++ b/target/xtensa/translate.c @@ -31,7 +31,6 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/exec-all.h" #include "tcg/tcg-op.h" #include "qemu/log.h" #include "qemu/qemu-print.h" diff --git a/target/xtensa/win_helper.c b/target/xtensa/win_helper.c index ec9ff44db0..4b25f8f4de 100644 --- a/target/xtensa/win_helper.c +++ b/target/xtensa/win_helper.c @@ -30,7 +30,6 @@ #include "cpu.h" #include "exec/helper-proto.h" #include "qemu/host-utils.h" -#include "exec/exec-all.h" static void copy_window_from_phys(CPUXtensaState *env, uint32_t window, uint32_t phys, uint32_t n) diff --git a/tcg/aarch64/tcg-target-con-set.h b/tcg/aarch64/tcg-target-con-set.h index 1281e5efc0..d0622e65fb 100644 --- a/tcg/aarch64/tcg-target-con-set.h +++ b/tcg/aarch64/tcg-target-con-set.h @@ -18,14 +18,16 @@ C_O1_I1(r, r) C_O1_I1(w, r) C_O1_I1(w, w) C_O1_I1(w, wr) -C_O1_I2(r, 0, rz) C_O1_I2(r, r, r) C_O1_I2(r, r, rA) C_O1_I2(r, r, rAL) C_O1_I2(r, r, rC) C_O1_I2(r, r, ri) C_O1_I2(r, r, rL) +C_O1_I2(r, rZ, rA) +C_O1_I2(r, rz, rMZ) C_O1_I2(r, rz, rz) +C_O1_I2(r, rZ, rZ) C_O1_I2(w, 0, w) C_O1_I2(w, w, w) C_O1_I2(w, w, wN) @@ -34,4 +36,3 @@ C_O1_I2(w, w, wZ) C_O1_I3(w, w, w, w) C_O1_I4(r, r, rC, rz, rz) C_O2_I1(r, r, r) -C_O2_I4(r, r, rz, rz, rA, rMZ) diff --git a/tcg/aarch64/tcg-target-has.h b/tcg/aarch64/tcg-target-has.h index 39f01c14cd..69e83efb69 100644 --- a/tcg/aarch64/tcg-target-has.h +++ b/tcg/aarch64/tcg-target-has.h @@ -13,64 +13,7 @@ #define have_lse2 (cpuinfo & CPUINFO_LSE2) /* optional instructions */ -#define TCG_TARGET_HAS_div_i32 1 -#define TCG_TARGET_HAS_rem_i32 1 -#define TCG_TARGET_HAS_ext8s_i32 1 -#define TCG_TARGET_HAS_ext16s_i32 1 -#define TCG_TARGET_HAS_ext8u_i32 1 -#define TCG_TARGET_HAS_ext16u_i32 1 -#define TCG_TARGET_HAS_bswap16_i32 1 -#define TCG_TARGET_HAS_bswap32_i32 1 -#define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_rot_i32 1 -#define TCG_TARGET_HAS_andc_i32 1 -#define TCG_TARGET_HAS_orc_i32 1 -#define TCG_TARGET_HAS_eqv_i32 1 -#define TCG_TARGET_HAS_nand_i32 0 -#define TCG_TARGET_HAS_nor_i32 0 -#define TCG_TARGET_HAS_clz_i32 1 -#define TCG_TARGET_HAS_ctz_i32 1 -#define TCG_TARGET_HAS_ctpop_i32 0 -#define TCG_TARGET_HAS_extract2_i32 1 -#define TCG_TARGET_HAS_negsetcond_i32 1 -#define TCG_TARGET_HAS_add2_i32 1 -#define TCG_TARGET_HAS_sub2_i32 1 -#define TCG_TARGET_HAS_mulu2_i32 0 -#define TCG_TARGET_HAS_muls2_i32 0 -#define TCG_TARGET_HAS_muluh_i32 0 -#define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_extr_i64_i32 0 -#define TCG_TARGET_HAS_qemu_st8_i32 0 - -#define TCG_TARGET_HAS_div_i64 1 -#define TCG_TARGET_HAS_rem_i64 1 -#define TCG_TARGET_HAS_ext8s_i64 1 -#define TCG_TARGET_HAS_ext16s_i64 1 -#define TCG_TARGET_HAS_ext32s_i64 1 -#define TCG_TARGET_HAS_ext8u_i64 1 -#define TCG_TARGET_HAS_ext16u_i64 1 -#define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_bswap16_i64 1 -#define TCG_TARGET_HAS_bswap32_i64 1 -#define TCG_TARGET_HAS_bswap64_i64 1 -#define TCG_TARGET_HAS_not_i64 1 -#define TCG_TARGET_HAS_rot_i64 1 -#define TCG_TARGET_HAS_andc_i64 1 -#define TCG_TARGET_HAS_orc_i64 1 -#define TCG_TARGET_HAS_eqv_i64 1 -#define TCG_TARGET_HAS_nand_i64 0 -#define TCG_TARGET_HAS_nor_i64 0 -#define TCG_TARGET_HAS_clz_i64 1 -#define TCG_TARGET_HAS_ctz_i64 1 -#define TCG_TARGET_HAS_ctpop_i64 0 -#define TCG_TARGET_HAS_extract2_i64 1 -#define TCG_TARGET_HAS_negsetcond_i64 1 -#define TCG_TARGET_HAS_add2_i64 1 -#define TCG_TARGET_HAS_sub2_i64 1 -#define TCG_TARGET_HAS_mulu2_i64 0 -#define TCG_TARGET_HAS_muls2_i64 0 -#define TCG_TARGET_HAS_muluh_i64 1 -#define TCG_TARGET_HAS_mulsh_i64 1 /* * Without FEAT_LSE2, we must use LDXP+STXP to implement atomic 128-bit load, diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index 4645242d85..4cb647cb34 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -508,7 +508,9 @@ typedef enum { /* Add/subtract with carry instructions. */ I3503_ADC = 0x1a000000, + I3503_ADCS = 0x3a000000, I3503_SBC = 0x5a000000, + I3503_SBCS = 0x7a000000, /* Conditional select instructions. */ I3506_CSEL = 0x1a800000, @@ -1347,70 +1349,37 @@ static inline void tcg_out_extr(TCGContext *s, TCGType ext, TCGReg rd, tcg_out_insn(s, 3403, EXTR, ext, rd, rn, rm, a); } -static inline void tcg_out_shl(TCGContext *s, TCGType ext, - TCGReg rd, TCGReg rn, unsigned int m) +static void tgen_cmp(TCGContext *s, TCGType ext, TCGCond cond, + TCGReg a, TCGReg b) { - int bits = ext ? 64 : 32; - int max = bits - 1; - tcg_out_ubfm(s, ext, rd, rn, (bits - m) & max, (max - m) & max); -} - -static inline void tcg_out_shr(TCGContext *s, TCGType ext, - TCGReg rd, TCGReg rn, unsigned int m) -{ - int max = ext ? 63 : 31; - tcg_out_ubfm(s, ext, rd, rn, m & max, max); -} - -static inline void tcg_out_sar(TCGContext *s, TCGType ext, - TCGReg rd, TCGReg rn, unsigned int m) -{ - int max = ext ? 63 : 31; - tcg_out_sbfm(s, ext, rd, rn, m & max, max); -} - -static inline void tcg_out_rotr(TCGContext *s, TCGType ext, - TCGReg rd, TCGReg rn, unsigned int m) -{ - int max = ext ? 63 : 31; - tcg_out_extr(s, ext, rd, rn, rn, m & max); -} - -static inline void tcg_out_rotl(TCGContext *s, TCGType ext, - TCGReg rd, TCGReg rn, unsigned int m) -{ - int max = ext ? 63 : 31; - tcg_out_extr(s, ext, rd, rn, rn, -m & max); + if (is_tst_cond(cond)) { + tcg_out_insn(s, 3510, ANDS, ext, TCG_REG_XZR, a, b); + } else { + tcg_out_insn(s, 3502, SUBS, ext, TCG_REG_XZR, a, b); + } } -static inline void tcg_out_dep(TCGContext *s, TCGType ext, TCGReg rd, - TCGReg rn, unsigned lsb, unsigned width) +static void tgen_cmpi(TCGContext *s, TCGType ext, TCGCond cond, + TCGReg a, tcg_target_long b) { - unsigned size = ext ? 64 : 32; - unsigned a = (size - lsb) & (size - 1); - unsigned b = width - 1; - tcg_out_bfm(s, ext, rd, rn, a, b); + if (is_tst_cond(cond)) { + tcg_out_logicali(s, I3404_ANDSI, ext, TCG_REG_XZR, a, b); + } else if (b >= 0) { + tcg_debug_assert(is_aimm(b)); + tcg_out_insn(s, 3401, SUBSI, ext, TCG_REG_XZR, a, b); + } else { + tcg_debug_assert(is_aimm(-b)); + tcg_out_insn(s, 3401, ADDSI, ext, TCG_REG_XZR, a, -b); + } } static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGCond cond, TCGReg a, tcg_target_long b, bool const_b) { - if (is_tst_cond(cond)) { - if (!const_b) { - tcg_out_insn(s, 3510, ANDS, ext, TCG_REG_XZR, a, b); - } else { - tcg_out_logicali(s, I3404_ANDSI, ext, TCG_REG_XZR, a, b); - } + if (const_b) { + tgen_cmpi(s, ext, cond, a, b); } else { - if (!const_b) { - tcg_out_insn(s, 3502, SUBS, ext, TCG_REG_XZR, a, b); - } else if (b >= 0) { - tcg_debug_assert(is_aimm(b)); - tcg_out_insn(s, 3401, SUBSI, ext, TCG_REG_XZR, a, b); - } else { - tcg_debug_assert(is_aimm(-b)); - tcg_out_insn(s, 3401, ADDSI, ext, TCG_REG_XZR, a, -b); - } + tgen_cmp(s, ext, cond, a, b); } } @@ -1438,7 +1407,7 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, tcg_out_call_int(s, target); } -static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l) +static void tcg_out_br(TCGContext *s, TCGLabel *l) { if (!l->has_value) { tcg_out_reloc(s, s->code_ptr, R_AARCH64_JUMP26, l, 0); @@ -1448,8 +1417,16 @@ static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l) } } -static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a, - TCGArg b, bool b_const, TCGLabel *l) +static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c, + TCGReg a, TCGReg b, TCGLabel *l) +{ + tgen_cmp(s, type, c, a, b); + tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0); + tcg_out_insn(s, 3202, B_C, c, 0); +} + +static void tgen_brcondi(TCGContext *s, TCGType ext, TCGCond c, + TCGReg a, tcg_target_long b, TCGLabel *l) { int tbit = -1; bool need_cmp = true; @@ -1458,14 +1435,14 @@ static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a, case TCG_COND_EQ: case TCG_COND_NE: /* cmp xN,0; b.ne L -> cbnz xN,L */ - if (b_const && b == 0) { + if (b == 0) { need_cmp = false; } break; case TCG_COND_LT: case TCG_COND_GE: /* cmp xN,0; b.mi L -> tbnz xN,63,L */ - if (b_const && b == 0) { + if (b == 0) { c = (c == TCG_COND_LT ? TCG_COND_TSTNE : TCG_COND_TSTEQ); tbit = ext ? 63 : 31; need_cmp = false; @@ -1474,14 +1451,14 @@ static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a, case TCG_COND_TSTEQ: case TCG_COND_TSTNE: /* tst xN,0xffffffff; b.ne L -> cbnz wN,L */ - if (b_const && b == UINT32_MAX) { + if (b == UINT32_MAX) { c = tcg_tst_eqne_cond(c); ext = TCG_TYPE_I32; need_cmp = false; break; } /* tst xN,1<<B; b.ne L -> tbnz xN,B,L */ - if (b_const && is_power_of_2(b)) { + if (is_power_of_2(b)) { tbit = ctz64(b); need_cmp = false; } @@ -1491,7 +1468,7 @@ static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a, } if (need_cmp) { - tcg_out_cmp(s, ext, c, a, b, b_const); + tgen_cmpi(s, ext, c, a, b); tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0); tcg_out_insn(s, 3202, B_C, c, 0); return; @@ -1524,6 +1501,12 @@ static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a, } } +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, rC), + .out_rr = tgen_brcond, + .out_ri = tgen_brcondi, +}; + static inline void tcg_out_rev(TCGContext *s, int ext, MemOp s_bits, TCGReg rd, TCGReg rn) { @@ -1592,67 +1575,7 @@ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn) tcg_out_mov(s, TCG_TYPE_I32, rd, rn); } -static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd, - TCGReg rn, int64_t aimm) -{ - if (aimm >= 0) { - tcg_out_insn(s, 3401, ADDI, ext, rd, rn, aimm); - } else { - tcg_out_insn(s, 3401, SUBI, ext, rd, rn, -aimm); - } -} - -static void tcg_out_addsub2(TCGContext *s, TCGType ext, TCGReg rl, - TCGReg rh, TCGReg al, TCGReg ah, - tcg_target_long bl, tcg_target_long bh, - bool const_bl, bool const_bh, bool sub) -{ - TCGReg orig_rl = rl; - AArch64Insn insn; - - if (rl == ah || (!const_bh && rl == bh)) { - rl = TCG_REG_TMP0; - } - - if (const_bl) { - if (bl < 0) { - bl = -bl; - insn = sub ? I3401_ADDSI : I3401_SUBSI; - } else { - insn = sub ? I3401_SUBSI : I3401_ADDSI; - } - - if (unlikely(al == TCG_REG_XZR)) { - /* ??? We want to allow al to be zero for the benefit of - negation via subtraction. However, that leaves open the - possibility of adding 0+const in the low part, and the - immediate add instructions encode XSP not XZR. Don't try - anything more elaborate here than loading another zero. */ - al = TCG_REG_TMP0; - tcg_out_movi(s, ext, al, 0); - } - tcg_out_insn_3401(s, insn, ext, rl, al, bl); - } else { - tcg_out_insn_3502(s, sub ? I3502_SUBS : I3502_ADDS, ext, rl, al, bl); - } - - insn = I3503_ADC; - if (const_bh) { - /* Note that the only two constants we support are 0 and -1, and - that SBC = rn + ~rm + c, so adc -1 is sbc 0, and vice-versa. */ - if ((bh != 0) ^ sub) { - insn = I3503_SBC; - } - bh = TCG_REG_XZR; - } else if (sub) { - insn = I3503_SBC; - } - tcg_out_insn_3503(s, insn, ext, rh, ah, bh); - - tcg_out_mov(s, ext, orig_rl, rl); -} - -static inline void tcg_out_mb(TCGContext *s, TCGArg a0) +static void tcg_out_mb(TCGContext *s, unsigned a0) { static const uint32_t sync[] = { [0 ... TCG_MO_ALL] = DMB_ISH | DMB_LD | DMB_ST, @@ -1664,37 +1587,6 @@ static inline void tcg_out_mb(TCGContext *s, TCGArg a0) tcg_out32(s, sync[a0 & TCG_MO_ALL]); } -static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d, - TCGReg a0, TCGArg b, bool const_b, bool is_ctz) -{ - TCGReg a1 = a0; - if (is_ctz) { - a1 = TCG_REG_TMP0; - tcg_out_insn(s, 3507, RBIT, ext, a1, a0); - } - if (const_b && b == (ext ? 64 : 32)) { - tcg_out_insn(s, 3507, CLZ, ext, d, a1); - } else { - AArch64Insn sel = I3506_CSEL; - - tcg_out_cmp(s, ext, TCG_COND_NE, a0, 0, 1); - tcg_out_insn(s, 3507, CLZ, ext, TCG_REG_TMP0, a1); - - if (const_b) { - if (b == -1) { - b = TCG_REG_XZR; - sel = I3506_CSINV; - } else if (b == 0) { - b = TCG_REG_XZR; - } else { - tcg_out_movi(s, ext, d, b); - b = d; - } - } - tcg_out_insn_3506(s, sel, ext, d, TCG_REG_TMP0, b, TCG_COND_NE); - } -} - typedef struct { TCGReg base; TCGReg index; @@ -1914,8 +1806,8 @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop, } } -static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType data_type) +static void tgen_qemu_ld(TCGContext *s, TCGType data_type, TCGReg data_reg, + TCGReg addr_reg, MemOpIdx oi) { TCGLabelQemuLdst *ldst; HostAddress h; @@ -1930,8 +1822,13 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, } } -static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType data_type) +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_qemu_ld, +}; + +static void tgen_qemu_st(TCGContext *s, TCGType data_type, TCGReg data_reg, + TCGReg addr_reg, MemOpIdx oi) { TCGLabelQemuLdst *ldst; HostAddress h; @@ -1946,6 +1843,11 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, } } +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out = tgen_qemu_st, +}; + static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi, TCGReg addr_reg, MemOpIdx oi, bool is_ld) { @@ -2048,6 +1950,28 @@ static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi, } } +static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr_reg, MemOpIdx oi) +{ + tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, true); +} + +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + .base.static_constraint = C_O2_I1(r, r, r), + .out = tgen_qemu_ld2, +}; + +static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr_reg, MemOpIdx oi) +{ + tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, false); +} + +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + .base.static_constraint = C_O0_I3(rz, rz, r), + .out = tgen_qemu_st2, +}; + static const tcg_insn_unit *tb_ret_addr; static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) @@ -2094,6 +2018,11 @@ static void tcg_out_goto_tb(TCGContext *s, int which) tcg_out_bti(s, BTI_J); } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + tcg_out_insn(s, 3207, BR, a0); +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { @@ -2115,402 +2044,859 @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, flush_idcache_range(jmp_rx, jmp_rw, 4); } -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType ext, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) + +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - /* Hoist the loads of the most common arguments. */ - TCGArg a0 = args[0]; - TCGArg a1 = args[1]; - TCGArg a2 = args[2]; - int c2 = const_args[2]; + tcg_out_insn(s, 3502, ADD, type, a0, a1, a2); +} - switch (opc) { - case INDEX_op_goto_ptr: - tcg_out_insn(s, 3207, BR, a0); - break; +static void tgen_addi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (a2 >= 0) { + tcg_out_insn(s, 3401, ADDI, type, a0, a1, a2); + } else { + tcg_out_insn(s, 3401, SUBI, type, a0, a1, -a2); + } +} - case INDEX_op_br: - tcg_out_goto_label(s, arg_label(a0)); - break; +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, rA), + .out_rrr = tgen_add, + .out_rri = tgen_addi, +}; - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - tcg_out_ldst(s, I3312_LDRB, a0, a1, a2, 0); - break; - case INDEX_op_ld8s_i32: - tcg_out_ldst(s, I3312_LDRSBW, a0, a1, a2, 0); - break; - case INDEX_op_ld8s_i64: - tcg_out_ldst(s, I3312_LDRSBX, a0, a1, a2, 0); - break; - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - tcg_out_ldst(s, I3312_LDRH, a0, a1, a2, 1); - break; - case INDEX_op_ld16s_i32: - tcg_out_ldst(s, I3312_LDRSHW, a0, a1, a2, 1); - break; - case INDEX_op_ld16s_i64: - tcg_out_ldst(s, I3312_LDRSHX, a0, a1, a2, 1); - break; - case INDEX_op_ld_i32: - case INDEX_op_ld32u_i64: - tcg_out_ldst(s, I3312_LDRW, a0, a1, a2, 2); - break; - case INDEX_op_ld32s_i64: - tcg_out_ldst(s, I3312_LDRSWX, a0, a1, a2, 2); - break; - case INDEX_op_ld_i64: - tcg_out_ldst(s, I3312_LDRX, a0, a1, a2, 3); - break; +static void tgen_addco(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3502, ADDS, type, a0, a1, a2); +} - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - tcg_out_ldst(s, I3312_STRB, a0, a1, a2, 0); - break; - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - tcg_out_ldst(s, I3312_STRH, a0, a1, a2, 1); +static void tgen_addco_imm(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (a2 >= 0) { + tcg_out_insn(s, 3401, ADDSI, type, a0, a1, a2); + } else { + tcg_out_insn(s, 3401, SUBSI, type, a0, a1, -a2); + } +} + +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_O1_I2(r, r, rA), + .out_rrr = tgen_addco, + .out_rri = tgen_addco_imm, +}; + +static void tgen_addci_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3503, ADC, type, a0, a1, a2); +} + +static void tgen_addci_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + /* + * Note that the only two constants we support are 0 and -1, and + * that SBC = rn + ~rm + c, so adc -1 is sbc 0, and vice-versa. + */ + if (a2) { + tcg_out_insn(s, 3503, SBC, type, a0, a1, TCG_REG_XZR); + } else { + tcg_out_insn(s, 3503, ADC, type, a0, a1, TCG_REG_XZR); + } +} + +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_O1_I2(r, rz, rMZ), + .out_rrr = tgen_addci_rrr, + .out_rri = tgen_addci_rri, +}; + +static void tgen_addcio(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3503, ADCS, type, a0, a1, a2); +} + +static void tgen_addcio_imm(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + /* Use SBCS w/0 for ADCS w/-1 -- see above. */ + if (a2) { + tcg_out_insn(s, 3503, SBCS, type, a0, a1, TCG_REG_XZR); + } else { + tcg_out_insn(s, 3503, ADCS, type, a0, a1, TCG_REG_XZR); + } +} + +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_O1_I2(r, rz, rMZ), + .out_rrr = tgen_addcio, + .out_rri = tgen_addcio_imm, +}; + +static void tcg_out_set_carry(TCGContext *s) +{ + tcg_out_insn(s, 3502, SUBS, TCG_TYPE_I32, + TCG_REG_XZR, TCG_REG_XZR, TCG_REG_XZR); +} + +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3510, AND, type, a0, a1, a2); +} + +static void tgen_andi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_logicali(s, I3404_ANDI, type, a0, a1, a2); +} + +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, r, rL), + .out_rrr = tgen_and, + .out_rri = tgen_andi, +}; + +static void tgen_andc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3510, BIC, type, a0, a1, a2); +} + +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_andc, +}; + +static void tgen_clz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_cmp(s, type, TCG_COND_NE, a1, 0, true); + tcg_out_insn(s, 3507, CLZ, type, TCG_REG_TMP0, a1); + tcg_out_insn(s, 3506, CSEL, type, a0, TCG_REG_TMP0, a2, TCG_COND_NE); +} + +static void tgen_clzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (a2 == (type == TCG_TYPE_I32 ? 32 : 64)) { + tcg_out_insn(s, 3507, CLZ, type, a0, a1); + return; + } + + tcg_out_cmp(s, type, TCG_COND_NE, a1, 0, true); + tcg_out_insn(s, 3507, CLZ, type, a0, a1); + + switch (a2) { + case -1: + tcg_out_insn(s, 3506, CSINV, type, a0, a0, TCG_REG_XZR, TCG_COND_NE); break; - case INDEX_op_st_i32: - case INDEX_op_st32_i64: - tcg_out_ldst(s, I3312_STRW, a0, a1, a2, 2); + case 0: + tcg_out_insn(s, 3506, CSEL, type, a0, a0, TCG_REG_XZR, TCG_COND_NE); break; - case INDEX_op_st_i64: - tcg_out_ldst(s, I3312_STRX, a0, a1, a2, 3); + default: + tcg_out_movi(s, type, TCG_REG_TMP0, a2); + tcg_out_insn(s, 3506, CSEL, type, a0, a0, TCG_REG_TMP0, TCG_COND_NE); break; + } +} - case INDEX_op_add_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_add_i64: - if (c2) { - tcg_out_addsubi(s, ext, a0, a1, a2); - } else { - tcg_out_insn(s, 3502, ADD, ext, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_O1_I2(r, r, rAL), + .out_rrr = tgen_clz, + .out_rri = tgen_clzi, +}; - case INDEX_op_sub_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_sub_i64: - if (c2) { - tcg_out_addsubi(s, ext, a0, a1, -a2); - } else { - tcg_out_insn(s, 3502, SUB, ext, a0, a1, a2); - } - break; +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_neg_i64: - case INDEX_op_neg_i32: - tcg_out_insn(s, 3502, SUB, ext, a0, TCG_REG_XZR, a1); - break; +static void tgen_ctz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3507, RBIT, type, TCG_REG_TMP0, a1); + tgen_clz(s, type, a0, TCG_REG_TMP0, a2); +} - case INDEX_op_and_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_and_i64: - if (c2) { - tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, a2); - } else { - tcg_out_insn(s, 3510, AND, ext, a0, a1, a2); - } - break; +static void tgen_ctzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_insn(s, 3507, RBIT, type, TCG_REG_TMP0, a1); + tgen_clzi(s, type, a0, TCG_REG_TMP0, a2); +} - case INDEX_op_andc_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_andc_i64: - if (c2) { - tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, ~a2); - } else { - tcg_out_insn(s, 3510, BIC, ext, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_O1_I2(r, r, rAL), + .out_rrr = tgen_ctz, + .out_rri = tgen_ctzi, +}; - case INDEX_op_or_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_or_i64: - if (c2) { - tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, a2); - } else { - tcg_out_insn(s, 3510, ORR, ext, a0, a1, a2); - } - break; +static void tgen_divs(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, SDIV, type, a0, a1, a2); +} - case INDEX_op_orc_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_orc_i64: - if (c2) { - tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, ~a2); - } else { - tcg_out_insn(s, 3510, ORN, ext, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divs, +}; - case INDEX_op_xor_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_xor_i64: - if (c2) { - tcg_out_logicali(s, I3404_EORI, ext, a0, a1, a2); - } else { - tcg_out_insn(s, 3510, EOR, ext, a0, a1, a2); - } - break; +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_eqv_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_eqv_i64: - if (c2) { - tcg_out_logicali(s, I3404_EORI, ext, a0, a1, ~a2); - } else { - tcg_out_insn(s, 3510, EON, ext, a0, a1, a2); - } - break; +static void tgen_divu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, UDIV, type, a0, a1, a2); +} - case INDEX_op_not_i64: - case INDEX_op_not_i32: - tcg_out_insn(s, 3510, ORN, ext, a0, TCG_REG_XZR, a1); - break; +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divu, +}; - case INDEX_op_mul_i64: - case INDEX_op_mul_i32: - tcg_out_insn(s, 3509, MADD, ext, a0, a1, a2, TCG_REG_XZR); - break; +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_div_i64: - case INDEX_op_div_i32: - tcg_out_insn(s, 3508, SDIV, ext, a0, a1, a2); - break; - case INDEX_op_divu_i64: - case INDEX_op_divu_i32: - tcg_out_insn(s, 3508, UDIV, ext, a0, a1, a2); - break; +static void tgen_eqv(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3510, EON, type, a0, a1, a2); +} - case INDEX_op_rem_i64: - case INDEX_op_rem_i32: - tcg_out_insn(s, 3508, SDIV, ext, TCG_REG_TMP0, a1, a2); - tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP0, a2, a1); - break; - case INDEX_op_remu_i64: - case INDEX_op_remu_i32: - tcg_out_insn(s, 3508, UDIV, ext, TCG_REG_TMP0, a1, a2); - tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP0, a2, a1); - break; +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_eqv, +}; - case INDEX_op_shl_i64: - case INDEX_op_shl_i32: - if (c2) { - tcg_out_shl(s, ext, a0, a1, a2); - } else { - tcg_out_insn(s, 3508, LSLV, ext, a0, a1, a2); - } - break; +static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_ubfm(s, TCG_TYPE_I64, a0, a1, 32, 63); +} - case INDEX_op_shr_i64: - case INDEX_op_shr_i32: - if (c2) { - tcg_out_shr(s, ext, a0, a1, a2); - } else { - tcg_out_insn(s, 3508, LSRV, ext, a0, a1, a2); - } - break; +static const TCGOutOpUnary outop_extrh_i64_i32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extrh_i64_i32, +}; - case INDEX_op_sar_i64: - case INDEX_op_sar_i32: - if (c2) { - tcg_out_sar(s, ext, a0, a1, a2); - } else { - tcg_out_insn(s, 3508, ASRV, ext, a0, a1, a2); - } - break; +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3509, MADD, type, a0, a1, a2, TCG_REG_XZR); +} - case INDEX_op_rotr_i64: - case INDEX_op_rotr_i32: - if (c2) { - tcg_out_rotr(s, ext, a0, a1, a2); - } else { - tcg_out_insn(s, 3508, RORV, ext, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_mul, +}; - case INDEX_op_rotl_i64: - case INDEX_op_rotl_i32: - if (c2) { - tcg_out_rotl(s, ext, a0, a1, a2); - } else { - tcg_out_insn(s, 3502, SUB, 0, TCG_REG_TMP0, TCG_REG_XZR, a2); - tcg_out_insn(s, 3508, RORV, ext, a0, a1, TCG_REG_TMP0); - } - break; +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_clz_i64: - case INDEX_op_clz_i32: - tcg_out_cltz(s, ext, a0, a1, a2, c2, false); - break; - case INDEX_op_ctz_i64: - case INDEX_op_ctz_i32: - tcg_out_cltz(s, ext, a0, a1, a2, c2, true); - break; +static TCGConstraintSetIndex cset_mulh(TCGType type, unsigned flags) +{ + return type == TCG_TYPE_I64 ? C_O1_I2(r, r, r) : C_NotImplemented; +} - case INDEX_op_brcond_i32: - a1 = (int32_t)a1; - /* FALLTHRU */ - case INDEX_op_brcond_i64: - tcg_out_brcond(s, ext, a2, a0, a1, const_args[1], arg_label(args[3])); - break; +static void tgen_mulsh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, SMULH, TCG_TYPE_I64, a0, a1, a2); +} - case INDEX_op_setcond_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_setcond_i64: - tcg_out_cmp(s, ext, args[3], a1, a2, c2); - /* Use CSET alias of CSINC Wd, WZR, WZR, invert(cond). */ - tcg_out_insn(s, 3506, CSINC, TCG_TYPE_I32, a0, TCG_REG_XZR, - TCG_REG_XZR, tcg_invert_cond(args[3])); - break; +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mulh, + .out_rrr = tgen_mulsh, +}; - case INDEX_op_negsetcond_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_negsetcond_i64: - tcg_out_cmp(s, ext, args[3], a1, a2, c2); - /* Use CSETM alias of CSINV Wd, WZR, WZR, invert(cond). */ - tcg_out_insn(s, 3506, CSINV, ext, a0, TCG_REG_XZR, - TCG_REG_XZR, tcg_invert_cond(args[3])); - break; +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_movcond_i32: - a2 = (int32_t)a2; - /* FALLTHRU */ - case INDEX_op_movcond_i64: - tcg_out_cmp(s, ext, args[5], a1, a2, c2); - tcg_out_insn(s, 3506, CSEL, ext, a0, args[3], args[4], args[5]); - break; +static void tgen_muluh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, UMULH, TCG_TYPE_I64, a0, a1, a2); +} - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, a0, a1, a2, ext); - break; - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, a0, a1, a2, ext); - break; - case INDEX_op_qemu_ld_i128: - tcg_out_qemu_ldst_i128(s, a0, a1, a2, args[3], true); - break; - case INDEX_op_qemu_st_i128: - tcg_out_qemu_ldst_i128(s, a0, a1, a2, args[3], false); - break; +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mulh, + .out_rrr = tgen_muluh, +}; - case INDEX_op_bswap64_i64: - tcg_out_rev(s, TCG_TYPE_I64, MO_64, a0, a1); - break; - case INDEX_op_bswap32_i64: - tcg_out_rev(s, TCG_TYPE_I32, MO_32, a0, a1); - if (a2 & TCG_BSWAP_OS) { - tcg_out_ext32s(s, a0, a0); - } - break; - case INDEX_op_bswap32_i32: - tcg_out_rev(s, TCG_TYPE_I32, MO_32, a0, a1); - break; - case INDEX_op_bswap16_i64: - case INDEX_op_bswap16_i32: - tcg_out_rev(s, TCG_TYPE_I32, MO_16, a0, a1); - if (a2 & TCG_BSWAP_OS) { - /* Output must be sign-extended. */ - tcg_out_ext16s(s, ext, a0, a0); - } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - /* Output must be zero-extended, but input isn't. */ - tcg_out_ext16u(s, a0, a0); - } - break; +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_deposit_i64: - case INDEX_op_deposit_i32: - tcg_out_dep(s, ext, a0, a2, args[3], args[4]); - break; +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_extract_i64: - case INDEX_op_extract_i32: - if (a2 == 0) { - uint64_t mask = MAKE_64BIT_MASK(0, args[3]); - tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, mask); - } else { - tcg_out_ubfm(s, ext, a0, a1, a2, a2 + args[3] - 1); - } - break; +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3510, ORR, type, a0, a1, a2); +} - case INDEX_op_sextract_i64: - case INDEX_op_sextract_i32: - tcg_out_sbfm(s, ext, a0, a1, a2, a2 + args[3] - 1); - break; +static void tgen_ori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_logicali(s, I3404_ORRI, type, a0, a1, a2); +} - case INDEX_op_extract2_i64: - case INDEX_op_extract2_i32: - tcg_out_extr(s, ext, a0, a2, a1, args[3]); - break; +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, r, rL), + .out_rrr = tgen_or, + .out_rri = tgen_ori, +}; - case INDEX_op_add2_i32: - tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, a2, args[3], - (int32_t)args[4], args[5], const_args[4], - const_args[5], false); - break; - case INDEX_op_add2_i64: - tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, a2, args[3], args[4], - args[5], const_args[4], const_args[5], false); - break; - case INDEX_op_sub2_i32: - tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, a2, args[3], - (int32_t)args[4], args[5], const_args[4], - const_args[5], true); - break; - case INDEX_op_sub2_i64: - tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, a2, args[3], args[4], - args[5], const_args[4], const_args[5], true); - break; +static void tgen_orc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3510, ORN, type, a0, a1, a2); +} - case INDEX_op_muluh_i64: - tcg_out_insn(s, 3508, UMULH, TCG_TYPE_I64, a0, a1, a2); - break; - case INDEX_op_mulsh_i64: - tcg_out_insn(s, 3508, SMULH, TCG_TYPE_I64, a0, a1, a2); - break; +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_orc, +}; - case INDEX_op_mb: - tcg_out_mb(s, a0); - break; +static void tgen_rems(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, SDIV, type, TCG_REG_TMP0, a1, a2); + tcg_out_insn(s, 3509, MSUB, type, a0, TCG_REG_TMP0, a2, a1); +} - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_mov_i64: - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - default: - g_assert_not_reached(); +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_rems, +}; + +static void tgen_remu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, UDIV, type, TCG_REG_TMP0, a1, a2); + tcg_out_insn(s, 3509, MSUB, type, a0, TCG_REG_TMP0, a2, a1); +} + +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_remu, +}; + +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_rotr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, RORV, type, a0, a1, a2); +} + +static void tgen_rotri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int max = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_extr(s, type, a0, a1, a1, a2 & max); +} + +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_rotr, + .out_rri = tgen_rotri, +}; + +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, ASRV, type, a0, a1, a2); +} + +static void tgen_sari(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int max = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_sbfm(s, type, a0, a1, a2 & max, max); +} + +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_sar, + .out_rri = tgen_sari, +}; + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, LSLV, type, a0, a1, a2); +} + +static void tgen_shli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int max = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_ubfm(s, type, a0, a1, -a2 & max, ~a2 & max); +} + +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shl, + .out_rri = tgen_shli, +}; + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3508, LSRV, type, a0, a1, a2); +} + +static void tgen_shri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int max = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_ubfm(s, type, a0, a1, a2 & max, max); +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shr, + .out_rri = tgen_shri, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3502, SUB, type, a0, a1, a2); +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_sub, +}; + +static void tgen_subbo_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3502, SUBS, type, a0, a1, a2); +} + +static void tgen_subbo_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (a2 >= 0) { + tcg_out_insn(s, 3401, SUBSI, type, a0, a1, a2); + } else { + tcg_out_insn(s, 3401, ADDSI, type, a0, a1, -a2); + } +} + +static void tgen_subbo_rir(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2) +{ + tgen_subbo_rrr(s, type, a0, TCG_REG_XZR, a2); +} + +static void tgen_subbo_rii(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, tcg_target_long a2) +{ + if (a2 == 0) { + tgen_subbo_rrr(s, type, a0, TCG_REG_XZR, TCG_REG_XZR); + return; + } + + /* + * We want to allow a1 to be zero for the benefit of negation via + * subtraction. However, that leaves open the possibility of + * adding 0 +/- const, and the immediate add/sub instructions + * encode XSP not XZR. Since we have 0 - non-zero, borrow is + * always set. + */ + tcg_out_movi(s, type, a0, -a2); + tcg_out_set_borrow(s); +} + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_O1_I2(r, rZ, rA), + .out_rrr = tgen_subbo_rrr, + .out_rri = tgen_subbo_rri, + .out_rir = tgen_subbo_rir, + .out_rii = tgen_subbo_rii, +}; + +static void tgen_subbi_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3503, SBC, type, a0, a1, a2); +} + +static void tgen_subbi_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_addci_rri(s, type, a0, a1, ~a2); +} + +static const TCGOutOpAddSubCarry outop_subbi = { + .base.static_constraint = C_O1_I2(r, rz, rMZ), + .out_rrr = tgen_subbi_rrr, + .out_rri = tgen_subbi_rri, +}; + +static void tgen_subbio_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3503, SBCS, type, a0, a1, a2); +} + +static void tgen_subbio_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_addcio_imm(s, type, a0, a1, ~a2); +} + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_O1_I2(r, rz, rMZ), + .out_rrr = tgen_subbio_rrr, + .out_rri = tgen_subbio_rri, +}; + +static void tcg_out_set_borrow(TCGContext *s) +{ + tcg_out_insn(s, 3502, ADDS, TCG_TYPE_I32, + TCG_REG_XZR, TCG_REG_XZR, TCG_REG_XZR); +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_insn(s, 3510, EOR, type, a0, a1, a2); +} + +static void tgen_xori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_logicali(s, I3404_EORI, type, a0, a1, a2); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, r, rL), + .out_rrr = tgen_xor, + .out_rri = tgen_xori, +}; + +static void tgen_bswap16(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_rev(s, TCG_TYPE_I32, MO_16, a0, a1); + if (flags & TCG_BSWAP_OS) { + /* Output must be sign-extended. */ + tcg_out_ext16s(s, type, a0, a0); + } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + /* Output must be zero-extended, but input isn't. */ + tcg_out_ext16u(s, a0, a0); + } +} + +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap16, +}; + +static void tgen_bswap32(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_rev(s, TCG_TYPE_I32, MO_32, a0, a1); + if (flags & TCG_BSWAP_OS) { + tcg_out_ext32s(s, a0, a0); + } +} + +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap32, +}; + +static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_rev(s, TCG_TYPE_I64, MO_64, a0, a1); +} + +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap64, +}; + +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_sub(s, type, a0, TCG_REG_XZR, a1); +} + +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_neg, +}; + +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_orc(s, type, a0, TCG_REG_XZR, a1); +} + +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_not, +}; + +static void tgen_cset(TCGContext *s, TCGCond cond, TCGReg ret) +{ + /* Use CSET alias of CSINC Wd, WZR, WZR, invert(cond). */ + tcg_out_insn(s, 3506, CSINC, TCG_TYPE_I32, ret, TCG_REG_XZR, + TCG_REG_XZR, tcg_invert_cond(cond)); +} + +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_cmp(s, type, cond, a1, a2); + tgen_cset(s, cond, a0); +} + +static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_cmpi(s, type, cond, a1, a2); + tgen_cset(s, cond, a0); +} + +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(r, r, rC), + .out_rrr = tgen_setcond, + .out_rri = tgen_setcondi, +}; + +static void tgen_csetm(TCGContext *s, TCGType ext, TCGCond cond, TCGReg ret) +{ + /* Use CSETM alias of CSINV Wd, WZR, WZR, invert(cond). */ + tcg_out_insn(s, 3506, CSINV, ext, ret, TCG_REG_XZR, + TCG_REG_XZR, tcg_invert_cond(cond)); +} + +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_cmp(s, type, cond, a1, a2); + tgen_csetm(s, type, cond, a0); +} + +static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_cmpi(s, type, cond, a1, a2); + tgen_csetm(s, type, cond, a0); +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(r, r, rC), + .out_rrr = tgen_negsetcond, + .out_rri = tgen_negsetcondi, +}; + +static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, + TCGArg vt, bool const_vt, TCGArg vf, bool const_vf) +{ + tcg_out_cmp(s, type, cond, c1, c2, const_c2); + tcg_out_insn(s, 3506, CSEL, type, ret, vt, vf, cond); +} + +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = C_O1_I4(r, r, rC, rz, rz), + .out = tgen_movcond, +}; + +static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned ofs, unsigned len) +{ + unsigned mask = type == TCG_TYPE_I32 ? 31 : 63; + + /* + * Since we can't support "0Z" as a constraint, we allow a1 in + * any register. Fix things up as if a matching constraint. + */ + if (a0 != a1) { + if (a0 == a2) { + tcg_out_mov(s, type, TCG_REG_TMP0, a2); + a2 = TCG_REG_TMP0; + } + tcg_out_mov(s, type, a0, a1); } + tcg_out_bfm(s, type, a0, a2, -ofs & mask, len - 1); +} + +static void tgen_depositi(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + tcg_target_long a2, unsigned ofs, unsigned len) +{ + tgen_andi(s, type, a0, a1, ~MAKE_64BIT_MASK(ofs, len)); } +static void tgen_depositz(TCGContext *s, TCGType type, TCGReg a0, TCGReg a2, + unsigned ofs, unsigned len) +{ + int max = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_ubfm(s, type, a0, a2, -ofs & max, len - 1); +} + +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_O1_I2(r, rZ, rZ), + .out_rrr = tgen_deposit, + .out_rri = tgen_depositi, + .out_rzr = tgen_depositz, +}; + +static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0) { + uint64_t mask = MAKE_64BIT_MASK(0, len); + tcg_out_logicali(s, I3404_ANDI, type, a0, a1, mask); + } else { + tcg_out_ubfm(s, type, a0, a1, ofs, ofs + len - 1); + } +} + +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extract, +}; + +static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + tcg_out_sbfm(s, type, a0, a1, ofs, ofs + len - 1); +} + +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_sextract, +}; + +static void tgen_extract2(TCGContext *s, TCGType type, TCGReg a0, + TCGReg a1, TCGReg a2, unsigned shr) +{ + tcg_out_extr(s, type, a0, a2, a1, shr); +} + +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_O1_I2(r, rz, rz), + .out_rrr = tgen_extract2, +}; + +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, I3312_LDRB, dest, base, offset, 0); +} + +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; + +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + AArch64Insn insn = type == TCG_TYPE_I32 ? I3312_LDRSBW : I3312_LDRSBX; + tcg_out_ldst(s, insn, dest, base, offset, 0); +} + +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; + +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, I3312_LDRH, dest, base, offset, 1); +} + +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; + +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + AArch64Insn insn = type == TCG_TYPE_I32 ? I3312_LDRSHW : I3312_LDRSHX; + tcg_out_ldst(s, insn, dest, base, offset, 1); +} + +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; + +static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, I3312_LDRW, dest, base, offset, 2); +} + +static const TCGOutOpLoad outop_ld32u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32u, +}; + +static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, I3312_LDRSWX, dest, base, offset, 2); +} + +static const TCGOutOpLoad outop_ld32s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32s, +}; + +static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, I3312_STRB, data, base, offset, 0); +} + +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st8_r, +}; + +static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, I3312_STRH, data, base, offset, 1); +} + +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st16_r, +}; + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tcg_out_st, +}; + static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg args[TCG_MAX_OP_ARGS], @@ -2955,148 +3341,6 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) { switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_ld8u_i32: - case INDEX_op_ld8s_i32: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16s_i32: - case INDEX_op_ld_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i64: - case INDEX_op_ld32u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld_i64: - case INDEX_op_neg_i32: - case INDEX_op_neg_i64: - case INDEX_op_not_i32: - case INDEX_op_not_i64: - case INDEX_op_bswap16_i32: - case INDEX_op_bswap32_i32: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: - case INDEX_op_ext8s_i32: - case INDEX_op_ext16s_i32: - case INDEX_op_ext8u_i32: - case INDEX_op_ext16u_i32: - case INDEX_op_ext8s_i64: - case INDEX_op_ext16s_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extract_i32: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i32: - case INDEX_op_sextract_i64: - return C_O1_I1(r, r); - - case INDEX_op_st8_i32: - case INDEX_op_st16_i32: - case INDEX_op_st_i32: - case INDEX_op_st8_i64: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: - return C_O0_I2(rz, r); - - case INDEX_op_add_i32: - case INDEX_op_add_i64: - case INDEX_op_sub_i32: - case INDEX_op_sub_i64: - return C_O1_I2(r, r, rA); - - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - case INDEX_op_negsetcond_i32: - case INDEX_op_negsetcond_i64: - return C_O1_I2(r, r, rC); - - case INDEX_op_mul_i32: - case INDEX_op_mul_i64: - case INDEX_op_div_i32: - case INDEX_op_div_i64: - case INDEX_op_divu_i32: - case INDEX_op_divu_i64: - case INDEX_op_rem_i32: - case INDEX_op_rem_i64: - case INDEX_op_remu_i32: - case INDEX_op_remu_i64: - case INDEX_op_muluh_i64: - case INDEX_op_mulsh_i64: - return C_O1_I2(r, r, r); - - case INDEX_op_and_i32: - case INDEX_op_and_i64: - case INDEX_op_or_i32: - case INDEX_op_or_i64: - case INDEX_op_xor_i32: - case INDEX_op_xor_i64: - case INDEX_op_andc_i32: - case INDEX_op_andc_i64: - case INDEX_op_orc_i32: - case INDEX_op_orc_i64: - case INDEX_op_eqv_i32: - case INDEX_op_eqv_i64: - return C_O1_I2(r, r, rL); - - case INDEX_op_shl_i32: - case INDEX_op_shr_i32: - case INDEX_op_sar_i32: - case INDEX_op_rotl_i32: - case INDEX_op_rotr_i32: - case INDEX_op_shl_i64: - case INDEX_op_shr_i64: - case INDEX_op_sar_i64: - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i64: - return C_O1_I2(r, r, ri); - - case INDEX_op_clz_i32: - case INDEX_op_ctz_i32: - case INDEX_op_clz_i64: - case INDEX_op_ctz_i64: - return C_O1_I2(r, r, rAL); - - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - return C_O0_I2(r, rC); - - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - return C_O1_I4(r, r, rC, rz, rz); - - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: - return C_O1_I1(r, r); - case INDEX_op_qemu_ld_i128: - return C_O2_I1(r, r, r); - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: - return C_O0_I2(rz, r); - case INDEX_op_qemu_st_i128: - return C_O0_I3(rz, rz, r); - - case INDEX_op_deposit_i32: - case INDEX_op_deposit_i64: - return C_O1_I2(r, 0, rz); - - case INDEX_op_extract2_i32: - case INDEX_op_extract2_i64: - return C_O1_I2(r, rz, rz); - - case INDEX_op_add2_i32: - case INDEX_op_add2_i64: - case INDEX_op_sub2_i32: - case INDEX_op_sub2_i64: - return C_O2_I4(r, r, rz, rz, rA, rMZ); - case INDEX_op_add_vec: case INDEX_op_sub_vec: case INDEX_op_mul_vec: diff --git a/tcg/arm/tcg-target-con-set.h b/tcg/arm/tcg-target-con-set.h index 229ae258ac..16b1193228 100644 --- a/tcg/arm/tcg-target-con-set.h +++ b/tcg/arm/tcg-target-con-set.h @@ -30,6 +30,9 @@ C_O1_I2(r, r, rI) C_O1_I2(r, r, rIK) C_O1_I2(r, r, rIN) C_O1_I2(r, r, ri) +C_O1_I2(r, rI, r) +C_O1_I2(r, rI, rIK) +C_O1_I2(r, rI, rIN) C_O1_I2(r, rZ, rZ) C_O1_I2(w, 0, w) C_O1_I2(w, w, w) @@ -42,5 +45,3 @@ C_O1_I4(r, r, rIN, rIK, 0) C_O2_I1(e, p, q) C_O2_I2(e, p, q, q) C_O2_I2(r, r, r, r) -C_O2_I4(r, r, r, r, rIN, rIK) -C_O2_I4(r, r, rI, rI, rIN, rIK) diff --git a/tcg/arm/tcg-target-has.h b/tcg/arm/tcg-target-has.h index e3510a8f7a..3bbbde5d59 100644 --- a/tcg/arm/tcg-target-has.h +++ b/tcg/arm/tcg-target-has.h @@ -24,34 +24,7 @@ extern bool use_neon_instructions; #endif /* optional instructions */ -#define TCG_TARGET_HAS_ext8s_i32 1 -#define TCG_TARGET_HAS_ext16s_i32 1 -#define TCG_TARGET_HAS_ext8u_i32 0 /* and r0, r1, #0xff */ -#define TCG_TARGET_HAS_ext16u_i32 1 -#define TCG_TARGET_HAS_bswap16_i32 1 -#define TCG_TARGET_HAS_bswap32_i32 1 -#define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_rot_i32 1 -#define TCG_TARGET_HAS_andc_i32 1 -#define TCG_TARGET_HAS_orc_i32 0 -#define TCG_TARGET_HAS_eqv_i32 0 -#define TCG_TARGET_HAS_nand_i32 0 -#define TCG_TARGET_HAS_nor_i32 0 -#define TCG_TARGET_HAS_clz_i32 1 -#define TCG_TARGET_HAS_ctz_i32 use_armv7_instructions -#define TCG_TARGET_HAS_ctpop_i32 0 -#define TCG_TARGET_HAS_extract2_i32 1 -#define TCG_TARGET_HAS_negsetcond_i32 1 -#define TCG_TARGET_HAS_mulu2_i32 1 -#define TCG_TARGET_HAS_muls2_i32 1 -#define TCG_TARGET_HAS_muluh_i32 0 -#define TCG_TARGET_HAS_mulsh_i32 0 -#define TCG_TARGET_HAS_div_i32 use_idiv_instructions -#define TCG_TARGET_HAS_rem_i32 0 -#define TCG_TARGET_HAS_qemu_st8_i32 0 - #define TCG_TARGET_HAS_qemu_ldst_i128 0 - #define TCG_TARGET_HAS_tst 1 #define TCG_TARGET_HAS_v64 use_neon_instructions diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index cec3d761d4..447e43583e 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -178,6 +178,8 @@ typedef enum { INSN_DMB_ISH = 0xf57ff05b, INSN_DMB_MCR = 0xee070fba, + INSN_MSRI_CPSR = 0x0360f000, + /* Architected nop introduced in v6k. */ /* ??? This is an MSR (imm) 0,0,0 insn. Anyone know if this also Just So Happened to do nothing on pre-v6k so that we @@ -874,22 +876,39 @@ static void tcg_out_dat_rI(TCGContext *s, ARMCond cond, ARMInsn opc, * Emit either the reg,imm or reg,reg form of a data-processing insn. * rhs must satisfy the "rIK" constraint. */ +static void tcg_out_dat_IK(TCGContext *s, ARMCond cond, ARMInsn opc, + ARMInsn opinv, TCGReg dst, TCGReg lhs, TCGArg rhs) +{ + int imm12 = encode_imm(rhs); + if (imm12 < 0) { + imm12 = encode_imm_nofail(~rhs); + opc = opinv; + } + tcg_out_dat_imm(s, cond, opc, dst, lhs, imm12); +} + static void tcg_out_dat_rIK(TCGContext *s, ARMCond cond, ARMInsn opc, ARMInsn opinv, TCGReg dst, TCGReg lhs, TCGArg rhs, bool rhs_is_const) { if (rhs_is_const) { - int imm12 = encode_imm(rhs); - if (imm12 < 0) { - imm12 = encode_imm_nofail(~rhs); - opc = opinv; - } - tcg_out_dat_imm(s, cond, opc, dst, lhs, imm12); + tcg_out_dat_IK(s, cond, opc, opinv, dst, lhs, rhs); } else { tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0)); } } +static void tcg_out_dat_IN(TCGContext *s, ARMCond cond, ARMInsn opc, + ARMInsn opneg, TCGReg dst, TCGReg lhs, TCGArg rhs) +{ + int imm12 = encode_imm(rhs); + if (imm12 < 0) { + imm12 = encode_imm_nofail(-rhs); + opc = opneg; + } + tcg_out_dat_imm(s, cond, opc, dst, lhs, imm12); +} + static void tcg_out_dat_rIN(TCGContext *s, ARMCond cond, ARMInsn opc, ARMInsn opneg, TCGReg dst, TCGReg lhs, TCGArg rhs, bool rhs_is_const) @@ -898,52 +917,12 @@ static void tcg_out_dat_rIN(TCGContext *s, ARMCond cond, ARMInsn opc, * rhs must satisfy the "rIN" constraint. */ if (rhs_is_const) { - int imm12 = encode_imm(rhs); - if (imm12 < 0) { - imm12 = encode_imm_nofail(-rhs); - opc = opneg; - } - tcg_out_dat_imm(s, cond, opc, dst, lhs, imm12); + tcg_out_dat_IN(s, cond, opc, opneg, dst, lhs, rhs); } else { tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0)); } } -static void tcg_out_mul32(TCGContext *s, ARMCond cond, TCGReg rd, - TCGReg rn, TCGReg rm) -{ - /* mul */ - tcg_out32(s, (cond << 28) | 0x90 | (rd << 16) | (rm << 8) | rn); -} - -static void tcg_out_umull32(TCGContext *s, ARMCond cond, TCGReg rd0, - TCGReg rd1, TCGReg rn, TCGReg rm) -{ - /* umull */ - tcg_out32(s, (cond << 28) | 0x00800090 | - (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn); -} - -static void tcg_out_smull32(TCGContext *s, ARMCond cond, TCGReg rd0, - TCGReg rd1, TCGReg rn, TCGReg rm) -{ - /* smull */ - tcg_out32(s, (cond << 28) | 0x00c00090 | - (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn); -} - -static void tcg_out_sdiv(TCGContext *s, ARMCond cond, - TCGReg rd, TCGReg rn, TCGReg rm) -{ - tcg_out32(s, 0x0710f010 | (cond << 28) | (rd << 16) | rn | (rm << 8)); -} - -static void tcg_out_udiv(TCGContext *s, ARMCond cond, - TCGReg rd, TCGReg rn, TCGReg rm) -{ - tcg_out32(s, 0x0730f010 | (cond << 28) | (rd << 16) | rn | (rm << 8)); -} - static void tcg_out_ext8s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn) { /* sxtb */ @@ -992,54 +971,40 @@ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn) g_assert_not_reached(); } -static void tcg_out_bswap16(TCGContext *s, ARMCond cond, - TCGReg rd, TCGReg rn, int flags) +static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned ofs, unsigned len) { - if (flags & TCG_BSWAP_OS) { - /* revsh */ - tcg_out32(s, 0x06ff0fb0 | (cond << 28) | (rd << 12) | rn); - return; - } - - /* rev16 */ - tcg_out32(s, 0x06bf0fb0 | (cond << 28) | (rd << 12) | rn); - if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - /* uxth */ - tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | rd); - } + /* bfi/bfc */ + tcg_out32(s, 0x07c00010 | (COND_AL << 28) | (a0 << 12) | a1 + | (ofs << 7) | ((ofs + len - 1) << 16)); } -static void tcg_out_bswap32(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn) +static void tgen_depositi(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + tcg_target_long a2, unsigned ofs, unsigned len) { - /* rev */ - tcg_out32(s, 0x06bf0f30 | (cond << 28) | (rd << 12) | rn); + /* bfi becomes bfc with rn == 15. */ + tgen_deposit(s, type, a0, a1, 15, ofs, len); } -static void tcg_out_deposit(TCGContext *s, ARMCond cond, TCGReg rd, - TCGArg a1, int ofs, int len, bool const_a1) -{ - if (const_a1) { - /* bfi becomes bfc with rn == 15. */ - a1 = 15; - } - /* bfi/bfc */ - tcg_out32(s, 0x07c00010 | (cond << 28) | (rd << 12) | a1 - | (ofs << 7) | ((ofs + len - 1) << 16)); -} +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_O1_I2(r, 0, rZ), + .out_rrr = tgen_deposit, + .out_rri = tgen_depositi, +}; -static void tcg_out_extract(TCGContext *s, ARMCond cond, TCGReg rd, - TCGReg rn, int ofs, int len) +static void tgen_extract(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn, + unsigned ofs, unsigned len) { /* According to gcc, AND can be faster. */ if (ofs == 0 && len <= 8) { - tcg_out_dat_imm(s, cond, ARITH_AND, rd, rn, + tcg_out_dat_imm(s, COND_AL, ARITH_AND, rd, rn, encode_imm_nofail((1 << len) - 1)); return; } if (use_armv7_instructions) { /* ubfx */ - tcg_out32(s, 0x07e00050 | (cond << 28) | (rd << 12) | rn + tcg_out32(s, 0x07e00050 | (COND_AL << 28) | (rd << 12) | rn | (ofs << 7) | ((len - 1) << 16)); return; } @@ -1048,23 +1013,30 @@ static void tcg_out_extract(TCGContext *s, ARMCond cond, TCGReg rd, switch (len) { case 8: /* uxtb */ - tcg_out32(s, 0x06ef0070 | (cond << 28) | (rd << 12) | (ofs << 7) | rn); + tcg_out32(s, 0x06ef0070 | (COND_AL << 28) | + (rd << 12) | (ofs << 7) | rn); break; case 16: /* uxth */ - tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | (ofs << 7) | rn); + tcg_out32(s, 0x06ff0070 | (COND_AL << 28) | + (rd << 12) | (ofs << 7) | rn); break; default: g_assert_not_reached(); } } -static void tcg_out_sextract(TCGContext *s, ARMCond cond, TCGReg rd, - TCGReg rn, int ofs, int len) +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extract, +}; + +static void tgen_sextract(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn, + unsigned ofs, unsigned len) { if (use_armv7_instructions) { /* sbfx */ - tcg_out32(s, 0x07a00050 | (cond << 28) | (rd << 12) | rn + tcg_out32(s, 0x07a00050 | (COND_AL << 28) | (rd << 12) | rn | (ofs << 7) | ((len - 1) << 16)); return; } @@ -1073,17 +1045,24 @@ static void tcg_out_sextract(TCGContext *s, ARMCond cond, TCGReg rd, switch (len) { case 8: /* sxtb */ - tcg_out32(s, 0x06af0070 | (cond << 28) | (rd << 12) | (ofs << 7) | rn); + tcg_out32(s, 0x06af0070 | (COND_AL << 28) | + (rd << 12) | (ofs << 7) | rn); break; case 16: /* sxth */ - tcg_out32(s, 0x06bf0070 | (cond << 28) | (rd << 12) | (ofs << 7) | rn); + tcg_out32(s, 0x06bf0070 | (COND_AL << 28) | + (rd << 12) | (ofs << 7) | rn); break; default: g_assert_not_reached(); } } +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_sextract, +}; + static void tcg_out_ld32u(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rn, int32_t offset) @@ -1105,66 +1084,6 @@ static void tcg_out_st32(TCGContext *s, ARMCond cond, tcg_out_st32_12(s, cond, rd, rn, offset); } -static void tcg_out_ld16u(TCGContext *s, ARMCond cond, - TCGReg rd, TCGReg rn, int32_t offset) -{ - if (offset > 0xff || offset < -0xff) { - tcg_out_movi32(s, cond, TCG_REG_TMP, offset); - tcg_out_ld16u_r(s, cond, rd, rn, TCG_REG_TMP); - } else - tcg_out_ld16u_8(s, cond, rd, rn, offset); -} - -static void tcg_out_ld16s(TCGContext *s, ARMCond cond, - TCGReg rd, TCGReg rn, int32_t offset) -{ - if (offset > 0xff || offset < -0xff) { - tcg_out_movi32(s, cond, TCG_REG_TMP, offset); - tcg_out_ld16s_r(s, cond, rd, rn, TCG_REG_TMP); - } else - tcg_out_ld16s_8(s, cond, rd, rn, offset); -} - -static void tcg_out_st16(TCGContext *s, ARMCond cond, - TCGReg rd, TCGReg rn, int32_t offset) -{ - if (offset > 0xff || offset < -0xff) { - tcg_out_movi32(s, cond, TCG_REG_TMP, offset); - tcg_out_st16_r(s, cond, rd, rn, TCG_REG_TMP); - } else - tcg_out_st16_8(s, cond, rd, rn, offset); -} - -static void tcg_out_ld8u(TCGContext *s, ARMCond cond, - TCGReg rd, TCGReg rn, int32_t offset) -{ - if (offset > 0xfff || offset < -0xfff) { - tcg_out_movi32(s, cond, TCG_REG_TMP, offset); - tcg_out_ld8_r(s, cond, rd, rn, TCG_REG_TMP); - } else - tcg_out_ld8_12(s, cond, rd, rn, offset); -} - -static void tcg_out_ld8s(TCGContext *s, ARMCond cond, - TCGReg rd, TCGReg rn, int32_t offset) -{ - if (offset > 0xff || offset < -0xff) { - tcg_out_movi32(s, cond, TCG_REG_TMP, offset); - tcg_out_ld8s_r(s, cond, rd, rn, TCG_REG_TMP); - } else - tcg_out_ld8s_8(s, cond, rd, rn, offset); -} - -static void tcg_out_st8(TCGContext *s, ARMCond cond, - TCGReg rd, TCGReg rn, int32_t offset) -{ - if (offset > 0xfff || offset < -0xfff) { - tcg_out_movi32(s, cond, TCG_REG_TMP, offset); - tcg_out_st8_r(s, cond, rd, rn, TCG_REG_TMP); - } else - tcg_out_st8_12(s, cond, rd, rn, offset); -} - /* * The _goto case is normally between TBs within the same code buffer, and * with the code buffer limited to 16MB we wouldn't need the long case. @@ -1224,7 +1143,12 @@ static void tcg_out_goto_label(TCGContext *s, ARMCond cond, TCGLabel *l) } } -static void tcg_out_mb(TCGContext *s, TCGArg a0) +static void tcg_out_br(TCGContext *s, TCGLabel *l) +{ + tcg_out_goto_label(s, COND_AL, l); +} + +static void tcg_out_mb(TCGContext *s, unsigned a0) { if (use_armv7_instructions) { tcg_out32(s, INSN_DMB_ISH); @@ -1233,44 +1157,53 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) } } -static TCGCond tcg_out_cmp(TCGContext *s, TCGCond cond, TCGReg a, - TCGArg b, int b_const) +static TCGCond tgen_cmp(TCGContext *s, TCGCond cond, TCGReg a, TCGReg b) +{ + if (is_tst_cond(cond)) { + tcg_out_dat_reg(s, COND_AL, ARITH_TST, 0, a, b, SHIFT_IMM_LSL(0)); + return tcg_tst_eqne_cond(cond); + } + tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0, a, b, SHIFT_IMM_LSL(0)); + return cond; +} + +static TCGCond tgen_cmpi(TCGContext *s, TCGCond cond, TCGReg a, TCGArg b) { + int imm12; + if (!is_tst_cond(cond)) { - tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0, a, b, b_const); + tcg_out_dat_IN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0, a, b); return cond; } - cond = tcg_tst_eqne_cond(cond); - if (b_const) { - int imm12 = encode_imm(b); - - /* - * The compare constraints allow rIN, but TST does not support N. - * Be prepared to load the constant into a scratch register. - */ - if (imm12 >= 0) { - tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, a, imm12); - return cond; - } + /* + * The compare constraints allow rIN, but TST does not support N. + * Be prepared to load the constant into a scratch register. + */ + imm12 = encode_imm(b); + if (imm12 >= 0) { + tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, a, imm12); + } else { tcg_out_movi32(s, COND_AL, TCG_REG_TMP, b); - b = TCG_REG_TMP; + tcg_out_dat_reg(s, COND_AL, ARITH_TST, 0, + a, TCG_REG_TMP, SHIFT_IMM_LSL(0)); } - tcg_out_dat_reg(s, COND_AL, ARITH_TST, 0, a, b, SHIFT_IMM_LSL(0)); - return cond; + return tcg_tst_eqne_cond(cond); } -static TCGCond tcg_out_cmp2(TCGContext *s, const TCGArg *args, - const int *const_args) +static TCGCond tcg_out_cmp(TCGContext *s, TCGCond cond, TCGReg a, + TCGArg b, int b_const) { - TCGReg al = args[0]; - TCGReg ah = args[1]; - TCGArg bl = args[2]; - TCGArg bh = args[3]; - TCGCond cond = args[4]; - int const_bl = const_args[2]; - int const_bh = const_args[3]; + if (b_const) { + return tgen_cmpi(s, cond, a, b); + } else { + return tgen_cmp(s, cond, a, b); + } +} +static TCGCond tcg_out_cmp2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, TCGArg bh, bool const_bh) +{ switch (cond) { case TCG_COND_EQ: case TCG_COND_NE: @@ -1653,8 +1586,42 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, } } -static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg addr, MemOpIdx oi, TCGType data_type) +static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + MemOp opc = get_memop(oi); + TCGLabelQemuLdst *ldst; + HostAddress h; + + ldst = prepare_host_addr(s, &h, addr, oi, true); + if (ldst) { + ldst->type = type; + ldst->datalo_reg = data; + ldst->datahi_reg = -1; + + /* + * This a conditional BL only to load a pointer within this + * opcode into LR for the slow path. We will not be using + * the value for a tail call. + */ + ldst->label_ptr[0] = s->code_ptr; + tcg_out_bl_imm(s, COND_NE, 0); + } + + tcg_out_qemu_ld_direct(s, opc, data, -1, h); + + if (ldst) { + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); + } +} + +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, q), + .out = tgen_qemu_ld, +}; + +static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) { MemOp opc = get_memop(oi); TCGLabelQemuLdst *ldst; @@ -1662,7 +1629,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, ldst = prepare_host_addr(s, &h, addr, oi, true); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = datalo; ldst->datahi_reg = datahi; @@ -1673,14 +1640,20 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, */ ldst->label_ptr[0] = s->code_ptr; tcg_out_bl_imm(s, COND_NE, 0); + } + + tcg_out_qemu_ld_direct(s, opc, datalo, datahi, h); - tcg_out_qemu_ld_direct(s, opc, datalo, datahi, h); + if (ldst) { ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); - } else { - tcg_out_qemu_ld_direct(s, opc, datalo, datahi, h); } } +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + .base.static_constraint = C_O2_I1(e, p, q), + .out = tgen_qemu_ld2, +}; + static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, TCGReg datahi, HostAddress h) { @@ -1738,8 +1711,38 @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, } } -static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg addr, MemOpIdx oi, TCGType data_type) +static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + MemOp opc = get_memop(oi); + TCGLabelQemuLdst *ldst; + HostAddress h; + + ldst = prepare_host_addr(s, &h, addr, oi, false); + if (ldst) { + ldst->type = type; + ldst->datalo_reg = data; + ldst->datahi_reg = -1; + + h.cond = COND_EQ; + tcg_out_qemu_st_direct(s, opc, data, -1, h); + + /* The conditional call is last, as we're going to return here. */ + ldst->label_ptr[0] = s->code_ptr; + tcg_out_bl_imm(s, COND_NE, 0); + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); + } else { + tcg_out_qemu_st_direct(s, opc, data, -1, h); + } +} + +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = C_O0_I2(q, q), + .out = tgen_qemu_st, +}; + +static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) { MemOp opc = get_memop(oi); TCGLabelQemuLdst *ldst; @@ -1747,7 +1750,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, ldst = prepare_host_addr(s, &h, addr, oi, false); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = datalo; ldst->datahi_reg = datahi; @@ -1763,6 +1766,11 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, } } +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + .base.static_constraint = C_O0_I3(Q, p, q), + .out = tgen_qemu_st2, +}; + static void tcg_out_epilogue(TCGContext *s); static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) @@ -1802,6 +1810,11 @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + tcg_out_b_reg(s, COND_AL, a0); +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { @@ -1821,393 +1834,816 @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, flush_idcache_range(jmp_rx, jmp_rw, 4); } -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) + +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - TCGArg a0, a1, a2, a3, a4, a5; - int c; + tcg_out_dat_reg(s, COND_AL, ARITH_ADD, a0, a1, a2, SHIFT_IMM_LSL(0)); +} - switch (opc) { - case INDEX_op_goto_ptr: - tcg_out_b_reg(s, COND_AL, args[0]); - break; - case INDEX_op_br: - tcg_out_goto_label(s, COND_AL, arg_label(args[0])); - break; +static void tgen_addi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_IN(s, COND_AL, ARITH_ADD, ARITH_SUB, a0, a1, a2); +} - case INDEX_op_ld8u_i32: - tcg_out_ld8u(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_ld8s_i32: - tcg_out_ld8s(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_ld16u_i32: - tcg_out_ld16u(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_ld16s_i32: - tcg_out_ld16s(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_ld_i32: - tcg_out_ld32u(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_st8_i32: - tcg_out_st8(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_st16_i32: - tcg_out_st16(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_st_i32: - tcg_out_st32(s, COND_AL, args[0], args[1], args[2]); - break; +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, rIN), + .out_rrr = tgen_add, + .out_rri = tgen_addi, +}; - case INDEX_op_movcond_i32: - /* Constraints mean that v2 is always in the same register as dest, - * so we only need to do "if condition passed, move v1 to dest". - */ - c = tcg_out_cmp(s, args[5], args[1], args[2], const_args[2]); - tcg_out_dat_rIK(s, tcg_cond_to_arm_cond[c], ARITH_MOV, - ARITH_MVN, args[0], 0, args[3], const_args[3]); - break; - case INDEX_op_add_i32: - tcg_out_dat_rIN(s, COND_AL, ARITH_ADD, ARITH_SUB, - args[0], args[1], args[2], const_args[2]); - break; - case INDEX_op_sub_i32: - if (const_args[1]) { - if (const_args[2]) { - tcg_out_movi32(s, COND_AL, args[0], args[1] - args[2]); - } else { - tcg_out_dat_rI(s, COND_AL, ARITH_RSB, - args[0], args[2], args[1], 1); - } - } else { - tcg_out_dat_rIN(s, COND_AL, ARITH_SUB, ARITH_ADD, - args[0], args[1], args[2], const_args[2]); - } - break; - case INDEX_op_and_i32: - tcg_out_dat_rIK(s, COND_AL, ARITH_AND, ARITH_BIC, - args[0], args[1], args[2], const_args[2]); - break; - case INDEX_op_andc_i32: - tcg_out_dat_rIK(s, COND_AL, ARITH_BIC, ARITH_AND, - args[0], args[1], args[2], const_args[2]); - break; - case INDEX_op_or_i32: - c = ARITH_ORR; - goto gen_arith; - case INDEX_op_xor_i32: - c = ARITH_EOR; - /* Fall through. */ - gen_arith: - tcg_out_dat_rI(s, COND_AL, c, args[0], args[1], args[2], const_args[2]); - break; - case INDEX_op_add2_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; - a3 = args[3], a4 = args[4], a5 = args[5]; - if (a0 == a3 || (a0 == a5 && !const_args[5])) { - a0 = TCG_REG_TMP; - } - tcg_out_dat_rIN(s, COND_AL, ARITH_ADD | TO_CPSR, ARITH_SUB | TO_CPSR, - a0, a2, a4, const_args[4]); - tcg_out_dat_rIK(s, COND_AL, ARITH_ADC, ARITH_SBC, - a1, a3, a5, const_args[5]); - tcg_out_mov_reg(s, COND_AL, args[0], a0); - break; - case INDEX_op_sub2_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; - a3 = args[3], a4 = args[4], a5 = args[5]; - if ((a0 == a3 && !const_args[3]) || (a0 == a5 && !const_args[5])) { - a0 = TCG_REG_TMP; - } - if (const_args[2]) { - if (const_args[4]) { - tcg_out_movi32(s, COND_AL, a0, a4); - a4 = a0; - } - tcg_out_dat_rI(s, COND_AL, ARITH_RSB | TO_CPSR, a0, a4, a2, 1); - } else { - tcg_out_dat_rIN(s, COND_AL, ARITH_SUB | TO_CPSR, - ARITH_ADD | TO_CPSR, a0, a2, a4, const_args[4]); - } - if (const_args[3]) { - if (const_args[5]) { - tcg_out_movi32(s, COND_AL, a1, a5); - a5 = a1; - } - tcg_out_dat_rI(s, COND_AL, ARITH_RSC, a1, a5, a3, 1); - } else { - tcg_out_dat_rIK(s, COND_AL, ARITH_SBC, ARITH_ADC, - a1, a3, a5, const_args[5]); - } - tcg_out_mov_reg(s, COND_AL, args[0], a0); - break; - case INDEX_op_neg_i32: - tcg_out_dat_imm(s, COND_AL, ARITH_RSB, args[0], args[1], 0); - break; - case INDEX_op_not_i32: - tcg_out_dat_reg(s, COND_AL, - ARITH_MVN, args[0], 0, args[1], SHIFT_IMM_LSL(0)); - break; - case INDEX_op_mul_i32: - tcg_out_mul32(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_mulu2_i32: - tcg_out_umull32(s, COND_AL, args[0], args[1], args[2], args[3]); - break; - case INDEX_op_muls2_i32: - tcg_out_smull32(s, COND_AL, args[0], args[1], args[2], args[3]); - break; - /* XXX: Perhaps args[2] & 0x1f is wrong */ - case INDEX_op_shl_i32: - c = const_args[2] ? - SHIFT_IMM_LSL(args[2] & 0x1f) : SHIFT_REG_LSL(args[2]); - goto gen_shift32; - case INDEX_op_shr_i32: - c = const_args[2] ? (args[2] & 0x1f) ? SHIFT_IMM_LSR(args[2] & 0x1f) : - SHIFT_IMM_LSL(0) : SHIFT_REG_LSR(args[2]); - goto gen_shift32; - case INDEX_op_sar_i32: - c = const_args[2] ? (args[2] & 0x1f) ? SHIFT_IMM_ASR(args[2] & 0x1f) : - SHIFT_IMM_LSL(0) : SHIFT_REG_ASR(args[2]); - goto gen_shift32; - case INDEX_op_rotr_i32: - c = const_args[2] ? (args[2] & 0x1f) ? SHIFT_IMM_ROR(args[2] & 0x1f) : - SHIFT_IMM_LSL(0) : SHIFT_REG_ROR(args[2]); - /* Fall through. */ - gen_shift32: - tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, args[1], c); - break; +static void tgen_addco(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_ADD | TO_CPSR, + a0, a1, a2, SHIFT_IMM_LSL(0)); +} - case INDEX_op_rotl_i32: - if (const_args[2]) { - tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, args[1], - ((0x20 - args[2]) & 0x1f) ? - SHIFT_IMM_ROR((0x20 - args[2]) & 0x1f) : - SHIFT_IMM_LSL(0)); - } else { - tcg_out_dat_imm(s, COND_AL, ARITH_RSB, TCG_REG_TMP, args[2], 0x20); - tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, args[1], - SHIFT_REG_ROR(TCG_REG_TMP)); - } - break; +static void tgen_addco_imm(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_IN(s, COND_AL, ARITH_ADD | TO_CPSR, ARITH_SUB | TO_CPSR, + a0, a1, a2); +} - case INDEX_op_ctz_i32: - tcg_out_dat_reg(s, COND_AL, INSN_RBIT, TCG_REG_TMP, 0, args[1], 0); - a1 = TCG_REG_TMP; - goto do_clz; - - case INDEX_op_clz_i32: - a1 = args[1]; - do_clz: - a0 = args[0]; - a2 = args[2]; - c = const_args[2]; - if (c && a2 == 32) { - tcg_out_dat_reg(s, COND_AL, INSN_CLZ, a0, 0, a1, 0); - break; - } +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_O1_I2(r, r, rIN), + .out_rrr = tgen_addco, + .out_rri = tgen_addco_imm, +}; + +static void tgen_addci(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_ADC, a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static void tgen_addci_imm(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_IK(s, COND_AL, ARITH_ADC, ARITH_SBC, a0, a1, a2); +} + +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_O1_I2(r, r, rIK), + .out_rrr = tgen_addci, + .out_rri = tgen_addci_imm, +}; + +static void tgen_addcio(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_ADC | TO_CPSR, + a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static void tgen_addcio_imm(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_IK(s, COND_AL, ARITH_ADC | TO_CPSR, ARITH_SBC | TO_CPSR, + a0, a1, a2); +} + +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_O1_I2(r, r, rIK), + .out_rrr = tgen_addcio, + .out_rri = tgen_addcio_imm, +}; + +/* Set C to @c; NZVQ all set to 0. */ +static void tcg_out_movi_apsr_c(TCGContext *s, bool c) +{ + int imm12 = encode_imm_nofail(c << 29); + tcg_out32(s, (COND_AL << 28) | INSN_MSRI_CPSR | 0x80000 | imm12); +} + +static void tcg_out_set_carry(TCGContext *s) +{ + tcg_out_movi_apsr_c(s, 1); +} + +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_AND, a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static void tgen_andi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_IK(s, COND_AL, ARITH_AND, ARITH_BIC, a0, a1, a2); +} + +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, r, rIK), + .out_rrr = tgen_and, + .out_rri = tgen_andi, +}; + +static void tgen_andc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_BIC, a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_andc, +}; + +static void tgen_clz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_imm(s, COND_AL, ARITH_CMP, 0, a1, 0); + tcg_out_dat_reg(s, COND_NE, INSN_CLZ, a0, 0, a1, 0); + tcg_out_mov_reg(s, COND_EQ, a0, a2); +} + +static void tgen_clzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (a2 == 32) { + tcg_out_dat_reg(s, COND_AL, INSN_CLZ, a0, 0, a1, 0); + } else { tcg_out_dat_imm(s, COND_AL, ARITH_CMP, 0, a1, 0); tcg_out_dat_reg(s, COND_NE, INSN_CLZ, a0, 0, a1, 0); - if (c || a0 != a2) { - tcg_out_dat_rIK(s, COND_EQ, ARITH_MOV, ARITH_MVN, a0, 0, a2, c); - } - break; + tcg_out_movi32(s, COND_EQ, a0, a2); + } +} - case INDEX_op_brcond_i32: - c = tcg_out_cmp(s, args[2], args[0], args[1], const_args[1]); - tcg_out_goto_label(s, tcg_cond_to_arm_cond[c], arg_label(args[3])); - break; - case INDEX_op_setcond_i32: - c = tcg_out_cmp(s, args[3], args[1], args[2], const_args[2]); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[c], - ARITH_MOV, args[0], 0, 1); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(c)], - ARITH_MOV, args[0], 0, 0); - break; - case INDEX_op_negsetcond_i32: - c = tcg_out_cmp(s, args[3], args[1], args[2], const_args[2]); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[c], - ARITH_MVN, args[0], 0, 0); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(c)], - ARITH_MOV, args[0], 0, 0); - break; +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_O1_I2(r, r, rIK), + .out_rrr = tgen_clz, + .out_rri = tgen_clzi, +}; - case INDEX_op_brcond2_i32: - c = tcg_out_cmp2(s, args, const_args); - tcg_out_goto_label(s, tcg_cond_to_arm_cond[c], arg_label(args[5])); - break; - case INDEX_op_setcond2_i32: - c = tcg_out_cmp2(s, args + 1, const_args + 1); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[c], ARITH_MOV, args[0], 0, 1); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(c)], - ARITH_MOV, args[0], 0, 0); - break; +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], TCG_TYPE_I32); - break; - case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args[0], args[1], args[2], args[3], TCG_TYPE_I64); - break; +static void tgen_ctz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, INSN_RBIT, TCG_REG_TMP, 0, a1, 0); + tgen_clz(s, TCG_TYPE_I32, a0, TCG_REG_TMP, a2); +} - case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args[0], -1, args[1], args[2], TCG_TYPE_I32); - break; - case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args[0], args[1], args[2], args[3], TCG_TYPE_I64); - break; +static void tgen_ctzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_reg(s, COND_AL, INSN_RBIT, TCG_REG_TMP, 0, a1, 0); + tgen_clzi(s, TCG_TYPE_I32, a0, TCG_REG_TMP, a2); +} - case INDEX_op_bswap16_i32: - tcg_out_bswap16(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_bswap32_i32: - tcg_out_bswap32(s, COND_AL, args[0], args[1]); - break; +static TCGConstraintSetIndex cset_ctz(TCGType type, unsigned flags) +{ + return use_armv7_instructions ? C_O1_I2(r, r, rIK) : C_NotImplemented; +} - case INDEX_op_deposit_i32: - tcg_out_deposit(s, COND_AL, args[0], args[2], - args[3], args[4], const_args[2]); - break; - case INDEX_op_extract_i32: - tcg_out_extract(s, COND_AL, args[0], args[1], args[2], args[3]); - break; - case INDEX_op_sextract_i32: - tcg_out_sextract(s, COND_AL, args[0], args[1], args[2], args[3]); - break; - case INDEX_op_extract2_i32: - /* ??? These optimization vs zero should be generic. */ - /* ??? But we can't substitute 2 for 1 in the opcode stream yet. */ - if (const_args[1]) { - if (const_args[2]) { - tcg_out_movi(s, TCG_TYPE_REG, args[0], 0); - } else { - tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, - args[2], SHIFT_IMM_LSL(32 - args[3])); - } - } else if (const_args[2]) { - tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, - args[1], SHIFT_IMM_LSR(args[3])); - } else { - /* We can do extract2 in 2 insns, vs the 3 required otherwise. */ - tcg_out_dat_reg(s, COND_AL, ARITH_MOV, TCG_REG_TMP, 0, - args[2], SHIFT_IMM_LSL(32 - args[3])); - tcg_out_dat_reg(s, COND_AL, ARITH_ORR, args[0], TCG_REG_TMP, - args[1], SHIFT_IMM_LSR(args[3])); - } - break; +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_ctz, + .out_rrr = tgen_ctz, + .out_rri = tgen_ctzi, +}; - case INDEX_op_div_i32: - tcg_out_sdiv(s, COND_AL, args[0], args[1], args[2]); - break; - case INDEX_op_divu_i32: - tcg_out_udiv(s, COND_AL, args[0], args[1], args[2]); - break; +static TCGConstraintSetIndex cset_idiv(TCGType type, unsigned flags) +{ + return use_idiv_instructions ? C_O1_I2(r, r, r) : C_NotImplemented; +} - case INDEX_op_mb: - tcg_out_mb(s, args[0]); - break; +static void tgen_divs(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + /* sdiv */ + tcg_out32(s, 0x0710f010 | (COND_AL << 28) | (a0 << 16) | a1 | (a2 << 8)); +} - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8u_i32: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16u_i32: - default: - g_assert_not_reached(); +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_idiv, + .out_rrr = tgen_divs, +}; + +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + /* udiv */ + tcg_out32(s, 0x0730f010 | (COND_AL << 28) | (a0 << 16) | a1 | (a2 << 8)); +} + +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_idiv, + .out_rrr = tgen_divu, +}; + +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + /* mul */ + tcg_out32(s, (COND_AL << 28) | 0x90 | (a0 << 16) | (a1 << 8) | a2); +} + +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_mul, +}; + +static void tgen_muls2(TCGContext *s, TCGType type, + TCGReg rd0, TCGReg rd1, TCGReg rn, TCGReg rm) +{ + /* smull */ + tcg_out32(s, (COND_AL << 28) | 0x00c00090 | + (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn); +} + +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_O2_I2(r, r, r, r), + .out_rrrr = tgen_muls2, +}; + +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_mulu2(TCGContext *s, TCGType type, + TCGReg rd0, TCGReg rd1, TCGReg rn, TCGReg rm) +{ + /* umull */ + tcg_out32(s, (COND_AL << 28) | 0x00800090 | + (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn); +} + +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_O2_I2(r, r, r, r), + .out_rrrr = tgen_mulu2, +}; + +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_ORR, a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static void tgen_ori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_imm(s, COND_AL, ARITH_ORR, a0, a1, encode_imm_nofail(a2)); +} + +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_or, + .out_rri = tgen_ori, +}; + +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_rotr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_REG_ROR(a2)); +} + +static void tgen_rotri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_IMM_ROR(a2 & 0x1f)); +} + +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_rotr, + .out_rri = tgen_rotri, +}; + +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_REG_ASR(a2)); +} + +static void tgen_sari(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, + SHIFT_IMM_ASR(a2 & 0x1f)); +} + +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_sar, + .out_rri = tgen_sari, +}; + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_REG_LSL(a2)); +} + +static void tgen_shli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, + SHIFT_IMM_LSL(a2 & 0x1f)); +} + +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shl, + .out_rri = tgen_shli, +}; + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_REG_LSR(a2)); +} + +static void tgen_shri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, + SHIFT_IMM_LSR(a2 & 0x1f)); +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shr, + .out_rri = tgen_shri, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_SUB, a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static void tgen_subfi(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2) +{ + tcg_out_dat_imm(s, COND_AL, ARITH_RSB, a0, a2, encode_imm_nofail(a1)); +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, rI, r), + .out_rrr = tgen_sub, + .out_rir = tgen_subfi, +}; + +static void tgen_subbo_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_SUB | TO_CPSR, + a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static void tgen_subbo_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_IN(s, COND_AL, ARITH_SUB | TO_CPSR, ARITH_ADD | TO_CPSR, + a0, a1, a2); +} + +static void tgen_subbo_rir(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2) +{ + tcg_out_dat_imm(s, COND_AL, ARITH_RSB | TO_CPSR, + a0, a2, encode_imm_nofail(a1)); +} + +static void tgen_subbo_rii(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, tcg_target_long a2) +{ + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, a2); + tgen_subbo_rir(s, TCG_TYPE_I32, a0, a1, TCG_REG_TMP); +} + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_O1_I2(r, rI, rIN), + .out_rrr = tgen_subbo_rrr, + .out_rri = tgen_subbo_rri, + .out_rir = tgen_subbo_rir, + .out_rii = tgen_subbo_rii, +}; + +static void tgen_subbi_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_SBC, + a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static void tgen_subbi_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_IK(s, COND_AL, ARITH_SBC, ARITH_ADC, a0, a1, a2); +} + +static void tgen_subbi_rir(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2) +{ + tcg_out_dat_imm(s, COND_AL, ARITH_RSC, a0, a2, encode_imm_nofail(a1)); +} + +static void tgen_subbi_rii(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, tcg_target_long a2) +{ + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, a2); + tgen_subbi_rir(s, TCG_TYPE_I32, a0, a1, TCG_REG_TMP); +} + +static const TCGOutOpAddSubCarry outop_subbi = { + .base.static_constraint = C_O1_I2(r, rI, rIK), + .out_rrr = tgen_subbi_rrr, + .out_rri = tgen_subbi_rri, + .out_rir = tgen_subbi_rir, + .out_rii = tgen_subbi_rii, +}; + +static void tgen_subbio_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_SBC | TO_CPSR, + a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static void tgen_subbio_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_IK(s, COND_AL, ARITH_SBC | TO_CPSR, ARITH_ADC | TO_CPSR, + a0, a1, a2); +} + +static void tgen_subbio_rir(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2) +{ + tcg_out_dat_imm(s, COND_AL, ARITH_RSC | TO_CPSR, + a0, a2, encode_imm_nofail(a1)); +} + +static void tgen_subbio_rii(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, tcg_target_long a2) +{ + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, a2); + tgen_subbio_rir(s, TCG_TYPE_I32, a0, a1, TCG_REG_TMP); +} + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_O1_I2(r, rI, rIK), + .out_rrr = tgen_subbio_rrr, + .out_rri = tgen_subbio_rri, + .out_rir = tgen_subbio_rir, + .out_rii = tgen_subbio_rii, +}; + +static void tcg_out_set_borrow(TCGContext *s) +{ + tcg_out_movi_apsr_c(s, 0); /* borrow = !carry */ +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_EOR, a0, a1, a2, SHIFT_IMM_LSL(0)); +} + +static void tgen_xori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_dat_imm(s, COND_AL, ARITH_EOR, a0, a1, encode_imm_nofail(a2)); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_xor, + .out_rri = tgen_xori, +}; + +static void tgen_bswap16(TCGContext *s, TCGType type, + TCGReg rd, TCGReg rn, unsigned flags) +{ + if (flags & TCG_BSWAP_OS) { + /* revsh */ + tcg_out32(s, 0x06ff0fb0 | (COND_AL << 28) | (rd << 12) | rn); + return; + } + + /* rev16 */ + tcg_out32(s, 0x06bf0fb0 | (COND_AL << 28) | (rd << 12) | rn); + if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + tcg_out_ext16u(s, rd, rd); } } +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap16, +}; + +static void tgen_bswap32(TCGContext *s, TCGType type, + TCGReg rd, TCGReg rn, unsigned flags) +{ + /* rev */ + tcg_out32(s, 0x06bf0f30 | (COND_AL << 28) | (rd << 12) | rn); +} + +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap32, +}; + +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_subfi(s, type, a0, 0, a1); +} + +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_neg, +}; + +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_dat_reg(s, COND_AL, ARITH_MVN, a0, 0, a1, SHIFT_IMM_LSL(0)); +} + +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_not, +}; + +static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, TCGReg a1, TCGLabel *l) +{ + cond = tgen_cmp(s, cond, a0, a1); + tcg_out_goto_label(s, tcg_cond_to_arm_cond[cond], l); +} + +static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, tcg_target_long a1, TCGLabel *l) +{ + cond = tgen_cmpi(s, cond, a0, a1); + tcg_out_goto_label(s, tcg_cond_to_arm_cond[cond], l); +} + +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, rIN), + .out_rr = tgen_brcond, + .out_ri = tgen_brcondi, +}; + +static void finish_setcond(TCGContext *s, TCGCond cond, TCGReg ret, bool neg) +{ + tcg_out_movi32(s, tcg_cond_to_arm_cond[tcg_invert_cond(cond)], ret, 0); + tcg_out_movi32(s, tcg_cond_to_arm_cond[cond], ret, neg ? -1 : 1); +} + +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + cond = tgen_cmp(s, cond, a1, a2); + finish_setcond(s, cond, a0, false); +} + +static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + cond = tgen_cmpi(s, cond, a1, a2); + finish_setcond(s, cond, a0, false); +} + +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(r, r, rIN), + .out_rrr = tgen_setcond, + .out_rri = tgen_setcondi, +}; + +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + cond = tgen_cmp(s, cond, a1, a2); + finish_setcond(s, cond, a0, true); +} + +static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + cond = tgen_cmpi(s, cond, a1, a2); + finish_setcond(s, cond, a0, true); +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(r, r, rIN), + .out_rrr = tgen_negsetcond, + .out_rri = tgen_negsetcondi, +}; + +static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, + TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf) +{ + cond = tcg_out_cmp(s, cond, c1, c2, const_c2); + tcg_out_dat_rIK(s, tcg_cond_to_arm_cond[cond], ARITH_MOV, ARITH_MVN, + ret, 0, vt, const_vt); +} + +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = C_O1_I4(r, r, rIN, rIK, 0), + .out = tgen_movcond, +}; + +static void tgen_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, TCGArg bh, bool const_bh, + TCGLabel *l) +{ + cond = tcg_out_cmp2(s, cond, al, ah, bl, const_bl, bh, const_bh); + tcg_out_goto_label(s, tcg_cond_to_arm_cond[cond], l); +} + +static const TCGOutOpBrcond2 outop_brcond2 = { + .base.static_constraint = C_O0_I4(r, r, rI, rI), + .out = tgen_brcond2, +}; + +static void tgen_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, + TCGArg bh, bool const_bh) +{ + cond = tcg_out_cmp2(s, cond, al, ah, bl, const_bl, bh, const_bh); + finish_setcond(s, cond, ret, false); +} + +static const TCGOutOpSetcond2 outop_setcond2 = { + .base.static_constraint = C_O1_I4(r, r, r, rI, rI), + .out = tgen_setcond2, +}; + +static void tgen_extract2(TCGContext *s, TCGType type, TCGReg a0, + TCGReg a1, TCGReg a2, unsigned shr) +{ + /* We can do extract2 in 2 insns, vs the 3 required otherwise. */ + tgen_shli(s, TCG_TYPE_I32, TCG_REG_TMP, a2, 32 - shr); + tcg_out_dat_reg(s, COND_AL, ARITH_ORR, a0, TCG_REG_TMP, + a1, SHIFT_IMM_LSR(shr)); +} + +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_extract2, +}; + +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg rd, + TCGReg rn, ptrdiff_t offset) +{ + if (offset > 0xfff || offset < -0xfff) { + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); + tcg_out_ld8_r(s, COND_AL, rd, rn, TCG_REG_TMP); + } else { + tcg_out_ld8_12(s, COND_AL, rd, rn, offset); + } +} + +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; + +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg rd, + TCGReg rn, ptrdiff_t offset) +{ + if (offset > 0xff || offset < -0xff) { + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); + tcg_out_ld8s_r(s, COND_AL, rd, rn, TCG_REG_TMP); + } else { + tcg_out_ld8s_8(s, COND_AL, rd, rn, offset); + } +} + +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; + +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg rd, + TCGReg rn, ptrdiff_t offset) +{ + if (offset > 0xff || offset < -0xff) { + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); + tcg_out_ld16u_r(s, COND_AL, rd, rn, TCG_REG_TMP); + } else { + tcg_out_ld16u_8(s, COND_AL, rd, rn, offset); + } +} + +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; + +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg rd, + TCGReg rn, ptrdiff_t offset) +{ + if (offset > 0xff || offset < -0xff) { + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); + tcg_out_ld16s_r(s, COND_AL, rd, rn, TCG_REG_TMP); + } else { + tcg_out_ld16s_8(s, COND_AL, rd, rn, offset); + } +} + +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; + +static void tgen_st8(TCGContext *s, TCGType type, TCGReg rd, + TCGReg rn, ptrdiff_t offset) +{ + if (offset > 0xfff || offset < -0xfff) { + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); + tcg_out_st8_r(s, COND_AL, rd, rn, TCG_REG_TMP); + } else { + tcg_out_st8_12(s, COND_AL, rd, rn, offset); + } +} + +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tgen_st8, +}; + +static void tgen_st16(TCGContext *s, TCGType type, TCGReg rd, + TCGReg rn, ptrdiff_t offset) +{ + if (offset > 0xff || offset < -0xff) { + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); + tcg_out_st16_r(s, COND_AL, rd, rn, TCG_REG_TMP); + } else { + tcg_out_st16_8(s, COND_AL, rd, rn, offset); + } +} + +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tgen_st16, +}; + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tcg_out_st, +}; + static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) { switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_ld8u_i32: - case INDEX_op_ld8s_i32: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16s_i32: - case INDEX_op_ld_i32: - case INDEX_op_neg_i32: - case INDEX_op_not_i32: - case INDEX_op_bswap16_i32: - case INDEX_op_bswap32_i32: - case INDEX_op_ext8s_i32: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16u_i32: - case INDEX_op_extract_i32: - case INDEX_op_sextract_i32: - return C_O1_I1(r, r); - - case INDEX_op_st8_i32: - case INDEX_op_st16_i32: - case INDEX_op_st_i32: - return C_O0_I2(r, r); - - case INDEX_op_add_i32: - case INDEX_op_sub_i32: - case INDEX_op_setcond_i32: - case INDEX_op_negsetcond_i32: - return C_O1_I2(r, r, rIN); - - case INDEX_op_and_i32: - case INDEX_op_andc_i32: - case INDEX_op_clz_i32: - case INDEX_op_ctz_i32: - return C_O1_I2(r, r, rIK); - - case INDEX_op_mul_i32: - case INDEX_op_div_i32: - case INDEX_op_divu_i32: - return C_O1_I2(r, r, r); - - case INDEX_op_mulu2_i32: - case INDEX_op_muls2_i32: - return C_O2_I2(r, r, r, r); - - case INDEX_op_or_i32: - case INDEX_op_xor_i32: - return C_O1_I2(r, r, rI); - - case INDEX_op_shl_i32: - case INDEX_op_shr_i32: - case INDEX_op_sar_i32: - case INDEX_op_rotl_i32: - case INDEX_op_rotr_i32: - return C_O1_I2(r, r, ri); - - case INDEX_op_brcond_i32: - return C_O0_I2(r, rIN); - case INDEX_op_deposit_i32: - return C_O1_I2(r, 0, rZ); - case INDEX_op_extract2_i32: - return C_O1_I2(r, rZ, rZ); - case INDEX_op_movcond_i32: - return C_O1_I4(r, r, rIN, rIK, 0); - case INDEX_op_add2_i32: - return C_O2_I4(r, r, r, r, rIN, rIK); - case INDEX_op_sub2_i32: - return C_O2_I4(r, r, rI, rI, rIN, rIK); - case INDEX_op_brcond2_i32: - return C_O0_I4(r, r, rI, rI); - case INDEX_op_setcond2_i32: - return C_O1_I4(r, r, r, rI, rI); - - case INDEX_op_qemu_ld_i32: - return C_O1_I1(r, q); - case INDEX_op_qemu_ld_i64: - return C_O2_I1(e, p, q); - case INDEX_op_qemu_st_i32: - return C_O0_I2(q, q); - case INDEX_op_qemu_st_i64: - return C_O0_I3(Q, p, q); - case INDEX_op_st_vec: return C_O0_I2(w, r); case INDEX_op_ld_vec: diff --git a/tcg/i386/tcg-target-con-set.h b/tcg/i386/tcg-target-con-set.h index 06e6521001..458d69c3c0 100644 --- a/tcg/i386/tcg-target-con-set.h +++ b/tcg/i386/tcg-target-con-set.h @@ -42,9 +42,10 @@ C_O1_I2(r, 0, reZ) C_O1_I2(r, 0, ri) C_O1_I2(r, 0, rI) C_O1_I2(r, L, L) +C_O1_I2(r, r, r) C_O1_I2(r, r, re) C_O1_I2(r, r, ri) -C_O1_I2(r, r, rI) +C_O1_I2(r, rO, re) C_O1_I2(x, x, x) C_N1_I2(r, r, r) C_N1_I2(r, r, rW) @@ -57,4 +58,3 @@ C_O2_I1(r, r, L) C_O2_I2(a, d, a, r) C_O2_I2(r, r, L, L) C_O2_I3(a, d, 0, 1, r) -C_N1_O1_I4(r, r, 0, 1, re, re) diff --git a/tcg/i386/tcg-target-con-str.h b/tcg/i386/tcg-target-con-str.h index 52142ab121..dbedff1f54 100644 --- a/tcg/i386/tcg-target-con-str.h +++ b/tcg/i386/tcg-target-con-str.h @@ -20,7 +20,7 @@ REGS('r', ALL_GENERAL_REGS) REGS('x', ALL_VECTOR_REGS) REGS('q', ALL_BYTEL_REGS) /* regs that can be used as a byte operand */ REGS('L', ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS) /* qemu_ld/st */ -REGS('s', ALL_BYTEL_REGS & ~SOFTMMU_RESERVE_REGS) /* qemu_st8_i32 data */ +REGS('s', ALL_BYTEL_REGS & ~SOFTMMU_RESERVE_REGS) /* qemu_st MO_8 data */ /* * Define constraint letters for constants: diff --git a/tcg/i386/tcg-target-has.h b/tcg/i386/tcg-target-has.h index 63768ff058..42647fabbd 100644 --- a/tcg/i386/tcg-target-has.h +++ b/tcg/i386/tcg-target-has.h @@ -26,66 +26,9 @@ #define have_avx512vbmi2 ((cpuinfo & CPUINFO_AVX512VBMI2) && have_avx512vl) /* optional instructions */ -#define TCG_TARGET_HAS_div2_i32 1 -#define TCG_TARGET_HAS_rot_i32 1 -#define TCG_TARGET_HAS_ext8s_i32 1 -#define TCG_TARGET_HAS_ext16s_i32 1 -#define TCG_TARGET_HAS_ext8u_i32 1 -#define TCG_TARGET_HAS_ext16u_i32 1 -#define TCG_TARGET_HAS_bswap16_i32 1 -#define TCG_TARGET_HAS_bswap32_i32 1 -#define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_andc_i32 have_bmi1 -#define TCG_TARGET_HAS_orc_i32 0 -#define TCG_TARGET_HAS_eqv_i32 0 -#define TCG_TARGET_HAS_nand_i32 0 -#define TCG_TARGET_HAS_nor_i32 0 -#define TCG_TARGET_HAS_clz_i32 1 -#define TCG_TARGET_HAS_ctz_i32 1 -#define TCG_TARGET_HAS_ctpop_i32 have_popcnt -#define TCG_TARGET_HAS_extract2_i32 1 -#define TCG_TARGET_HAS_negsetcond_i32 1 -#define TCG_TARGET_HAS_add2_i32 1 -#define TCG_TARGET_HAS_sub2_i32 1 -#define TCG_TARGET_HAS_mulu2_i32 1 -#define TCG_TARGET_HAS_muls2_i32 1 -#define TCG_TARGET_HAS_muluh_i32 0 -#define TCG_TARGET_HAS_mulsh_i32 0 - #if TCG_TARGET_REG_BITS == 64 /* Keep 32-bit values zero-extended in a register. */ #define TCG_TARGET_HAS_extr_i64_i32 1 -#define TCG_TARGET_HAS_div2_i64 1 -#define TCG_TARGET_HAS_rot_i64 1 -#define TCG_TARGET_HAS_ext8s_i64 1 -#define TCG_TARGET_HAS_ext16s_i64 1 -#define TCG_TARGET_HAS_ext32s_i64 1 -#define TCG_TARGET_HAS_ext8u_i64 1 -#define TCG_TARGET_HAS_ext16u_i64 1 -#define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_bswap16_i64 1 -#define TCG_TARGET_HAS_bswap32_i64 1 -#define TCG_TARGET_HAS_bswap64_i64 1 -#define TCG_TARGET_HAS_not_i64 1 -#define TCG_TARGET_HAS_andc_i64 have_bmi1 -#define TCG_TARGET_HAS_orc_i64 0 -#define TCG_TARGET_HAS_eqv_i64 0 -#define TCG_TARGET_HAS_nand_i64 0 -#define TCG_TARGET_HAS_nor_i64 0 -#define TCG_TARGET_HAS_clz_i64 1 -#define TCG_TARGET_HAS_ctz_i64 1 -#define TCG_TARGET_HAS_ctpop_i64 have_popcnt -#define TCG_TARGET_HAS_extract2_i64 1 -#define TCG_TARGET_HAS_negsetcond_i64 1 -#define TCG_TARGET_HAS_add2_i64 1 -#define TCG_TARGET_HAS_sub2_i64 1 -#define TCG_TARGET_HAS_mulu2_i64 1 -#define TCG_TARGET_HAS_muls2_i64 1 -#define TCG_TARGET_HAS_muluh_i64 0 -#define TCG_TARGET_HAS_mulsh_i64 0 -#define TCG_TARGET_HAS_qemu_st8_i32 0 -#else -#define TCG_TARGET_HAS_qemu_st8_i32 1 #endif #define TCG_TARGET_HAS_qemu_ldst_i128 \ diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index 33d303a123..09fce27b06 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -424,6 +424,7 @@ static bool tcg_target_const_match(int64_t val, int ct, #define OPC_SHLX (0xf7 | P_EXT38 | P_DATA16) #define OPC_SHRX (0xf7 | P_EXT38 | P_SIMDF2) #define OPC_SHRD_Ib (0xac | P_EXT) +#define OPC_STC (0xf9) #define OPC_TESTB (0x84) #define OPC_TESTL (0x85) #define OPC_TZCNT (0xbc | P_EXT | P_SIMDF3) @@ -1092,7 +1093,7 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, { tcg_target_long diff; - if (arg == 0) { + if (arg == 0 && !s->carry_live) { tgen_arithr(s, ARITH_XOR, ret, ret); return; } @@ -1167,7 +1168,7 @@ static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val) } } -static inline void tcg_out_mb(TCGContext *s, TCGArg a0) +static void tcg_out_mb(TCGContext *s, unsigned a0) { /* Given the strength of x86 memory ordering, we only need care for store-load ordering. Experimentally, "lock orl $0,0(%esp)" is @@ -1545,6 +1546,11 @@ static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, bool small) } } +static void tcg_out_br(TCGContext *s, TCGLabel *l) +{ + tcg_out_jxx(s, JCC_JMP, l, 0); +} + static int tcg_out_cmp(TCGContext *s, TCGCond cond, TCGArg arg1, TCGArg arg2, int const_arg2, int rexw) { @@ -1642,47 +1648,78 @@ static void tcg_out_brcond(TCGContext *s, int rexw, TCGCond cond, tcg_out_jxx(s, jcc, label, small); } -#if TCG_TARGET_REG_BITS == 32 -static void tcg_out_brcond2(TCGContext *s, const TCGArg *args, - const int *const_args, bool small) +static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, TCGReg arg2, TCGLabel *label) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_brcond(s, rexw, cond, arg1, arg2, false, label, false); +} + +static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, tcg_target_long arg2, TCGLabel *label) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_brcond(s, rexw, cond, arg1, arg2, true, label, false); +} + +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, reT), + .out_rr = tgen_brcond, + .out_ri = tgen_brcondi, +}; + +static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, + TCGReg ah, TCGArg bl, bool blconst, + TCGArg bh, bool bhconst, + TCGLabel *label_this, bool small) { TCGLabel *label_next = gen_new_label(); - TCGLabel *label_this = arg_label(args[5]); - TCGCond cond = args[4]; switch (cond) { case TCG_COND_EQ: case TCG_COND_TSTEQ: tcg_out_brcond(s, 0, tcg_invert_cond(cond), - args[0], args[2], const_args[2], label_next, 1); - tcg_out_brcond(s, 0, cond, args[1], args[3], const_args[3], - label_this, small); + al, bl, blconst, label_next, true); + tcg_out_brcond(s, 0, cond, ah, bh, bhconst, label_this, small); break; case TCG_COND_NE: case TCG_COND_TSTNE: - tcg_out_brcond(s, 0, cond, args[0], args[2], const_args[2], - label_this, small); - tcg_out_brcond(s, 0, cond, args[1], args[3], const_args[3], - label_this, small); + tcg_out_brcond(s, 0, cond, al, bl, blconst, label_this, small); + tcg_out_brcond(s, 0, cond, ah, bh, bhconst, label_this, small); break; default: - tcg_out_brcond(s, 0, tcg_high_cond(cond), args[1], - args[3], const_args[3], label_this, small); + tcg_out_brcond(s, 0, tcg_high_cond(cond), + ah, bh, bhconst, label_this, small); tcg_out_jxx(s, JCC_JNE, label_next, 1); - tcg_out_brcond(s, 0, tcg_unsigned_cond(cond), args[0], - args[2], const_args[2], label_this, small); + tcg_out_brcond(s, 0, tcg_unsigned_cond(cond), + al, bl, blconst, label_this, small); break; } tcg_out_label(s, label_next); } + +static void tgen_brcond2(TCGContext *s, TCGCond cond, TCGReg al, + TCGReg ah, TCGArg bl, bool blconst, + TCGArg bh, bool bhconst, TCGLabel *l) +{ + tcg_out_brcond2(s, cond, al, ah, bl, blconst, bh, bhconst, l, false); +} + +#if TCG_TARGET_REG_BITS != 32 +__attribute__((unused)) #endif +static const TCGOutOpBrcond2 outop_brcond2 = { + .base.static_constraint = C_O0_I4(r, r, ri, ri), + .out = tgen_brcond2, +}; -static void tcg_out_setcond(TCGContext *s, int rexw, TCGCond cond, - TCGArg dest, TCGArg arg1, TCGArg arg2, - int const_arg2, bool neg) +static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGArg arg2, + bool const_arg2, bool neg) { + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; int cmp_rexw = rexw; bool inv = false; bool cleared; @@ -1757,7 +1794,7 @@ static void tcg_out_setcond(TCGContext *s, int rexw, TCGCond cond, case TCG_COND_LT: /* If arg2 is 0, extract the sign bit. */ if (const_arg2 && arg2 == 0) { - tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, dest, arg1); + tcg_out_mov(s, type, dest, arg1); if (inv) { tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, dest); } @@ -1793,49 +1830,89 @@ static void tcg_out_setcond(TCGContext *s, int rexw, TCGCond cond, } } -#if TCG_TARGET_REG_BITS == 32 -static void tcg_out_setcond2(TCGContext *s, const TCGArg *args, - const int *const_args) +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, false); +} + +static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, false); +} + +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(q, r, reT), + .out_rrr = tgen_setcond, + .out_rri = tgen_setcondi, +}; + +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, true); +} + +static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) { - TCGArg new_args[6]; - TCGLabel *label_true, *label_over; + tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, true); +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(q, r, reT), + .out_rrr = tgen_negsetcond, + .out_rri = tgen_negsetcondi, +}; - memcpy(new_args, args+1, 5*sizeof(TCGArg)); +static void tgen_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, + TCGArg bh, bool const_bh) +{ + TCGLabel *label_over = gen_new_label(); - if (args[0] == args[1] || args[0] == args[2] - || (!const_args[3] && args[0] == args[3]) - || (!const_args[4] && args[0] == args[4])) { - /* When the destination overlaps with one of the argument - registers, don't do anything tricky. */ - label_true = gen_new_label(); - label_over = gen_new_label(); + if (ret == al || ret == ah + || (!const_bl && ret == bl) + || (!const_bh && ret == bh)) { + /* + * When the destination overlaps with one of the argument + * registers, don't do anything tricky. + */ + TCGLabel *label_true = gen_new_label(); - new_args[5] = label_arg(label_true); - tcg_out_brcond2(s, new_args, const_args+1, 1); + tcg_out_brcond2(s, cond, al, ah, bl, const_bl, + bh, const_bh, label_true, true); - tcg_out_movi(s, TCG_TYPE_I32, args[0], 0); + tcg_out_movi(s, TCG_TYPE_I32, ret, 0); tcg_out_jxx(s, JCC_JMP, label_over, 1); tcg_out_label(s, label_true); - tcg_out_movi(s, TCG_TYPE_I32, args[0], 1); - tcg_out_label(s, label_over); + tcg_out_movi(s, TCG_TYPE_I32, ret, 1); } else { - /* When the destination does not overlap one of the arguments, - clear the destination first, jump if cond false, and emit an - increment in the true case. This results in smaller code. */ - - tcg_out_movi(s, TCG_TYPE_I32, args[0], 0); + /* + * When the destination does not overlap one of the arguments, + * clear the destination first, jump if cond false, and emit an + * increment in the true case. This results in smaller code. + */ + tcg_out_movi(s, TCG_TYPE_I32, ret, 0); - label_over = gen_new_label(); - new_args[4] = tcg_invert_cond(new_args[4]); - new_args[5] = label_arg(label_over); - tcg_out_brcond2(s, new_args, const_args+1, 1); + tcg_out_brcond2(s, tcg_invert_cond(cond), al, ah, bl, const_bl, + bh, const_bh, label_over, true); - tgen_arithi(s, ARITH_ADD, args[0], 1, 0); - tcg_out_label(s, label_over); + tgen_arithi(s, ARITH_ADD, ret, 1, 0); } + tcg_out_label(s, label_over); } + +#if TCG_TARGET_REG_BITS != 32 +__attribute__((unused)) #endif +static const TCGOutOpSetcond2 outop_setcond2 = { + .base.static_constraint = C_O1_I4(r, r, r, ri, ri), + .out = tgen_setcond2, +}; static void tcg_out_cmov(TCGContext *s, int jcc, int rexw, TCGReg dest, TCGReg v1) @@ -1843,57 +1920,20 @@ static void tcg_out_cmov(TCGContext *s, int jcc, int rexw, tcg_out_modrm(s, OPC_CMOVCC | jcc | rexw, dest, v1); } -static void tcg_out_movcond(TCGContext *s, int rexw, TCGCond cond, - TCGReg dest, TCGReg c1, TCGArg c2, int const_c2, - TCGReg v1) +static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg c1, TCGArg c2, bool const_c2, + TCGArg vt, bool const_vt, + TCGArg vf, bool consf_vf) { + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; int jcc = tcg_out_cmp(s, cond, c1, c2, const_c2, rexw); - tcg_out_cmov(s, jcc, rexw, dest, v1); + tcg_out_cmov(s, jcc, rexw, dest, vt); } -static void tcg_out_ctz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, - TCGArg arg2, bool const_a2) -{ - if (have_bmi1) { - tcg_out_modrm(s, OPC_TZCNT + rexw, dest, arg1); - if (const_a2) { - tcg_debug_assert(arg2 == (rexw ? 64 : 32)); - } else { - tcg_debug_assert(dest != arg2); - tcg_out_cmov(s, JCC_JB, rexw, dest, arg2); - } - } else { - tcg_debug_assert(dest != arg2); - tcg_out_modrm(s, OPC_BSF + rexw, dest, arg1); - tcg_out_cmov(s, JCC_JE, rexw, dest, arg2); - } -} - -static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, - TCGArg arg2, bool const_a2) -{ - if (have_lzcnt) { - tcg_out_modrm(s, OPC_LZCNT + rexw, dest, arg1); - if (const_a2) { - tcg_debug_assert(arg2 == (rexw ? 64 : 32)); - } else { - tcg_debug_assert(dest != arg2); - tcg_out_cmov(s, JCC_JB, rexw, dest, arg2); - } - } else { - tcg_debug_assert(!const_a2); - tcg_debug_assert(dest != arg1); - tcg_debug_assert(dest != arg2); - - /* Recall that the output of BSR is the index not the count. */ - tcg_out_modrm(s, OPC_BSR + rexw, dest, arg1); - tgen_arithi(s, ARITH_XOR + rexw, dest, rexw ? 63 : 31, 0); - - /* Since we have destroyed the flags from BSR, we have to re-test. */ - int jcc = tcg_out_cmp(s, TCG_COND_EQ, arg1, 0, 1, rexw); - tcg_out_cmov(s, jcc, rexw, dest, arg2); - } -} +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = C_O1_I4(r, r, reT, r, 0), + .out = tgen_movcond, +}; static void tcg_out_branch(TCGContext *s, int call, const tcg_insn_unit *dest) { @@ -2382,23 +2422,50 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, } } -static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg addr, MemOpIdx oi, TCGType data_type) +static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + TCGLabelQemuLdst *ldst; + HostAddress h; + + ldst = prepare_host_addr(s, &h, addr, oi, true); + tcg_out_qemu_ld_direct(s, data, -1, h, type, get_memop(oi)); + + if (ldst) { + ldst->type = type; + ldst->datalo_reg = data; + ldst->datahi_reg = -1; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); + } +} + +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, L), + .out = tgen_qemu_ld, +}; + +static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) { TCGLabelQemuLdst *ldst; HostAddress h; ldst = prepare_host_addr(s, &h, addr, oi, true); - tcg_out_qemu_ld_direct(s, datalo, datahi, h, data_type, get_memop(oi)); + tcg_out_qemu_ld_direct(s, datalo, datahi, h, type, get_memop(oi)); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = datalo; ldst->datahi_reg = datahi; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + .base.static_constraint = C_O2_I1(r, r, L), + .out = tgen_qemu_ld2, +}; + static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, HostAddress h, MemOp memop) { @@ -2417,7 +2484,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, switch (memop & MO_SIZE) { case MO_8: - /* This is handled with constraints on INDEX_op_qemu_st8_i32. */ + /* This is handled with constraints in cset_qemu_st(). */ tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || datalo < 4); tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + h.seg, datalo, h.base, h.index, 0, h.ofs); @@ -2509,8 +2576,38 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, } } -static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg addr, MemOpIdx oi, TCGType data_type) +static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + TCGLabelQemuLdst *ldst; + HostAddress h; + + ldst = prepare_host_addr(s, &h, addr, oi, false); + tcg_out_qemu_st_direct(s, data, -1, h, get_memop(oi)); + + if (ldst) { + ldst->type = type; + ldst->datalo_reg = data; + ldst->datahi_reg = -1; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); + } +} + +static TCGConstraintSetIndex cset_qemu_st(TCGType type, unsigned flags) +{ + return flags == MO_8 ? C_O0_I2(s, L) : C_O0_I2(L, L); +} + +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = + TCG_TARGET_REG_BITS == 32 ? C_Dynamic : C_O0_I2(L, L), + .base.dynamic_constraint = + TCG_TARGET_REG_BITS == 32 ? cset_qemu_st : NULL, + .out = tgen_qemu_st, +}; + +static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) { TCGLabelQemuLdst *ldst; HostAddress h; @@ -2519,13 +2616,18 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, tcg_out_qemu_st_direct(s, datalo, datahi, h, get_memop(oi)); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = datalo; ldst->datahi_reg = datahi; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + .base.static_constraint = C_O0_I3(L, L, L), + .out = tgen_qemu_st2, +}; + static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) { /* Reuse the zeroing that exists for goto_ptr. */ @@ -2553,6 +2655,12 @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + /* Jump to the given host address (could be epilogue) */ + tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0); +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { @@ -2562,480 +2670,938 @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, /* no need to flush icache explicitly */ } -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) + +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - TCGArg a0, a1, a2; - int c, const_a2, vexop, rexw; + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; -#if TCG_TARGET_REG_BITS == 64 -# define OP_32_64(x) \ - case glue(glue(INDEX_op_, x), _i64): \ - case glue(glue(INDEX_op_, x), _i32) -#else -# define OP_32_64(x) \ - case glue(glue(INDEX_op_, x), _i32) -#endif + if (a0 == a1) { + tgen_arithr(s, ARITH_ADD + rexw, a0, a2); + } else if (a0 == a2) { + tgen_arithr(s, ARITH_ADD + rexw, a0, a1); + } else { + tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, 0); + } +} - /* Hoist the loads of the most common arguments. */ - a0 = args[0]; - a1 = args[1]; - a2 = args[2]; - const_a2 = const_args[2]; - rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; +static void tgen_addi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; - switch (opc) { - case INDEX_op_goto_ptr: - /* jmp to the given host address (could be epilogue) */ - tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0); - break; - case INDEX_op_br: - tcg_out_jxx(s, JCC_JMP, arg_label(a0), 0); - break; - OP_32_64(ld8u): - /* Note that we can ignore REXW for the zero-extend to 64-bit. */ - tcg_out_modrm_offset(s, OPC_MOVZBL, a0, a1, a2); - break; - OP_32_64(ld8s): - tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, a0, a1, a2); - break; - OP_32_64(ld16u): - /* Note that we can ignore REXW for the zero-extend to 64-bit. */ - tcg_out_modrm_offset(s, OPC_MOVZWL, a0, a1, a2); - break; - OP_32_64(ld16s): - tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, a0, a1, a2); - break; -#if TCG_TARGET_REG_BITS == 64 - case INDEX_op_ld32u_i64: -#endif - case INDEX_op_ld_i32: - tcg_out_ld(s, TCG_TYPE_I32, a0, a1, a2); - break; + if (a0 == a1) { + tgen_arithi(s, ARITH_ADD + rexw, a0, a2, false); + } else { + tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, -1, 0, a2); + } +} + +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, re), + .out_rrr = tgen_add, + .out_rri = tgen_addi, +}; + +static void tgen_addco(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithr(s, ARITH_ADD + rexw, a0, a2); +} + +static void tgen_addco_imm(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithi(s, ARITH_ADD + rexw, a0, a2, true); +} + +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_O1_I2(r, 0, re), + .out_rrr = tgen_addco, + .out_rri = tgen_addco_imm, +}; + +static void tgen_addcio(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithr(s, ARITH_ADC + rexw, a0, a2); +} + +static void tgen_addcio_imm(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithi(s, ARITH_ADC + rexw, a0, a2, true); +} + +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_O1_I2(r, 0, re), + .out_rrr = tgen_addcio, + .out_rri = tgen_addcio_imm, +}; + +static void tgen_addci_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + /* Because "0O" is not a valid constraint, we must match ourselves. */ + if (a0 == a2) { + tgen_addcio(s, type, a0, a0, a1); + } else { + tcg_out_mov(s, type, a0, a1); + tgen_addcio(s, type, a0, a0, a2); + } +} + +static void tgen_addci_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_mov(s, type, a0, a1); + tgen_addcio_imm(s, type, a0, a0, a2); +} + +static void tgen_addci_rir(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2) +{ + tgen_addci_rri(s, type, a0, a2, a1); +} + +static void tgen_addci_rii(TCGContext *s, TCGType type, TCGReg a0, + tcg_target_long a1, tcg_target_long a2) +{ + if (a2 == 0) { + /* Implement 0 + 0 + C with -(x - x - c). */ + tgen_arithr(s, ARITH_SBB, a0, a0); + tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_NEG, a0); + } else { + tcg_out_movi(s, type, a0, a2); + tgen_addcio_imm(s, type, a0, a0, a1); + } +} + +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_O1_I2(r, rO, re), + .out_rrr = tgen_addci_rrr, + .out_rri = tgen_addci_rri, + .out_rir = tgen_addci_rir, + .out_rii = tgen_addci_rii, +}; + +static void tcg_out_set_carry(TCGContext *s) +{ + tcg_out8(s, OPC_STC); +} + +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithr(s, ARITH_AND + rexw, a0, a2); +} + +static void tgen_andi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithi(s, ARITH_AND + rexw, a0, a2, false); +} + +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, 0, reZ), + .out_rrr = tgen_and, + .out_rri = tgen_andi, +}; + +static void tgen_andc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_vex_modrm(s, OPC_ANDN + rexw, a0, a2, a1); +} + +static TCGConstraintSetIndex cset_andc(TCGType type, unsigned flags) +{ + return have_bmi1 ? C_O1_I2(r, r, r) : C_NotImplemented; +} + +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_andc, + .out_rrr = tgen_andc, +}; + +static void tgen_clz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + int jcc; + + if (have_lzcnt) { + tcg_out_modrm(s, OPC_LZCNT + rexw, a0, a1); + jcc = JCC_JB; + } else { + /* Recall that the output of BSR is the index not the count. */ + tcg_out_modrm(s, OPC_BSR + rexw, a0, a1); + tgen_arithi(s, ARITH_XOR + rexw, a0, rexw ? 63 : 31, 0); + + /* Since we have destroyed the flags from BSR, we have to re-test. */ + jcc = tcg_out_cmp(s, TCG_COND_EQ, a1, 0, 1, rexw); + } + tcg_out_cmov(s, jcc, rexw, a0, a2); +} + +static void tgen_clzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_LZCNT + rexw, a0, a1); +} + +static TCGConstraintSetIndex cset_clz(TCGType type, unsigned flags) +{ + return have_lzcnt ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); +} + +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_clz, + .out_rrr = tgen_clz, + .out_rri = tgen_clzi, +}; + +static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_POPCNT + rexw, a0, a1); +} + +static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags) +{ + return have_popcnt ? C_O1_I1(r, r) : C_NotImplemented; +} + +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_ctpop, + .out_rr = tgen_ctpop, +}; + +static void tgen_ctz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + int jcc; + + if (have_bmi1) { + tcg_out_modrm(s, OPC_TZCNT + rexw, a0, a1); + jcc = JCC_JB; + } else { + tcg_out_modrm(s, OPC_BSF + rexw, a0, a1); + jcc = JCC_JE; + } + tcg_out_cmov(s, jcc, rexw, a0, a2); +} + +static void tgen_ctzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_TZCNT + rexw, a0, a1); +} + +static TCGConstraintSetIndex cset_ctz(TCGType type, unsigned flags) +{ + return have_bmi1 ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); +} + +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_ctz, + .out_rrr = tgen_ctz, + .out_rri = tgen_ctzi, +}; + +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divs2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a4) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, a4); +} + +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_O2_I3(a, d, 0, 1, r), + .out_rr01r = tgen_divs2, +}; + +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divu2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a4) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, a4); +} + +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_O2_I3(a, d, 0, 1, r), + .out_rr01r = tgen_divu2, +}; + +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_NotImplemented, +}; - OP_32_64(st8): - if (const_args[0]) { - tcg_out_modrm_offset(s, OPC_MOVB_EvIz, 0, a1, a2); - tcg_out8(s, a0); - } else { - tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R, a0, a1, a2); - } - break; - OP_32_64(st16): - if (const_args[0]) { - tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16, 0, a1, a2); - tcg_out16(s, a0); - } else { - tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16, a0, a1, a2); - } - break; #if TCG_TARGET_REG_BITS == 64 - case INDEX_op_st32_i64: -#endif - case INDEX_op_st_i32: - if (const_args[0]) { - tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, a1, a2); - tcg_out32(s, a0); - } else { - tcg_out_st(s, TCG_TYPE_I32, a0, a1, a2); - } - break; +static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32); +} - OP_32_64(add): - /* For 3-operand addition, use LEA. */ - if (a0 != a1) { - TCGArg c3 = 0; - if (const_a2) { - c3 = a2, a2 = -1; - } else if (a0 == a2) { - /* Watch out for dest = src + dest, since we've removed - the matching constraint on the add. */ - tgen_arithr(s, ARITH_ADD + rexw, a0, a1); - break; - } +static const TCGOutOpUnary outop_extrh_i64_i32 = { + .base.static_constraint = C_O1_I1(r, 0), + .out_rr = tgen_extrh_i64_i32, +}; +#endif /* TCG_TARGET_REG_BITS == 64 */ - tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, c3); - break; - } - c = ARITH_ADD; - goto gen_arith; - OP_32_64(sub): - c = ARITH_SUB; - goto gen_arith; - OP_32_64(and): - c = ARITH_AND; - goto gen_arith; - OP_32_64(or): - c = ARITH_OR; - goto gen_arith; - OP_32_64(xor): - c = ARITH_XOR; - goto gen_arith; - gen_arith: - if (const_a2) { - tgen_arithi(s, c + rexw, a0, a2, 0); - } else { - tgen_arithr(s, c + rexw, a0, a2); - } - break; +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, a0, a2); +} - OP_32_64(andc): - if (const_a2) { - tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1); - tgen_arithi(s, ARITH_AND + rexw, a0, ~a2, 0); - } else { - tcg_out_vex_modrm(s, OPC_ANDN + rexw, a0, a2, a1); - } - break; +static void tgen_muli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; - OP_32_64(mul): - if (const_a2) { - int32_t val; - val = a2; - if (val == (int8_t)val) { - tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, a0, a0); - tcg_out8(s, val); - } else { - tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, a0, a0); - tcg_out32(s, val); - } - } else { - tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, a0, a2); - } - break; + if (a2 == (int8_t)a2) { + tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, a0, a0); + tcg_out8(s, a2); + } else { + tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, a0, a0); + tcg_out32(s, a2); + } +} - OP_32_64(div2): - tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, args[4]); - break; - OP_32_64(divu2): - tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, args[4]); - break; +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_O1_I2(r, 0, re), + .out_rrr = tgen_mul, + .out_rri = tgen_muli, +}; - OP_32_64(shl): - /* For small constant 3-operand shift, use LEA. */ - if (const_a2 && a0 != a1 && (a2 - 1) < 3) { - if (a2 - 1 == 0) { - /* shl $1,a1,a0 -> lea (a1,a1),a0 */ - tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a1, 0, 0); - } else { - /* shl $n,a1,a0 -> lea 0(,a1,n),a0 */ - tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, -1, a1, a2, 0); - } - break; - } - c = SHIFT_SHL; - vexop = OPC_SHLX; - goto gen_shift_maybe_vex; - OP_32_64(shr): - c = SHIFT_SHR; - vexop = OPC_SHRX; - goto gen_shift_maybe_vex; - OP_32_64(sar): - c = SHIFT_SAR; - vexop = OPC_SARX; - goto gen_shift_maybe_vex; - OP_32_64(rotl): - c = SHIFT_ROL; - goto gen_shift; - OP_32_64(rotr): - c = SHIFT_ROR; - goto gen_shift; - gen_shift_maybe_vex: - if (have_bmi2) { - if (!const_a2) { - tcg_out_vex_modrm(s, vexop + rexw, a0, a2, a1); - break; - } - tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1); - } - /* FALLTHRU */ - gen_shift: - if (const_a2) { - tcg_out_shifti(s, c + rexw, a0, a2); - } else { - tcg_out_modrm(s, OPC_SHIFT_cl + rexw, c, a0); - } - break; +static void tgen_muls2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, a3); +} - OP_32_64(ctz): - tcg_out_ctz(s, rexw, args[0], args[1], args[2], const_args[2]); - break; - OP_32_64(clz): - tcg_out_clz(s, rexw, args[0], args[1], args[2], const_args[2]); - break; - OP_32_64(ctpop): - tcg_out_modrm(s, OPC_POPCNT + rexw, a0, a1); - break; +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_O2_I2(a, d, a, r), + .out_rrrr = tgen_muls2, +}; - OP_32_64(brcond): - tcg_out_brcond(s, rexw, a2, a0, a1, const_args[1], - arg_label(args[3]), 0); - break; - OP_32_64(setcond): - tcg_out_setcond(s, rexw, args[3], a0, a1, a2, const_a2, false); - break; - OP_32_64(negsetcond): - tcg_out_setcond(s, rexw, args[3], a0, a1, a2, const_a2, true); - break; - OP_32_64(movcond): - tcg_out_movcond(s, rexw, args[5], a0, a1, a2, const_a2, args[3]); - break; +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_NotImplemented, +}; - OP_32_64(bswap16): - if (a2 & TCG_BSWAP_OS) { - /* Output must be sign-extended. */ - if (rexw) { - tcg_out_bswap64(s, a0); - tcg_out_shifti(s, SHIFT_SAR + rexw, a0, 48); - } else { - tcg_out_bswap32(s, a0); - tcg_out_shifti(s, SHIFT_SAR, a0, 16); - } - } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - /* Output must be zero-extended, but input isn't. */ - tcg_out_bswap32(s, a0); - tcg_out_shifti(s, SHIFT_SHR, a0, 16); - } else { - tcg_out_rolw_8(s, a0); - } - break; - OP_32_64(bswap32): - tcg_out_bswap32(s, a0); - if (rexw && (a2 & TCG_BSWAP_OS)) { - tcg_out_ext32s(s, a0, a0); - } - break; +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_NotImplemented, +}; - OP_32_64(neg): - tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, a0); - break; - OP_32_64(not): - tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0); - break; +static void tgen_mulu2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, a3); +} - case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, a0, -1, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_ld_i64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_qemu_ld(s, a0, -1, a1, a2, TCG_TYPE_I64); - } else { - tcg_out_qemu_ld(s, a0, a1, a2, args[3], TCG_TYPE_I64); - } - break; - case INDEX_op_qemu_ld_i128: - tcg_debug_assert(TCG_TARGET_REG_BITS == 64); - tcg_out_qemu_ld(s, a0, a1, a2, args[3], TCG_TYPE_I128); - break; +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_O2_I2(a, d, a, r), + .out_rrrr = tgen_mulu2, +}; - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st8_i32: - tcg_out_qemu_st(s, a0, -1, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_st_i64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_qemu_st(s, a0, -1, a1, a2, TCG_TYPE_I64); - } else { - tcg_out_qemu_st(s, a0, a1, a2, args[3], TCG_TYPE_I64); - } - break; - case INDEX_op_qemu_st_i128: - tcg_debug_assert(TCG_TARGET_REG_BITS == 64); - tcg_out_qemu_st(s, a0, a1, a2, args[3], TCG_TYPE_I128); - break; +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_NotImplemented, +}; - OP_32_64(mulu2): - tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]); - break; - OP_32_64(muls2): - tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, args[3]); - break; - OP_32_64(add2): - if (const_args[4]) { - tgen_arithi(s, ARITH_ADD + rexw, a0, args[4], 1); - } else { - tgen_arithr(s, ARITH_ADD + rexw, a0, args[4]); - } - if (const_args[5]) { - tgen_arithi(s, ARITH_ADC + rexw, a1, args[5], 1); - } else { - tgen_arithr(s, ARITH_ADC + rexw, a1, args[5]); - } - break; - OP_32_64(sub2): - if (const_args[4]) { - tgen_arithi(s, ARITH_SUB + rexw, a0, args[4], 1); - } else { - tgen_arithr(s, ARITH_SUB + rexw, a0, args[4]); - } - if (const_args[5]) { - tgen_arithi(s, ARITH_SBB + rexw, a1, args[5], 1); +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithr(s, ARITH_OR + rexw, a0, a2); +} + +static void tgen_ori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithi(s, ARITH_OR + rexw, a0, a2, false); +} + +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, 0, re), + .out_rrr = tgen_or, + .out_rri = tgen_ori, +}; + +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_rotl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_ROL, a0); +} + +static void tgen_rotli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_shifti(s, SHIFT_ROL + rexw, a0, a2); +} + +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_O1_I2(r, 0, ci), + .out_rrr = tgen_rotl, + .out_rri = tgen_rotli, +}; + +static void tgen_rotr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_ROR, a0); +} + +static void tgen_rotri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_shifti(s, SHIFT_ROR + rexw, a0, a2); +} + +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_O1_I2(r, 0, ci), + .out_rrr = tgen_rotr, + .out_rri = tgen_rotri, +}; + +static TCGConstraintSetIndex cset_shift(TCGType type, unsigned flags) +{ + return have_bmi2 ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ci); +} + +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + if (have_bmi2) { + tcg_out_vex_modrm(s, OPC_SARX + rexw, a0, a2, a1); + } else { + tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_SAR, a0); + } +} + +static void tgen_sari(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + + tcg_out_mov(s, type, a0, a1); + tcg_out_shifti(s, SHIFT_SAR + rexw, a0, a2); +} + +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_shift, + .out_rrr = tgen_sar, + .out_rri = tgen_sari, +}; + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + if (have_bmi2) { + tcg_out_vex_modrm(s, OPC_SHLX + rexw, a0, a2, a1); + } else { + tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_SHL, a0); + } +} + +static void tgen_shli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + + /* For small constant 3-operand shift, use LEA. */ + if (a0 != a1 && a2 >= 1 && a2 <= 3) { + if (a2 == 1) { + /* shl $1,a1,a0 -> lea (a1,a1),a0 */ + tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a1, 0, 0); } else { - tgen_arithr(s, ARITH_SBB + rexw, a1, args[5]); + /* shl $n,a1,a0 -> lea 0(,a1,n),a0 */ + tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, -1, a1, a2, 0); } - break; + return; + } + tcg_out_mov(s, type, a0, a1); + tcg_out_shifti(s, SHIFT_SHL + rexw, a0, a2); +} -#if TCG_TARGET_REG_BITS == 32 - case INDEX_op_brcond2_i32: - tcg_out_brcond2(s, args, const_args, 0); - break; - case INDEX_op_setcond2_i32: - tcg_out_setcond2(s, args, const_args); - break; -#else /* TCG_TARGET_REG_BITS == 64 */ - case INDEX_op_ld32s_i64: - tcg_out_modrm_offset(s, OPC_MOVSLQ, a0, a1, a2); - break; - case INDEX_op_ld_i64: - tcg_out_ld(s, TCG_TYPE_I64, a0, a1, a2); - break; - case INDEX_op_st_i64: - if (const_args[0]) { - tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW, 0, a1, a2); - tcg_out32(s, a0); +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_shift, + .out_rrr = tgen_shl, + .out_rri = tgen_shli, +}; + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + if (have_bmi2) { + tcg_out_vex_modrm(s, OPC_SHRX + rexw, a0, a2, a1); + } else { + tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_SHR, a0); + } +} + +static void tgen_shri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + + tcg_out_mov(s, type, a0, a1); + tcg_out_shifti(s, SHIFT_SHR + rexw, a0, a2); +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_shift, + .out_rrr = tgen_shr, + .out_rri = tgen_shri, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithr(s, ARITH_SUB + rexw, a0, a2); +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, 0, r), + .out_rrr = tgen_sub, +}; + +static void tgen_subbo_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithi(s, ARITH_SUB + rexw, a0, a2, 1); +} + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_O1_I2(r, 0, re), + .out_rrr = tgen_sub, + .out_rri = tgen_subbo_rri, +}; + +static void tgen_subbio_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithr(s, ARITH_SBB + rexw, a0, a2); +} + +static void tgen_subbio_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithi(s, ARITH_SBB + rexw, a0, a2, 1); +} + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_O1_I2(r, 0, re), + .out_rrr = tgen_subbio_rrr, + .out_rri = tgen_subbio_rri, +}; + +#define outop_subbi outop_subbio + +static void tcg_out_set_borrow(TCGContext *s) +{ + tcg_out8(s, OPC_STC); +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithr(s, ARITH_XOR + rexw, a0, a2); +} + +static void tgen_xori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tgen_arithi(s, ARITH_XOR + rexw, a0, a2, false); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, 0, re), + .out_rrr = tgen_xor, + .out_rri = tgen_xori, +}; + +static void tgen_bswap16(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + + if (flags & TCG_BSWAP_OS) { + /* Output must be sign-extended. */ + if (rexw) { + tcg_out_bswap64(s, a0); + tcg_out_shifti(s, SHIFT_SAR + rexw, a0, 48); } else { - tcg_out_st(s, TCG_TYPE_I64, a0, a1, a2); + tcg_out_bswap32(s, a0); + tcg_out_shifti(s, SHIFT_SAR, a0, 16); } - break; + } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + /* Output must be zero-extended, but input isn't. */ + tcg_out_bswap32(s, a0); + tcg_out_shifti(s, SHIFT_SHR, a0, 16); + } else { + tcg_out_rolw_8(s, a0); + } +} - case INDEX_op_bswap64_i64: - tcg_out_bswap64(s, a0); - break; - case INDEX_op_extrh_i64_i32: - tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32); - break; +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_O1_I1(r, 0), + .out_rr = tgen_bswap16, +}; + +static void tgen_bswap32(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_bswap32(s, a0); + if (flags & TCG_BSWAP_OS) { + tcg_out_ext32s(s, a0, a0); + } +} + +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_O1_I1(r, 0), + .out_rr = tgen_bswap32, +}; + +#if TCG_TARGET_REG_BITS == 64 +static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_bswap64(s, a0); +} + +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_O1_I1(r, 0), + .out_rr = tgen_bswap64, +}; #endif - OP_32_64(deposit): - if (args[3] == 0 && args[4] == 8) { - /* load bits 0..7 */ - if (const_a2) { - tcg_out_opc(s, OPC_MOVB_Ib | P_REXB_RM | LOWREGMASK(a0), - 0, a0, 0); - tcg_out8(s, a2); - } else { - tcg_out_modrm(s, OPC_MOVB_EvGv | P_REXB_R | P_REXB_RM, a2, a0); - } - } else if (TCG_TARGET_REG_BITS == 32 && args[3] == 8 && args[4] == 8) { - /* load bits 8..15 */ - if (const_a2) { - tcg_out8(s, OPC_MOVB_Ib + a0 + 4); - tcg_out8(s, a2); - } else { - tcg_out_modrm(s, OPC_MOVB_EvGv, a2, a0 + 4); - } - } else if (args[3] == 0 && args[4] == 16) { - /* load bits 0..15 */ - if (const_a2) { - tcg_out_opc(s, OPC_MOVL_Iv | P_DATA16 | LOWREGMASK(a0), - 0, a0, 0); - tcg_out16(s, a2); - } else { - tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, a2, a0); - } - } else { - g_assert_not_reached(); - } - break; +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, a0); +} - case INDEX_op_extract_i64: - if (a2 + args[3] == 32) { - if (a2 == 0) { - tcg_out_ext32u(s, a0, a1); - break; - } - /* This is a 32-bit zero-extending right shift. */ - tcg_out_mov(s, TCG_TYPE_I32, a0, a1); - tcg_out_shifti(s, SHIFT_SHR, a0, a2); - break; - } - /* FALLTHRU */ - case INDEX_op_extract_i32: - if (a2 == 0 && args[3] == 8) { +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, 0), + .out_rr = tgen_neg, +}; + +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0); +} + +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_O1_I1(r, 0), + .out_rr = tgen_not, +}; + +static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned ofs, unsigned len) +{ + if (ofs == 0 && len == 8) { + tcg_out_modrm(s, OPC_MOVB_EvGv | P_REXB_R | P_REXB_RM, a2, a0); + } else if (ofs == 0 && len == 16) { + tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, a2, a0); + } else if (TCG_TARGET_REG_BITS == 32 && ofs == 8 && len == 8) { + tcg_out_modrm(s, OPC_MOVB_EvGv, a2, a0 + 4); + } else { + g_assert_not_reached(); + } +} + +static void tgen_depositi(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + tcg_target_long a2, unsigned ofs, unsigned len) +{ + if (ofs == 0 && len == 8) { + tcg_out_opc(s, OPC_MOVB_Ib | P_REXB_RM | LOWREGMASK(a0), 0, a0, 0); + tcg_out8(s, a2); + } else if (ofs == 0 && len == 16) { + tcg_out_opc(s, OPC_MOVL_Iv | P_DATA16 | LOWREGMASK(a0), 0, a0, 0); + tcg_out16(s, a2); + } else if (TCG_TARGET_REG_BITS == 32 && ofs == 8 && len == 8) { + tcg_out8(s, OPC_MOVB_Ib + a0 + 4); + tcg_out8(s, a2); + } else { + g_assert_not_reached(); + } +} + +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_O1_I2(q, 0, qi), + .out_rrr = tgen_deposit, + .out_rri = tgen_depositi, +}; + +static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0) { + switch (len) { + case 8: tcg_out_ext8u(s, a0, a1); - } else if (a2 == 0 && args[3] == 16) { + return; + case 16: tcg_out_ext16u(s, a0, a1); - } else if (a2 == 8 && args[3] == 8) { - /* - * On the off-chance that we can use the high-byte registers. - * Otherwise we emit the same ext16 + shift pattern that we - * would have gotten from the normal tcg-op.c expansion. - */ - if (a1 < 4 && a0 < 8) { - tcg_out_modrm(s, OPC_MOVZBL, a0, a1 + 4); - } else { - tcg_out_ext16u(s, a0, a1); - tcg_out_shifti(s, SHIFT_SHR, a0, 8); - } + return; + case 32: + tcg_out_ext32u(s, a0, a1); + return; + } + } else if (TCG_TARGET_REG_BITS == 64 && ofs + len == 32) { + /* This is a 32-bit zero-extending right shift. */ + tcg_out_mov(s, TCG_TYPE_I32, a0, a1); + tcg_out_shifti(s, SHIFT_SHR, a0, ofs); + return; + } else if (ofs == 8 && len == 8) { + /* + * On the off-chance that we can use the high-byte registers. + * Otherwise we emit the same ext16 + shift pattern that we + * would have gotten from the normal tcg-op.c expansion. + */ + if (a1 < 4 && (TCG_TARGET_REG_BITS == 32 || a0 < 8)) { + tcg_out_modrm(s, OPC_MOVZBL, a0, a1 + 4); } else { - g_assert_not_reached(); + tcg_out_ext16u(s, a0, a1); + tcg_out_shifti(s, SHIFT_SHR, a0, 8); } - break; + return; + } + g_assert_not_reached(); +} + +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extract, +}; - case INDEX_op_sextract_i64: - if (a2 == 0 && args[3] == 8) { - tcg_out_ext8s(s, TCG_TYPE_I64, a0, a1); - } else if (a2 == 0 && args[3] == 16) { - tcg_out_ext16s(s, TCG_TYPE_I64, a0, a1); - } else if (a2 == 0 && args[3] == 32) { +static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0) { + switch (len) { + case 8: + tcg_out_ext8s(s, type, a0, a1); + return; + case 16: + tcg_out_ext16s(s, type, a0, a1); + return; + case 32: tcg_out_ext32s(s, a0, a1); - } else { - g_assert_not_reached(); + return; } - break; - - case INDEX_op_sextract_i32: - if (a2 == 0 && args[3] == 8) { - tcg_out_ext8s(s, TCG_TYPE_I32, a0, a1); - } else if (a2 == 0 && args[3] == 16) { - tcg_out_ext16s(s, TCG_TYPE_I32, a0, a1); - } else if (a2 == 8 && args[3] == 8) { - if (a1 < 4 && a0 < 8) { - tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4); - } else { - tcg_out_ext16s(s, TCG_TYPE_I32, a0, a1); - tcg_out_shifti(s, SHIFT_SAR, a0, 8); - } + } else if (ofs == 8 && len == 8) { + if (type == TCG_TYPE_I32 && a1 < 4 && a0 < 8) { + tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4); } else { - g_assert_not_reached(); + tcg_out_ext16s(s, type, a0, a1); + tgen_sari(s, type, a0, a0, 8); } - break; + return; + } + g_assert_not_reached(); +} - OP_32_64(extract2): - /* Note that SHRD outputs to the r/m operand. */ - tcg_out_modrm(s, OPC_SHRD_Ib + rexw, a2, a0); - tcg_out8(s, args[3]); - break; +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_sextract, +}; - case INDEX_op_mb: - tcg_out_mb(s, a0); - break; - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_mov_i64: - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - default: - g_assert_not_reached(); - } +static void tgen_extract2(TCGContext *s, TCGType type, TCGReg a0, + TCGReg a1, TCGReg a2, unsigned shr) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + + /* Note that SHRD outputs to the r/m operand. */ + tcg_out_modrm(s, OPC_SHRD_Ib + rexw, a2, a0); + tcg_out8(s, shr); +} -#undef OP_32_64 +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_O1_I2(r, 0, r), + .out_rrr = tgen_extract2, +}; + +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_modrm_offset(s, OPC_MOVZBL, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; + +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; + +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_modrm_offset(s, OPC_MOVZWL, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; + +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; + tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, dest, base, offset); } +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; + +#if TCG_TARGET_REG_BITS == 64 +static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_modrm_offset(s, OPC_MOVL_GvEv, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32u, +}; + +static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_modrm_offset(s, OPC_MOVSLQ, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32s, +}; +#endif + +static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R, data, base, offset); +} + +static void tgen_st8_i(TCGContext *s, TCGType type, tcg_target_long data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_modrm_offset(s, OPC_MOVB_EvIz, 0, base, offset); + tcg_out8(s, data); +} + +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(qi, r), + .out_r = tgen_st8_r, + .out_i = tgen_st8_i, +}; + +static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16, data, base, offset); +} + +static void tgen_st16_i(TCGContext *s, TCGType type, tcg_target_long data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16, 0, base, offset); + tcg_out16(s, data); +} + +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(ri, r), + .out_r = tgen_st16_r, + .out_i = tgen_st16_i, +}; + +static void tgen_st_i(TCGContext *s, TCGType type, tcg_target_long data, + TCGReg base, ptrdiff_t offset) +{ + bool ok = tcg_out_sti(s, type, data, base, offset); + tcg_debug_assert(ok); +} + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(re, r), + .out_r = tcg_out_st, + .out_i = tgen_st_i, +}; + static int const umin_insn[4] = { OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_VPMINUQ }; @@ -3581,182 +4147,6 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) { switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i32: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i32: - case INDEX_op_ld16s_i64: - case INDEX_op_ld_i32: - case INDEX_op_ld32u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld_i64: - return C_O1_I1(r, r); - - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - return C_O0_I2(qi, r); - - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - case INDEX_op_st_i32: - case INDEX_op_st32_i64: - return C_O0_I2(ri, r); - - case INDEX_op_st_i64: - return C_O0_I2(re, r); - - case INDEX_op_add_i32: - case INDEX_op_add_i64: - return C_O1_I2(r, r, re); - - case INDEX_op_sub_i32: - case INDEX_op_sub_i64: - case INDEX_op_mul_i32: - case INDEX_op_mul_i64: - case INDEX_op_or_i32: - case INDEX_op_or_i64: - case INDEX_op_xor_i32: - case INDEX_op_xor_i64: - return C_O1_I2(r, 0, re); - - case INDEX_op_and_i32: - case INDEX_op_and_i64: - return C_O1_I2(r, 0, reZ); - - case INDEX_op_andc_i32: - case INDEX_op_andc_i64: - return C_O1_I2(r, r, rI); - - case INDEX_op_shl_i32: - case INDEX_op_shl_i64: - case INDEX_op_shr_i32: - case INDEX_op_shr_i64: - case INDEX_op_sar_i32: - case INDEX_op_sar_i64: - return have_bmi2 ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ci); - - case INDEX_op_rotl_i32: - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i32: - case INDEX_op_rotr_i64: - return C_O1_I2(r, 0, ci); - - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - return C_O0_I2(r, reT); - - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i32: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: - case INDEX_op_neg_i32: - case INDEX_op_neg_i64: - case INDEX_op_not_i32: - case INDEX_op_not_i64: - case INDEX_op_extrh_i64_i32: - return C_O1_I1(r, 0); - - case INDEX_op_ext8s_i32: - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - return C_O1_I1(r, q); - - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - case INDEX_op_extract_i32: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i32: - case INDEX_op_sextract_i64: - case INDEX_op_ctpop_i32: - case INDEX_op_ctpop_i64: - return C_O1_I1(r, r); - - case INDEX_op_extract2_i32: - case INDEX_op_extract2_i64: - return C_O1_I2(r, 0, r); - - case INDEX_op_deposit_i32: - case INDEX_op_deposit_i64: - return C_O1_I2(q, 0, qi); - - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - case INDEX_op_negsetcond_i32: - case INDEX_op_negsetcond_i64: - return C_O1_I2(q, r, reT); - - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - return C_O1_I4(r, r, reT, r, 0); - - case INDEX_op_div2_i32: - case INDEX_op_div2_i64: - case INDEX_op_divu2_i32: - case INDEX_op_divu2_i64: - return C_O2_I3(a, d, 0, 1, r); - - case INDEX_op_mulu2_i32: - case INDEX_op_mulu2_i64: - case INDEX_op_muls2_i32: - case INDEX_op_muls2_i64: - return C_O2_I2(a, d, a, r); - - case INDEX_op_add2_i32: - case INDEX_op_add2_i64: - case INDEX_op_sub2_i32: - case INDEX_op_sub2_i64: - return C_N1_O1_I4(r, r, 0, 1, re, re); - - case INDEX_op_ctz_i32: - case INDEX_op_ctz_i64: - return have_bmi1 ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); - - case INDEX_op_clz_i32: - case INDEX_op_clz_i64: - return have_lzcnt ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); - - case INDEX_op_qemu_ld_i32: - return C_O1_I1(r, L); - - case INDEX_op_qemu_st_i32: - return C_O0_I2(L, L); - case INDEX_op_qemu_st8_i32: - return C_O0_I2(s, L); - - case INDEX_op_qemu_ld_i64: - return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) : C_O2_I1(r, r, L); - - case INDEX_op_qemu_st_i64: - return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) : C_O0_I3(L, L, L); - - case INDEX_op_qemu_ld_i128: - tcg_debug_assert(TCG_TARGET_REG_BITS == 64); - return C_O2_I1(r, r, L); - case INDEX_op_qemu_st_i128: - tcg_debug_assert(TCG_TARGET_REG_BITS == 64); - return C_O0_I3(L, L, L); - - case INDEX_op_brcond2_i32: - return C_O0_I4(r, r, ri, ri); - - case INDEX_op_setcond2_i32: - return C_O1_I4(r, r, r, ri, ri); - case INDEX_op_ld_vec: case INDEX_op_dupm_vec: return C_O1_I1(x, r); diff --git a/tcg/loongarch64/tcg-target-con-set.h b/tcg/loongarch64/tcg-target-con-set.h index 8afaee9476..fd731c0c0f 100644 --- a/tcg/loongarch64/tcg-target-con-set.h +++ b/tcg/loongarch64/tcg-target-con-set.h @@ -16,25 +16,22 @@ */ C_O0_I1(r) C_O0_I2(rz, r) -C_O0_I2(rz, rz) +C_O0_I2(r, rz) C_O0_I2(w, r) C_O0_I3(r, r, r) C_O1_I1(r, r) C_O1_I1(w, r) C_O1_I1(w, w) -C_O1_I2(r, r, rC) +C_O1_I2(r, r, r) C_O1_I2(r, r, ri) C_O1_I2(r, r, rI) C_O1_I2(r, r, rJ) C_O1_I2(r, r, rU) C_O1_I2(r, r, rW) C_O1_I2(r, 0, rz) -C_O1_I2(r, rz, ri) -C_O1_I2(r, rz, rJ) -C_O1_I2(r, rz, rz) C_O1_I2(w, w, w) C_O1_I2(w, w, wM) C_O1_I2(w, w, wA) C_O1_I3(w, w, w, w) -C_O1_I4(r, rz, rJ, rz, rz) +C_O1_I4(r, r, rJ, rz, rz) C_N2_I1(r, r, r) diff --git a/tcg/loongarch64/tcg-target-con-str.h b/tcg/loongarch64/tcg-target-con-str.h index 99759120b4..e5e57452d6 100644 --- a/tcg/loongarch64/tcg-target-con-str.h +++ b/tcg/loongarch64/tcg-target-con-str.h @@ -23,7 +23,6 @@ REGS('w', ALL_VECTOR_REGS) CONST('I', TCG_CT_CONST_S12) CONST('J', TCG_CT_CONST_S32) CONST('U', TCG_CT_CONST_U12) -CONST('C', TCG_CT_CONST_C12) CONST('W', TCG_CT_CONST_WSZ) CONST('M', TCG_CT_CONST_VCMP) CONST('A', TCG_CT_CONST_VADD) diff --git a/tcg/loongarch64/tcg-target-has.h b/tcg/loongarch64/tcg-target-has.h index 188b00799f..32abc6f457 100644 --- a/tcg/loongarch64/tcg-target-has.h +++ b/tcg/loongarch64/tcg-target-has.h @@ -9,68 +9,8 @@ #include "host/cpuinfo.h" -/* optional instructions */ -#define TCG_TARGET_HAS_negsetcond_i32 0 -#define TCG_TARGET_HAS_div_i32 1 -#define TCG_TARGET_HAS_rem_i32 1 -#define TCG_TARGET_HAS_div2_i32 0 -#define TCG_TARGET_HAS_rot_i32 1 -#define TCG_TARGET_HAS_extract2_i32 0 -#define TCG_TARGET_HAS_add2_i32 0 -#define TCG_TARGET_HAS_sub2_i32 0 -#define TCG_TARGET_HAS_mulu2_i32 0 -#define TCG_TARGET_HAS_muls2_i32 0 -#define TCG_TARGET_HAS_muluh_i32 1 -#define TCG_TARGET_HAS_mulsh_i32 1 -#define TCG_TARGET_HAS_ext8s_i32 1 -#define TCG_TARGET_HAS_ext16s_i32 1 -#define TCG_TARGET_HAS_ext8u_i32 1 -#define TCG_TARGET_HAS_ext16u_i32 1 -#define TCG_TARGET_HAS_bswap16_i32 1 -#define TCG_TARGET_HAS_bswap32_i32 1 -#define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_andc_i32 1 -#define TCG_TARGET_HAS_orc_i32 1 -#define TCG_TARGET_HAS_eqv_i32 0 -#define TCG_TARGET_HAS_nand_i32 0 -#define TCG_TARGET_HAS_nor_i32 1 -#define TCG_TARGET_HAS_clz_i32 1 -#define TCG_TARGET_HAS_ctz_i32 1 -#define TCG_TARGET_HAS_ctpop_i32 0 -#define TCG_TARGET_HAS_qemu_st8_i32 0 - /* 64-bit operations */ -#define TCG_TARGET_HAS_negsetcond_i64 0 -#define TCG_TARGET_HAS_div_i64 1 -#define TCG_TARGET_HAS_rem_i64 1 -#define TCG_TARGET_HAS_div2_i64 0 -#define TCG_TARGET_HAS_rot_i64 1 -#define TCG_TARGET_HAS_extract2_i64 0 #define TCG_TARGET_HAS_extr_i64_i32 1 -#define TCG_TARGET_HAS_ext8s_i64 1 -#define TCG_TARGET_HAS_ext16s_i64 1 -#define TCG_TARGET_HAS_ext32s_i64 1 -#define TCG_TARGET_HAS_ext8u_i64 1 -#define TCG_TARGET_HAS_ext16u_i64 1 -#define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_bswap16_i64 1 -#define TCG_TARGET_HAS_bswap32_i64 1 -#define TCG_TARGET_HAS_bswap64_i64 1 -#define TCG_TARGET_HAS_not_i64 1 -#define TCG_TARGET_HAS_andc_i64 1 -#define TCG_TARGET_HAS_orc_i64 1 -#define TCG_TARGET_HAS_eqv_i64 0 -#define TCG_TARGET_HAS_nand_i64 0 -#define TCG_TARGET_HAS_nor_i64 1 -#define TCG_TARGET_HAS_clz_i64 1 -#define TCG_TARGET_HAS_ctz_i64 1 -#define TCG_TARGET_HAS_ctpop_i64 0 -#define TCG_TARGET_HAS_add2_i64 0 -#define TCG_TARGET_HAS_sub2_i64 0 -#define TCG_TARGET_HAS_mulu2_i64 0 -#define TCG_TARGET_HAS_muls2_i64 0 -#define TCG_TARGET_HAS_muluh_i64 1 -#define TCG_TARGET_HAS_mulsh_i64 1 #define TCG_TARGET_HAS_qemu_ldst_i128 (cpuinfo & CPUINFO_LSX) diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index cbd7642b58..e5580d69a8 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -176,10 +176,9 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define TCG_CT_CONST_S12 0x100 #define TCG_CT_CONST_S32 0x200 #define TCG_CT_CONST_U12 0x400 -#define TCG_CT_CONST_C12 0x800 -#define TCG_CT_CONST_WSZ 0x1000 -#define TCG_CT_CONST_VCMP 0x2000 -#define TCG_CT_CONST_VADD 0x4000 +#define TCG_CT_CONST_WSZ 0x800 +#define TCG_CT_CONST_VCMP 0x1000 +#define TCG_CT_CONST_VADD 0x2000 #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) #define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32) @@ -205,18 +204,27 @@ static bool tcg_target_const_match(int64_t val, int ct, if ((ct & TCG_CT_CONST_U12) && val >= 0 && val <= 0xfff) { return true; } - if ((ct & TCG_CT_CONST_C12) && ~val >= 0 && ~val <= 0xfff) { - return true; - } if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) { return true; } - int64_t vec_val = sextract64(val, 0, 8 << vece); - if ((ct & TCG_CT_CONST_VCMP) && -0x10 <= vec_val && vec_val <= 0x1f) { - return true; - } - if ((ct & TCG_CT_CONST_VADD) && -0x1f <= vec_val && vec_val <= 0x1f) { - return true; + if (ct & (TCG_CT_CONST_VCMP | TCG_CT_CONST_VADD)) { + int64_t vec_val = sextract64(val, 0, 8 << vece); + if (ct & TCG_CT_CONST_VCMP) { + switch (cond) { + case TCG_COND_EQ: + case TCG_COND_LE: + case TCG_COND_LT: + return -0x10 <= vec_val && vec_val <= 0x0f; + case TCG_COND_LEU: + case TCG_COND_LTU: + return 0x00 <= vec_val && vec_val <= 0x1f; + default: + return false; + } + } + if ((ct & TCG_CT_CONST_VADD) && -0x1f <= vec_val && vec_val <= 0x1f) { + return true; + } } return false; } @@ -293,7 +301,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, * TCG intrinsics */ -static void tcg_out_mb(TCGContext *s, TCGArg a0) +static void tcg_out_mb(TCGContext *s, unsigned a0) { /* Baseline LoongArch only has the full barrier, unfortunately. */ tcg_out_opc_dbar(s, 0); @@ -538,28 +546,6 @@ static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg) tcg_out_ext32s(s, ret, arg); } -static void tcg_out_clzctz(TCGContext *s, LoongArchInsn opc, - TCGReg a0, TCGReg a1, TCGReg a2, - bool c2, bool is_32bit) -{ - if (c2) { - /* - * Fast path: semantics already satisfied due to constraint and - * insn behavior, single instruction is enough. - */ - tcg_debug_assert(a2 == (is_32bit ? 32 : 64)); - /* all clz/ctz insns belong to DJ-format */ - tcg_out32(s, encode_dj_insn(opc, a0, a1)); - return; - } - - tcg_out32(s, encode_dj_insn(opc, TCG_REG_TMP0, a1)); - /* a0 = a1 ? REG_TMP0 : a2 */ - tcg_out_opc_maskeqz(s, TCG_REG_TMP0, TCG_REG_TMP0, a1); - tcg_out_opc_masknez(s, a0, a2, a1); - tcg_out_opc_or(s, a0, TCG_REG_TMP0, a0); -} - #define SETCOND_INV TCG_TARGET_NB_REGS #define SETCOND_NEZ (SETCOND_INV << 1) #define SETCOND_FLAGS (SETCOND_INV | SETCOND_NEZ) @@ -660,14 +646,29 @@ static int tcg_out_setcond_int(TCGContext *s, TCGCond cond, TCGReg ret, } static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg arg1, tcg_target_long arg2, bool c2) + TCGReg arg1, tcg_target_long arg2, + bool c2, bool neg) { int tmpflags = tcg_out_setcond_int(s, cond, ret, arg1, arg2, c2); + TCGReg tmp = tmpflags & ~SETCOND_FLAGS; - if (tmpflags != ret) { - TCGReg tmp = tmpflags & ~SETCOND_FLAGS; - + if (neg) { + /* If intermediate result is zero/non-zero: test != 0. */ + if (tmpflags & SETCOND_NEZ) { + tcg_out_opc_sltu(s, ret, TCG_REG_ZERO, tmp); + tmp = ret; + } + /* Produce the 0/-1 result. */ + if (tmpflags & SETCOND_INV) { + tcg_out_opc_addi_d(s, ret, tmp, -1); + } else { + tcg_out_opc_sub_d(s, ret, TCG_REG_ZERO, tmp); + } + } else { switch (tmpflags & SETCOND_FLAGS) { + case 0: + tcg_debug_assert(tmp == ret); + break; case SETCOND_INV: /* Intermediate result is boolean: simply invert. */ tcg_out_opc_xori(s, ret, tmp, 1); @@ -686,11 +687,47 @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, } } -static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg c1, tcg_target_long c2, bool const2, - TCGReg v1, TCGReg v2) +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_setcond(s, cond, dest, arg1, arg2, false, false); +} + +static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tcg_out_setcond(s, cond, dest, arg1, arg2, true, false); +} + +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_setcond, + .out_rri = tgen_setcondi, +}; + +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_setcond(s, cond, dest, arg1, arg2, false, true); +} + +static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tcg_out_setcond(s, cond, dest, arg1, arg2, true, true); +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_negsetcond, + .out_rri = tgen_negsetcondi, +}; + +static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, + TCGArg v1, bool const_v1, TCGArg v2, bool const_v2) { - int tmpflags = tcg_out_setcond_int(s, cond, TCG_REG_TMP0, c1, c2, const2); + int tmpflags = tcg_out_setcond_int(s, cond, TCG_REG_TMP0, c1, c2, const_c2); TCGReg t; /* Standardize the test below to t != 0. */ @@ -710,10 +747,21 @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, } } +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = C_O1_I4(r, r, rJ, rz, rz), + .out = tgen_movcond, +}; + /* * Branch helpers */ +static void tcg_out_br(TCGContext *s, TCGLabel *l) +{ + tcg_out_reloc(s, s->code_ptr, R_LOONGARCH_BR_SD10K16, l, 0); + tcg_out_opc_b(s, 0); +} + static const struct { LoongArchInsn op; bool swap; @@ -730,8 +778,8 @@ static const struct { [TCG_COND_GTU] = { OPC_BGTU, false } }; -static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, - TCGReg arg2, TCGLabel *l) +static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, TCGReg arg2, TCGLabel *l) { LoongArchInsn op = tcg_brcond_to_loongarch[cond].op; @@ -748,6 +796,11 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, tcg_out32(s, encode_djsk16_insn(op, arg1, arg2, 0)); } +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, rz), + .out_rr = tgen_brcond, +}; + static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; @@ -1114,22 +1167,27 @@ static void tcg_out_qemu_ld_indexed(TCGContext *s, MemOp opc, TCGType type, } } -static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType data_type) +static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data_reg, + TCGReg addr_reg, MemOpIdx oi) { TCGLabelQemuLdst *ldst; HostAddress h; ldst = prepare_host_addr(s, &h, addr_reg, oi, true); - tcg_out_qemu_ld_indexed(s, get_memop(oi), data_type, data_reg, h); + tcg_out_qemu_ld_indexed(s, get_memop(oi), type, data_reg, h); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = data_reg; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_qemu_ld, +}; + static void tcg_out_qemu_st_indexed(TCGContext *s, MemOp opc, TCGReg rd, HostAddress h) { @@ -1154,8 +1212,8 @@ static void tcg_out_qemu_st_indexed(TCGContext *s, MemOp opc, } } -static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType data_type) +static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data_reg, + TCGReg addr_reg, MemOpIdx oi) { TCGLabelQemuLdst *ldst; HostAddress h; @@ -1164,12 +1222,17 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_out_qemu_st_indexed(s, get_memop(oi), data_reg, h); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = data_reg; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out = tgen_qemu_st, +}; + static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg data_lo, TCGReg data_hi, TCGReg addr_reg, MemOpIdx oi, bool is_ld) { @@ -1217,6 +1280,28 @@ static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg data_lo, TCGReg data_hi } } +static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr_reg, MemOpIdx oi) +{ + tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, true); +} + +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + .base.static_constraint = C_N2_I1(r, r, r), + .out = tgen_qemu_ld2, +}; + +static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr_reg, MemOpIdx oi) +{ + tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, false); +} + +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + .base.static_constraint = C_O0_I3(r, r, r), + .out = tgen_qemu_st2, +}; + /* * Entry-points */ @@ -1254,6 +1339,11 @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + tcg_out_opc_jirl(s, TCG_REG_ZERO, a0, 0); +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { @@ -1274,445 +1364,684 @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, flush_idcache_range(jmp_rx, jmp_rw, 4); } -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) -{ - TCGArg a0 = args[0]; - TCGArg a1 = args[1]; - TCGArg a2 = args[2]; - TCGArg a3 = args[3]; - int c2 = const_args[2]; - - switch (opc) { - case INDEX_op_mb: - tcg_out_mb(s, a0); - break; - case INDEX_op_goto_ptr: - tcg_out_opc_jirl(s, TCG_REG_ZERO, a0, 0); - break; +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_add_w(s, a0, a1, a2); + } else { + tcg_out_opc_add_d(s, a0, a1, a2); + } +} - case INDEX_op_br: - tcg_out_reloc(s, s->code_ptr, R_LOONGARCH_BR_SD10K16, arg_label(a0), - 0); - tcg_out_opc_b(s, 0); - break; +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_add, + .out_rri = tcg_out_addi, +}; - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); - break; +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_extrh_i64_i32: - tcg_out_opc_srai_d(s, a0, a1, 32); - break; +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_not_i32: - case INDEX_op_not_i64: - tcg_out_opc_nor(s, a0, a1, TCG_REG_ZERO); - break; +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_nor_i32: - case INDEX_op_nor_i64: - if (c2) { - tcg_out_opc_ori(s, a0, a1, a2); - tcg_out_opc_nor(s, a0, a0, TCG_REG_ZERO); - } else { - tcg_out_opc_nor(s, a0, a1, a2); - } - break; +static void tcg_out_set_carry(TCGContext *s) +{ + g_assert_not_reached(); +} - case INDEX_op_andc_i32: - case INDEX_op_andc_i64: - if (c2) { - /* guaranteed to fit due to constraint */ - tcg_out_opc_andi(s, a0, a1, ~a2); - } else { - tcg_out_opc_andn(s, a0, a1, a2); - } - break; +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_and(s, a0, a1, a2); +} - case INDEX_op_orc_i32: - case INDEX_op_orc_i64: - if (c2) { - /* guaranteed to fit due to constraint */ - tcg_out_opc_ori(s, a0, a1, ~a2); - } else { - tcg_out_opc_orn(s, a0, a1, a2); - } - break; +static void tgen_andi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_opc_andi(s, a0, a1, a2); +} - case INDEX_op_and_i32: - case INDEX_op_and_i64: - if (c2) { - tcg_out_opc_andi(s, a0, a1, a2); - } else { - tcg_out_opc_and(s, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, r, rU), + .out_rrr = tgen_and, + .out_rri = tgen_andi, +}; - case INDEX_op_or_i32: - case INDEX_op_or_i64: - if (c2) { - tcg_out_opc_ori(s, a0, a1, a2); - } else { - tcg_out_opc_or(s, a0, a1, a2); - } - break; +static void tgen_andc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_andn(s, a0, a1, a2); +} - case INDEX_op_xor_i32: - case INDEX_op_xor_i64: - if (c2) { - tcg_out_opc_xori(s, a0, a1, a2); - } else { - tcg_out_opc_xor(s, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_andc, +}; - case INDEX_op_extract_i32: - if (a2 == 0 && args[3] <= 12) { - tcg_out_opc_andi(s, a0, a1, (1 << args[3]) - 1); - } else { - tcg_out_opc_bstrpick_w(s, a0, a1, a2, a2 + args[3] - 1); - } - break; - case INDEX_op_extract_i64: - if (a2 == 0 && args[3] <= 12) { - tcg_out_opc_andi(s, a0, a1, (1 << args[3]) - 1); - } else { - tcg_out_opc_bstrpick_d(s, a0, a1, a2, a2 + args[3] - 1); - } - break; +static void tgen_clzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + /* a2 is constrained to exactly the type width. */ + if (type == TCG_TYPE_I32) { + tcg_out_opc_clz_w(s, a0, a1); + } else { + tcg_out_opc_clz_d(s, a0, a1); + } +} - case INDEX_op_sextract_i64: - if (a2 + args[3] == 32) { - if (a2 == 0) { - tcg_out_ext32s(s, a0, a1); - } else { - tcg_out_opc_srai_w(s, a0, a1, a2); - } - break; - } - /* FALLTHRU */ - case INDEX_op_sextract_i32: - if (a2 == 0 && args[3] == 8) { - tcg_out_ext8s(s, TCG_TYPE_REG, a0, a1); - } else if (a2 == 0 && args[3] == 16) { - tcg_out_ext16s(s, TCG_TYPE_REG, a0, a1); - } else { - g_assert_not_reached(); - } - break; +static void tgen_clz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_clzi(s, type, TCG_REG_TMP0, a1, /* ignored */ 0); + /* a0 = a1 ? REG_TMP0 : a2 */ + tcg_out_opc_maskeqz(s, TCG_REG_TMP0, TCG_REG_TMP0, a1); + tcg_out_opc_masknez(s, a0, a2, a1); + tcg_out_opc_or(s, a0, a0, TCG_REG_TMP0); +} - case INDEX_op_deposit_i32: - tcg_out_opc_bstrins_w(s, a0, a2, args[3], args[3] + args[4] - 1); - break; - case INDEX_op_deposit_i64: - tcg_out_opc_bstrins_d(s, a0, a2, args[3], args[3] + args[4] - 1); - break; +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_O1_I2(r, r, rW), + .out_rrr = tgen_clz, + .out_rri = tgen_clzi, +}; - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - tcg_out_opc_revb_2h(s, a0, a1); - if (a2 & TCG_BSWAP_OS) { - tcg_out_ext16s(s, TCG_TYPE_REG, a0, a0); - } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - tcg_out_ext16u(s, a0, a0); - } - break; +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_bswap32_i32: - /* All 32-bit values are computed sign-extended in the register. */ - a2 = TCG_BSWAP_OS; - /* fallthrough */ - case INDEX_op_bswap32_i64: - tcg_out_opc_revb_2w(s, a0, a1); - if (a2 & TCG_BSWAP_OS) { - tcg_out_ext32s(s, a0, a0); - } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - tcg_out_ext32u(s, a0, a0); - } - break; +static void tgen_ctzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + /* a2 is constrained to exactly the type width. */ + if (type == TCG_TYPE_I32) { + tcg_out_opc_ctz_w(s, a0, a1); + } else { + tcg_out_opc_ctz_d(s, a0, a1); + } +} - case INDEX_op_bswap64_i64: - tcg_out_opc_revb_d(s, a0, a1); - break; +static void tgen_ctz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_ctzi(s, type, TCG_REG_TMP0, a1, /* ignored */ 0); + /* a0 = a1 ? REG_TMP0 : a2 */ + tcg_out_opc_maskeqz(s, TCG_REG_TMP0, TCG_REG_TMP0, a1); + tcg_out_opc_masknez(s, a0, a2, a1); + tcg_out_opc_or(s, a0, a0, TCG_REG_TMP0); +} - case INDEX_op_clz_i32: - tcg_out_clzctz(s, OPC_CLZ_W, a0, a1, a2, c2, true); - break; - case INDEX_op_clz_i64: - tcg_out_clzctz(s, OPC_CLZ_D, a0, a1, a2, c2, false); - break; +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_O1_I2(r, r, rW), + .out_rrr = tgen_ctz, + .out_rri = tgen_ctzi, +}; - case INDEX_op_ctz_i32: - tcg_out_clzctz(s, OPC_CTZ_W, a0, a1, a2, c2, true); - break; - case INDEX_op_ctz_i64: - tcg_out_clzctz(s, OPC_CTZ_D, a0, a1, a2, c2, false); - break; +static void tgen_divs(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_div_w(s, a0, a1, a2); + } else { + tcg_out_opc_div_d(s, a0, a1, a2); + } +} - case INDEX_op_shl_i32: - if (c2) { - tcg_out_opc_slli_w(s, a0, a1, a2 & 0x1f); - } else { - tcg_out_opc_sll_w(s, a0, a1, a2); - } - break; - case INDEX_op_shl_i64: - if (c2) { - tcg_out_opc_slli_d(s, a0, a1, a2 & 0x3f); - } else { - tcg_out_opc_sll_d(s, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divs, +}; - case INDEX_op_shr_i32: - if (c2) { - tcg_out_opc_srli_w(s, a0, a1, a2 & 0x1f); - } else { - tcg_out_opc_srl_w(s, a0, a1, a2); - } - break; - case INDEX_op_shr_i64: - if (c2) { - tcg_out_opc_srli_d(s, a0, a1, a2 & 0x3f); - } else { - tcg_out_opc_srl_d(s, a0, a1, a2); - } - break; +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_sar_i32: - if (c2) { - tcg_out_opc_srai_w(s, a0, a1, a2 & 0x1f); - } else { - tcg_out_opc_sra_w(s, a0, a1, a2); - } - break; - case INDEX_op_sar_i64: - if (c2) { - tcg_out_opc_srai_d(s, a0, a1, a2 & 0x3f); - } else { - tcg_out_opc_sra_d(s, a0, a1, a2); - } - break; +static void tgen_divu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_div_wu(s, a0, a1, a2); + } else { + tcg_out_opc_div_du(s, a0, a1, a2); + } +} - case INDEX_op_rotl_i32: - /* transform into equivalent rotr/rotri */ - if (c2) { - tcg_out_opc_rotri_w(s, a0, a1, (32 - a2) & 0x1f); - } else { - tcg_out_opc_sub_w(s, TCG_REG_TMP0, TCG_REG_ZERO, a2); - tcg_out_opc_rotr_w(s, a0, a1, TCG_REG_TMP0); - } - break; - case INDEX_op_rotl_i64: - /* transform into equivalent rotr/rotri */ - if (c2) { - tcg_out_opc_rotri_d(s, a0, a1, (64 - a2) & 0x3f); - } else { - tcg_out_opc_sub_w(s, TCG_REG_TMP0, TCG_REG_ZERO, a2); - tcg_out_opc_rotr_d(s, a0, a1, TCG_REG_TMP0); - } - break; +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divu, +}; - case INDEX_op_rotr_i32: - if (c2) { - tcg_out_opc_rotri_w(s, a0, a1, a2 & 0x1f); - } else { - tcg_out_opc_rotr_w(s, a0, a1, a2); - } - break; - case INDEX_op_rotr_i64: - if (c2) { - tcg_out_opc_rotri_d(s, a0, a1, a2 & 0x3f); - } else { - tcg_out_opc_rotr_d(s, a0, a1, a2); - } - break; +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_add_i32: - if (c2) { - tcg_out_addi(s, TCG_TYPE_I32, a0, a1, a2); - } else { - tcg_out_opc_add_w(s, a0, a1, a2); - } - break; - case INDEX_op_add_i64: - if (c2) { - tcg_out_addi(s, TCG_TYPE_I64, a0, a1, a2); - } else { - tcg_out_opc_add_d(s, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_sub_i32: - if (c2) { - tcg_out_addi(s, TCG_TYPE_I32, a0, a1, -a2); - } else { - tcg_out_opc_sub_w(s, a0, a1, a2); - } - break; - case INDEX_op_sub_i64: - if (c2) { - tcg_out_addi(s, TCG_TYPE_I64, a0, a1, -a2); - } else { - tcg_out_opc_sub_d(s, a0, a1, a2); - } - break; +static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_opc_srai_d(s, a0, a1, 32); +} - case INDEX_op_neg_i32: - tcg_out_opc_sub_w(s, a0, TCG_REG_ZERO, a1); - break; - case INDEX_op_neg_i64: - tcg_out_opc_sub_d(s, a0, TCG_REG_ZERO, a1); - break; +static const TCGOutOpUnary outop_extrh_i64_i32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extrh_i64_i32, +}; - case INDEX_op_mul_i32: +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { tcg_out_opc_mul_w(s, a0, a1, a2); - break; - case INDEX_op_mul_i64: + } else { tcg_out_opc_mul_d(s, a0, a1, a2); - break; + } +} + +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_mul, +}; + +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_mulsh_i32: +static void tgen_mulsh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { tcg_out_opc_mulh_w(s, a0, a1, a2); - break; - case INDEX_op_mulsh_i64: + } else { tcg_out_opc_mulh_d(s, a0, a1, a2); - break; + } +} + +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_mulsh, +}; - case INDEX_op_muluh_i32: +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_muluh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { tcg_out_opc_mulh_wu(s, a0, a1, a2); - break; - case INDEX_op_muluh_i64: + } else { tcg_out_opc_mulh_du(s, a0, a1, a2); - break; + } +} - case INDEX_op_div_i32: - tcg_out_opc_div_w(s, a0, a1, a2); - break; - case INDEX_op_div_i64: - tcg_out_opc_div_d(s, a0, a1, a2); - break; +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_muluh, +}; - case INDEX_op_divu_i32: - tcg_out_opc_div_wu(s, a0, a1, a2); - break; - case INDEX_op_divu_i64: - tcg_out_opc_div_du(s, a0, a1, a2); - break; +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_nor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_nor(s, a0, a1, a2); +} + +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_nor, +}; + +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_or(s, a0, a1, a2); +} + +static void tgen_ori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_opc_ori(s, a0, a1, a2); +} + +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, r, rU), + .out_rrr = tgen_or, + .out_rri = tgen_ori, +}; + +static void tgen_orc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_orn(s, a0, a1, a2); +} + +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_orc, +}; - case INDEX_op_rem_i32: +static void tgen_rems(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { tcg_out_opc_mod_w(s, a0, a1, a2); - break; - case INDEX_op_rem_i64: + } else { tcg_out_opc_mod_d(s, a0, a1, a2); - break; + } +} + +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_rems, +}; - case INDEX_op_remu_i32: +static void tgen_remu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { tcg_out_opc_mod_wu(s, a0, a1, a2); - break; - case INDEX_op_remu_i64: + } else { tcg_out_opc_mod_du(s, a0, a1, a2); - break; + } +} - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - tcg_out_setcond(s, args[3], a0, a1, a2, c2); - break; +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_remu, +}; - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - tcg_out_movcond(s, args[5], a0, a1, a2, c2, args[3], args[4]); - break; +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_ld8s_i32: - case INDEX_op_ld8s_i64: - tcg_out_ldst(s, OPC_LD_B, a0, a1, a2); - break; - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - tcg_out_ldst(s, OPC_LD_BU, a0, a1, a2); - break; - case INDEX_op_ld16s_i32: - case INDEX_op_ld16s_i64: - tcg_out_ldst(s, OPC_LD_H, a0, a1, a2); - break; - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - tcg_out_ldst(s, OPC_LD_HU, a0, a1, a2); - break; - case INDEX_op_ld_i32: - case INDEX_op_ld32s_i64: - tcg_out_ldst(s, OPC_LD_W, a0, a1, a2); - break; - case INDEX_op_ld32u_i64: - tcg_out_ldst(s, OPC_LD_WU, a0, a1, a2); - break; - case INDEX_op_ld_i64: - tcg_out_ldst(s, OPC_LD_D, a0, a1, a2); - break; +static void tgen_rotr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_rotr_w(s, a0, a1, a2); + } else { + tcg_out_opc_rotr_d(s, a0, a1, a2); + } +} - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - tcg_out_ldst(s, OPC_ST_B, a0, a1, a2); - break; - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - tcg_out_ldst(s, OPC_ST_H, a0, a1, a2); - break; - case INDEX_op_st_i32: - case INDEX_op_st32_i64: - tcg_out_ldst(s, OPC_ST_W, a0, a1, a2); - break; - case INDEX_op_st_i64: - tcg_out_ldst(s, OPC_ST_D, a0, a1, a2); - break; +static void tgen_rotri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_rotri_w(s, a0, a1, a2 & 0x1f); + } else { + tcg_out_opc_rotri_d(s, a0, a1, a2 & 0x3f); + } +} - case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); - break; - case INDEX_op_qemu_ld_i128: - tcg_out_qemu_ldst_i128(s, a0, a1, a2, a3, true); - break; - case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); - break; - case INDEX_op_qemu_st_i128: - tcg_out_qemu_ldst_i128(s, a0, a1, a2, a3, false); - break; +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_rotr, + .out_rri = tgen_rotri, +}; - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_mov_i64: - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - default: - g_assert_not_reached(); +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_sra_w(s, a0, a1, a2); + } else { + tcg_out_opc_sra_d(s, a0, a1, a2); } } +static void tgen_sari(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_srai_w(s, a0, a1, a2 & 0x1f); + } else { + tcg_out_opc_srai_d(s, a0, a1, a2 & 0x3f); + } +} + +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_sar, + .out_rri = tgen_sari, +}; + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_sll_w(s, a0, a1, a2); + } else { + tcg_out_opc_sll_d(s, a0, a1, a2); + } +} + +static void tgen_shli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_slli_w(s, a0, a1, a2 & 0x1f); + } else { + tcg_out_opc_slli_d(s, a0, a1, a2 & 0x3f); + } +} + +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shl, + .out_rri = tgen_shli, +}; + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_srl_w(s, a0, a1, a2); + } else { + tcg_out_opc_srl_d(s, a0, a1, a2); + } +} + +static void tgen_shri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_srli_w(s, a0, a1, a2 & 0x1f); + } else { + tcg_out_opc_srli_d(s, a0, a1, a2 & 0x3f); + } +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shr, + .out_rri = tgen_shri, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_sub_w(s, a0, a1, a2); + } else { + tcg_out_opc_sub_d(s, a0, a1, a2); + } +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_sub, +}; + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpAddSubCarry outop_subbi = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_NotImplemented, +}; + +static void tcg_out_set_borrow(TCGContext *s) +{ + g_assert_not_reached(); +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_xor(s, a0, a1, a2); +} + +static void tgen_xori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_opc_xori(s, a0, a1, a2); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, r, rU), + .out_rrr = tgen_xor, + .out_rri = tgen_xori, +}; + +static void tgen_bswap16(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_opc_revb_2h(s, a0, a1); + if (flags & TCG_BSWAP_OS) { + tcg_out_ext16s(s, TCG_TYPE_REG, a0, a0); + } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + tcg_out_ext16u(s, a0, a0); + } +} + +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap16, +}; + +static void tgen_bswap32(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_opc_revb_2w(s, a0, a1); + + /* All 32-bit values are computed sign-extended in the register. */ + if (type == TCG_TYPE_I32 || (flags & TCG_BSWAP_OS)) { + tcg_out_ext32s(s, a0, a0); + } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + tcg_out_ext32u(s, a0, a0); + } +} + +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap32, +}; + +static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_opc_revb_d(s, a0, a1); +} + +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap64, +}; + +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_sub(s, type, a0, TCG_REG_ZERO, a1); +} + +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_neg, +}; + +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_nor(s, type, a0, a1, TCG_REG_ZERO); +} + +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_not, +}; + +static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned ofs, unsigned len) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_bstrins_w(s, a0, a2, ofs, ofs + len - 1); + } else { + tcg_out_opc_bstrins_d(s, a0, a2, ofs, ofs + len - 1); + } +} + +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_O1_I2(r, 0, rz), + .out_rrr = tgen_deposit, +}; + +static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0 && len <= 12) { + tcg_out_opc_andi(s, a0, a1, (1 << len) - 1); + } else if (type == TCG_TYPE_I32) { + tcg_out_opc_bstrpick_w(s, a0, a1, ofs, ofs + len - 1); + } else { + tcg_out_opc_bstrpick_d(s, a0, a1, ofs, ofs + len - 1); + } +} + +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extract, +}; + +static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0) { + switch (len) { + case 8: + tcg_out_ext8s(s, type, a0, a1); + return; + case 16: + tcg_out_ext16s(s, type, a0, a1); + return; + case 32: + tcg_out_ext32s(s, a0, a1); + return; + } + } else if (ofs + len == 32) { + tcg_out_opc_srai_w(s, a0, a1, ofs); + return; + } + g_assert_not_reached(); +} + +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_sextract, +}; + +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LD_BU, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; + +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LD_B, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; + +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LD_HU, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; + +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LD_H, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; + +static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LD_WU, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32u, +}; + +static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LD_W, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32s, +}; + +static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_ST_B, data, base, offset); +} + +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st8_r, +}; + +static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_ST_H, data, base, offset); +} + +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st16_r, +}; + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tcg_out_st, +}; + static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg rd, TCGReg rs) { @@ -2027,28 +2356,22 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, * Try vseqi/vslei/vslti */ int64_t value = sextract64(a2, 0, 8 << vece); - if ((cond == TCG_COND_EQ || - cond == TCG_COND_LE || - cond == TCG_COND_LT) && - (-0x10 <= value && value <= 0x0f)) { + switch (cond) { + case TCG_COND_EQ: + case TCG_COND_LE: + case TCG_COND_LT: insn = cmp_vec_imm_insn[cond][lasx][vece]; tcg_out32(s, encode_vdvjsk5_insn(insn, a0, a1, value)); break; - } else if ((cond == TCG_COND_LEU || - cond == TCG_COND_LTU) && - (0x00 <= value && value <= 0x1f)) { + case TCG_COND_LEU: + case TCG_COND_LTU: insn = cmp_vec_imm_insn[cond][lasx][vece]; tcg_out32(s, encode_vdvjuk5_insn(insn, a0, a1, value)); break; + default: + g_assert_not_reached(); } - - /* - * Fallback to: - * dupi_vec temp, a2 - * cmp_vec a0, a1, temp, cond - */ - tcg_out_dupi_vec(s, type, vece, TCG_VEC_TMP0, a2); - a2 = TCG_VEC_TMP0; + break; } insn = cmp_vec_insn[cond][lasx][vece]; @@ -2213,150 +2536,6 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) { switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i32: - case INDEX_op_st_i64: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: - return C_O0_I2(rz, r); - - case INDEX_op_qemu_ld_i128: - return C_N2_I1(r, r, r); - - case INDEX_op_qemu_st_i128: - return C_O0_I3(r, r, r); - - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - return C_O0_I2(rz, rz); - - case INDEX_op_ext8s_i32: - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - case INDEX_op_extrh_i64_i32: - case INDEX_op_ext_i32_i64: - case INDEX_op_neg_i32: - case INDEX_op_neg_i64: - case INDEX_op_not_i32: - case INDEX_op_not_i64: - case INDEX_op_extract_i32: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i32: - case INDEX_op_sextract_i64: - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i32: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: - case INDEX_op_ld8s_i32: - case INDEX_op_ld8s_i64: - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld16s_i32: - case INDEX_op_ld16s_i64: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld32u_i64: - case INDEX_op_ld_i32: - case INDEX_op_ld_i64: - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: - return C_O1_I1(r, r); - - case INDEX_op_andc_i32: - case INDEX_op_andc_i64: - case INDEX_op_orc_i32: - case INDEX_op_orc_i64: - /* - * LoongArch insns for these ops don't have reg-imm forms, but we - * can express using andi/ori if ~constant satisfies - * TCG_CT_CONST_U12. - */ - return C_O1_I2(r, r, rC); - - case INDEX_op_shl_i32: - case INDEX_op_shl_i64: - case INDEX_op_shr_i32: - case INDEX_op_shr_i64: - case INDEX_op_sar_i32: - case INDEX_op_sar_i64: - case INDEX_op_rotl_i32: - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i32: - case INDEX_op_rotr_i64: - return C_O1_I2(r, r, ri); - - case INDEX_op_add_i32: - return C_O1_I2(r, r, ri); - case INDEX_op_add_i64: - return C_O1_I2(r, r, rJ); - - case INDEX_op_and_i32: - case INDEX_op_and_i64: - case INDEX_op_nor_i32: - case INDEX_op_nor_i64: - case INDEX_op_or_i32: - case INDEX_op_or_i64: - case INDEX_op_xor_i32: - case INDEX_op_xor_i64: - /* LoongArch reg-imm bitops have their imms ZERO-extended */ - return C_O1_I2(r, r, rU); - - case INDEX_op_clz_i32: - case INDEX_op_clz_i64: - case INDEX_op_ctz_i32: - case INDEX_op_ctz_i64: - return C_O1_I2(r, r, rW); - - case INDEX_op_deposit_i32: - case INDEX_op_deposit_i64: - /* Must deposit into the same register as input */ - return C_O1_I2(r, 0, rz); - - case INDEX_op_sub_i32: - case INDEX_op_setcond_i32: - return C_O1_I2(r, rz, ri); - case INDEX_op_sub_i64: - case INDEX_op_setcond_i64: - return C_O1_I2(r, rz, rJ); - - case INDEX_op_mul_i32: - case INDEX_op_mul_i64: - case INDEX_op_mulsh_i32: - case INDEX_op_mulsh_i64: - case INDEX_op_muluh_i32: - case INDEX_op_muluh_i64: - case INDEX_op_div_i32: - case INDEX_op_div_i64: - case INDEX_op_divu_i32: - case INDEX_op_divu_i64: - case INDEX_op_rem_i32: - case INDEX_op_rem_i64: - case INDEX_op_remu_i32: - case INDEX_op_remu_i64: - return C_O1_I2(r, rz, rz); - - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - return C_O1_I4(r, rz, rJ, rz, rz); - case INDEX_op_ld_vec: case INDEX_op_dupm_vec: case INDEX_op_dup_vec: diff --git a/tcg/mips/tcg-target-con-set.h b/tcg/mips/tcg-target-con-set.h index 06ab04cc4d..5304691dc1 100644 --- a/tcg/mips/tcg-target-con-set.h +++ b/tcg/mips/tcg-target-con-set.h @@ -10,12 +10,10 @@ * tcg-target-con-str.h; the constraint combination is inclusive or. */ C_O0_I1(r) +C_O0_I2(r, rz) C_O0_I2(rz, r) -C_O0_I2(rz, rz) -C_O0_I3(rz, r, r) C_O0_I3(rz, rz, r) -C_O0_I4(rz, rz, rz, rz) -C_O0_I4(rz, rz, r, r) +C_O0_I4(r, r, rz, rz) C_O1_I1(r, r) C_O1_I2(r, 0, rz) C_O1_I2(r, r, r) @@ -23,11 +21,10 @@ C_O1_I2(r, r, ri) C_O1_I2(r, r, rI) C_O1_I2(r, r, rIK) C_O1_I2(r, r, rJ) +C_O1_I2(r, r, rz) C_O1_I2(r, r, rzW) -C_O1_I2(r, rz, rN) -C_O1_I2(r, rz, rz) -C_O1_I4(r, rz, rz, rz, 0) -C_O1_I4(r, rz, rz, rz, rz) +C_O1_I4(r, r, rz, rz, 0) +C_O1_I4(r, r, rz, rz, rz) +C_O1_I4(r, r, r, rz, rz) C_O2_I1(r, r, r) C_O2_I2(r, r, r, r) -C_O2_I4(r, r, rz, rz, rN, rN) diff --git a/tcg/mips/tcg-target-con-str.h b/tcg/mips/tcg-target-con-str.h index dfe2b156df..db2b225e4a 100644 --- a/tcg/mips/tcg-target-con-str.h +++ b/tcg/mips/tcg-target-con-str.h @@ -17,5 +17,4 @@ REGS('r', ALL_GENERAL_REGS) CONST('I', TCG_CT_CONST_U16) CONST('J', TCG_CT_CONST_S16) CONST('K', TCG_CT_CONST_P2M1) -CONST('N', TCG_CT_CONST_N16) CONST('W', TCG_CT_CONST_WSZ) diff --git a/tcg/mips/tcg-target-has.h b/tcg/mips/tcg-target-has.h index df6960fe9a..b9eb338528 100644 --- a/tcg/mips/tcg-target-has.h +++ b/tcg/mips/tcg-target-has.h @@ -39,77 +39,13 @@ extern bool use_mips32r2_instructions; #endif /* optional instructions */ -#define TCG_TARGET_HAS_div_i32 1 -#define TCG_TARGET_HAS_rem_i32 1 -#define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_nor_i32 1 -#define TCG_TARGET_HAS_andc_i32 0 -#define TCG_TARGET_HAS_orc_i32 0 -#define TCG_TARGET_HAS_eqv_i32 0 -#define TCG_TARGET_HAS_nand_i32 0 -#define TCG_TARGET_HAS_mulu2_i32 (!use_mips32r6_instructions) -#define TCG_TARGET_HAS_muls2_i32 (!use_mips32r6_instructions) -#define TCG_TARGET_HAS_muluh_i32 1 -#define TCG_TARGET_HAS_mulsh_i32 1 -#define TCG_TARGET_HAS_bswap16_i32 1 -#define TCG_TARGET_HAS_bswap32_i32 1 -#define TCG_TARGET_HAS_negsetcond_i32 0 - #if TCG_TARGET_REG_BITS == 64 -#define TCG_TARGET_HAS_add2_i32 0 -#define TCG_TARGET_HAS_sub2_i32 0 #define TCG_TARGET_HAS_extr_i64_i32 1 -#define TCG_TARGET_HAS_div_i64 1 -#define TCG_TARGET_HAS_rem_i64 1 -#define TCG_TARGET_HAS_not_i64 1 -#define TCG_TARGET_HAS_nor_i64 1 -#define TCG_TARGET_HAS_andc_i64 0 -#define TCG_TARGET_HAS_orc_i64 0 -#define TCG_TARGET_HAS_eqv_i64 0 -#define TCG_TARGET_HAS_nand_i64 0 -#define TCG_TARGET_HAS_add2_i64 0 -#define TCG_TARGET_HAS_sub2_i64 0 -#define TCG_TARGET_HAS_mulu2_i64 (!use_mips32r6_instructions) -#define TCG_TARGET_HAS_muls2_i64 (!use_mips32r6_instructions) -#define TCG_TARGET_HAS_muluh_i64 1 -#define TCG_TARGET_HAS_mulsh_i64 1 #define TCG_TARGET_HAS_ext32s_i64 1 #define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_negsetcond_i64 0 #endif /* optional instructions detected at runtime */ -#define TCG_TARGET_HAS_extract2_i32 0 -#define TCG_TARGET_HAS_ext8s_i32 use_mips32r2_instructions -#define TCG_TARGET_HAS_ext16s_i32 use_mips32r2_instructions -#define TCG_TARGET_HAS_rot_i32 use_mips32r2_instructions -#define TCG_TARGET_HAS_clz_i32 use_mips32r2_instructions -#define TCG_TARGET_HAS_ctz_i32 0 -#define TCG_TARGET_HAS_ctpop_i32 0 -#define TCG_TARGET_HAS_qemu_st8_i32 0 - -#if TCG_TARGET_REG_BITS == 64 -#define TCG_TARGET_HAS_bswap16_i64 1 -#define TCG_TARGET_HAS_bswap32_i64 1 -#define TCG_TARGET_HAS_bswap64_i64 1 -#define TCG_TARGET_HAS_extract2_i64 0 -#define TCG_TARGET_HAS_ext8s_i64 use_mips32r2_instructions -#define TCG_TARGET_HAS_ext16s_i64 use_mips32r2_instructions -#define TCG_TARGET_HAS_rot_i64 use_mips32r2_instructions -#define TCG_TARGET_HAS_clz_i64 use_mips32r2_instructions -#define TCG_TARGET_HAS_ctz_i64 0 -#define TCG_TARGET_HAS_ctpop_i64 0 -#endif - -/* optional instructions automatically implemented */ -#define TCG_TARGET_HAS_ext8u_i32 0 /* andi rt, rs, 0xff */ -#define TCG_TARGET_HAS_ext16u_i32 0 /* andi rt, rs, 0xffff */ - -#if TCG_TARGET_REG_BITS == 64 -#define TCG_TARGET_HAS_ext8u_i64 0 /* andi rt, rs, 0xff */ -#define TCG_TARGET_HAS_ext16u_i64 0 /* andi rt, rs, 0xffff */ -#endif - #define TCG_TARGET_HAS_qemu_ldst_i128 0 #define TCG_TARGET_HAS_tst 0 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index f8c105ba37..2c0457e588 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -187,8 +187,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, #define TCG_CT_CONST_U16 0x100 /* Unsigned 16-bit: 0 - 0xffff. */ #define TCG_CT_CONST_S16 0x200 /* Signed 16-bit: -32768 - 32767 */ #define TCG_CT_CONST_P2M1 0x400 /* Power of 2 minus 1. */ -#define TCG_CT_CONST_N16 0x800 /* "Negatable" 16-bit: -32767 - 32767 */ -#define TCG_CT_CONST_WSZ 0x1000 /* word size */ +#define TCG_CT_CONST_WSZ 0x800 /* word size */ #define ALL_GENERAL_REGS 0xffffffffu @@ -207,8 +206,6 @@ static bool tcg_target_const_match(int64_t val, int ct, return 1; } else if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) { return 1; - } else if ((ct & TCG_CT_CONST_N16) && val >= -32767 && val <= 32767) { - return 1; } else if ((ct & TCG_CT_CONST_P2M1) && use_mips32r2_instructions && is_p2m1(val)) { return 1; @@ -647,7 +644,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type, static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) { - tcg_debug_assert(TCG_TARGET_HAS_ext8s_i32); + tcg_debug_assert(use_mips32r2_instructions); tcg_out_opc_reg(s, OPC_SEB, rd, TCG_REG_ZERO, rs); } @@ -658,7 +655,7 @@ static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs) static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) { - tcg_debug_assert(TCG_TARGET_HAS_ext16s_i32); + tcg_debug_assert(use_mips32r2_instructions); tcg_out_opc_reg(s, OPC_SEH, rd, TCG_REG_ZERO, rs); } @@ -702,39 +699,6 @@ static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, g_assert_not_reached(); } -static void tcg_out_bswap16(TCGContext *s, TCGReg ret, TCGReg arg, int flags) -{ - /* ret and arg can't be register tmp0 */ - tcg_debug_assert(ret != TCG_TMP0); - tcg_debug_assert(arg != TCG_TMP0); - - /* With arg = abcd: */ - if (use_mips32r2_instructions) { - tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg); /* badc */ - if (flags & TCG_BSWAP_OS) { - tcg_out_opc_reg(s, OPC_SEH, ret, 0, ret); /* ssdc */ - } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xffff); /* 00dc */ - } - return; - } - - tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8); /* 0abc */ - if (!(flags & TCG_BSWAP_IZ)) { - tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP0, TCG_TMP0, 0x00ff); /* 000c */ - } - if (flags & TCG_BSWAP_OS) { - tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24); /* d000 */ - tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16); /* ssd0 */ - } else { - tcg_out_opc_sa(s, OPC_SLL, ret, arg, 8); /* bcd0 */ - if (flags & TCG_BSWAP_OZ) { - tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xff00); /* 00d0 */ - } - } - tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0); /* ssdc */ -} - static void tcg_out_bswap_subr(TCGContext *s, const tcg_insn_unit *sub) { if (!tcg_out_opc_jmp(s, OPC_JAL, sub)) { @@ -743,39 +707,6 @@ static void tcg_out_bswap_subr(TCGContext *s, const tcg_insn_unit *sub) } } -static void tcg_out_bswap32(TCGContext *s, TCGReg ret, TCGReg arg, int flags) -{ - if (use_mips32r2_instructions) { - tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg); - tcg_out_opc_sa(s, OPC_ROTR, ret, ret, 16); - if (flags & TCG_BSWAP_OZ) { - tcg_out_opc_bf(s, OPC_DEXT, ret, ret, 31, 0); - } - } else { - if (flags & TCG_BSWAP_OZ) { - tcg_out_bswap_subr(s, bswap32u_addr); - } else { - tcg_out_bswap_subr(s, bswap32_addr); - } - /* delay slot -- never omit the insn, like tcg_out_mov might. */ - tcg_out_opc_reg(s, OPC_OR, TCG_TMP0, arg, TCG_REG_ZERO); - tcg_out_mov(s, TCG_TYPE_I32, ret, TCG_TMP3); - } -} - -static void tcg_out_bswap64(TCGContext *s, TCGReg ret, TCGReg arg) -{ - if (use_mips32r2_instructions) { - tcg_out_opc_reg(s, OPC_DSBH, ret, 0, arg); - tcg_out_opc_reg(s, OPC_DSHD, ret, 0, ret); - } else { - tcg_out_bswap_subr(s, bswap64_addr); - /* delay slot -- never omit the insn, like tcg_out_mov might. */ - tcg_out_opc_reg(s, OPC_OR, TCG_TMP0, arg, TCG_REG_ZERO); - tcg_out_mov(s, TCG_TYPE_I32, ret, TCG_TMP3); - } -} - static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg) { tcg_debug_assert(TCG_TARGET_REG_BITS == 64); @@ -831,55 +762,6 @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, return false; } -static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh, TCGReg al, - TCGReg ah, TCGArg bl, TCGArg bh, bool cbl, - bool cbh, bool is_sub) -{ - TCGReg th = TCG_TMP1; - - /* If we have a negative constant such that negating it would - make the high part zero, we can (usually) eliminate one insn. */ - if (cbl && cbh && bh == -1 && bl != 0) { - bl = -bl; - bh = 0; - is_sub = !is_sub; - } - - /* By operating on the high part first, we get to use the final - carry operation to move back from the temporary. */ - if (!cbh) { - tcg_out_opc_reg(s, (is_sub ? OPC_SUBU : OPC_ADDU), th, ah, bh); - } else if (bh != 0 || ah == rl) { - tcg_out_opc_imm(s, OPC_ADDIU, th, ah, (is_sub ? -bh : bh)); - } else { - th = ah; - } - - /* Note that tcg optimization should eliminate the bl == 0 case. */ - if (is_sub) { - if (cbl) { - tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, al, bl); - tcg_out_opc_imm(s, OPC_ADDIU, rl, al, -bl); - } else { - tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, al, bl); - tcg_out_opc_reg(s, OPC_SUBU, rl, al, bl); - } - tcg_out_opc_reg(s, OPC_SUBU, rh, th, TCG_TMP0); - } else { - if (cbl) { - tcg_out_opc_imm(s, OPC_ADDIU, rl, al, bl); - tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, rl, bl); - } else if (rl == al && rl == bl) { - tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, al, TCG_TARGET_REG_BITS - 1); - tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl); - } else { - tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl); - tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, rl, (rl == bl ? al : bl)); - } - tcg_out_opc_reg(s, OPC_ADDU, rh, th, TCG_TMP0); - } -} - #define SETCOND_INV TCG_TARGET_NB_REGS #define SETCOND_NEZ (SETCOND_INV << 1) #define SETCOND_FLAGS (SETCOND_INV | SETCOND_NEZ) @@ -952,15 +834,44 @@ static void tcg_out_setcond_end(TCGContext *s, TCGReg ret, int tmpflags) } } -static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg arg1, TCGReg arg2) +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg arg1, TCGReg arg2) { int tmpflags = tcg_out_setcond_int(s, cond, ret, arg1, arg2); tcg_out_setcond_end(s, ret, tmpflags); } -static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, - TCGReg arg2, TCGLabel *l) +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(r, r, rz), + .out_rrr = tgen_setcond, +}; + +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg arg1, TCGReg arg2) +{ + int tmpflags = tcg_out_setcond_int(s, cond, ret, arg1, arg2); + TCGReg tmp = tmpflags & ~SETCOND_FLAGS; + + /* If intermediate result is zero/non-zero: test != 0. */ + if (tmpflags & SETCOND_NEZ) { + tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, tmp); + tmp = ret; + } + /* Produce the 0/-1 result. */ + if (tmpflags & SETCOND_INV) { + tcg_out_opc_imm(s, OPC_ADDIU, ret, tmp, -1); + } else { + tcg_out_opc_reg(s, OPC_SUBU, ret, TCG_REG_ZERO, tmp); + } +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(r, r, rz), + .out_rrr = tgen_negsetcond, +}; + +static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, TCGReg arg2, TCGLabel *l) { static const MIPSInsn b_zero[16] = { [TCG_COND_LT] = OPC_BLTZ, @@ -1005,6 +916,16 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, tcg_out_nop(s); } +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, rz), + .out_rr = tgen_brcond, +}; + +void tcg_out_br(TCGContext *s, TCGLabel *l) +{ + tgen_brcond(s, TCG_TYPE_I32, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO, l); +} + static int tcg_out_setcond2_int(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) { @@ -1022,25 +943,37 @@ static int tcg_out_setcond2_int(TCGContext *s, TCGCond cond, TCGReg ret, break; default: - tcg_out_setcond(s, TCG_COND_EQ, TCG_TMP0, ah, bh); - tcg_out_setcond(s, tcg_unsigned_cond(cond), TCG_TMP1, al, bl); + tgen_setcond(s, TCG_TYPE_I32, TCG_COND_EQ, TCG_TMP0, ah, bh); + tgen_setcond(s, TCG_TYPE_I32, tcg_unsigned_cond(cond), + TCG_TMP1, al, bl); tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, TCG_TMP0); - tcg_out_setcond(s, tcg_high_cond(cond), TCG_TMP0, ah, bh); + tgen_setcond(s, TCG_TYPE_I32, tcg_high_cond(cond), TCG_TMP0, ah, bh); tcg_out_opc_reg(s, OPC_OR, ret, TCG_TMP0, TCG_TMP1); break; } return ret | flags; } -static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) +static void tgen_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, + TCGArg bh, bool const_bh) { int tmpflags = tcg_out_setcond2_int(s, cond, ret, al, ah, bl, bh); tcg_out_setcond_end(s, ret, tmpflags); } -static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, - TCGReg bl, TCGReg bh, TCGLabel *l) +#if TCG_TARGET_REG_BITS != 32 +__attribute__((unused)) +#endif +static const TCGOutOpSetcond2 outop_setcond2 = { + .base.static_constraint = C_O1_I4(r, r, r, rz, rz), + .out = tgen_setcond2, +}; + +static void tgen_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, + TCGArg bh, bool const_bh, TCGLabel *l) { int tmpflags = tcg_out_setcond2_int(s, cond, TCG_TMP0, al, ah, bl, bh); TCGReg tmp = tmpflags & ~SETCOND_FLAGS; @@ -1051,8 +984,17 @@ static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, tcg_out_nop(s); } -static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg c1, TCGReg c2, TCGReg v1, TCGReg v2) +#if TCG_TARGET_REG_BITS != 32 +__attribute__((unused)) +#endif +static const TCGOutOpBrcond2 outop_brcond2 = { + .base.static_constraint = C_O0_I4(r, r, rz, rz), + .out = tgen_brcond2, +}; + +static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, + TCGArg v1, bool const_v1, TCGArg v2, bool const_v2) { int tmpflags; bool eqz; @@ -1098,6 +1040,13 @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, } } +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = (use_mips32r6_instructions + ? C_O1_I4(r, r, rz, rz, rz) + : C_O1_I4(r, r, rz, rz, 0)), + .out = tgen_movcond, +}; + static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { /* @@ -1438,29 +1387,66 @@ static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, } } -static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg addr, MemOpIdx oi, TCGType data_type) +static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + MemOp opc = get_memop(oi); + TCGLabelQemuLdst *ldst; + HostAddress h; + + ldst = prepare_host_addr(s, &h, addr, oi, true); + + if (use_mips32r6_instructions || h.aa.align >= (opc & MO_SIZE)) { + tcg_out_qemu_ld_direct(s, data, 0, h.base, opc, type); + } else { + tcg_out_qemu_ld_unalign(s, data, 0, h.base, opc, type); + } + + if (ldst) { + ldst->type = type; + ldst->datalo_reg = data; + ldst->datahi_reg = 0; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); + } +} + +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_qemu_ld, +}; + +static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) { MemOp opc = get_memop(oi); TCGLabelQemuLdst *ldst; HostAddress h; + tcg_debug_assert(TCG_TARGET_REG_BITS == 32); ldst = prepare_host_addr(s, &h, addr, oi, true); if (use_mips32r6_instructions || h.aa.align >= (opc & MO_SIZE)) { - tcg_out_qemu_ld_direct(s, datalo, datahi, h.base, opc, data_type); + tcg_out_qemu_ld_direct(s, datalo, datahi, h.base, opc, type); } else { - tcg_out_qemu_ld_unalign(s, datalo, datahi, h.base, opc, data_type); + tcg_out_qemu_ld_unalign(s, datalo, datahi, h.base, opc, type); } if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = datalo; ldst->datahi_reg = datahi; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + /* Ensure that the mips32 code is compiled but discarded for mips64. */ + .base.static_constraint = + TCG_TARGET_REG_BITS == 32 ? C_O2_I1(r, r, r) : C_NotImplemented, + .out = + TCG_TARGET_REG_BITS == 32 ? tgen_qemu_ld2 : NULL, +}; + static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, TCGReg base, MemOp opc) { @@ -1524,13 +1510,42 @@ static void tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi, } } -static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg addr, MemOpIdx oi, TCGType data_type) +static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + MemOp opc = get_memop(oi); + TCGLabelQemuLdst *ldst; + HostAddress h; + + ldst = prepare_host_addr(s, &h, addr, oi, false); + + if (use_mips32r6_instructions || h.aa.align >= (opc & MO_SIZE)) { + tcg_out_qemu_st_direct(s, data, 0, h.base, opc); + } else { + tcg_out_qemu_st_unalign(s, data, 0, h.base, opc); + } + + if (ldst) { + ldst->type = type; + ldst->datalo_reg = data; + ldst->datahi_reg = 0; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); + } +} + +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out = tgen_qemu_st, +}; + +static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) { MemOp opc = get_memop(oi); TCGLabelQemuLdst *ldst; HostAddress h; + tcg_debug_assert(TCG_TARGET_REG_BITS == 32); ldst = prepare_host_addr(s, &h, addr, oi, false); if (use_mips32r6_instructions || h.aa.align >= (opc & MO_SIZE)) { @@ -1540,14 +1555,22 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, } if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = datalo; ldst->datahi_reg = datahi; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } -static void tcg_out_mb(TCGContext *s, TCGArg a0) +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + /* Ensure that the mips32 code is compiled but discarded for mips64. */ + .base.static_constraint = + TCG_TARGET_REG_BITS == 32 ? C_O0_I3(rz, rz, r) : C_NotImplemented, + .out = + TCG_TARGET_REG_BITS == 32 ? tgen_qemu_st2 : NULL, +}; + +static void tcg_out_mb(TCGContext *s, unsigned a0) { static const MIPSInsn sync[] = { /* Note that SYNC_MB is a slightly weaker than SYNC 0, @@ -1563,33 +1586,6 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) tcg_out32(s, sync[a0 & TCG_MO_ALL]); } -static void tcg_out_clz(TCGContext *s, MIPSInsn opcv2, MIPSInsn opcv6, - int width, TCGReg a0, TCGReg a1, TCGArg a2) -{ - if (use_mips32r6_instructions) { - if (a2 == width) { - tcg_out_opc_reg(s, opcv6, a0, a1, 0); - } else { - tcg_out_opc_reg(s, opcv6, TCG_TMP0, a1, 0); - tcg_out_movcond(s, TCG_COND_EQ, a0, a1, 0, a2, TCG_TMP0); - } - } else { - if (a2 == width) { - tcg_out_opc_reg(s, opcv2, a0, a1, a1); - } else if (a0 == a2) { - tcg_out_opc_reg(s, opcv2, TCG_TMP0, a1, a1); - tcg_out_opc_reg(s, OPC_MOVN, a0, TCG_TMP0, a1); - } else if (a0 != a1) { - tcg_out_opc_reg(s, opcv2, a0, a1, a1); - tcg_out_opc_reg(s, OPC_MOVZ, a0, a2, a1); - } else { - tcg_out_opc_reg(s, opcv2, TCG_TMP0, a1, a1); - tcg_out_opc_reg(s, OPC_MOVZ, TCG_TMP0, a2, a1); - tcg_out_mov(s, TCG_TYPE_REG, a0, TCG_TMP0); - } - } -} - static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) { TCGReg base = TCG_REG_ZERO; @@ -1649,616 +1645,809 @@ static void tcg_out_goto_tb(TCGContext *s, int which) } } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + tcg_out_opc_reg(s, OPC_JR, 0, a0, 0); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, a0); + } else { + tcg_out_nop(s); + } +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { /* Always indirect, nothing to do */ } -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) + +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - MIPSInsn i1, i2; - TCGArg a0, a1, a2; - int c2; + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_ADDU : OPC_DADDU; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} - a0 = args[0]; - a1 = args[1]; - a2 = args[2]; - c2 = const_args[2]; +static void tgen_addi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_ADDIU : OPC_DADDIU; + tcg_out_opc_imm(s, insn, a0, a1, a2); +} - switch (opc) { - case INDEX_op_goto_ptr: - /* jmp to the given host address (could be epilogue) */ - tcg_out_opc_reg(s, OPC_JR, 0, a0, 0); - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, a0); +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_add, + .out_rri = tgen_addi, +}; + +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_NotImplemented, +}; + +static void tcg_out_set_carry(TCGContext *s) +{ + g_assert_not_reached(); +} + +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_AND, a0, a1, a2); +} + +static void tgen_andi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + int msb; + + if (a2 == (uint16_t)a2) { + tcg_out_opc_imm(s, OPC_ANDI, a0, a1, a2); + return; + } + + tcg_debug_assert(use_mips32r2_instructions); + tcg_debug_assert(is_p2m1(a2)); + msb = ctz64(~a2) - 1; + if (type == TCG_TYPE_I32) { + tcg_out_opc_bf(s, OPC_EXT, a0, a1, msb, 0); + } else { + tcg_out_opc_bf64(s, OPC_DEXT, OPC_DEXTM, OPC_DEXTU, a0, a1, msb, 0); + } +} + +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, r, rIK), + .out_rrr = tgen_and, + .out_rri = tgen_andi, +}; + +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_clz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (use_mips32r6_instructions) { + MIPSInsn opcv6 = type == TCG_TYPE_I32 ? OPC_CLZ_R6 : OPC_DCLZ_R6; + tcg_out_opc_reg(s, opcv6, TCG_TMP0, a1, 0); + tgen_movcond(s, TCG_TYPE_REG, TCG_COND_EQ, a0, a1, a2, false, + TCG_TMP0, false, TCG_REG_ZERO, false); + } else { + MIPSInsn opcv2 = type == TCG_TYPE_I32 ? OPC_CLZ : OPC_DCLZ; + if (a0 == a2) { + tcg_out_opc_reg(s, opcv2, TCG_TMP0, a1, a1); + tcg_out_opc_reg(s, OPC_MOVN, a0, TCG_TMP0, a1); + } else if (a0 != a1) { + tcg_out_opc_reg(s, opcv2, a0, a1, a1); + tcg_out_opc_reg(s, OPC_MOVZ, a0, a2, a1); } else { - tcg_out_nop(s); + tcg_out_opc_reg(s, opcv2, TCG_TMP0, a1, a1); + tcg_out_opc_reg(s, OPC_MOVZ, TCG_TMP0, a2, a1); + tcg_out_mov(s, type, a0, TCG_TMP0); } - break; - case INDEX_op_br: - tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO, - arg_label(a0)); - break; + } +} - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - i1 = OPC_LBU; - goto do_ldst; - case INDEX_op_ld8s_i32: - case INDEX_op_ld8s_i64: - i1 = OPC_LB; - goto do_ldst; - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - i1 = OPC_LHU; - goto do_ldst; - case INDEX_op_ld16s_i32: - case INDEX_op_ld16s_i64: - i1 = OPC_LH; - goto do_ldst; - case INDEX_op_ld_i32: - case INDEX_op_ld32s_i64: - i1 = OPC_LW; - goto do_ldst; - case INDEX_op_ld32u_i64: - i1 = OPC_LWU; - goto do_ldst; - case INDEX_op_ld_i64: - i1 = OPC_LD; - goto do_ldst; - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - i1 = OPC_SB; - goto do_ldst; - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - i1 = OPC_SH; - goto do_ldst; - case INDEX_op_st_i32: - case INDEX_op_st32_i64: - i1 = OPC_SW; - goto do_ldst; - case INDEX_op_st_i64: - i1 = OPC_SD; - do_ldst: - tcg_out_ldst(s, i1, a0, a1, a2); - break; +static void tgen_clzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (a2 == 0) { + tgen_clz(s, type, a0, a1, TCG_REG_ZERO); + } else if (use_mips32r6_instructions) { + MIPSInsn opcv6 = type == TCG_TYPE_I32 ? OPC_CLZ_R6 : OPC_DCLZ_R6; + tcg_out_opc_reg(s, opcv6, a0, a1, 0); + } else { + MIPSInsn opcv2 = type == TCG_TYPE_I32 ? OPC_CLZ : OPC_DCLZ; + tcg_out_opc_reg(s, opcv2, a0, a1, a1); + } +} - case INDEX_op_add_i32: - i1 = OPC_ADDU, i2 = OPC_ADDIU; - goto do_binary; - case INDEX_op_add_i64: - i1 = OPC_DADDU, i2 = OPC_DADDIU; - goto do_binary; - case INDEX_op_or_i32: - case INDEX_op_or_i64: - i1 = OPC_OR, i2 = OPC_ORI; - goto do_binary; - case INDEX_op_xor_i32: - case INDEX_op_xor_i64: - i1 = OPC_XOR, i2 = OPC_XORI; - do_binary: - if (c2) { - tcg_out_opc_imm(s, i2, a0, a1, a2); - break; - } - do_binaryv: - tcg_out_opc_reg(s, i1, a0, a1, a2); - break; +static TCGConstraintSetIndex cset_clz(TCGType type, unsigned flags) +{ + return use_mips32r2_instructions ? C_O1_I2(r, r, rzW) : C_NotImplemented; +} - case INDEX_op_sub_i32: - i1 = OPC_SUBU, i2 = OPC_ADDIU; - goto do_subtract; - case INDEX_op_sub_i64: - i1 = OPC_DSUBU, i2 = OPC_DADDIU; - do_subtract: - if (c2) { - tcg_out_opc_imm(s, i2, a0, a1, -a2); - break; - } - goto do_binaryv; - case INDEX_op_and_i32: - if (c2 && a2 != (uint16_t)a2) { - int msb = ctz32(~a2) - 1; - tcg_debug_assert(use_mips32r2_instructions); - tcg_debug_assert(is_p2m1(a2)); - tcg_out_opc_bf(s, OPC_EXT, a0, a1, msb, 0); - break; - } - i1 = OPC_AND, i2 = OPC_ANDI; - goto do_binary; - case INDEX_op_and_i64: - if (c2 && a2 != (uint16_t)a2) { - int msb = ctz64(~a2) - 1; - tcg_debug_assert(use_mips32r2_instructions); - tcg_debug_assert(is_p2m1(a2)); - tcg_out_opc_bf64(s, OPC_DEXT, OPC_DEXTM, OPC_DEXTU, a0, a1, msb, 0); - break; - } - i1 = OPC_AND, i2 = OPC_ANDI; - goto do_binary; - case INDEX_op_nor_i32: - case INDEX_op_nor_i64: - i1 = OPC_NOR; - goto do_binaryv; - - case INDEX_op_mul_i32: - if (use_mips32_instructions) { - tcg_out_opc_reg(s, OPC_MUL, a0, a1, a2); - break; - } - i1 = OPC_MULT, i2 = OPC_MFLO; - goto do_hilo1; - case INDEX_op_mulsh_i32: - if (use_mips32r6_instructions) { - tcg_out_opc_reg(s, OPC_MUH, a0, a1, a2); - break; - } - i1 = OPC_MULT, i2 = OPC_MFHI; - goto do_hilo1; - case INDEX_op_muluh_i32: - if (use_mips32r6_instructions) { - tcg_out_opc_reg(s, OPC_MUHU, a0, a1, a2); - break; - } - i1 = OPC_MULTU, i2 = OPC_MFHI; - goto do_hilo1; - case INDEX_op_div_i32: - if (use_mips32r6_instructions) { +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_clz, + .out_rrr = tgen_clz, + .out_rri = tgen_clzi, +}; + +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divs(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (use_mips32r6_instructions) { + if (type == TCG_TYPE_I32) { tcg_out_opc_reg(s, OPC_DIV_R6, a0, a1, a2); - break; + } else { + tcg_out_opc_reg(s, OPC_DDIV_R6, a0, a1, a2); } - i1 = OPC_DIV, i2 = OPC_MFLO; - goto do_hilo1; - case INDEX_op_divu_i32: - if (use_mips32r6_instructions) { + } else { + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_DIV : OPC_DDIV; + tcg_out_opc_reg(s, insn, 0, a1, a2); + tcg_out_opc_reg(s, OPC_MFLO, a0, 0, 0); + } +} + +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divs, +}; + +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (use_mips32r6_instructions) { + if (type == TCG_TYPE_I32) { tcg_out_opc_reg(s, OPC_DIVU_R6, a0, a1, a2); - break; - } - i1 = OPC_DIVU, i2 = OPC_MFLO; - goto do_hilo1; - case INDEX_op_rem_i32: - if (use_mips32r6_instructions) { - tcg_out_opc_reg(s, OPC_MOD, a0, a1, a2); - break; + } else { + tcg_out_opc_reg(s, OPC_DDIVU_R6, a0, a1, a2); } - i1 = OPC_DIV, i2 = OPC_MFHI; - goto do_hilo1; - case INDEX_op_remu_i32: - if (use_mips32r6_instructions) { - tcg_out_opc_reg(s, OPC_MODU, a0, a1, a2); - break; + } else { + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_DIVU : OPC_DDIVU; + tcg_out_opc_reg(s, insn, 0, a1, a2); + tcg_out_opc_reg(s, OPC_MFLO, a0, 0, 0); + } +} + +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divu, +}; + +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_NotImplemented, +}; + +#if TCG_TARGET_REG_BITS == 64 +static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_dsra(s, a0, a1, 32); +} + +static const TCGOutOpUnary outop_extrh_i64_i32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extrh_i64_i32, +}; +#endif + +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + MIPSInsn insn; + + if (type == TCG_TYPE_I32) { + if (use_mips32_instructions) { + tcg_out_opc_reg(s, OPC_MUL, a0, a1, a2); + return; } - i1 = OPC_DIVU, i2 = OPC_MFHI; - goto do_hilo1; - case INDEX_op_mul_i64: + insn = OPC_MULT; + } else { if (use_mips32r6_instructions) { tcg_out_opc_reg(s, OPC_DMUL, a0, a1, a2); - break; - } - i1 = OPC_DMULT, i2 = OPC_MFLO; - goto do_hilo1; - case INDEX_op_mulsh_i64: - if (use_mips32r6_instructions) { - tcg_out_opc_reg(s, OPC_DMUH, a0, a1, a2); - break; - } - i1 = OPC_DMULT, i2 = OPC_MFHI; - goto do_hilo1; - case INDEX_op_muluh_i64: - if (use_mips32r6_instructions) { - tcg_out_opc_reg(s, OPC_DMUHU, a0, a1, a2); - break; - } - i1 = OPC_DMULTU, i2 = OPC_MFHI; - goto do_hilo1; - case INDEX_op_div_i64: - if (use_mips32r6_instructions) { - tcg_out_opc_reg(s, OPC_DDIV_R6, a0, a1, a2); - break; - } - i1 = OPC_DDIV, i2 = OPC_MFLO; - goto do_hilo1; - case INDEX_op_divu_i64: - if (use_mips32r6_instructions) { - tcg_out_opc_reg(s, OPC_DDIVU_R6, a0, a1, a2); - break; + return; } - i1 = OPC_DDIVU, i2 = OPC_MFLO; - goto do_hilo1; - case INDEX_op_rem_i64: - if (use_mips32r6_instructions) { + insn = OPC_DMULT; + } + tcg_out_opc_reg(s, insn, 0, a1, a2); + tcg_out_opc_reg(s, OPC_MFLO, a0, 0, 0); +} + +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_mul, +}; + +static void tgen_muls2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_MULT : OPC_DMULT; + tcg_out_opc_reg(s, insn, 0, a2, a3); + tcg_out_opc_reg(s, OPC_MFLO, a0, 0, 0); + tcg_out_opc_reg(s, OPC_MFHI, a1, 0, 0); +} + +static TCGConstraintSetIndex cset_mul2(TCGType type, unsigned flags) +{ + return use_mips32r6_instructions ? C_NotImplemented : C_O2_I2(r, r, r, r); +} + +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mul2, + .out_rrrr = tgen_muls2, +}; + +static void tgen_mulsh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (use_mips32r6_instructions) { + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_MUH : OPC_DMUH; + tcg_out_opc_reg(s, insn, a0, a1, a2); + } else { + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_MULT : OPC_DMULT; + tcg_out_opc_reg(s, insn, 0, a1, a2); + tcg_out_opc_reg(s, OPC_MFHI, a0, 0, 0); + } +} + +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_mulsh, +}; + +static void tgen_mulu2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_MULTU : OPC_DMULTU; + tcg_out_opc_reg(s, insn, 0, a2, a3); + tcg_out_opc_reg(s, OPC_MFLO, a0, 0, 0); + tcg_out_opc_reg(s, OPC_MFHI, a1, 0, 0); +} + +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mul2, + .out_rrrr = tgen_mulu2, +}; + +static void tgen_muluh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (use_mips32r6_instructions) { + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_MUHU : OPC_DMUHU; + tcg_out_opc_reg(s, insn, a0, a1, a2); + } else { + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_MULTU : OPC_DMULTU; + tcg_out_opc_reg(s, insn, 0, a1, a2); + tcg_out_opc_reg(s, OPC_MFHI, a0, 0, 0); + } +} + +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_muluh, +}; + +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_nor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_NOR, a0, a1, a2); +} + +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_nor, +}; + +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_OR, a0, a1, a2); +} + +static void tgen_ori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_opc_imm(s, OPC_ORI, a0, a1, a2); +} + +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_or, + .out_rri = tgen_ori, +}; + +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_rems(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (use_mips32r6_instructions) { + if (type == TCG_TYPE_I32) { + tcg_out_opc_reg(s, OPC_MOD, a0, a1, a2); + } else { tcg_out_opc_reg(s, OPC_DMOD, a0, a1, a2); - break; } - i1 = OPC_DDIV, i2 = OPC_MFHI; - goto do_hilo1; - case INDEX_op_remu_i64: - if (use_mips32r6_instructions) { + } else { + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_DIV : OPC_DDIV; + tcg_out_opc_reg(s, insn, 0, a1, a2); + tcg_out_opc_reg(s, OPC_MFHI, a0, 0, 0); + } +} + +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_rems, +}; + +static void tgen_remu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (use_mips32r6_instructions) { + if (type == TCG_TYPE_I32) { + tcg_out_opc_reg(s, OPC_MODU, a0, a1, a2); + } else { tcg_out_opc_reg(s, OPC_DMODU, a0, a1, a2); - break; } - i1 = OPC_DDIVU, i2 = OPC_MFHI; - do_hilo1: - tcg_out_opc_reg(s, i1, 0, a1, a2); - tcg_out_opc_reg(s, i2, a0, 0, 0); - break; + } else { + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_DIVU : OPC_DDIVU; + tcg_out_opc_reg(s, insn, 0, a1, a2); + tcg_out_opc_reg(s, OPC_MFHI, a0, 0, 0); + } +} - case INDEX_op_muls2_i32: - i1 = OPC_MULT; - goto do_hilo2; - case INDEX_op_mulu2_i32: - i1 = OPC_MULTU; - goto do_hilo2; - case INDEX_op_muls2_i64: - i1 = OPC_DMULT; - goto do_hilo2; - case INDEX_op_mulu2_i64: - i1 = OPC_DMULTU; - do_hilo2: - tcg_out_opc_reg(s, i1, 0, a2, args[3]); - tcg_out_opc_reg(s, OPC_MFLO, a0, 0, 0); - tcg_out_opc_reg(s, OPC_MFHI, a1, 0, 0); - break; +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_remu, +}; - case INDEX_op_neg_i32: - i1 = OPC_SUBU; - goto do_unary; - case INDEX_op_neg_i64: - i1 = OPC_DSUBU; - goto do_unary; - case INDEX_op_not_i32: - case INDEX_op_not_i64: - i1 = OPC_NOR; - goto do_unary; - do_unary: - tcg_out_opc_reg(s, i1, a0, TCG_REG_ZERO, a1); - break; +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - tcg_out_bswap16(s, a0, a1, a2); - break; - case INDEX_op_bswap32_i32: - tcg_out_bswap32(s, a0, a1, 0); - break; - case INDEX_op_bswap32_i64: - tcg_out_bswap32(s, a0, a1, a2); - break; - case INDEX_op_bswap64_i64: - tcg_out_bswap64(s, a0, a1); - break; - case INDEX_op_extrh_i64_i32: - tcg_out_dsra(s, a0, a1, 32); - break; +static TCGConstraintSetIndex cset_rotr(TCGType type, unsigned flags) +{ + return use_mips32r2_instructions ? C_O1_I2(r, r, ri) : C_NotImplemented; +} - case INDEX_op_sar_i32: - i1 = OPC_SRAV, i2 = OPC_SRA; - goto do_shift; - case INDEX_op_shl_i32: - i1 = OPC_SLLV, i2 = OPC_SLL; - goto do_shift; - case INDEX_op_shr_i32: - i1 = OPC_SRLV, i2 = OPC_SRL; - goto do_shift; - case INDEX_op_rotr_i32: - i1 = OPC_ROTRV, i2 = OPC_ROTR; - do_shift: - if (c2) { - tcg_out_opc_sa(s, i2, a0, a1, a2); - break; - } - do_shiftv: - tcg_out_opc_reg(s, i1, a0, a2, a1); - break; - case INDEX_op_rotl_i32: - if (c2) { - tcg_out_opc_sa(s, OPC_ROTR, a0, a1, 32 - a2); - } else { - tcg_out_opc_reg(s, OPC_SUBU, TCG_TMP0, TCG_REG_ZERO, a2); - tcg_out_opc_reg(s, OPC_ROTRV, a0, TCG_TMP0, a1); - } - break; - case INDEX_op_sar_i64: - if (c2) { - tcg_out_dsra(s, a0, a1, a2); - break; - } - i1 = OPC_DSRAV; - goto do_shiftv; - case INDEX_op_shl_i64: - if (c2) { - tcg_out_dsll(s, a0, a1, a2); - break; +static void tgen_rotr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_ROTRV : OPC_DROTRV; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static void tgen_rotri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_sa(s, OPC_ROTR, a0, a1, a2); + } else { + tcg_out_opc_sa64(s, OPC_DROTR, OPC_DROTR32, a0, a1, a2); + } +} + +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_rotr, + .out_rrr = tgen_rotr, + .out_rri = tgen_rotri, +}; + +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_SRAV : OPC_DSRAV; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static void tgen_sari(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_sa(s, OPC_SRA, a0, a1, a2); + } else { + tcg_out_dsra(s, a0, a1, a2); + } +} + +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_sar, + .out_rri = tgen_sari, +}; + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_SLLV : OPC_DSLLV; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static void tgen_shli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_sa(s, OPC_SLL, a0, a1, a2); + } else { + tcg_out_dsll(s, a0, a1, a2); + } +} + +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shl, + .out_rri = tgen_shli, +}; + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_SRLV : OPC_DSRLV; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static void tgen_shri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_sa(s, OPC_SRL, a0, a1, a2); + } else { + tcg_out_dsrl(s, a0, a1, a2); + } +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shr, + .out_rri = tgen_shri, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + MIPSInsn insn = type == TCG_TYPE_I32 ? OPC_SUBU : OPC_DSUBU; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_sub, +}; + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpAddSubCarry outop_subbi = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_NotImplemented, +}; + +static void tcg_out_set_borrow(TCGContext *s) +{ + g_assert_not_reached(); +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_XOR, a0, a1, a2); +} + +static void tgen_xori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_opc_imm(s, OPC_XORI, a0, a1, a2); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_xor, + .out_rri = tgen_xori, +}; + +static void tgen_bswap16(TCGContext *s, TCGType type, + TCGReg ret, TCGReg arg, unsigned flags) +{ + /* With arg = abcd: */ + if (use_mips32r2_instructions) { + tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg); /* badc */ + if (flags & TCG_BSWAP_OS) { + tcg_out_opc_reg(s, OPC_SEH, ret, 0, ret); /* ssdc */ + } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xffff); /* 00dc */ } - i1 = OPC_DSLLV; - goto do_shiftv; - case INDEX_op_shr_i64: - if (c2) { - tcg_out_dsrl(s, a0, a1, a2); - break; + return; + } + + tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8); /* 0abc */ + if (!(flags & TCG_BSWAP_IZ)) { + tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP0, TCG_TMP0, 0x00ff); /* 000c */ + } + if (flags & TCG_BSWAP_OS) { + tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24); /* d000 */ + tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16); /* ssd0 */ + } else { + tcg_out_opc_sa(s, OPC_SLL, ret, arg, 8); /* bcd0 */ + if (flags & TCG_BSWAP_OZ) { + tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xff00); /* 00d0 */ } - i1 = OPC_DSRLV; - goto do_shiftv; - case INDEX_op_rotr_i64: - if (c2) { - tcg_out_opc_sa64(s, OPC_DROTR, OPC_DROTR32, a0, a1, a2); - break; + } + tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0); /* ssdc */ +} + +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap16, +}; + +static void tgen_bswap32(TCGContext *s, TCGType type, + TCGReg ret, TCGReg arg, unsigned flags) +{ + if (use_mips32r2_instructions) { + tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg); + tcg_out_opc_sa(s, OPC_ROTR, ret, ret, 16); + if (flags & TCG_BSWAP_OZ) { + tcg_out_opc_bf(s, OPC_DEXT, ret, ret, 31, 0); } - i1 = OPC_DROTRV; - goto do_shiftv; - case INDEX_op_rotl_i64: - if (c2) { - tcg_out_opc_sa64(s, OPC_DROTR, OPC_DROTR32, a0, a1, 64 - a2); + } else { + if (flags & TCG_BSWAP_OZ) { + tcg_out_bswap_subr(s, bswap32u_addr); } else { - tcg_out_opc_reg(s, OPC_DSUBU, TCG_TMP0, TCG_REG_ZERO, a2); - tcg_out_opc_reg(s, OPC_DROTRV, a0, TCG_TMP0, a1); + tcg_out_bswap_subr(s, bswap32_addr); } - break; + /* delay slot -- never omit the insn, like tcg_out_mov might. */ + tcg_out_opc_reg(s, OPC_OR, TCG_TMP0, arg, TCG_REG_ZERO); + tcg_out_mov(s, TCG_TYPE_I32, ret, TCG_TMP3); + } +} - case INDEX_op_clz_i32: - tcg_out_clz(s, OPC_CLZ, OPC_CLZ_R6, 32, a0, a1, a2); - break; - case INDEX_op_clz_i64: - tcg_out_clz(s, OPC_DCLZ, OPC_DCLZ_R6, 64, a0, a1, a2); - break; +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap32, +}; - case INDEX_op_deposit_i32: - tcg_out_opc_bf(s, OPC_INS, a0, a2, args[3] + args[4] - 1, args[3]); - break; - case INDEX_op_deposit_i64: +#if TCG_TARGET_REG_BITS == 64 +static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) +{ + if (use_mips32r2_instructions) { + tcg_out_opc_reg(s, OPC_DSBH, ret, 0, arg); + tcg_out_opc_reg(s, OPC_DSHD, ret, 0, ret); + } else { + tcg_out_bswap_subr(s, bswap64_addr); + /* delay slot -- never omit the insn, like tcg_out_mov might. */ + tcg_out_opc_reg(s, OPC_OR, TCG_TMP0, arg, TCG_REG_ZERO); + tcg_out_mov(s, TCG_TYPE_I32, ret, TCG_TMP3); + } +} + +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap64, +}; +#endif /* TCG_TARGET_REG_BITS == 64 */ + +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_sub(s, type, a0, TCG_REG_ZERO, a1); +} + +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_neg, +}; + +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_nor(s, type, a0, TCG_REG_ZERO, a1); +} + +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_not, +}; + +static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned ofs, unsigned len) +{ + if (type == TCG_TYPE_I32) { + tcg_out_opc_bf(s, OPC_INS, a0, a2, ofs + len - 1, ofs); + } else { tcg_out_opc_bf64(s, OPC_DINS, OPC_DINSM, OPC_DINSU, a0, a2, - args[3] + args[4] - 1, args[3]); - break; + ofs + len - 1, ofs); + } +} - case INDEX_op_extract_i32: - if (a2 == 0 && args[3] <= 16) { - tcg_out_opc_imm(s, OPC_ANDI, a0, a1, (1 << args[3]) - 1); - } else { - tcg_out_opc_bf(s, OPC_EXT, a0, a1, args[3] - 1, a2); - } - break; - case INDEX_op_extract_i64: - if (a2 == 0 && args[3] <= 16) { - tcg_out_opc_imm(s, OPC_ANDI, a0, a1, (1 << args[3]) - 1); - } else { - tcg_out_opc_bf64(s, OPC_DEXT, OPC_DEXTM, OPC_DEXTU, - a0, a1, args[3] - 1, a2); - } - break; +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_O1_I2(r, 0, rz), + .out_rrr = tgen_deposit, +}; + +static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0 && len <= 16) { + tcg_out_opc_imm(s, OPC_ANDI, a0, a1, (1 << len) - 1); + } else if (type == TCG_TYPE_I32) { + tcg_out_opc_bf(s, OPC_EXT, a0, a1, len - 1, ofs); + } else { + tcg_out_opc_bf64(s, OPC_DEXT, OPC_DEXTM, OPC_DEXTU, + a0, a1, len - 1, ofs); + } +} - case INDEX_op_sextract_i64: - if (a2 == 0 && args[3] == 32) { +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extract, +}; + +static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0) { + switch (len) { + case 8: + tcg_out_ext8s(s, type, a0, a1); + return; + case 16: + tcg_out_ext16s(s, type, a0, a1); + return; + case 32: tcg_out_ext32s(s, a0, a1); - break; - } - /* FALLTHRU */ - case INDEX_op_sextract_i32: - if (a2 == 0 && args[3] == 8) { - tcg_out_ext8s(s, TCG_TYPE_REG, a0, a1); - } else if (a2 == 0 && args[3] == 16) { - tcg_out_ext16s(s, TCG_TYPE_REG, a0, a1); - } else { - g_assert_not_reached(); + return; } - break; + } + g_assert_not_reached(); +} - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); - break; - case INDEX_op_brcond2_i32: - tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], arg_label(args[5])); - break; +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_sextract, +}; - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - tcg_out_movcond(s, args[5], a0, a1, a2, args[3], args[4]); - break; +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - tcg_out_setcond(s, args[3], a0, a1, a2); - break; - case INDEX_op_setcond2_i32: - tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]); - break; +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LBU, dest, base, offset); +} - case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, a0, 0, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_ld_i64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_qemu_ld(s, a0, 0, a1, a2, TCG_TYPE_I64); - } else { - tcg_out_qemu_ld(s, a0, a1, a2, args[3], TCG_TYPE_I64); - } - break; +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; - case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, a0, 0, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_st_i64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_qemu_st(s, a0, 0, a1, a2, TCG_TYPE_I64); - } else { - tcg_out_qemu_st(s, a0, a1, a2, args[3], TCG_TYPE_I64); - } - break; +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LB, dest, base, offset); +} - case INDEX_op_add2_i32: - tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], - const_args[4], const_args[5], false); - break; - case INDEX_op_sub2_i32: - tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], - const_args[4], const_args[5], true); - break; +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; - case INDEX_op_mb: - tcg_out_mb(s, a0); - break; - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_mov_i64: - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - default: - g_assert_not_reached(); - } +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LHU, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; + +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LH, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; + +#if TCG_TARGET_REG_BITS == 64 +static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LWU, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32u, +}; + +static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LW, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32s, +}; +#endif + +static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_SB, data, base, offset); +} + +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st8_r, +}; + +static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_SH, data, base, offset); } +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st16_r, +}; + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tcg_out_st, +}; + + static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) { - switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_ld8u_i32: - case INDEX_op_ld8s_i32: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16s_i32: - case INDEX_op_ld_i32: - case INDEX_op_neg_i32: - case INDEX_op_not_i32: - case INDEX_op_bswap16_i32: - case INDEX_op_bswap32_i32: - case INDEX_op_ext8s_i32: - case INDEX_op_ext16s_i32: - case INDEX_op_extract_i32: - case INDEX_op_sextract_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld32u_i64: - case INDEX_op_ld_i64: - case INDEX_op_neg_i64: - case INDEX_op_not_i64: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: - case INDEX_op_ext8s_i64: - case INDEX_op_ext16s_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - case INDEX_op_extrh_i64_i32: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i64: - return C_O1_I1(r, r); - - case INDEX_op_st8_i32: - case INDEX_op_st16_i32: - case INDEX_op_st_i32: - case INDEX_op_st8_i64: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: - return C_O0_I2(rz, r); - - case INDEX_op_add_i32: - case INDEX_op_add_i64: - return C_O1_I2(r, r, rJ); - case INDEX_op_sub_i32: - case INDEX_op_sub_i64: - return C_O1_I2(r, rz, rN); - case INDEX_op_mul_i32: - case INDEX_op_mulsh_i32: - case INDEX_op_muluh_i32: - case INDEX_op_div_i32: - case INDEX_op_divu_i32: - case INDEX_op_rem_i32: - case INDEX_op_remu_i32: - case INDEX_op_nor_i32: - case INDEX_op_setcond_i32: - case INDEX_op_mul_i64: - case INDEX_op_mulsh_i64: - case INDEX_op_muluh_i64: - case INDEX_op_div_i64: - case INDEX_op_divu_i64: - case INDEX_op_rem_i64: - case INDEX_op_remu_i64: - case INDEX_op_nor_i64: - case INDEX_op_setcond_i64: - return C_O1_I2(r, rz, rz); - case INDEX_op_muls2_i32: - case INDEX_op_mulu2_i32: - case INDEX_op_muls2_i64: - case INDEX_op_mulu2_i64: - return C_O2_I2(r, r, r, r); - case INDEX_op_and_i32: - case INDEX_op_and_i64: - return C_O1_I2(r, r, rIK); - case INDEX_op_or_i32: - case INDEX_op_xor_i32: - case INDEX_op_or_i64: - case INDEX_op_xor_i64: - return C_O1_I2(r, r, rI); - case INDEX_op_shl_i32: - case INDEX_op_shr_i32: - case INDEX_op_sar_i32: - case INDEX_op_rotr_i32: - case INDEX_op_rotl_i32: - case INDEX_op_shl_i64: - case INDEX_op_shr_i64: - case INDEX_op_sar_i64: - case INDEX_op_rotr_i64: - case INDEX_op_rotl_i64: - return C_O1_I2(r, r, ri); - case INDEX_op_clz_i32: - case INDEX_op_clz_i64: - return C_O1_I2(r, r, rzW); - - case INDEX_op_deposit_i32: - case INDEX_op_deposit_i64: - return C_O1_I2(r, 0, rz); - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - return C_O0_I2(rz, rz); - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - return (use_mips32r6_instructions - ? C_O1_I4(r, rz, rz, rz, rz) - : C_O1_I4(r, rz, rz, rz, 0)); - case INDEX_op_add2_i32: - case INDEX_op_sub2_i32: - return C_O2_I4(r, r, rz, rz, rN, rN); - case INDEX_op_setcond2_i32: - return C_O1_I4(r, rz, rz, rz, rz); - case INDEX_op_brcond2_i32: - return C_O0_I4(rz, rz, rz, rz); - - case INDEX_op_qemu_ld_i32: - return C_O1_I1(r, r); - case INDEX_op_qemu_st_i32: - return C_O0_I2(rz, r); - case INDEX_op_qemu_ld_i64: - return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O2_I1(r, r, r); - case INDEX_op_qemu_st_i64: - return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(rz, r) : C_O0_I3(rz, rz, r); - - default: - return C_NotImplemented; - } + return C_NotImplemented; } static const int tcg_target_callee_save_regs[] = { diff --git a/tcg/optimize.c b/tcg/optimize.c index f922f86a1d..10a76c5461 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -30,14 +30,6 @@ #include "tcg-internal.h" #include "tcg-has.h" -#define CASE_OP_32_64(x) \ - glue(glue(case INDEX_op_, x), _i32): \ - glue(glue(case INDEX_op_, x), _i64) - -#define CASE_OP_32_64_VEC(x) \ - glue(glue(case INDEX_op_, x), _i32): \ - glue(glue(case INDEX_op_, x), _i64): \ - glue(glue(case INDEX_op_, x), _vec) typedef struct MemCopyInfo { IntervalTreeNode itree; @@ -66,6 +58,7 @@ typedef struct OptContext { /* In flight values from optimization. */ TCGType type; + int carry_state; /* -1 = non-constant, {0,1} = constant carry-in */ } OptContext; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -344,6 +337,18 @@ static TCGArg arg_new_temp(OptContext *ctx) return temp_arg(ts); } +static TCGOp *opt_insert_after(OptContext *ctx, TCGOp *op, + TCGOpcode opc, unsigned narg) +{ + return tcg_op_insert_after(ctx->tcg, op, opc, ctx->type, narg); +} + +static TCGOp *opt_insert_before(OptContext *ctx, TCGOp *op, + TCGOpcode opc, unsigned narg) +{ + return tcg_op_insert_before(ctx->tcg, op, opc, ctx->type, narg); +} + static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); @@ -363,10 +368,8 @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) switch (ctx->type) { case TCG_TYPE_I32: - new_op = INDEX_op_mov_i32; - break; case TCG_TYPE_I64: - new_op = INDEX_op_mov_i64; + new_op = INDEX_op_mov; break; case TCG_TYPE_V64: case TCG_TYPE_V128: @@ -409,162 +412,163 @@ static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, return tcg_opt_gen_mov(ctx, op, dst, arg_new_constant(ctx, val)); } -static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) +static uint64_t do_constant_folding_2(TCGOpcode op, TCGType type, + uint64_t x, uint64_t y) { uint64_t l64, h64; switch (op) { - CASE_OP_32_64(add): + case INDEX_op_add: return x + y; - CASE_OP_32_64(sub): + case INDEX_op_sub: return x - y; - CASE_OP_32_64(mul): + case INDEX_op_mul: return x * y; - CASE_OP_32_64_VEC(and): + case INDEX_op_and: + case INDEX_op_and_vec: return x & y; - CASE_OP_32_64_VEC(or): + case INDEX_op_or: + case INDEX_op_or_vec: return x | y; - CASE_OP_32_64_VEC(xor): + case INDEX_op_xor: + case INDEX_op_xor_vec: return x ^ y; - case INDEX_op_shl_i32: - return (uint32_t)x << (y & 31); - - case INDEX_op_shl_i64: + case INDEX_op_shl: + if (type == TCG_TYPE_I32) { + return (uint32_t)x << (y & 31); + } return (uint64_t)x << (y & 63); - case INDEX_op_shr_i32: - return (uint32_t)x >> (y & 31); - - case INDEX_op_shr_i64: + case INDEX_op_shr: + if (type == TCG_TYPE_I32) { + return (uint32_t)x >> (y & 31); + } return (uint64_t)x >> (y & 63); - case INDEX_op_sar_i32: - return (int32_t)x >> (y & 31); - - case INDEX_op_sar_i64: + case INDEX_op_sar: + if (type == TCG_TYPE_I32) { + return (int32_t)x >> (y & 31); + } return (int64_t)x >> (y & 63); - case INDEX_op_rotr_i32: - return ror32(x, y & 31); - - case INDEX_op_rotr_i64: + case INDEX_op_rotr: + if (type == TCG_TYPE_I32) { + return ror32(x, y & 31); + } return ror64(x, y & 63); - case INDEX_op_rotl_i32: - return rol32(x, y & 31); - - case INDEX_op_rotl_i64: + case INDEX_op_rotl: + if (type == TCG_TYPE_I32) { + return rol32(x, y & 31); + } return rol64(x, y & 63); - CASE_OP_32_64_VEC(not): + case INDEX_op_not: + case INDEX_op_not_vec: return ~x; - CASE_OP_32_64(neg): + case INDEX_op_neg: return -x; - CASE_OP_32_64_VEC(andc): + case INDEX_op_andc: + case INDEX_op_andc_vec: return x & ~y; - CASE_OP_32_64_VEC(orc): + case INDEX_op_orc: + case INDEX_op_orc_vec: return x | ~y; - CASE_OP_32_64_VEC(eqv): + case INDEX_op_eqv: + case INDEX_op_eqv_vec: return ~(x ^ y); - CASE_OP_32_64_VEC(nand): + case INDEX_op_nand: + case INDEX_op_nand_vec: return ~(x & y); - CASE_OP_32_64_VEC(nor): + case INDEX_op_nor: + case INDEX_op_nor_vec: return ~(x | y); - case INDEX_op_clz_i32: - return (uint32_t)x ? clz32(x) : y; - - case INDEX_op_clz_i64: + case INDEX_op_clz: + if (type == TCG_TYPE_I32) { + return (uint32_t)x ? clz32(x) : y; + } return x ? clz64(x) : y; - case INDEX_op_ctz_i32: - return (uint32_t)x ? ctz32(x) : y; - - case INDEX_op_ctz_i64: + case INDEX_op_ctz: + if (type == TCG_TYPE_I32) { + return (uint32_t)x ? ctz32(x) : y; + } return x ? ctz64(x) : y; - case INDEX_op_ctpop_i32: - return ctpop32(x); - - case INDEX_op_ctpop_i64: - return ctpop64(x); - - CASE_OP_32_64(ext8s): - return (int8_t)x; + case INDEX_op_ctpop: + return type == TCG_TYPE_I32 ? ctpop32(x) : ctpop64(x); - CASE_OP_32_64(ext16s): - return (int16_t)x; - - CASE_OP_32_64(ext8u): - return (uint8_t)x; - - CASE_OP_32_64(ext16u): - return (uint16_t)x; - - CASE_OP_32_64(bswap16): + case INDEX_op_bswap16: x = bswap16(x); return y & TCG_BSWAP_OS ? (int16_t)x : x; - CASE_OP_32_64(bswap32): + case INDEX_op_bswap32: x = bswap32(x); return y & TCG_BSWAP_OS ? (int32_t)x : x; - case INDEX_op_bswap64_i64: + case INDEX_op_bswap64: return bswap64(x); case INDEX_op_ext_i32_i64: - case INDEX_op_ext32s_i64: return (int32_t)x; case INDEX_op_extu_i32_i64: case INDEX_op_extrl_i64_i32: - case INDEX_op_ext32u_i64: return (uint32_t)x; case INDEX_op_extrh_i64_i32: return (uint64_t)x >> 32; - case INDEX_op_muluh_i32: - return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32; - case INDEX_op_mulsh_i32: - return ((int64_t)(int32_t)x * (int32_t)y) >> 32; - - case INDEX_op_muluh_i64: + case INDEX_op_muluh: + if (type == TCG_TYPE_I32) { + return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32; + } mulu64(&l64, &h64, x, y); return h64; - case INDEX_op_mulsh_i64: + + case INDEX_op_mulsh: + if (type == TCG_TYPE_I32) { + return ((int64_t)(int32_t)x * (int32_t)y) >> 32; + } muls64(&l64, &h64, x, y); return h64; - case INDEX_op_div_i32: + case INDEX_op_divs: /* Avoid crashing on divide by zero, otherwise undefined. */ - return (int32_t)x / ((int32_t)y ? : 1); - case INDEX_op_divu_i32: - return (uint32_t)x / ((uint32_t)y ? : 1); - case INDEX_op_div_i64: + if (type == TCG_TYPE_I32) { + return (int32_t)x / ((int32_t)y ? : 1); + } return (int64_t)x / ((int64_t)y ? : 1); - case INDEX_op_divu_i64: + + case INDEX_op_divu: + if (type == TCG_TYPE_I32) { + return (uint32_t)x / ((uint32_t)y ? : 1); + } return (uint64_t)x / ((uint64_t)y ? : 1); - case INDEX_op_rem_i32: - return (int32_t)x % ((int32_t)y ? : 1); - case INDEX_op_remu_i32: - return (uint32_t)x % ((uint32_t)y ? : 1); - case INDEX_op_rem_i64: + case INDEX_op_rems: + if (type == TCG_TYPE_I32) { + return (int32_t)x % ((int32_t)y ? : 1); + } return (int64_t)x % ((int64_t)y ? : 1); - case INDEX_op_remu_i64: + + case INDEX_op_remu: + if (type == TCG_TYPE_I32) { + return (uint32_t)x % ((uint32_t)y ? : 1); + } return (uint64_t)x % ((uint64_t)y ? : 1); default: @@ -575,7 +579,7 @@ static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) static uint64_t do_constant_folding(TCGOpcode op, TCGType type, uint64_t x, uint64_t y) { - uint64_t res = do_constant_folding_2(op, x, y); + uint64_t res = do_constant_folding_2(op, type, x, y); if (type == TCG_TYPE_I32) { res = (int32_t)res; } @@ -725,12 +729,18 @@ static int do_constant_folding_cond(TCGType type, TCGArg x, #define NO_DEST temp_arg(NULL) +static int pref_commutative(TempOptInfo *ti) +{ + /* Slight preference for non-zero constants second. */ + return !ti_is_const(ti) ? 0 : ti_const_val(ti) ? 3 : 2; +} + static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2) { TCGArg a1 = *p1, a2 = *p2; int sum = 0; - sum += arg_is_const(a1); - sum -= arg_is_const(a2); + sum += pref_commutative(arg_info(a1)); + sum -= pref_commutative(arg_info(a2)); /* Prefer the constant in second argument, and then the form op a, a, b, which is better handled on non-RISC hosts. */ @@ -745,10 +755,10 @@ static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2) static bool swap_commutative2(TCGArg *p1, TCGArg *p2) { int sum = 0; - sum += arg_is_const(p1[0]); - sum += arg_is_const(p1[1]); - sum -= arg_is_const(p2[0]); - sum -= arg_is_const(p2[1]); + sum += pref_commutative(arg_info(p1[0])); + sum += pref_commutative(arg_info(p1[1])); + sum -= pref_commutative(arg_info(p2[0])); + sum -= pref_commutative(arg_info(p2[1])); if (sum > 0) { TCGArg t; t = p1[0], p1[0] = p2[0], p2[0] = t; @@ -806,9 +816,7 @@ static int do_constant_folding_cond1(OptContext *ctx, TCGOp *op, TCGArg dest, /* Expand to AND with a temporary if no backend support. */ if (!TCG_TARGET_HAS_tst) { - TCGOpcode and_opc = (ctx->type == TCG_TYPE_I32 - ? INDEX_op_and_i32 : INDEX_op_and_i64); - TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, and_opc, 3); + TCGOp *op2 = opt_insert_before(ctx, op, INDEX_op_and, 3); TCGArg tmp = arg_new_temp(ctx); op2->args[0] = tmp; @@ -901,8 +909,8 @@ static int do_constant_folding_cond2(OptContext *ctx, TCGOp *op, TCGArg *args) /* Expand to AND with a temporary if no backend support. */ if (!TCG_TARGET_HAS_tst && is_tst_cond(c)) { - TCGOp *op1 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3); - TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3); + TCGOp *op1 = opt_insert_before(ctx, op, INDEX_op_and, 3); + TCGOp *op2 = opt_insert_before(ctx, op, INDEX_op_and, 3); TCGArg t1 = arg_new_temp(ctx); TCGArg t2 = arg_new_temp(ctx); @@ -1101,12 +1109,9 @@ static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx) switch (ctx->type) { case TCG_TYPE_I32: - not_op = INDEX_op_not_i32; - have_not = TCG_TARGET_HAS_not_i32; - break; case TCG_TYPE_I64: - not_op = INDEX_op_not_i64; - have_not = TCG_TARGET_HAS_not_i64; + not_op = INDEX_op_not; + have_not = tcg_op_supported(INDEX_op_not, ctx->type, 0); break; case TCG_TYPE_V64: case TCG_TYPE_V128: @@ -1197,8 +1202,10 @@ static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) * 3) those that produce information about the result value. */ +static bool fold_addco(OptContext *ctx, TCGOp *op); static bool fold_or(OptContext *ctx, TCGOp *op); static bool fold_orc(OptContext *ctx, TCGOp *op); +static bool fold_subbo(OptContext *ctx, TCGOp *op); static bool fold_xor(OptContext *ctx, TCGOp *op); static bool fold_add(OptContext *ctx, TCGOp *op) @@ -1220,80 +1227,168 @@ static bool fold_add_vec(OptContext *ctx, TCGOp *op) return finish_folding(ctx, op); } -static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) +static void squash_prev_carryout(OptContext *ctx, TCGOp *op) { - bool a_const = arg_is_const(op->args[2]) && arg_is_const(op->args[3]); - bool b_const = arg_is_const(op->args[4]) && arg_is_const(op->args[5]); + TempOptInfo *t2; - if (a_const && b_const) { - uint64_t al = arg_info(op->args[2])->val; - uint64_t ah = arg_info(op->args[3])->val; - uint64_t bl = arg_info(op->args[4])->val; - uint64_t bh = arg_info(op->args[5])->val; - TCGArg rl, rh; - TCGOp *op2; + op = QTAILQ_PREV(op, link); + switch (op->opc) { + case INDEX_op_addco: + op->opc = INDEX_op_add; + fold_add(ctx, op); + break; + case INDEX_op_addcio: + op->opc = INDEX_op_addci; + break; + case INDEX_op_addc1o: + op->opc = INDEX_op_add; + t2 = arg_info(op->args[2]); + if (ti_is_const(t2)) { + op->args[2] = arg_new_constant(ctx, ti_const_val(t2) + 1); + /* Perform other constant folding, if needed. */ + fold_add(ctx, op); + } else { + TCGArg ret = op->args[0]; + op = opt_insert_after(ctx, op, INDEX_op_add, 3); + op->args[0] = ret; + op->args[1] = ret; + op->args[2] = arg_new_constant(ctx, 1); + } + break; + default: + g_assert_not_reached(); + } +} - if (ctx->type == TCG_TYPE_I32) { - uint64_t a = deposit64(al, 32, 32, ah); - uint64_t b = deposit64(bl, 32, 32, bh); +static bool fold_addci(OptContext *ctx, TCGOp *op) +{ + fold_commutative(ctx, op); - if (add) { - a += b; - } else { - a -= b; - } + if (ctx->carry_state < 0) { + return finish_folding(ctx, op); + } + + squash_prev_carryout(ctx, op); + op->opc = INDEX_op_add; + + if (ctx->carry_state > 0) { + TempOptInfo *t2 = arg_info(op->args[2]); - al = sextract64(a, 0, 32); - ah = sextract64(a, 32, 32); + /* + * Propagate the known carry-in into a constant, if possible. + * Otherwise emit a second add +1. + */ + if (ti_is_const(t2)) { + op->args[2] = arg_new_constant(ctx, ti_const_val(t2) + 1); } else { - Int128 a = int128_make128(al, ah); - Int128 b = int128_make128(bl, bh); + TCGOp *op2 = opt_insert_before(ctx, op, INDEX_op_add, 3); - if (add) { - a = int128_add(a, b); - } else { - a = int128_sub(a, b); - } + op2->args[0] = op->args[0]; + op2->args[1] = op->args[1]; + op2->args[2] = op->args[2]; + fold_add(ctx, op2); - al = int128_getlo(a); - ah = int128_gethi(a); + op->args[1] = op->args[0]; + op->args[2] = arg_new_constant(ctx, 1); } + } - rl = op->args[0]; - rh = op->args[1]; + ctx->carry_state = -1; + return fold_add(ctx, op); +} - /* The proper opcode is supplied by tcg_opt_gen_mov. */ - op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2); +static bool fold_addcio(OptContext *ctx, TCGOp *op) +{ + TempOptInfo *t1, *t2; + int carry_out = -1; + uint64_t sum, max; - tcg_opt_gen_movi(ctx, op, rl, al); - tcg_opt_gen_movi(ctx, op2, rh, ah); - return true; + fold_commutative(ctx, op); + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + + /* + * The z_mask value is >= the maximum value that can be represented + * with the known zero bits. So adding the z_mask values will not + * overflow if and only if the true values cannot overflow. + */ + if (!uadd64_overflow(t1->z_mask, t2->z_mask, &sum) && + !uadd64_overflow(sum, ctx->carry_state != 0, &sum)) { + carry_out = 0; } - /* Fold sub2 r,x,i to add2 r,x,-i */ - if (!add && b_const) { - uint64_t bl = arg_info(op->args[4])->val; - uint64_t bh = arg_info(op->args[5])->val; + if (ctx->carry_state < 0) { + ctx->carry_state = carry_out; + return finish_folding(ctx, op); + } - /* Negate the two parts without assembling and disassembling. */ - bl = -bl; - bh = ~bh + !bl; + squash_prev_carryout(ctx, op); + if (ctx->carry_state == 0) { + goto do_addco; + } - op->opc = (ctx->type == TCG_TYPE_I32 - ? INDEX_op_add2_i32 : INDEX_op_add2_i64); - op->args[4] = arg_new_constant(ctx, bl); - op->args[5] = arg_new_constant(ctx, bh); + /* Propagate the known carry-in into a constant, if possible. */ + max = ctx->type == TCG_TYPE_I32 ? UINT32_MAX : UINT64_MAX; + if (ti_is_const(t2)) { + uint64_t v = ti_const_val(t2) & max; + if (v < max) { + op->args[2] = arg_new_constant(ctx, v + 1); + goto do_addco; + } + /* max + known carry in produces known carry out. */ + carry_out = 1; } + if (ti_is_const(t1)) { + uint64_t v = ti_const_val(t1) & max; + if (v < max) { + op->args[1] = arg_new_constant(ctx, v + 1); + goto do_addco; + } + carry_out = 1; + } + + /* Adjust the opcode to remember the known carry-in. */ + op->opc = INDEX_op_addc1o; + ctx->carry_state = carry_out; return finish_folding(ctx, op); + + do_addco: + op->opc = INDEX_op_addco; + return fold_addco(ctx, op); } -static bool fold_add2(OptContext *ctx, TCGOp *op) +static bool fold_addco(OptContext *ctx, TCGOp *op) { - /* Note that the high and low parts may be independently swapped. */ - swap_commutative(op->args[0], &op->args[2], &op->args[4]); - swap_commutative(op->args[1], &op->args[3], &op->args[5]); + TempOptInfo *t1, *t2; + int carry_out = -1; + uint64_t ign; - return fold_addsub2(ctx, op, true); + fold_commutative(ctx, op); + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + + if (ti_is_const(t2)) { + uint64_t v2 = ti_const_val(t2); + + if (ti_is_const(t1)) { + uint64_t v1 = ti_const_val(t1); + /* Given sign-extension of z_mask for I32, we need not truncate. */ + carry_out = uadd64_overflow(v1, v2, &ign); + } else if (v2 == 0) { + carry_out = 0; + } + } else { + /* + * The z_mask value is >= the maximum value that can be represented + * with the known zero bits. So adding the z_mask values will not + * overflow if and only if the true values cannot overflow. + */ + if (!uadd64_overflow(t1->z_mask, t2->z_mask, &ign)) { + carry_out = 0; + } + } + ctx->carry_state = carry_out; + return finish_folding(ctx, op); } static bool fold_and(OptContext *ctx, TCGOp *op) @@ -1348,6 +1443,25 @@ static bool fold_andc(OptContext *ctx, TCGOp *op) t2 = arg_info(op->args[2]); z_mask = t1->z_mask; + if (ti_is_const(t2)) { + /* Fold andc r,x,i to and r,x,~i. */ + switch (ctx->type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + op->opc = INDEX_op_and; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + op->opc = INDEX_op_and_vec; + break; + default: + g_assert_not_reached(); + } + op->args[2] = arg_new_constant(ctx, ~ti_const_val(t2)); + return fold_and(ctx, op); + } + /* * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer anything from it. @@ -1504,14 +1618,14 @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) break; do_brcond_low: - op->opc = INDEX_op_brcond_i32; + op->opc = INDEX_op_brcond; op->args[1] = op->args[2]; op->args[2] = cond; op->args[3] = label; return fold_brcond(ctx, op); do_brcond_high: - op->opc = INDEX_op_brcond_i32; + op->opc = INDEX_op_brcond; op->args[0] = op->args[1]; op->args[1] = op->args[3]; op->args[2] = cond; @@ -1547,17 +1661,15 @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) z_mask = t1->z_mask; switch (op->opc) { - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: + case INDEX_op_bswap16: z_mask = bswap16(z_mask); sign = INT16_MIN; break; - case INDEX_op_bswap32_i32: - case INDEX_op_bswap32_i64: + case INDEX_op_bswap32: z_mask = bswap32(z_mask); sign = INT32_MIN; break; - case INDEX_op_bswap64_i64: + case INDEX_op_bswap64: z_mask = bswap64(z_mask); sign = INT64_MIN; break; @@ -1713,8 +1825,7 @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) TempOptInfo *t2 = arg_info(op->args[2]); int ofs = op->args[3]; int len = op->args[4]; - int width; - TCGOpcode and_opc; + int width = 8 * tcg_type_size(ctx->type); uint64_t z_mask, s_mask; if (ti_is_const(t1) && ti_is_const(t2)) { @@ -1723,24 +1834,11 @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) ti_const_val(t2))); } - switch (ctx->type) { - case TCG_TYPE_I32: - and_opc = INDEX_op_and_i32; - width = 32; - break; - case TCG_TYPE_I64: - and_opc = INDEX_op_and_i64; - width = 64; - break; - default: - g_assert_not_reached(); - } - /* Inserting a value into zero at offset 0. */ if (ti_is_const_val(t1, 0) && ofs == 0) { uint64_t mask = MAKE_64BIT_MASK(0, len); - op->opc = and_opc; + op->opc = INDEX_op_and; op->args[1] = op->args[2]; op->args[2] = arg_new_constant(ctx, mask); return fold_and(ctx, op); @@ -1750,7 +1848,7 @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) if (ti_is_const_val(t2, 0)) { uint64_t mask = deposit64(-1, ofs, len, 0); - op->opc = and_opc; + op->opc = INDEX_op_and; op->args[2] = arg_new_constant(ctx, mask); return fold_and(ctx, op); } @@ -1803,6 +1901,7 @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { uint64_t s_mask; + TempOptInfo *t1, *t2; if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, -1) || @@ -1810,8 +1909,28 @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) return true; } - s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; + t2 = arg_info(op->args[2]); + if (ti_is_const(t2)) { + /* Fold eqv r,x,i to xor r,x,~i. */ + switch (ctx->type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + op->opc = INDEX_op_xor; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + op->opc = INDEX_op_xor_vec; + break; + default: + g_assert_not_reached(); + } + op->args[2] = arg_new_constant(ctx, ~ti_const_val(t2)); + return fold_xor(ctx, op); + } + + t1 = arg_info(op->args[1]); + s_mask = t1->s_mask & t2->s_mask; return fold_masks_s(ctx, op, s_mask); } @@ -1843,12 +1962,12 @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) uint64_t v2 = arg_info(op->args[2])->val; int shr = op->args[3]; - if (op->opc == INDEX_op_extract2_i64) { - v1 >>= shr; - v2 <<= 64 - shr; - } else { + if (ctx->type == TCG_TYPE_I32) { v1 = (uint32_t)v1 >> shr; v2 = (uint64_t)((int32_t)v2 << (32 - shr)); + } else { + v1 >>= shr; + v2 <<= 64 - shr; } return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2); } @@ -1857,8 +1976,7 @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) static bool fold_exts(OptContext *ctx, TCGOp *op) { - uint64_t s_mask_old, s_mask, z_mask; - bool type_change = false; + uint64_t s_mask, z_mask; TempOptInfo *t1; if (fold_const1(ctx, op)) { @@ -1868,72 +1986,38 @@ static bool fold_exts(OptContext *ctx, TCGOp *op) t1 = arg_info(op->args[1]); z_mask = t1->z_mask; s_mask = t1->s_mask; - s_mask_old = s_mask; switch (op->opc) { - CASE_OP_32_64(ext8s): - s_mask |= INT8_MIN; - z_mask = (int8_t)z_mask; - break; - CASE_OP_32_64(ext16s): - s_mask |= INT16_MIN; - z_mask = (int16_t)z_mask; - break; case INDEX_op_ext_i32_i64: - type_change = true; - QEMU_FALLTHROUGH; - case INDEX_op_ext32s_i64: s_mask |= INT32_MIN; z_mask = (int32_t)z_mask; break; default: g_assert_not_reached(); } - - if (!type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) { - return true; - } - return fold_masks_zs(ctx, op, z_mask, s_mask); } static bool fold_extu(OptContext *ctx, TCGOp *op) { - uint64_t z_mask_old, z_mask; - bool type_change = false; + uint64_t z_mask; if (fold_const1(ctx, op)) { return true; } - z_mask_old = z_mask = arg_info(op->args[1])->z_mask; - + z_mask = arg_info(op->args[1])->z_mask; switch (op->opc) { - CASE_OP_32_64(ext8u): - z_mask = (uint8_t)z_mask; - break; - CASE_OP_32_64(ext16u): - z_mask = (uint16_t)z_mask; - break; case INDEX_op_extrl_i64_i32: case INDEX_op_extu_i32_i64: - type_change = true; - QEMU_FALLTHROUGH; - case INDEX_op_ext32u_i64: z_mask = (uint32_t)z_mask; break; case INDEX_op_extrh_i64_i32: - type_change = true; z_mask >>= 32; break; default: g_assert_not_reached(); } - - if (!type_change && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) { - return true; - } - return fold_masks_z(ctx, op, z_mask); } @@ -1999,42 +2083,20 @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) if (ti_is_const(tt) && ti_is_const(ft)) { uint64_t tv = ti_const_val(tt); uint64_t fv = ti_const_val(ft); - TCGOpcode opc, negopc = 0; TCGCond cond = op->args[5]; - switch (ctx->type) { - case TCG_TYPE_I32: - opc = INDEX_op_setcond_i32; - if (TCG_TARGET_HAS_negsetcond_i32) { - negopc = INDEX_op_negsetcond_i32; - } - tv = (int32_t)tv; - fv = (int32_t)fv; - break; - case TCG_TYPE_I64: - opc = INDEX_op_setcond_i64; - if (TCG_TARGET_HAS_negsetcond_i64) { - negopc = INDEX_op_negsetcond_i64; - } - break; - default: - g_assert_not_reached(); - } - if (tv == 1 && fv == 0) { - op->opc = opc; + op->opc = INDEX_op_setcond; op->args[3] = cond; } else if (fv == 1 && tv == 0) { - op->opc = opc; + op->opc = INDEX_op_setcond; + op->args[3] = tcg_invert_cond(cond); + } else if (tv == -1 && fv == 0) { + op->opc = INDEX_op_negsetcond; + op->args[3] = cond; + } else if (fv == -1 && tv == 0) { + op->opc = INDEX_op_negsetcond; op->args[3] = tcg_invert_cond(cond); - } else if (negopc) { - if (tv == -1 && fv == 0) { - op->opc = negopc; - op->args[3] = cond; - } else if (fv == -1 && tv == 0) { - op->opc = negopc; - op->args[3] = tcg_invert_cond(cond); - } } } @@ -2072,21 +2134,23 @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) TCGOp *op2; switch (op->opc) { - case INDEX_op_mulu2_i32: - l = (uint64_t)(uint32_t)a * (uint32_t)b; - h = (int32_t)(l >> 32); - l = (int32_t)l; - break; - case INDEX_op_muls2_i32: - l = (int64_t)(int32_t)a * (int32_t)b; - h = l >> 32; - l = (int32_t)l; - break; - case INDEX_op_mulu2_i64: - mulu64(&l, &h, a, b); + case INDEX_op_mulu2: + if (ctx->type == TCG_TYPE_I32) { + l = (uint64_t)(uint32_t)a * (uint32_t)b; + h = (int32_t)(l >> 32); + l = (int32_t)l; + } else { + mulu64(&l, &h, a, b); + } break; - case INDEX_op_muls2_i64: - muls64(&l, &h, a, b); + case INDEX_op_muls2: + if (ctx->type == TCG_TYPE_I32) { + l = (int64_t)(int32_t)a * (int32_t)b; + h = l >> 32; + l = (int32_t)l; + } else { + muls64(&l, &h, a, b); + } break; default: g_assert_not_reached(); @@ -2096,7 +2160,7 @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) rh = op->args[1]; /* The proper opcode is supplied by tcg_opt_gen_mov. */ - op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2); + op2 = opt_insert_before(ctx, op, 0, 2); tcg_opt_gen_movi(ctx, op, rl, l); tcg_opt_gen_movi(ctx, op2, rh, h); @@ -2176,6 +2240,7 @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { uint64_t s_mask; + TempOptInfo *t1, *t2; if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, -1) || @@ -2184,8 +2249,28 @@ static bool fold_orc(OptContext *ctx, TCGOp *op) return true; } - s_mask = arg_info(op->args[1])->s_mask - & arg_info(op->args[2])->s_mask; + t2 = arg_info(op->args[2]); + if (ti_is_const(t2)) { + /* Fold orc r,x,i to or r,x,~i. */ + switch (ctx->type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + op->opc = INDEX_op_or; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + op->opc = INDEX_op_or_vec; + break; + default: + g_assert_not_reached(); + } + op->args[2] = arg_new_constant(ctx, ~ti_const_val(t2)); + return fold_or(ctx, op); + } + + t1 = arg_info(op->args[1]); + s_mask = t1->s_mask & t2->s_mask; return fold_masks_s(ctx, op, s_mask); } @@ -2300,34 +2385,17 @@ static int fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) break; } if (convert) { - TCGOpcode add_opc, xor_opc, neg_opc; - if (!inv && !neg) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); } - switch (ctx->type) { - case TCG_TYPE_I32: - add_opc = INDEX_op_add_i32; - neg_opc = INDEX_op_neg_i32; - xor_opc = INDEX_op_xor_i32; - break; - case TCG_TYPE_I64: - add_opc = INDEX_op_add_i64; - neg_opc = INDEX_op_neg_i64; - xor_opc = INDEX_op_xor_i64; - break; - default: - g_assert_not_reached(); - } - if (!inv) { - op->opc = neg_opc; + op->opc = INDEX_op_neg; } else if (neg) { - op->opc = add_opc; + op->opc = INDEX_op_add; op->args[2] = arg_new_constant(ctx, -1); } else { - op->opc = xor_opc; + op->opc = INDEX_op_xor; op->args[2] = arg_new_constant(ctx, 1); } return -1; @@ -2338,8 +2406,6 @@ static int fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg) static void fold_setcond_tst_pow2(OptContext *ctx, TCGOp *op, bool neg) { - TCGOpcode and_opc, sub_opc, xor_opc, neg_opc, shr_opc; - TCGOpcode uext_opc = 0, sext_opc = 0; TCGCond cond = op->args[3]; TCGArg ret, src1, src2; TCGOp *op2; @@ -2358,77 +2424,46 @@ static void fold_setcond_tst_pow2(OptContext *ctx, TCGOp *op, bool neg) } sh = ctz64(val); - switch (ctx->type) { - case TCG_TYPE_I32: - and_opc = INDEX_op_and_i32; - sub_opc = INDEX_op_sub_i32; - xor_opc = INDEX_op_xor_i32; - shr_opc = INDEX_op_shr_i32; - neg_opc = INDEX_op_neg_i32; - if (TCG_TARGET_extract_valid(TCG_TYPE_I32, sh, 1)) { - uext_opc = INDEX_op_extract_i32; - } - if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, sh, 1)) { - sext_opc = INDEX_op_sextract_i32; - } - break; - case TCG_TYPE_I64: - and_opc = INDEX_op_and_i64; - sub_opc = INDEX_op_sub_i64; - xor_opc = INDEX_op_xor_i64; - shr_opc = INDEX_op_shr_i64; - neg_opc = INDEX_op_neg_i64; - if (TCG_TARGET_extract_valid(TCG_TYPE_I64, sh, 1)) { - uext_opc = INDEX_op_extract_i64; - } - if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, sh, 1)) { - sext_opc = INDEX_op_sextract_i64; - } - break; - default: - g_assert_not_reached(); - } - ret = op->args[0]; src1 = op->args[1]; inv = cond == TCG_COND_TSTEQ; - if (sh && sext_opc && neg && !inv) { - op->opc = sext_opc; + if (sh && neg && !inv && TCG_TARGET_sextract_valid(ctx->type, sh, 1)) { + op->opc = INDEX_op_sextract; op->args[1] = src1; op->args[2] = sh; op->args[3] = 1; return; - } else if (sh && uext_opc) { - op->opc = uext_opc; + } else if (sh && TCG_TARGET_extract_valid(ctx->type, sh, 1)) { + op->opc = INDEX_op_extract; op->args[1] = src1; op->args[2] = sh; op->args[3] = 1; } else { if (sh) { - op2 = tcg_op_insert_before(ctx->tcg, op, shr_opc, 3); + op2 = opt_insert_before(ctx, op, INDEX_op_shr, 3); op2->args[0] = ret; op2->args[1] = src1; op2->args[2] = arg_new_constant(ctx, sh); src1 = ret; } - op->opc = and_opc; + op->opc = INDEX_op_and; op->args[1] = src1; op->args[2] = arg_new_constant(ctx, 1); } if (neg && inv) { - op2 = tcg_op_insert_after(ctx->tcg, op, sub_opc, 3); + op2 = opt_insert_after(ctx, op, INDEX_op_add, 3); op2->args[0] = ret; op2->args[1] = ret; - op2->args[2] = arg_new_constant(ctx, 1); + op2->args[2] = arg_new_constant(ctx, -1); } else if (inv) { - op2 = tcg_op_insert_after(ctx->tcg, op, xor_opc, 3); + op2 = opt_insert_after(ctx, op, INDEX_op_xor, 3); op2->args[0] = ret; op2->args[1] = ret; op2->args[2] = arg_new_constant(ctx, 1); } else if (neg) { - op2 = tcg_op_insert_after(ctx->tcg, op, neg_opc, 2); + op2 = opt_insert_after(ctx, op, INDEX_op_neg, 2); op2->args[0] = ret; op2->args[1] = ret; } @@ -2540,14 +2575,14 @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) do_setcond_low: op->args[2] = op->args[3]; op->args[3] = cond; - op->opc = INDEX_op_setcond_i32; + op->opc = INDEX_op_setcond; return fold_setcond(ctx, op); do_setcond_high: op->args[1] = op->args[2]; op->args[2] = op->args[4]; op->args[3] = cond; - op->opc = INDEX_op_setcond_i32; + op->opc = INDEX_op_setcond; return fold_setcond(ctx, op); } @@ -2607,13 +2642,13 @@ static bool fold_shift(OptContext *ctx, TCGOp *op) } switch (op->opc) { - CASE_OP_32_64(sar): + case INDEX_op_sar: /* * Arithmetic right shift will not reduce the number of * input sign repetitions. */ return fold_masks_s(ctx, op, s_mask); - CASE_OP_32_64(shr): + case INDEX_op_shr: /* * If the sign bit is known zero, then logical right shift * will not reduce the number of input sign repetitions. @@ -2640,11 +2675,8 @@ static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op) switch (ctx->type) { case TCG_TYPE_I32: - neg_op = INDEX_op_neg_i32; - have_neg = true; - break; case TCG_TYPE_I64: - neg_op = INDEX_op_neg_i64; + neg_op = INDEX_op_neg; have_neg = true; break; case TCG_TYPE_V64: @@ -2689,16 +2721,149 @@ static bool fold_sub(OptContext *ctx, TCGOp *op) if (arg_is_const(op->args[2])) { uint64_t val = arg_info(op->args[2])->val; - op->opc = (ctx->type == TCG_TYPE_I32 - ? INDEX_op_add_i32 : INDEX_op_add_i64); + op->opc = INDEX_op_add; op->args[2] = arg_new_constant(ctx, -val); } return finish_folding(ctx, op); } -static bool fold_sub2(OptContext *ctx, TCGOp *op) +static void squash_prev_borrowout(OptContext *ctx, TCGOp *op) +{ + TempOptInfo *t2; + + op = QTAILQ_PREV(op, link); + switch (op->opc) { + case INDEX_op_subbo: + op->opc = INDEX_op_sub; + fold_sub(ctx, op); + break; + case INDEX_op_subbio: + op->opc = INDEX_op_subbi; + break; + case INDEX_op_subb1o: + t2 = arg_info(op->args[2]); + if (ti_is_const(t2)) { + op->opc = INDEX_op_add; + op->args[2] = arg_new_constant(ctx, -(ti_const_val(t2) + 1)); + /* Perform other constant folding, if needed. */ + fold_add(ctx, op); + } else { + TCGArg ret = op->args[0]; + op->opc = INDEX_op_sub; + op = opt_insert_after(ctx, op, INDEX_op_add, 3); + op->args[0] = ret; + op->args[1] = ret; + op->args[2] = arg_new_constant(ctx, -1); + } + break; + default: + g_assert_not_reached(); + } +} + +static bool fold_subbi(OptContext *ctx, TCGOp *op) +{ + TempOptInfo *t2; + int borrow_in = ctx->carry_state; + + if (borrow_in < 0) { + return finish_folding(ctx, op); + } + ctx->carry_state = -1; + + squash_prev_borrowout(ctx, op); + if (borrow_in == 0) { + op->opc = INDEX_op_sub; + return fold_sub(ctx, op); + } + + /* + * Propagate the known carry-in into any constant, then negate to + * transform from sub to add. If there is no constant, emit a + * separate add -1. + */ + t2 = arg_info(op->args[2]); + if (ti_is_const(t2)) { + op->args[2] = arg_new_constant(ctx, -(ti_const_val(t2) + 1)); + } else { + TCGOp *op2 = opt_insert_before(ctx, op, INDEX_op_sub, 3); + + op2->args[0] = op->args[0]; + op2->args[1] = op->args[1]; + op2->args[2] = op->args[2]; + fold_sub(ctx, op2); + + op->args[1] = op->args[0]; + op->args[2] = arg_new_constant(ctx, -1); + } + op->opc = INDEX_op_add; + return fold_add(ctx, op); +} + +static bool fold_subbio(OptContext *ctx, TCGOp *op) +{ + TempOptInfo *t1, *t2; + int borrow_out = -1; + + if (ctx->carry_state < 0) { + return finish_folding(ctx, op); + } + + squash_prev_borrowout(ctx, op); + if (ctx->carry_state == 0) { + goto do_subbo; + } + + t1 = arg_info(op->args[1]); + t2 = arg_info(op->args[2]); + + /* Propagate the known borrow-in into a constant, if possible. */ + if (ti_is_const(t2)) { + uint64_t max = ctx->type == TCG_TYPE_I32 ? UINT32_MAX : UINT64_MAX; + uint64_t v = ti_const_val(t2) & max; + + if (v < max) { + op->args[2] = arg_new_constant(ctx, v + 1); + goto do_subbo; + } + /* subtracting max + 1 produces known borrow out. */ + borrow_out = 1; + } + if (ti_is_const(t1)) { + uint64_t v = ti_const_val(t1); + if (v != 0) { + op->args[2] = arg_new_constant(ctx, v - 1); + goto do_subbo; + } + } + + /* Adjust the opcode to remember the known carry-in. */ + op->opc = INDEX_op_subb1o; + ctx->carry_state = borrow_out; + return finish_folding(ctx, op); + + do_subbo: + op->opc = INDEX_op_subbo; + return fold_subbo(ctx, op); +} + +static bool fold_subbo(OptContext *ctx, TCGOp *op) { - return fold_addsub2(ctx, op, false); + TempOptInfo *t1 = arg_info(op->args[1]); + TempOptInfo *t2 = arg_info(op->args[2]); + int borrow_out = -1; + + if (ti_is_const(t2)) { + uint64_t v2 = ti_const_val(t2); + if (v2 == 0) { + borrow_out = 0; + } else if (ti_is_const(t1)) { + uint64_t v1 = ti_const_val(t1); + borrow_out = v1 < v2; + } + } + ctx->carry_state = borrow_out; + return finish_folding(ctx, op); } static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) @@ -2707,22 +2872,22 @@ static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) /* We can't do any folding with a load, but we can record bits. */ switch (op->opc) { - CASE_OP_32_64(ld8s): + case INDEX_op_ld8s: s_mask = INT8_MIN; break; - CASE_OP_32_64(ld8u): + case INDEX_op_ld8u: z_mask = MAKE_64BIT_MASK(0, 8); break; - CASE_OP_32_64(ld16s): + case INDEX_op_ld16s: s_mask = INT16_MIN; break; - CASE_OP_32_64(ld16u): + case INDEX_op_ld16u: z_mask = MAKE_64BIT_MASK(0, 16); break; - case INDEX_op_ld32s_i64: + case INDEX_op_ld32s: s_mask = INT32_MIN; break; - case INDEX_op_ld32u_i64: + case INDEX_op_ld32u: z_mask = MAKE_64BIT_MASK(0, 32); break; default: @@ -2765,19 +2930,16 @@ static bool fold_tcg_st(OptContext *ctx, TCGOp *op) } switch (op->opc) { - CASE_OP_32_64(st8): + case INDEX_op_st8: lm1 = 0; break; - CASE_OP_32_64(st16): + case INDEX_op_st16: lm1 = 1; break; - case INDEX_op_st32_i64: - case INDEX_op_st_i32: + case INDEX_op_st32: lm1 = 3; break; - case INDEX_op_st_i64: - lm1 = 7; - break; + case INDEX_op_st: case INDEX_op_st_vec: lm1 = tcg_type_size(ctx->type) - 1; break; @@ -2881,44 +3043,52 @@ void tcg_optimize(TCGContext *s) * Sorted alphabetically by opcode as much as possible. */ switch (opc) { - CASE_OP_32_64(add): + case INDEX_op_add: done = fold_add(&ctx, op); break; case INDEX_op_add_vec: done = fold_add_vec(&ctx, op); break; - CASE_OP_32_64(add2): - done = fold_add2(&ctx, op); + case INDEX_op_addci: + done = fold_addci(&ctx, op); + break; + case INDEX_op_addcio: + done = fold_addcio(&ctx, op); + break; + case INDEX_op_addco: + done = fold_addco(&ctx, op); break; - CASE_OP_32_64_VEC(and): + case INDEX_op_and: + case INDEX_op_and_vec: done = fold_and(&ctx, op); break; - CASE_OP_32_64_VEC(andc): + case INDEX_op_andc: + case INDEX_op_andc_vec: done = fold_andc(&ctx, op); break; - CASE_OP_32_64(brcond): + case INDEX_op_brcond: done = fold_brcond(&ctx, op); break; case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; - CASE_OP_32_64(bswap16): - CASE_OP_32_64(bswap32): - case INDEX_op_bswap64_i64: + case INDEX_op_bswap16: + case INDEX_op_bswap32: + case INDEX_op_bswap64: done = fold_bswap(&ctx, op); break; - CASE_OP_32_64(clz): - CASE_OP_32_64(ctz): + case INDEX_op_clz: + case INDEX_op_ctz: done = fold_count_zeros(&ctx, op); break; - CASE_OP_32_64(ctpop): + case INDEX_op_ctpop: done = fold_ctpop(&ctx, op); break; - CASE_OP_32_64(deposit): + case INDEX_op_deposit: done = fold_deposit(&ctx, op); break; - CASE_OP_32_64(div): - CASE_OP_32_64(divu): + case INDEX_op_divs: + case INDEX_op_divu: done = fold_divide(&ctx, op); break; case INDEX_op_dup_vec: @@ -2927,123 +3097,114 @@ void tcg_optimize(TCGContext *s) case INDEX_op_dup2_vec: done = fold_dup2(&ctx, op); break; - CASE_OP_32_64_VEC(eqv): + case INDEX_op_eqv: + case INDEX_op_eqv_vec: done = fold_eqv(&ctx, op); break; - CASE_OP_32_64(extract): + case INDEX_op_extract: done = fold_extract(&ctx, op); break; - CASE_OP_32_64(extract2): + case INDEX_op_extract2: done = fold_extract2(&ctx, op); break; - CASE_OP_32_64(ext8s): - CASE_OP_32_64(ext16s): - case INDEX_op_ext32s_i64: case INDEX_op_ext_i32_i64: done = fold_exts(&ctx, op); break; - CASE_OP_32_64(ext8u): - CASE_OP_32_64(ext16u): - case INDEX_op_ext32u_i64: case INDEX_op_extu_i32_i64: case INDEX_op_extrl_i64_i32: case INDEX_op_extrh_i64_i32: done = fold_extu(&ctx, op); break; - CASE_OP_32_64(ld8s): - CASE_OP_32_64(ld8u): - CASE_OP_32_64(ld16s): - CASE_OP_32_64(ld16u): - case INDEX_op_ld32s_i64: - case INDEX_op_ld32u_i64: + case INDEX_op_ld8s: + case INDEX_op_ld8u: + case INDEX_op_ld16s: + case INDEX_op_ld16u: + case INDEX_op_ld32s: + case INDEX_op_ld32u: done = fold_tcg_ld(&ctx, op); break; - case INDEX_op_ld_i32: - case INDEX_op_ld_i64: + case INDEX_op_ld: case INDEX_op_ld_vec: done = fold_tcg_ld_memcopy(&ctx, op); break; - CASE_OP_32_64(st8): - CASE_OP_32_64(st16): - case INDEX_op_st32_i64: + case INDEX_op_st8: + case INDEX_op_st16: + case INDEX_op_st32: done = fold_tcg_st(&ctx, op); break; - case INDEX_op_st_i32: - case INDEX_op_st_i64: + case INDEX_op_st: case INDEX_op_st_vec: done = fold_tcg_st_memcopy(&ctx, op); break; case INDEX_op_mb: done = fold_mb(&ctx, op); break; - CASE_OP_32_64_VEC(mov): + case INDEX_op_mov: + case INDEX_op_mov_vec: done = fold_mov(&ctx, op); break; - CASE_OP_32_64(movcond): + case INDEX_op_movcond: done = fold_movcond(&ctx, op); break; - CASE_OP_32_64(mul): + case INDEX_op_mul: done = fold_mul(&ctx, op); break; - CASE_OP_32_64(mulsh): - CASE_OP_32_64(muluh): + case INDEX_op_mulsh: + case INDEX_op_muluh: done = fold_mul_highpart(&ctx, op); break; - CASE_OP_32_64(muls2): - CASE_OP_32_64(mulu2): + case INDEX_op_muls2: + case INDEX_op_mulu2: done = fold_multiply2(&ctx, op); break; - CASE_OP_32_64_VEC(nand): + case INDEX_op_nand: + case INDEX_op_nand_vec: done = fold_nand(&ctx, op); break; - CASE_OP_32_64(neg): + case INDEX_op_neg: done = fold_neg(&ctx, op); break; - CASE_OP_32_64_VEC(nor): + case INDEX_op_nor: + case INDEX_op_nor_vec: done = fold_nor(&ctx, op); break; - CASE_OP_32_64_VEC(not): + case INDEX_op_not: + case INDEX_op_not_vec: done = fold_not(&ctx, op); break; - CASE_OP_32_64_VEC(or): + case INDEX_op_or: + case INDEX_op_or_vec: done = fold_or(&ctx, op); break; - CASE_OP_32_64_VEC(orc): + case INDEX_op_orc: + case INDEX_op_orc_vec: done = fold_orc(&ctx, op); break; - case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_ld: done = fold_qemu_ld_1reg(&ctx, op); break; - case INDEX_op_qemu_ld_i64: - if (TCG_TARGET_REG_BITS == 64) { - done = fold_qemu_ld_1reg(&ctx, op); - break; - } - QEMU_FALLTHROUGH; - case INDEX_op_qemu_ld_i128: + case INDEX_op_qemu_ld2: done = fold_qemu_ld_2reg(&ctx, op); break; - case INDEX_op_qemu_st8_i32: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: - case INDEX_op_qemu_st_i128: + case INDEX_op_qemu_st: + case INDEX_op_qemu_st2: done = fold_qemu_st(&ctx, op); break; - CASE_OP_32_64(rem): - CASE_OP_32_64(remu): + case INDEX_op_rems: + case INDEX_op_remu: done = fold_remainder(&ctx, op); break; - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - CASE_OP_32_64(sar): - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): + case INDEX_op_rotl: + case INDEX_op_rotr: + case INDEX_op_sar: + case INDEX_op_shl: + case INDEX_op_shr: done = fold_shift(&ctx, op); break; - CASE_OP_32_64(setcond): + case INDEX_op_setcond: done = fold_setcond(&ctx, op); break; - CASE_OP_32_64(negsetcond): + case INDEX_op_negsetcond: done = fold_negsetcond(&ctx, op); break; case INDEX_op_setcond2_i32: @@ -3058,19 +3219,26 @@ void tcg_optimize(TCGContext *s) case INDEX_op_bitsel_vec: done = fold_bitsel_vec(&ctx, op); break; - CASE_OP_32_64(sextract): + case INDEX_op_sextract: done = fold_sextract(&ctx, op); break; - CASE_OP_32_64(sub): + case INDEX_op_sub: done = fold_sub(&ctx, op); break; + case INDEX_op_subbi: + done = fold_subbi(&ctx, op); + break; + case INDEX_op_subbio: + done = fold_subbio(&ctx, op); + break; + case INDEX_op_subbo: + done = fold_subbo(&ctx, op); + break; case INDEX_op_sub_vec: done = fold_sub_vec(&ctx, op); break; - CASE_OP_32_64(sub2): - done = fold_sub2(&ctx, op); - break; - CASE_OP_32_64_VEC(xor): + case INDEX_op_xor: + case INDEX_op_xor_vec: done = fold_xor(&ctx, op); break; case INDEX_op_set_label: diff --git a/tcg/perf.c b/tcg/perf.c index 412a987d95..4e8d2c1bee 100644 --- a/tcg/perf.c +++ b/tcg/perf.c @@ -313,7 +313,7 @@ void perf_report_code(uint64_t guest_pc, TranslationBlock *tb, const void *start) { struct debuginfo_query *q; - size_t insn, start_words; + size_t insn; uint64_t *gen_insn_data; if (!perfmap && !jitdump) { @@ -329,11 +329,10 @@ void perf_report_code(uint64_t guest_pc, TranslationBlock *tb, /* Query debuginfo for each guest instruction. */ gen_insn_data = tcg_ctx->gen_insn_data; - start_words = tcg_ctx->insn_start_words; for (insn = 0; insn < tb->icount; insn++) { /* FIXME: This replicates the restore_state_to_opc() logic. */ - q[insn].address = gen_insn_data[insn * start_words + 0]; + q[insn].address = gen_insn_data[insn * INSN_START_WORDS + 0]; if (tb_cflags(tb) & CF_PCREL) { q[insn].address |= (guest_pc & qemu_target_page_mask()); } diff --git a/tcg/ppc/tcg-target-con-set.h b/tcg/ppc/tcg-target-con-set.h index 453abde6c1..da7a383bff 100644 --- a/tcg/ppc/tcg-target-con-set.h +++ b/tcg/ppc/tcg-target-con-set.h @@ -15,29 +15,29 @@ C_O0_I2(r, rC) C_O0_I2(v, r) C_O0_I3(r, r, r) C_O0_I3(o, m, r) -C_O0_I4(r, r, ri, ri) +C_O0_I4(r, r, rU, rC) C_O0_I4(r, r, r, r) C_O1_I1(r, r) C_O1_I1(v, r) C_O1_I1(v, v) C_O1_I1(v, vr) C_O1_I2(r, 0, rZ) -C_O1_I2(r, rI, ri) -C_O1_I2(r, rI, rT) +C_O1_I2(r, rI, r) C_O1_I2(r, r, r) C_O1_I2(r, r, ri) C_O1_I2(r, r, rC) C_O1_I2(r, r, rI) C_O1_I2(r, r, rT) C_O1_I2(r, r, rU) +C_O1_I2(r, r, rZM) C_O1_I2(r, r, rZW) +C_O1_I2(r, rI, rN) +C_O1_I2(r, rZM, rZM) C_O1_I2(v, v, v) C_O1_I3(v, v, v, v) C_O1_I4(v, v, v, vZM, v) C_O1_I4(r, r, rC, rZ, rZ) -C_O1_I4(r, r, r, ri, ri) +C_O1_I4(r, r, r, rU, rC) C_O2_I1(r, r, r) C_N1O1_I1(o, m, r) C_O2_I2(r, r, r, r) -C_O2_I4(r, r, rI, rZM, r, r) -C_O2_I4(r, r, r, r, rI, rZM) diff --git a/tcg/ppc/tcg-target-con-str.h b/tcg/ppc/tcg-target-con-str.h index 16b687216e..faf92da47f 100644 --- a/tcg/ppc/tcg-target-con-str.h +++ b/tcg/ppc/tcg-target-con-str.h @@ -19,6 +19,7 @@ REGS('v', ALL_VECTOR_REGS) CONST('C', TCG_CT_CONST_CMP) CONST('I', TCG_CT_CONST_S16) CONST('M', TCG_CT_CONST_MONE) +CONST('N', TCG_CT_CONST_N16) CONST('T', TCG_CT_CONST_S32) CONST('U', TCG_CT_CONST_U32) CONST('W', TCG_CT_CONST_WSZ) diff --git a/tcg/ppc/tcg-target-has.h b/tcg/ppc/tcg-target-has.h index 6db91f78ce..81ec5aece7 100644 --- a/tcg/ppc/tcg-target-has.h +++ b/tcg/ppc/tcg-target-has.h @@ -16,68 +16,9 @@ #define have_altivec (cpuinfo & CPUINFO_ALTIVEC) #define have_vsx (cpuinfo & CPUINFO_VSX) -/* optional instructions automatically implemented */ -#define TCG_TARGET_HAS_ext8u_i32 0 /* andi */ -#define TCG_TARGET_HAS_ext16u_i32 0 - /* optional instructions */ -#define TCG_TARGET_HAS_div_i32 1 -#define TCG_TARGET_HAS_rem_i32 have_isa_3_00 -#define TCG_TARGET_HAS_rot_i32 1 -#define TCG_TARGET_HAS_ext8s_i32 1 -#define TCG_TARGET_HAS_ext16s_i32 1 -#define TCG_TARGET_HAS_bswap16_i32 1 -#define TCG_TARGET_HAS_bswap32_i32 1 -#define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_andc_i32 1 -#define TCG_TARGET_HAS_orc_i32 1 -#define TCG_TARGET_HAS_eqv_i32 1 -#define TCG_TARGET_HAS_nand_i32 1 -#define TCG_TARGET_HAS_nor_i32 1 -#define TCG_TARGET_HAS_clz_i32 1 -#define TCG_TARGET_HAS_ctz_i32 have_isa_3_00 -#define TCG_TARGET_HAS_ctpop_i32 have_isa_2_06 -#define TCG_TARGET_HAS_extract2_i32 0 -#define TCG_TARGET_HAS_negsetcond_i32 1 -#define TCG_TARGET_HAS_mulu2_i32 0 -#define TCG_TARGET_HAS_muls2_i32 0 -#define TCG_TARGET_HAS_muluh_i32 1 -#define TCG_TARGET_HAS_mulsh_i32 1 -#define TCG_TARGET_HAS_qemu_st8_i32 0 - #if TCG_TARGET_REG_BITS == 64 -#define TCG_TARGET_HAS_add2_i32 0 -#define TCG_TARGET_HAS_sub2_i32 0 #define TCG_TARGET_HAS_extr_i64_i32 0 -#define TCG_TARGET_HAS_div_i64 1 -#define TCG_TARGET_HAS_rem_i64 have_isa_3_00 -#define TCG_TARGET_HAS_rot_i64 1 -#define TCG_TARGET_HAS_ext8s_i64 1 -#define TCG_TARGET_HAS_ext16s_i64 1 -#define TCG_TARGET_HAS_ext32s_i64 1 -#define TCG_TARGET_HAS_ext8u_i64 0 -#define TCG_TARGET_HAS_ext16u_i64 0 -#define TCG_TARGET_HAS_ext32u_i64 0 -#define TCG_TARGET_HAS_bswap16_i64 1 -#define TCG_TARGET_HAS_bswap32_i64 1 -#define TCG_TARGET_HAS_bswap64_i64 1 -#define TCG_TARGET_HAS_not_i64 1 -#define TCG_TARGET_HAS_andc_i64 1 -#define TCG_TARGET_HAS_orc_i64 1 -#define TCG_TARGET_HAS_eqv_i64 1 -#define TCG_TARGET_HAS_nand_i64 1 -#define TCG_TARGET_HAS_nor_i64 1 -#define TCG_TARGET_HAS_clz_i64 1 -#define TCG_TARGET_HAS_ctz_i64 have_isa_3_00 -#define TCG_TARGET_HAS_ctpop_i64 have_isa_2_06 -#define TCG_TARGET_HAS_extract2_i64 0 -#define TCG_TARGET_HAS_negsetcond_i64 1 -#define TCG_TARGET_HAS_add2_i64 1 -#define TCG_TARGET_HAS_sub2_i64 1 -#define TCG_TARGET_HAS_mulu2_i64 0 -#define TCG_TARGET_HAS_muls2_i64 0 -#define TCG_TARGET_HAS_muluh_i64 1 -#define TCG_TARGET_HAS_mulsh_i64 1 #endif #define TCG_TARGET_HAS_qemu_ldst_i128 \ diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 822925a19b..2e94778104 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -89,14 +89,15 @@ /* Shorthand for size of a register. */ #define SZR (TCG_TARGET_REG_BITS / 8) -#define TCG_CT_CONST_S16 0x100 -#define TCG_CT_CONST_U16 0x200 -#define TCG_CT_CONST_S32 0x400 -#define TCG_CT_CONST_U32 0x800 -#define TCG_CT_CONST_ZERO 0x1000 -#define TCG_CT_CONST_MONE 0x2000 -#define TCG_CT_CONST_WSZ 0x4000 -#define TCG_CT_CONST_CMP 0x8000 +#define TCG_CT_CONST_S16 0x00100 +#define TCG_CT_CONST_U16 0x00200 +#define TCG_CT_CONST_N16 0x00400 +#define TCG_CT_CONST_S32 0x00800 +#define TCG_CT_CONST_U32 0x01000 +#define TCG_CT_CONST_ZERO 0x02000 +#define TCG_CT_CONST_MONE 0x04000 +#define TCG_CT_CONST_WSZ 0x08000 +#define TCG_CT_CONST_CMP 0x10000 #define ALL_GENERAL_REGS 0xffffffffu #define ALL_VECTOR_REGS 0xffffffff00000000ull @@ -342,6 +343,9 @@ static bool tcg_target_const_match(int64_t sval, int ct, if ((ct & TCG_CT_CONST_U16) && uval == (uint16_t)uval) { return 1; } + if ((ct & TCG_CT_CONST_N16) && -sval == (int16_t)-sval) { + return 1; + } if ((ct & TCG_CT_CONST_S32) && sval == (int32_t)sval) { return 1; } @@ -1012,111 +1016,6 @@ static void tcg_out_addpcis(TCGContext *s, TCGReg dst, intptr_t imm) tcg_out32(s, ADDPCIS | RT(dst) | (d1 << 16) | (d0 << 6) | d2); } -static void tcg_out_bswap16(TCGContext *s, TCGReg dst, TCGReg src, int flags) -{ - TCGReg tmp = dst == src ? TCG_REG_R0 : dst; - - if (have_isa_3_10) { - tcg_out32(s, BRH | RA(dst) | RS(src)); - if (flags & TCG_BSWAP_OS) { - tcg_out_ext16s(s, TCG_TYPE_REG, dst, dst); - } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - tcg_out_ext16u(s, dst, dst); - } - return; - } - - /* - * In the following, - * dep(a, b, m) -> (a & ~m) | (b & m) - * - * Begin with: src = xxxxabcd - */ - /* tmp = rol32(src, 24) & 0x000000ff = 0000000c */ - tcg_out_rlw(s, RLWINM, tmp, src, 24, 24, 31); - /* tmp = dep(tmp, rol32(src, 8), 0x0000ff00) = 000000dc */ - tcg_out_rlw(s, RLWIMI, tmp, src, 8, 16, 23); - - if (flags & TCG_BSWAP_OS) { - tcg_out_ext16s(s, TCG_TYPE_REG, dst, tmp); - } else { - tcg_out_mov(s, TCG_TYPE_REG, dst, tmp); - } -} - -static void tcg_out_bswap32(TCGContext *s, TCGReg dst, TCGReg src, int flags) -{ - TCGReg tmp = dst == src ? TCG_REG_R0 : dst; - - if (have_isa_3_10) { - tcg_out32(s, BRW | RA(dst) | RS(src)); - if (flags & TCG_BSWAP_OS) { - tcg_out_ext32s(s, dst, dst); - } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - tcg_out_ext32u(s, dst, dst); - } - return; - } - - /* - * Stolen from gcc's builtin_bswap32. - * In the following, - * dep(a, b, m) -> (a & ~m) | (b & m) - * - * Begin with: src = xxxxabcd - */ - /* tmp = rol32(src, 8) & 0xffffffff = 0000bcda */ - tcg_out_rlw(s, RLWINM, tmp, src, 8, 0, 31); - /* tmp = dep(tmp, rol32(src, 24), 0xff000000) = 0000dcda */ - tcg_out_rlw(s, RLWIMI, tmp, src, 24, 0, 7); - /* tmp = dep(tmp, rol32(src, 24), 0x0000ff00) = 0000dcba */ - tcg_out_rlw(s, RLWIMI, tmp, src, 24, 16, 23); - - if (flags & TCG_BSWAP_OS) { - tcg_out_ext32s(s, dst, tmp); - } else { - tcg_out_mov(s, TCG_TYPE_REG, dst, tmp); - } -} - -static void tcg_out_bswap64(TCGContext *s, TCGReg dst, TCGReg src) -{ - TCGReg t0 = dst == src ? TCG_REG_R0 : dst; - TCGReg t1 = dst == src ? dst : TCG_REG_R0; - - if (have_isa_3_10) { - tcg_out32(s, BRD | RA(dst) | RS(src)); - return; - } - - /* - * In the following, - * dep(a, b, m) -> (a & ~m) | (b & m) - * - * Begin with: src = abcdefgh - */ - /* t0 = rol32(src, 8) & 0xffffffff = 0000fghe */ - tcg_out_rlw(s, RLWINM, t0, src, 8, 0, 31); - /* t0 = dep(t0, rol32(src, 24), 0xff000000) = 0000hghe */ - tcg_out_rlw(s, RLWIMI, t0, src, 24, 0, 7); - /* t0 = dep(t0, rol32(src, 24), 0x0000ff00) = 0000hgfe */ - tcg_out_rlw(s, RLWIMI, t0, src, 24, 16, 23); - - /* t0 = rol64(t0, 32) = hgfe0000 */ - tcg_out_rld(s, RLDICL, t0, t0, 32, 0); - /* t1 = rol64(src, 32) = efghabcd */ - tcg_out_rld(s, RLDICL, t1, src, 32, 0); - - /* t0 = dep(t0, rol32(t1, 24), 0xffffffff) = hgfebcda */ - tcg_out_rlw(s, RLWIMI, t0, t1, 8, 0, 31); - /* t0 = dep(t0, rol32(t1, 24), 0xff000000) = hgfedcda */ - tcg_out_rlw(s, RLWIMI, t0, t1, 24, 0, 7); - /* t0 = dep(t0, rol32(t1, 24), 0x0000ff00) = hgfedcba */ - tcg_out_rlw(s, RLWIMI, t0, t1, 24, 16, 23); - - tcg_out_mov(s, TCG_TYPE_REG, dst, t0); -} - /* Emit a move into ret of arg, if it can be done in one insn. */ static bool tcg_out_movi_one(TCGContext *s, TCGReg ret, tcg_target_long arg) { @@ -1777,9 +1676,8 @@ static void tcg_out_test(TCGContext *s, TCGReg dest, TCGReg arg1, TCGArg arg2, } static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2, - int const_arg2, int cr, TCGType type) + bool const_arg2, int cr, TCGType type) { - int imm; uint32_t op; tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32); @@ -1796,18 +1694,15 @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2, case TCG_COND_EQ: case TCG_COND_NE: if (const_arg2) { - if ((int16_t) arg2 == arg2) { + if ((int16_t)arg2 == arg2) { op = CMPI; - imm = 1; - break; - } else if ((uint16_t) arg2 == arg2) { - op = CMPLI; - imm = 1; break; } + tcg_debug_assert((uint16_t)arg2 == arg2); + op = CMPLI; + break; } op = CMPL; - imm = 0; break; case TCG_COND_TSTEQ: @@ -1821,14 +1716,11 @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2, case TCG_COND_LE: case TCG_COND_GT: if (const_arg2) { - if ((int16_t) arg2 == arg2) { - op = CMPI; - imm = 1; - break; - } + tcg_debug_assert((int16_t)arg2 == arg2); + op = CMPI; + break; } op = CMP; - imm = 0; break; case TCG_COND_LTU: @@ -1836,30 +1728,20 @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2, case TCG_COND_LEU: case TCG_COND_GTU: if (const_arg2) { - if ((uint16_t) arg2 == arg2) { - op = CMPLI; - imm = 1; - break; - } + tcg_debug_assert((uint16_t)arg2 == arg2); + op = CMPLI; + break; } op = CMPL; - imm = 0; break; default: g_assert_not_reached(); } op |= BF(cr) | ((type == TCG_TYPE_I64) << 21); - - if (imm) { - tcg_out32(s, op | RA(arg1) | (arg2 & 0xffff)); - } else { - if (const_arg2) { - tcg_out_movi(s, type, TCG_REG_R0, arg2); - arg2 = TCG_REG_R0; - } - tcg_out32(s, op | RA(arg1) | RB(arg2)); - } + op |= RA(arg1); + op |= const_arg2 ? arg2 & 0xffff : RB(arg2); + tcg_out32(s, op); } static void tcg_out_setcond_eq0(TCGContext *s, TCGType type, @@ -1926,8 +1808,8 @@ static TCGReg tcg_gen_setcond_xor(TCGContext *s, TCGReg arg1, TCGArg arg2, } static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, - TCGArg arg0, TCGArg arg1, TCGArg arg2, - int const_arg2, bool neg) + TCGReg arg0, TCGReg arg1, TCGArg arg2, + bool const_arg2, bool neg) { int sh; bool inv; @@ -2072,6 +1954,54 @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, } } +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, false); +} + +static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, false); +} + +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(r, r, rC), + .out_rrr = tgen_setcond, + .out_rri = tgen_setcondi, +}; + +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, true); +} + +static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, true); +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(r, r, rC), + .out_rrr = tgen_negsetcond, + .out_rri = tgen_negsetcondi, +}; + +void tcg_out_br(TCGContext *s, TCGLabel *l) +{ + uint32_t insn = B; + + if (l->has_value) { + insn |= reloc_pc24_val(tcg_splitwx_to_rx(s->code_ptr), l->u.value_ptr); + } else { + tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0); + } + tcg_out32(s, insn); +} + static void tcg_out_bc(TCGContext *s, TCGCond cond, int bd) { tcg_out32(s, tcg_to_bc[cond] | bd); @@ -2088,17 +2018,29 @@ static void tcg_out_bc_lab(TCGContext *s, TCGCond cond, TCGLabel *l) tcg_out_bc(s, cond, bd); } -static void tcg_out_brcond(TCGContext *s, TCGCond cond, - TCGArg arg1, TCGArg arg2, int const_arg2, - TCGLabel *l, TCGType type) +static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, TCGReg arg2, TCGLabel *l) +{ + tcg_out_cmp(s, cond, arg1, arg2, false, 0, type); + tcg_out_bc_lab(s, cond, l); +} + +static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, tcg_target_long arg2, TCGLabel *l) { - tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 0, type); + tcg_out_cmp(s, cond, arg1, arg2, true, 0, type); tcg_out_bc_lab(s, cond, l); } -static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond, - TCGArg dest, TCGArg c1, TCGArg c2, TCGArg v1, - TCGArg v2, bool const_c2) +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, rC), + .out_rr = tgen_brcond, + .out_ri = tgen_brcondi, +}; + +static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg c1, TCGArg c2, bool const_c2, + TCGArg v1, bool const_v1, TCGArg v2, bool const_v2) { /* If for some reason both inputs are zero, don't produce bad code. */ if (v1 == 0 && v2 == 0) { @@ -2144,6 +2086,11 @@ static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond, } } +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = C_O1_I4(r, r, rC, rZ, rZ), + .out = tgen_movcond, +}; + static void tcg_out_cntxz(TCGContext *s, TCGType type, uint32_t opc, TCGArg a0, TCGArg a1, TCGArg a2, bool const_a2) { @@ -2170,8 +2117,8 @@ static void tcg_out_cntxz(TCGContext *s, TCGType type, uint32_t opc, } } -static void tcg_out_cmp2(TCGContext *s, const TCGArg *args, - const int *const_args) +static void tcg_out_cmp2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, + TCGArg bl, bool blconst, TCGArg bh, bool bhconst) { static const struct { uint8_t bit1, bit2; } bits[] = { [TCG_COND_LT ] = { CR_LT, CR_LT }, @@ -2184,18 +2131,9 @@ static void tcg_out_cmp2(TCGContext *s, const TCGArg *args, [TCG_COND_GEU] = { CR_GT, CR_LT }, }; - TCGCond cond = args[4], cond2; - TCGArg al, ah, bl, bh; - int blconst, bhconst; + TCGCond cond2; int op, bit1, bit2; - al = args[0]; - ah = args[1]; - bl = args[2]; - bh = args[3]; - blconst = const_args[2]; - bhconst = const_args[3]; - switch (cond) { case TCG_COND_EQ: op = CRAND; @@ -2247,22 +2185,42 @@ static void tcg_out_cmp2(TCGContext *s, const TCGArg *args, } } -static void tcg_out_setcond2(TCGContext *s, const TCGArg *args, - const int *const_args) +static void tgen_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, + TCGArg bh, bool const_bh) { - tcg_out_cmp2(s, args + 1, const_args + 1); + tcg_out_cmp2(s, cond, al, ah, bl, const_bl, bh, const_bh); tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(0)); - tcg_out_rlw(s, RLWINM, args[0], TCG_REG_R0, CR_EQ + 0*4 + 1, 31, 31); + tcg_out_rlw(s, RLWINM, ret, TCG_REG_R0, CR_EQ + 0*4 + 1, 31, 31); } -static void tcg_out_brcond2(TCGContext *s, const TCGArg *args, - const int *const_args) +#if TCG_TARGET_REG_BITS != 32 +__attribute__((unused)) +#endif +static const TCGOutOpSetcond2 outop_setcond2 = { + .base.static_constraint = C_O1_I4(r, r, r, rU, rC), + .out = tgen_setcond2, +}; + +static void tgen_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, + TCGArg bh, bool const_bh, TCGLabel *l) { - tcg_out_cmp2(s, args, const_args); - tcg_out_bc_lab(s, TCG_COND_EQ, arg_label(args[5])); + assert(TCG_TARGET_REG_BITS == 32); + tcg_out_cmp2(s, cond, al, ah, bl, const_bl, bh, const_bh); + tcg_out_bc_lab(s, TCG_COND_EQ, l); } -static void tcg_out_mb(TCGContext *s, TCGArg a0) +#if TCG_TARGET_REG_BITS != 32 +__attribute__((unused)) +#endif +static const TCGOutOpBrcond2 outop_brcond2 = { + .base.static_constraint = C_O0_I4(r, r, rU, rC), + .out = tgen_brcond2, +}; + +static void tcg_out_mb(TCGContext *s, unsigned a0) { uint32_t insn; @@ -2737,6 +2695,60 @@ static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi, } } +static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + tcg_out_qemu_ld(s, data, -1, addr, oi, type); +} + +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_qemu_ld, +}; + +static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) +{ + if (TCG_TARGET_REG_BITS == 32) { + tcg_out_qemu_ld(s, datalo, datahi, addr, oi, type); + } else { + tcg_out_qemu_ldst_i128(s, datalo, datahi, addr, oi, true); + } +} + +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + .base.static_constraint = + TCG_TARGET_REG_BITS == 64 ? C_N1O1_I1(o, m, r) : C_O2_I1(r, r, r), + .out = tgen_qemu_ld2, +}; + +static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + tcg_out_qemu_st(s, data, -1, addr, oi, type); +} + +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = C_O0_I2(r, r), + .out = tgen_qemu_st, +}; + +static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) +{ + if (TCG_TARGET_REG_BITS == 32) { + tcg_out_qemu_st(s, datalo, datahi, addr, oi, type); + } else { + tcg_out_qemu_ldst_i128(s, datalo, datahi, addr, oi, false); + } +} + +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + .base.static_constraint = + TCG_TARGET_REG_BITS == 64 ? C_O0_I3(o, m, r) : C_O0_I3(r, r, r), + .out = tgen_qemu_st2, +}; + static void tcg_out_nop_fill(tcg_insn_unit *p, int count) { int i; @@ -2885,6 +2897,13 @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + tcg_out32(s, MTSPR | RS(a0) | CTR); + tcg_out32(s, ADDI | TAI(TCG_REG_R3, 0, 0)); + tcg_out32(s, BCCTR | BO_ALWAYS); +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { @@ -2902,595 +2921,913 @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, flush_idcache_range(jmp_rx, jmp_rw, 4); } -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) + +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - TCGArg a0, a1, a2; + tcg_out32(s, ADD | TAB(a0, a1, a2)); +} - switch (opc) { - case INDEX_op_goto_ptr: - tcg_out32(s, MTSPR | RS(args[0]) | CTR); - tcg_out32(s, ADDI | TAI(TCG_REG_R3, 0, 0)); - tcg_out32(s, BCCTR | BO_ALWAYS); - break; - case INDEX_op_br: - { - TCGLabel *l = arg_label(args[0]); - uint32_t insn = B; - - if (l->has_value) { - insn |= reloc_pc24_val(tcg_splitwx_to_rx(s->code_ptr), - l->u.value_ptr); - } else { - tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0); - } - tcg_out32(s, insn); - } - break; - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]); - break; - case INDEX_op_ld8s_i32: - case INDEX_op_ld8s_i64: - tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]); - tcg_out_ext8s(s, TCG_TYPE_REG, args[0], args[0]); - break; - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - tcg_out_mem_long(s, LHZ, LHZX, args[0], args[1], args[2]); - break; - case INDEX_op_ld16s_i32: - case INDEX_op_ld16s_i64: - tcg_out_mem_long(s, LHA, LHAX, args[0], args[1], args[2]); - break; - case INDEX_op_ld_i32: - case INDEX_op_ld32u_i64: - tcg_out_mem_long(s, LWZ, LWZX, args[0], args[1], args[2]); - break; - case INDEX_op_ld32s_i64: - tcg_out_mem_long(s, LWA, LWAX, args[0], args[1], args[2]); - break; - case INDEX_op_ld_i64: - tcg_out_mem_long(s, LD, LDX, args[0], args[1], args[2]); - break; - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - tcg_out_mem_long(s, STB, STBX, args[0], args[1], args[2]); - break; - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - tcg_out_mem_long(s, STH, STHX, args[0], args[1], args[2]); - break; - case INDEX_op_st_i32: - case INDEX_op_st32_i64: - tcg_out_mem_long(s, STW, STWX, args[0], args[1], args[2]); - break; - case INDEX_op_st_i64: - tcg_out_mem_long(s, STD, STDX, args[0], args[1], args[2]); - break; +static void tgen_addi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2); +} - case INDEX_op_add_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - do_addi_32: - tcg_out_mem_long(s, ADDI, ADD, a0, a1, (int32_t)a2); - } else { - tcg_out32(s, ADD | TAB(a0, a1, a2)); - } - break; - case INDEX_op_sub_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[1]) { - if (const_args[2]) { - tcg_out_movi(s, TCG_TYPE_I32, a0, a1 - a2); - } else { - tcg_out32(s, SUBFIC | TAI(a0, a2, a1)); - } - } else if (const_args[2]) { - a2 = -a2; - goto do_addi_32; - } else { - tcg_out32(s, SUBF | TAB(a0, a2, a1)); - } - break; +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, rT), + .out_rrr = tgen_add, + .out_rri = tgen_addi, +}; - case INDEX_op_and_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_andi32(s, a0, a1, a2); - } else { - tcg_out32(s, AND | SAB(a1, a0, a2)); - } - break; - case INDEX_op_and_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_andi64(s, a0, a1, a2); - } else { - tcg_out32(s, AND | SAB(a1, a0, a2)); - } - break; - case INDEX_op_or_i64: - case INDEX_op_or_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_ori32(s, a0, a1, a2); - } else { - tcg_out32(s, OR | SAB(a1, a0, a2)); - } - break; - case INDEX_op_xor_i64: - case INDEX_op_xor_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_xori32(s, a0, a1, a2); - } else { - tcg_out32(s, XOR | SAB(a1, a0, a2)); - } - break; - case INDEX_op_andc_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_andi32(s, a0, a1, ~a2); - } else { - tcg_out32(s, ANDC | SAB(a1, a0, a2)); - } - break; - case INDEX_op_andc_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_andi64(s, a0, a1, ~a2); - } else { - tcg_out32(s, ANDC | SAB(a1, a0, a2)); - } - break; - case INDEX_op_orc_i32: - if (const_args[2]) { - tcg_out_ori32(s, args[0], args[1], ~args[2]); - break; - } - /* FALLTHRU */ - case INDEX_op_orc_i64: - tcg_out32(s, ORC | SAB(args[1], args[0], args[2])); - break; - case INDEX_op_eqv_i32: - if (const_args[2]) { - tcg_out_xori32(s, args[0], args[1], ~args[2]); - break; - } - /* FALLTHRU */ - case INDEX_op_eqv_i64: - tcg_out32(s, EQV | SAB(args[1], args[0], args[2])); - break; - case INDEX_op_nand_i32: - case INDEX_op_nand_i64: - tcg_out32(s, NAND | SAB(args[1], args[0], args[2])); - break; - case INDEX_op_nor_i32: - case INDEX_op_nor_i64: - tcg_out32(s, NOR | SAB(args[1], args[0], args[2])); - break; +static void tgen_addco_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, ADDC | TAB(a0, a1, a2)); +} - case INDEX_op_clz_i32: - tcg_out_cntxz(s, TCG_TYPE_I32, CNTLZW, args[0], args[1], - args[2], const_args[2]); - break; - case INDEX_op_ctz_i32: - tcg_out_cntxz(s, TCG_TYPE_I32, CNTTZW, args[0], args[1], - args[2], const_args[2]); - break; - case INDEX_op_ctpop_i32: - tcg_out32(s, CNTPOPW | SAB(args[1], args[0], 0)); - break; +static void tgen_addco_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out32(s, ADDIC | TAI(a0, a1, a2)); +} - case INDEX_op_clz_i64: - tcg_out_cntxz(s, TCG_TYPE_I64, CNTLZD, args[0], args[1], - args[2], const_args[2]); - break; - case INDEX_op_ctz_i64: - tcg_out_cntxz(s, TCG_TYPE_I64, CNTTZD, args[0], args[1], - args[2], const_args[2]); - break; - case INDEX_op_ctpop_i64: - tcg_out32(s, CNTPOPD | SAB(args[1], args[0], 0)); - break; +static TCGConstraintSetIndex cset_addco(TCGType type, unsigned flags) +{ + /* + * Note that the CA bit is defined based on the word size of the + * environment. So in 64-bit mode it's always carry-out of bit 63. + * The fallback code using deposit works just as well for TCG_TYPE_I32. + */ + return type == TCG_TYPE_REG ? C_O1_I2(r, r, rI) : C_NotImplemented; +} - case INDEX_op_mul_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out32(s, MULLI | TAI(a0, a1, a2)); - } else { - tcg_out32(s, MULLW | TAB(a0, a1, a2)); - } - break; +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addco, + .out_rrr = tgen_addco_rrr, + .out_rri = tgen_addco_rri, +}; - case INDEX_op_div_i32: - tcg_out32(s, DIVW | TAB(args[0], args[1], args[2])); - break; +static void tgen_addcio_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, ADDE | TAB(a0, a1, a2)); +} - case INDEX_op_divu_i32: - tcg_out32(s, DIVWU | TAB(args[0], args[1], args[2])); - break; +static void tgen_addcio_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out32(s, (a2 ? ADDME : ADDZE) | RT(a0) | RA(a1)); +} - case INDEX_op_rem_i32: - tcg_out32(s, MODSW | TAB(args[0], args[1], args[2])); - break; +static TCGConstraintSetIndex cset_addcio(TCGType type, unsigned flags) +{ + return type == TCG_TYPE_REG ? C_O1_I2(r, r, rZM) : C_NotImplemented; +} - case INDEX_op_remu_i32: - tcg_out32(s, MODUW | TAB(args[0], args[1], args[2])); - break; +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addcio, + .out_rrr = tgen_addcio_rrr, + .out_rri = tgen_addcio_rri, +}; - case INDEX_op_shl_i32: - if (const_args[2]) { - /* Limit immediate shift count lest we create an illegal insn. */ - tcg_out_shli32(s, args[0], args[1], args[2] & 31); - } else { - tcg_out32(s, SLW | SAB(args[1], args[0], args[2])); - } - break; - case INDEX_op_shr_i32: - if (const_args[2]) { - /* Limit immediate shift count lest we create an illegal insn. */ - tcg_out_shri32(s, args[0], args[1], args[2] & 31); - } else { - tcg_out32(s, SRW | SAB(args[1], args[0], args[2])); - } - break; - case INDEX_op_sar_i32: - if (const_args[2]) { - tcg_out_sari32(s, args[0], args[1], args[2]); - } else { - tcg_out32(s, SRAW | SAB(args[1], args[0], args[2])); - } - break; - case INDEX_op_rotl_i32: - if (const_args[2]) { - tcg_out_rlw(s, RLWINM, args[0], args[1], args[2], 0, 31); - } else { - tcg_out32(s, RLWNM | SAB(args[1], args[0], args[2]) - | MB(0) | ME(31)); - } - break; - case INDEX_op_rotr_i32: - if (const_args[2]) { - tcg_out_rlw(s, RLWINM, args[0], args[1], 32 - args[2], 0, 31); - } else { - tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 32)); - tcg_out32(s, RLWNM | SAB(args[1], args[0], TCG_REG_R0) - | MB(0) | ME(31)); - } - break; +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addcio, + .out_rrr = tgen_addcio_rrr, + .out_rri = tgen_addcio_rri, +}; - case INDEX_op_brcond_i32: - tcg_out_brcond(s, args[2], args[0], args[1], const_args[1], - arg_label(args[3]), TCG_TYPE_I32); - break; - case INDEX_op_brcond_i64: - tcg_out_brcond(s, args[2], args[0], args[1], const_args[1], - arg_label(args[3]), TCG_TYPE_I64); - break; - case INDEX_op_brcond2_i32: - tcg_out_brcond2(s, args, const_args); - break; +static void tcg_out_set_carry(TCGContext *s) +{ + tcg_out32(s, SUBFC | TAB(TCG_REG_R0, TCG_REG_R0, TCG_REG_R0)); +} - case INDEX_op_neg_i32: - case INDEX_op_neg_i64: - tcg_out32(s, NEG | RT(args[0]) | RA(args[1])); - break; +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, AND | SAB(a1, a0, a2)); +} - case INDEX_op_not_i32: - case INDEX_op_not_i64: - tcg_out32(s, NOR | SAB(args[1], args[0], args[1])); - break; +static void tgen_andi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_andi32(s, a0, a1, a2); + } else { + tcg_out_andi64(s, a0, a1, a2); + } +} - case INDEX_op_add_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - do_addi_64: - tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2); - } else { - tcg_out32(s, ADD | TAB(a0, a1, a2)); - } - break; - case INDEX_op_sub_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[1]) { - if (const_args[2]) { - tcg_out_movi(s, TCG_TYPE_I64, a0, a1 - a2); - } else { - tcg_out32(s, SUBFIC | TAI(a0, a2, a1)); - } - } else if (const_args[2]) { - a2 = -a2; - goto do_addi_64; - } else { - tcg_out32(s, SUBF | TAB(a0, a2, a1)); - } - break; +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_and, + .out_rri = tgen_andi, +}; - case INDEX_op_shl_i64: - if (const_args[2]) { - /* Limit immediate shift count lest we create an illegal insn. */ - tcg_out_shli64(s, args[0], args[1], args[2] & 63); - } else { - tcg_out32(s, SLD | SAB(args[1], args[0], args[2])); - } - break; - case INDEX_op_shr_i64: - if (const_args[2]) { - /* Limit immediate shift count lest we create an illegal insn. */ - tcg_out_shri64(s, args[0], args[1], args[2] & 63); - } else { - tcg_out32(s, SRD | SAB(args[1], args[0], args[2])); - } - break; - case INDEX_op_sar_i64: - if (const_args[2]) { - tcg_out_sari64(s, args[0], args[1], args[2]); - } else { - tcg_out32(s, SRAD | SAB(args[1], args[0], args[2])); - } - break; - case INDEX_op_rotl_i64: - if (const_args[2]) { - tcg_out_rld(s, RLDICL, args[0], args[1], args[2], 0); - } else { - tcg_out32(s, RLDCL | SAB(args[1], args[0], args[2]) | MB64(0)); - } - break; - case INDEX_op_rotr_i64: - if (const_args[2]) { - tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 0); - } else { - tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 64)); - tcg_out32(s, RLDCL | SAB(args[1], args[0], TCG_REG_R0) | MB64(0)); - } - break; +static void tgen_andc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, ANDC | SAB(a1, a0, a2)); +} - case INDEX_op_mul_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out32(s, MULLI | TAI(a0, a1, a2)); - } else { - tcg_out32(s, MULLD | TAB(a0, a1, a2)); - } - break; - case INDEX_op_div_i64: - tcg_out32(s, DIVD | TAB(args[0], args[1], args[2])); - break; - case INDEX_op_divu_i64: - tcg_out32(s, DIVDU | TAB(args[0], args[1], args[2])); - break; - case INDEX_op_rem_i64: - tcg_out32(s, MODSD | TAB(args[0], args[1], args[2])); - break; - case INDEX_op_remu_i64: - tcg_out32(s, MODUD | TAB(args[0], args[1], args[2])); - break; +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_andc, +}; - case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], TCG_TYPE_I32); - break; - case INDEX_op_qemu_ld_i64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], TCG_TYPE_I64); - } else { - tcg_out_qemu_ld(s, args[0], args[1], args[2], - args[3], TCG_TYPE_I64); - } - break; - case INDEX_op_qemu_ld_i128: - tcg_debug_assert(TCG_TARGET_REG_BITS == 64); - tcg_out_qemu_ldst_i128(s, args[0], args[1], args[2], args[3], true); - break; +static void tgen_clz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? CNTLZW : CNTLZD; + tcg_out_cntxz(s, type, insn, a0, a1, a2, false); +} - case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args[0], -1, args[1], args[2], TCG_TYPE_I32); - break; - case INDEX_op_qemu_st_i64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_qemu_st(s, args[0], -1, args[1], args[2], TCG_TYPE_I64); - } else { - tcg_out_qemu_st(s, args[0], args[1], args[2], - args[3], TCG_TYPE_I64); - } - break; - case INDEX_op_qemu_st_i128: - tcg_debug_assert(TCG_TARGET_REG_BITS == 64); - tcg_out_qemu_ldst_i128(s, args[0], args[1], args[2], args[3], false); - break; +static void tgen_clzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? CNTLZW : CNTLZD; + tcg_out_cntxz(s, type, insn, a0, a1, a2, true); +} - case INDEX_op_setcond_i32: - tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2], - const_args[2], false); - break; - case INDEX_op_setcond_i64: - tcg_out_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], args[2], - const_args[2], false); - break; - case INDEX_op_negsetcond_i32: - tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2], - const_args[2], true); - break; - case INDEX_op_negsetcond_i64: - tcg_out_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], args[2], - const_args[2], true); - break; - case INDEX_op_setcond2_i32: - tcg_out_setcond2(s, args, const_args); - break; +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_O1_I2(r, r, rZW), + .out_rrr = tgen_clz, + .out_rri = tgen_clzi, +}; - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - tcg_out_bswap16(s, args[0], args[1], args[2]); - break; - case INDEX_op_bswap32_i32: - tcg_out_bswap32(s, args[0], args[1], 0); - break; - case INDEX_op_bswap32_i64: - tcg_out_bswap32(s, args[0], args[1], args[2]); - break; - case INDEX_op_bswap64_i64: - tcg_out_bswap64(s, args[0], args[1]); - break; +static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + uint32_t insn = type == TCG_TYPE_I32 ? CNTPOPW : CNTPOPD; + tcg_out32(s, insn | SAB(a1, a0, 0)); +} - case INDEX_op_deposit_i32: - if (const_args[2]) { - uint32_t mask = ((2u << (args[4] - 1)) - 1) << args[3]; - tcg_out_andi32(s, args[0], args[0], ~mask); - } else { - tcg_out_rlw(s, RLWIMI, args[0], args[2], args[3], - 32 - args[3] - args[4], 31 - args[3]); - } - break; - case INDEX_op_deposit_i64: - if (const_args[2]) { - uint64_t mask = ((2ull << (args[4] - 1)) - 1) << args[3]; - tcg_out_andi64(s, args[0], args[0], ~mask); - } else { - tcg_out_rld(s, RLDIMI, args[0], args[2], args[3], - 64 - args[3] - args[4]); - } - break; +static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags) +{ + return have_isa_2_06 ? C_O1_I1(r, r) : C_NotImplemented; +} - case INDEX_op_extract_i32: - if (args[2] == 0 && args[3] <= 16) { - tcg_out32(s, ANDI | SAI(args[1], args[0], (1 << args[3]) - 1)); - break; - } - tcg_out_rlw(s, RLWINM, args[0], args[1], - 32 - args[2], 32 - args[3], 31); - break; - case INDEX_op_extract_i64: - if (args[2] == 0 && args[3] <= 16) { - tcg_out32(s, ANDI | SAI(args[1], args[0], (1 << args[3]) - 1)); - break; - } - tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 64 - args[3]); - break; +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_ctpop, + .out_rr = tgen_ctpop, +}; - case INDEX_op_sextract_i64: - if (args[2] + args[3] == 32) { - if (args[2] == 0) { - tcg_out_ext32s(s, args[0], args[1]); - } else { - tcg_out_sari32(s, args[0], args[1], args[2]); - } - break; - } - /* FALLTHRU */ - case INDEX_op_sextract_i32: - if (args[2] == 0 && args[3] == 8) { - tcg_out_ext8s(s, TCG_TYPE_I32, args[0], args[1]); - } else if (args[2] == 0 && args[3] == 16) { - tcg_out_ext16s(s, TCG_TYPE_I32, args[0], args[1]); - } else { - g_assert_not_reached(); - } - break; +static void tgen_ctz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? CNTTZW : CNTTZD; + tcg_out_cntxz(s, type, insn, a0, a1, a2, false); +} - case INDEX_op_movcond_i32: - tcg_out_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1], args[2], - args[3], args[4], const_args[2]); - break; - case INDEX_op_movcond_i64: - tcg_out_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1], args[2], - args[3], args[4], const_args[2]); - break; +static void tgen_ctzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? CNTTZW : CNTTZD; + tcg_out_cntxz(s, type, insn, a0, a1, a2, true); +} + +static TCGConstraintSetIndex cset_ctz(TCGType type, unsigned flags) +{ + return have_isa_3_00 ? C_O1_I2(r, r, rZW) : C_NotImplemented; +} + +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_ctz, + .out_rrr = tgen_ctz, + .out_rri = tgen_ctzi, +}; + +static void tgen_eqv(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, EQV | SAB(a1, a0, a2)); +} #if TCG_TARGET_REG_BITS == 64 - case INDEX_op_add2_i64: -#else - case INDEX_op_add2_i32: +static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_shri64(s, a0, a1, 32); +} + +static const TCGOutOpUnary outop_extrh_i64_i32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extrh_i64_i32, +}; #endif - /* Note that the CA bit is defined based on the word size of the - environment. So in 64-bit mode it's always carry-out of bit 63. - The fallback code using deposit works just as well for 32-bit. */ - a0 = args[0], a1 = args[1]; - if (a0 == args[3] || (!const_args[5] && a0 == args[5])) { - a0 = TCG_REG_R0; - } - if (const_args[4]) { - tcg_out32(s, ADDIC | TAI(a0, args[2], args[4])); - } else { - tcg_out32(s, ADDC | TAB(a0, args[2], args[4])); + +static void tgen_divs(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? DIVW : DIVD; + tcg_out32(s, insn | TAB(a0, a1, a2)); +} + +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divs, +}; + +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? DIVWU : DIVDU; + tcg_out32(s, insn | TAB(a0, a1, a2)); +} + +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divu, +}; + +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_eqv, +}; + +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? MULLW : MULLD; + tcg_out32(s, insn | TAB(a0, a1, a2)); +} + +static void tgen_muli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out32(s, MULLI | TAI(a0, a1, a2)); +} + +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_mul, + .out_rri = tgen_muli, +}; + +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_mulsh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? MULHW : MULHD; + tcg_out32(s, insn | TAB(a0, a1, a2)); +} + +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_mulsh, +}; + +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_muluh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? MULHWU : MULHDU; + tcg_out32(s, insn | TAB(a0, a1, a2)); +} + +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_muluh, +}; + +static void tgen_nand(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, NAND | SAB(a1, a0, a2)); +} + +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_nand, +}; + +static void tgen_nor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, NOR | SAB(a1, a0, a2)); +} + +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_nor, +}; + +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, OR | SAB(a1, a0, a2)); +} + +static void tgen_ori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_ori32(s, a0, a1, a2); +} + +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, r, rU), + .out_rrr = tgen_or, + .out_rri = tgen_ori, +}; + +static void tgen_orc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, ORC | SAB(a1, a0, a2)); +} + +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_orc, +}; + +static TCGConstraintSetIndex cset_mod(TCGType type, unsigned flags) +{ + return have_isa_3_00 ? C_O1_I2(r, r, r) : C_NotImplemented; +} + +static void tgen_rems(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? MODSW : MODSD; + tcg_out32(s, insn | TAB(a0, a1, a2)); +} + +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mod, + .out_rrr = tgen_rems, +}; + +static void tgen_remu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? MODUW : MODUD; + tcg_out32(s, insn | TAB(a0, a1, a2)); +} + +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mod, + .out_rrr = tgen_remu, +}; + +static void tgen_rotl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out32(s, RLWNM | SAB(a1, a0, a2) | MB(0) | ME(31)); + } else { + tcg_out32(s, RLDCL | SAB(a1, a0, a2) | MB64(0)); + } +} + +static void tgen_rotli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_rlw(s, RLWINM, a0, a1, a2, 0, 31); + } else { + tcg_out_rld(s, RLDICL, a0, a1, a2, 0); + } +} + +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_rotl, + .out_rri = tgen_rotli, +}; + +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? SRAW : SRAD; + tcg_out32(s, insn | SAB(a1, a0, a2)); +} + +static void tgen_sari(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + /* Limit immediate shift count lest we create an illegal insn. */ + if (type == TCG_TYPE_I32) { + tcg_out_sari32(s, a0, a1, a2 & 31); + } else { + tcg_out_sari64(s, a0, a1, a2 & 63); + } +} + +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_sar, + .out_rri = tgen_sari, +}; + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? SLW : SLD; + tcg_out32(s, insn | SAB(a1, a0, a2)); +} + +static void tgen_shli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + /* Limit immediate shift count lest we create an illegal insn. */ + if (type == TCG_TYPE_I32) { + tcg_out_shli32(s, a0, a1, a2 & 31); + } else { + tcg_out_shli64(s, a0, a1, a2 & 63); + } +} + +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shl, + .out_rri = tgen_shli, +}; + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? SRW : SRD; + tcg_out32(s, insn | SAB(a1, a0, a2)); +} + +static void tgen_shri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + /* Limit immediate shift count lest we create an illegal insn. */ + if (type == TCG_TYPE_I32) { + tcg_out_shri32(s, a0, a1, a2 & 31); + } else { + tcg_out_shri64(s, a0, a1, a2 & 63); + } +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shr, + .out_rri = tgen_shri, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, SUBF | TAB(a0, a2, a1)); +} + +static void tgen_subfi(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2) +{ + tcg_out32(s, SUBFIC | TAI(a0, a2, a1)); +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, rI, r), + .out_rrr = tgen_sub, + .out_rir = tgen_subfi, +}; + +static void tgen_subbo_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, SUBFC | TAB(a0, a2, a1)); +} + +static void tgen_subbo_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (a2 == 0) { + tcg_out_movi(s, type, TCG_REG_R0, 0); + tgen_subbo_rrr(s, type, a0, a1, TCG_REG_R0); + } else { + tgen_addco_rri(s, type, a0, a1, -a2); + } +} + +/* The underlying insn for subfi is subfic. */ +#define tgen_subbo_rir tgen_subfi + +static void tgen_subbo_rii(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, tcg_target_long a2) +{ + tcg_out_movi(s, type, TCG_REG_R0, a2); + tgen_subbo_rir(s, type, a0, a1, TCG_REG_R0); +} + +static TCGConstraintSetIndex cset_subbo(TCGType type, unsigned flags) +{ + /* Recall that the CA bit is defined based on the host word size. */ + return type == TCG_TYPE_REG ? C_O1_I2(r, rI, rN) : C_NotImplemented; +} + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_subbo, + .out_rrr = tgen_subbo_rrr, + .out_rri = tgen_subbo_rri, + .out_rir = tgen_subbo_rir, + .out_rii = tgen_subbo_rii, +}; + +static void tgen_subbio_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, SUBFE | TAB(a0, a2, a1)); +} + +static void tgen_subbio_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_addcio_rri(s, type, a0, a1, ~a2); +} + +static void tgen_subbio_rir(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2) +{ + tcg_debug_assert(a1 == 0 || a1 == -1); + tcg_out32(s, (a1 ? SUBFME : SUBFZE) | RT(a0) | RA(a2)); +} + +static void tgen_subbio_rii(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, tcg_target_long a2) +{ + tcg_out_movi(s, type, TCG_REG_R0, a2); + tgen_subbio_rir(s, type, a0, a1, TCG_REG_R0); +} + +static TCGConstraintSetIndex cset_subbio(TCGType type, unsigned flags) +{ + return type == TCG_TYPE_REG ? C_O1_I2(r, rZM, rZM) : C_NotImplemented; +} + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_subbio, + .out_rrr = tgen_subbio_rrr, + .out_rri = tgen_subbio_rri, + .out_rir = tgen_subbio_rir, + .out_rii = tgen_subbio_rii, +}; + +#define outop_subbi outop_subbio + +static void tcg_out_set_borrow(TCGContext *s) +{ + /* borrow = !carry */ + tcg_out32(s, ADDIC | TAI(TCG_REG_R0, TCG_REG_R0, 0)); +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, XOR | SAB(a1, a0, a2)); +} + +static void tgen_xori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_xori32(s, a0, a1, a2); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, r, rU), + .out_rrr = tgen_xor, + .out_rri = tgen_xori, +}; + +static void tgen_bswap16(TCGContext *s, TCGType type, + TCGReg dst, TCGReg src, unsigned flags) +{ + TCGReg tmp = dst == src ? TCG_REG_R0 : dst; + + if (have_isa_3_10) { + tcg_out32(s, BRH | RA(dst) | RS(src)); + if (flags & TCG_BSWAP_OS) { + tcg_out_ext16s(s, TCG_TYPE_REG, dst, dst); + } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + tcg_out_ext16u(s, dst, dst); } - if (const_args[5]) { - tcg_out32(s, (args[5] ? ADDME : ADDZE) | RT(a1) | RA(args[3])); - } else { - tcg_out32(s, ADDE | TAB(a1, args[3], args[5])); + return; + } + + /* + * In the following, + * dep(a, b, m) -> (a & ~m) | (b & m) + * + * Begin with: src = xxxxabcd + */ + /* tmp = rol32(src, 24) & 0x000000ff = 0000000c */ + tcg_out_rlw(s, RLWINM, tmp, src, 24, 24, 31); + /* tmp = dep(tmp, rol32(src, 8), 0x0000ff00) = 000000dc */ + tcg_out_rlw(s, RLWIMI, tmp, src, 8, 16, 23); + + if (flags & TCG_BSWAP_OS) { + tcg_out_ext16s(s, TCG_TYPE_REG, dst, tmp); + } else { + tcg_out_mov(s, TCG_TYPE_REG, dst, tmp); + } +} + +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap16, +}; + +static void tgen_bswap32(TCGContext *s, TCGType type, + TCGReg dst, TCGReg src, unsigned flags) +{ + TCGReg tmp = dst == src ? TCG_REG_R0 : dst; + + if (have_isa_3_10) { + tcg_out32(s, BRW | RA(dst) | RS(src)); + if (flags & TCG_BSWAP_OS) { + tcg_out_ext32s(s, dst, dst); + } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + tcg_out_ext32u(s, dst, dst); } - if (a0 != args[0]) { - tcg_out_mov(s, TCG_TYPE_REG, args[0], a0); + return; + } + + /* + * Stolen from gcc's builtin_bswap32. + * In the following, + * dep(a, b, m) -> (a & ~m) | (b & m) + * + * Begin with: src = xxxxabcd + */ + /* tmp = rol32(src, 8) & 0xffffffff = 0000bcda */ + tcg_out_rlw(s, RLWINM, tmp, src, 8, 0, 31); + /* tmp = dep(tmp, rol32(src, 24), 0xff000000) = 0000dcda */ + tcg_out_rlw(s, RLWIMI, tmp, src, 24, 0, 7); + /* tmp = dep(tmp, rol32(src, 24), 0x0000ff00) = 0000dcba */ + tcg_out_rlw(s, RLWIMI, tmp, src, 24, 16, 23); + + if (flags & TCG_BSWAP_OS) { + tcg_out_ext32s(s, dst, tmp); + } else { + tcg_out_mov(s, TCG_TYPE_REG, dst, tmp); + } +} + +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap32, +}; + +#if TCG_TARGET_REG_BITS == 64 +static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg dst, TCGReg src) +{ + TCGReg t0 = dst == src ? TCG_REG_R0 : dst; + TCGReg t1 = dst == src ? dst : TCG_REG_R0; + + if (have_isa_3_10) { + tcg_out32(s, BRD | RA(dst) | RS(src)); + return; + } + + /* + * In the following, + * dep(a, b, m) -> (a & ~m) | (b & m) + * + * Begin with: src = abcdefgh + */ + /* t0 = rol32(src, 8) & 0xffffffff = 0000fghe */ + tcg_out_rlw(s, RLWINM, t0, src, 8, 0, 31); + /* t0 = dep(t0, rol32(src, 24), 0xff000000) = 0000hghe */ + tcg_out_rlw(s, RLWIMI, t0, src, 24, 0, 7); + /* t0 = dep(t0, rol32(src, 24), 0x0000ff00) = 0000hgfe */ + tcg_out_rlw(s, RLWIMI, t0, src, 24, 16, 23); + + /* t0 = rol64(t0, 32) = hgfe0000 */ + tcg_out_rld(s, RLDICL, t0, t0, 32, 0); + /* t1 = rol64(src, 32) = efghabcd */ + tcg_out_rld(s, RLDICL, t1, src, 32, 0); + + /* t0 = dep(t0, rol32(t1, 24), 0xffffffff) = hgfebcda */ + tcg_out_rlw(s, RLWIMI, t0, t1, 8, 0, 31); + /* t0 = dep(t0, rol32(t1, 24), 0xff000000) = hgfedcda */ + tcg_out_rlw(s, RLWIMI, t0, t1, 24, 0, 7); + /* t0 = dep(t0, rol32(t1, 24), 0x0000ff00) = hgfedcba */ + tcg_out_rlw(s, RLWIMI, t0, t1, 24, 16, 23); + + tcg_out_mov(s, TCG_TYPE_REG, dst, t0); +} + +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap64, +}; +#endif /* TCG_TARGET_REG_BITS == 64 */ + +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out32(s, NEG | RT(a0) | RA(a1)); +} + +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_neg, +}; + +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_nor(s, type, a0, a1, a1); +} + +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_not, +}; + +static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned ofs, unsigned len) +{ + if (type == TCG_TYPE_I32) { + tcg_out_rlw(s, RLWIMI, a0, a2, ofs, 32 - ofs - len, 31 - ofs); + } else { + tcg_out_rld(s, RLDIMI, a0, a2, ofs, 64 - ofs - len); + } +} + +static void tgen_depositi(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + tcg_target_long a2, unsigned ofs, unsigned len) +{ + tgen_andi(s, type, a0, a1, ~MAKE_64BIT_MASK(ofs, len)); +} + +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_O1_I2(r, 0, rZ), + .out_rrr = tgen_deposit, + .out_rri = tgen_depositi, +}; + +static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0 && len <= 16) { + tgen_andi(s, TCG_TYPE_I32, a0, a1, (1 << len) - 1); + } else if (type == TCG_TYPE_I32) { + tcg_out_rlw(s, RLWINM, a0, a1, 32 - ofs, 32 - len, 31); + } else { + tcg_out_rld(s, RLDICL, a0, a1, 64 - ofs, 64 - len); + } +} + +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extract, +}; + +static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0) { + switch (len) { + case 8: + tcg_out_ext8s(s, type, a0, a1); + return; + case 16: + tcg_out_ext16s(s, type, a0, a1); + return; + case 32: + tcg_out_ext32s(s, a0, a1); + return; } - break; + } else if (ofs + len == 32) { + tcg_out_sari32(s, a0, a1, ofs); + return; + } + g_assert_not_reached(); +} + +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_sextract, +}; + +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem_long(s, LBZ, LBZX, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; + +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tgen_ld8u(s, type, dest, base, offset); + tcg_out_ext8s(s, type, dest, dest); +} + +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; + +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem_long(s, LHZ, LHZX, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; + +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem_long(s, LHA, LHAX, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; #if TCG_TARGET_REG_BITS == 64 - case INDEX_op_sub2_i64: -#else - case INDEX_op_sub2_i32: +static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem_long(s, LWZ, LWZX, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32u, +}; + +static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem_long(s, LWA, LWAX, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32s, +}; #endif - a0 = args[0], a1 = args[1]; - if (a0 == args[5] || (!const_args[3] && a0 == args[3])) { - a0 = TCG_REG_R0; - } - if (const_args[2]) { - tcg_out32(s, SUBFIC | TAI(a0, args[4], args[2])); - } else { - tcg_out32(s, SUBFC | TAB(a0, args[4], args[2])); - } - if (const_args[3]) { - tcg_out32(s, (args[3] ? SUBFME : SUBFZE) | RT(a1) | RA(args[5])); - } else { - tcg_out32(s, SUBFE | TAB(a1, args[5], args[3])); - } - if (a0 != args[0]) { - tcg_out_mov(s, TCG_TYPE_REG, args[0], a0); - } - break; - case INDEX_op_muluh_i32: - tcg_out32(s, MULHWU | TAB(args[0], args[1], args[2])); - break; - case INDEX_op_mulsh_i32: - tcg_out32(s, MULHW | TAB(args[0], args[1], args[2])); - break; - case INDEX_op_muluh_i64: - tcg_out32(s, MULHDU | TAB(args[0], args[1], args[2])); - break; - case INDEX_op_mulsh_i64: - tcg_out32(s, MULHD | TAB(args[0], args[1], args[2])); - break; +static void tgen_st8(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem_long(s, STB, STBX, data, base, offset); +} - case INDEX_op_mb: - tcg_out_mb(s, args[0]); - break; +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tgen_st8, +}; - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_mov_i64: - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - default: - g_assert_not_reached(); - } +static void tgen_st16(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem_long(s, STH, STHX, data, base, offset); } +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tgen_st16, +}; + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tcg_out_st, +}; + + int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) { switch (opc) { @@ -4098,154 +4435,11 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) { switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_ld8u_i32: - case INDEX_op_ld8s_i32: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16s_i32: - case INDEX_op_ld_i32: - case INDEX_op_ctpop_i32: - case INDEX_op_neg_i32: - case INDEX_op_not_i32: - case INDEX_op_ext8s_i32: - case INDEX_op_ext16s_i32: - case INDEX_op_bswap16_i32: - case INDEX_op_bswap32_i32: - case INDEX_op_extract_i32: - case INDEX_op_sextract_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i64: - case INDEX_op_ld32u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld_i64: - case INDEX_op_ctpop_i64: - case INDEX_op_neg_i64: - case INDEX_op_not_i64: - case INDEX_op_ext8s_i64: - case INDEX_op_ext16s_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i64: - return C_O1_I1(r, r); - - case INDEX_op_st8_i32: - case INDEX_op_st16_i32: - case INDEX_op_st_i32: - case INDEX_op_st8_i64: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: + case INDEX_op_qemu_st: return C_O0_I2(r, r); - - case INDEX_op_add_i32: - case INDEX_op_and_i32: - case INDEX_op_or_i32: - case INDEX_op_xor_i32: - case INDEX_op_andc_i32: - case INDEX_op_orc_i32: - case INDEX_op_eqv_i32: - case INDEX_op_shl_i32: - case INDEX_op_shr_i32: - case INDEX_op_sar_i32: - case INDEX_op_rotl_i32: - case INDEX_op_rotr_i32: - case INDEX_op_and_i64: - case INDEX_op_andc_i64: - case INDEX_op_shl_i64: - case INDEX_op_shr_i64: - case INDEX_op_sar_i64: - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i64: - return C_O1_I2(r, r, ri); - - case INDEX_op_mul_i32: - case INDEX_op_mul_i64: - return C_O1_I2(r, r, rI); - - case INDEX_op_div_i32: - case INDEX_op_divu_i32: - case INDEX_op_rem_i32: - case INDEX_op_remu_i32: - case INDEX_op_nand_i32: - case INDEX_op_nor_i32: - case INDEX_op_muluh_i32: - case INDEX_op_mulsh_i32: - case INDEX_op_orc_i64: - case INDEX_op_eqv_i64: - case INDEX_op_nand_i64: - case INDEX_op_nor_i64: - case INDEX_op_div_i64: - case INDEX_op_divu_i64: - case INDEX_op_rem_i64: - case INDEX_op_remu_i64: - case INDEX_op_mulsh_i64: - case INDEX_op_muluh_i64: - return C_O1_I2(r, r, r); - - case INDEX_op_sub_i32: - return C_O1_I2(r, rI, ri); - case INDEX_op_add_i64: - return C_O1_I2(r, r, rT); - case INDEX_op_or_i64: - case INDEX_op_xor_i64: - return C_O1_I2(r, r, rU); - case INDEX_op_sub_i64: - return C_O1_I2(r, rI, rT); - case INDEX_op_clz_i32: - case INDEX_op_ctz_i32: - case INDEX_op_clz_i64: - case INDEX_op_ctz_i64: - return C_O1_I2(r, r, rZW); - - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - return C_O0_I2(r, rC); - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - case INDEX_op_negsetcond_i32: - case INDEX_op_negsetcond_i64: - return C_O1_I2(r, r, rC); - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - return C_O1_I4(r, r, rC, rZ, rZ); - - case INDEX_op_deposit_i32: - case INDEX_op_deposit_i64: - return C_O1_I2(r, 0, rZ); - case INDEX_op_brcond2_i32: - return C_O0_I4(r, r, ri, ri); - case INDEX_op_setcond2_i32: - return C_O1_I4(r, r, r, ri, ri); - case INDEX_op_add2_i64: - case INDEX_op_add2_i32: - return C_O2_I4(r, r, r, r, rI, rZM); - case INDEX_op_sub2_i64: - case INDEX_op_sub2_i32: - return C_O2_I4(r, r, rI, rZM, r, r); - - case INDEX_op_qemu_ld_i32: - return C_O1_I1(r, r); - case INDEX_op_qemu_ld_i64: - return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O2_I1(r, r, r); - - case INDEX_op_qemu_st_i32: - return C_O0_I2(r, r); - case INDEX_op_qemu_st_i64: - return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) : C_O0_I3(r, r, r); - - case INDEX_op_qemu_ld_i128: - return C_N1O1_I1(o, m, r); - case INDEX_op_qemu_st_i128: - return C_O0_I3(o, m, r); + case INDEX_op_qemu_st2: + return TCG_TARGET_REG_BITS == 64 + ? C_O0_I3(o, m, r) : C_O0_I3(r, r, r); case INDEX_op_add_vec: case INDEX_op_sub_vec: diff --git a/tcg/riscv/tcg-target-con-set.h b/tcg/riscv/tcg-target-con-set.h index e92e815491..0fc26d3f98 100644 --- a/tcg/riscv/tcg-target-con-set.h +++ b/tcg/riscv/tcg-target-con-set.h @@ -11,16 +11,13 @@ */ C_O0_I1(r) C_O0_I2(rz, r) -C_O0_I2(rz, rz) +C_O0_I2(r, rz) C_O1_I1(r, r) +C_O1_I2(r, r, r) C_O1_I2(r, r, ri) C_O1_I2(r, r, rI) -C_O1_I2(r, r, rJ) -C_O1_I2(r, rz, rN) -C_O1_I2(r, rz, rz) C_N1_I2(r, r, rM) C_O1_I4(r, r, rI, rM, rM) -C_O2_I4(r, r, rz, rz, rM, rM) C_O0_I2(v, r) C_O1_I1(v, r) C_O1_I1(v, v) diff --git a/tcg/riscv/tcg-target-con-str.h b/tcg/riscv/tcg-target-con-str.h index 2f9700638c..c04e15ddfa 100644 --- a/tcg/riscv/tcg-target-con-str.h +++ b/tcg/riscv/tcg-target-con-str.h @@ -16,8 +16,6 @@ REGS('v', ALL_VECTOR_REGS) * CONST(letter, TCG_CT_CONST_* bit set) */ CONST('I', TCG_CT_CONST_S12) -CONST('J', TCG_CT_CONST_J12) CONST('K', TCG_CT_CONST_S5) CONST('L', TCG_CT_CONST_CMP_VI) -CONST('N', TCG_CT_CONST_N12) CONST('M', TCG_CT_CONST_M12) diff --git a/tcg/riscv/tcg-target-has.h b/tcg/riscv/tcg-target-has.h index 98081084f2..aef10c2d9d 100644 --- a/tcg/riscv/tcg-target-has.h +++ b/tcg/riscv/tcg-target-has.h @@ -10,69 +10,8 @@ #include "host/cpuinfo.h" /* optional instructions */ -#define TCG_TARGET_HAS_negsetcond_i32 1 -#define TCG_TARGET_HAS_div_i32 1 -#define TCG_TARGET_HAS_rem_i32 1 -#define TCG_TARGET_HAS_div2_i32 0 -#define TCG_TARGET_HAS_rot_i32 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_extract2_i32 0 -#define TCG_TARGET_HAS_add2_i32 1 -#define TCG_TARGET_HAS_sub2_i32 1 -#define TCG_TARGET_HAS_mulu2_i32 0 -#define TCG_TARGET_HAS_muls2_i32 0 -#define TCG_TARGET_HAS_muluh_i32 0 -#define TCG_TARGET_HAS_mulsh_i32 0 -#define TCG_TARGET_HAS_ext8s_i32 1 -#define TCG_TARGET_HAS_ext16s_i32 1 -#define TCG_TARGET_HAS_ext8u_i32 1 -#define TCG_TARGET_HAS_ext16u_i32 1 -#define TCG_TARGET_HAS_bswap16_i32 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_bswap32_i32 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_andc_i32 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_orc_i32 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_eqv_i32 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_nand_i32 0 -#define TCG_TARGET_HAS_nor_i32 0 -#define TCG_TARGET_HAS_clz_i32 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_ctz_i32 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_ctpop_i32 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_qemu_st8_i32 0 - -#define TCG_TARGET_HAS_negsetcond_i64 1 -#define TCG_TARGET_HAS_div_i64 1 -#define TCG_TARGET_HAS_rem_i64 1 -#define TCG_TARGET_HAS_div2_i64 0 -#define TCG_TARGET_HAS_rot_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_extract2_i64 0 #define TCG_TARGET_HAS_extr_i64_i32 1 -#define TCG_TARGET_HAS_ext8s_i64 1 -#define TCG_TARGET_HAS_ext16s_i64 1 -#define TCG_TARGET_HAS_ext32s_i64 1 -#define TCG_TARGET_HAS_ext8u_i64 1 -#define TCG_TARGET_HAS_ext16u_i64 1 -#define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_bswap16_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_bswap32_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_bswap64_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_not_i64 1 -#define TCG_TARGET_HAS_andc_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_orc_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_eqv_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_nand_i64 0 -#define TCG_TARGET_HAS_nor_i64 0 -#define TCG_TARGET_HAS_clz_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_ctz_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_ctpop_i64 (cpuinfo & CPUINFO_ZBB) -#define TCG_TARGET_HAS_add2_i64 1 -#define TCG_TARGET_HAS_sub2_i64 1 -#define TCG_TARGET_HAS_mulu2_i64 0 -#define TCG_TARGET_HAS_muls2_i64 0 -#define TCG_TARGET_HAS_muluh_i64 1 -#define TCG_TARGET_HAS_mulsh_i64 1 - #define TCG_TARGET_HAS_qemu_ldst_i128 0 - #define TCG_TARGET_HAS_tst 0 /* vector instructions */ diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index f7e1ca5a56..f9417d15f7 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -113,11 +113,9 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) } #define TCG_CT_CONST_S12 0x100 -#define TCG_CT_CONST_N12 0x200 -#define TCG_CT_CONST_M12 0x400 -#define TCG_CT_CONST_J12 0x800 -#define TCG_CT_CONST_S5 0x1000 -#define TCG_CT_CONST_CMP_VI 0x2000 +#define TCG_CT_CONST_M12 0x200 +#define TCG_CT_CONST_S5 0x400 +#define TCG_CT_CONST_CMP_VI 0x800 #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) #define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32) @@ -402,28 +400,14 @@ static bool tcg_target_const_match(int64_t val, int ct, return 1; } /* - * Sign extended from 12 bits, negated: [-0x7ff, 0x800]. - * Used for subtraction, where a constant must be handled by ADDI. - */ - if ((ct & TCG_CT_CONST_N12) && val >= -0x7ff && val <= 0x800) { - return 1; - } - /* * Sign extended from 12 bits, +/- matching: [-0x7ff, 0x7ff]. - * Used by addsub2 and movcond, which may need the negative value, + * Used by movcond, which may need the negative value, * and requires the modified constant to be representable. */ if ((ct & TCG_CT_CONST_M12) && val >= -0x7ff && val <= 0x7ff) { return 1; } /* - * Inverse of sign extended from 12 bits: ~[-0x800, 0x7ff]. - * Used to map ANDN back to ANDI, etc. - */ - if ((ct & TCG_CT_CONST_J12) && ~val >= -0x800 && ~val <= 0x7ff) { - return 1; - } - /* * Sign extended from 5 bits: [-0x10, 0x0f]. * Used for vector-immediate. */ @@ -1089,67 +1073,6 @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, return false; } -static void tcg_out_addsub2(TCGContext *s, - TCGReg rl, TCGReg rh, - TCGReg al, TCGReg ah, - TCGArg bl, TCGArg bh, - bool cbl, bool cbh, bool is_sub, bool is32bit) -{ - const RISCVInsn opc_add = is32bit ? OPC_ADDW : OPC_ADD; - const RISCVInsn opc_addi = is32bit ? OPC_ADDIW : OPC_ADDI; - const RISCVInsn opc_sub = is32bit ? OPC_SUBW : OPC_SUB; - TCGReg th = TCG_REG_TMP1; - - /* If we have a negative constant such that negating it would - make the high part zero, we can (usually) eliminate one insn. */ - if (cbl && cbh && bh == -1 && bl != 0) { - bl = -bl; - bh = 0; - is_sub = !is_sub; - } - - /* By operating on the high part first, we get to use the final - carry operation to move back from the temporary. */ - if (!cbh) { - tcg_out_opc_reg(s, (is_sub ? opc_sub : opc_add), th, ah, bh); - } else if (bh != 0 || ah == rl) { - tcg_out_opc_imm(s, opc_addi, th, ah, (is_sub ? -bh : bh)); - } else { - th = ah; - } - - /* Note that tcg optimization should eliminate the bl == 0 case. */ - if (is_sub) { - if (cbl) { - tcg_out_opc_imm(s, OPC_SLTIU, TCG_REG_TMP0, al, bl); - tcg_out_opc_imm(s, opc_addi, rl, al, -bl); - } else { - tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_TMP0, al, bl); - tcg_out_opc_reg(s, opc_sub, rl, al, bl); - } - tcg_out_opc_reg(s, opc_sub, rh, th, TCG_REG_TMP0); - } else { - if (cbl) { - tcg_out_opc_imm(s, opc_addi, rl, al, bl); - tcg_out_opc_imm(s, OPC_SLTIU, TCG_REG_TMP0, rl, bl); - } else if (al == bl) { - /* - * If the input regs overlap, this is a simple doubling - * and carry-out is the input msb. This special case is - * required when the output reg overlaps the input, - * but we might as well use it always. - */ - tcg_out_opc_imm(s, OPC_SLTI, TCG_REG_TMP0, al, 0); - tcg_out_opc_reg(s, opc_add, rl, al, al); - } else { - tcg_out_opc_reg(s, opc_add, rl, al, bl); - tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_TMP0, - rl, (rl == bl ? al : bl)); - } - tcg_out_opc_reg(s, opc_add, rh, th, TCG_REG_TMP0); - } -} - static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg dst, TCGReg src) { @@ -1184,6 +1107,12 @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, tcg_out_dup_vec(s, type, vece, dst, TCG_REG_TMP0); } +static void tcg_out_br(TCGContext *s, TCGLabel *l) +{ + tcg_out_reloc(s, s->code_ptr, R_RISCV_JAL, l, 0); + tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); +} + static const struct { RISCVInsn op; bool swap; @@ -1200,8 +1129,8 @@ static const struct { [TCG_COND_GTU] = { OPC_BLTU, true } }; -static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, - TCGReg arg2, TCGLabel *l) +static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, TCGReg arg2, TCGLabel *l) { RISCVInsn op = tcg_brcond_to_riscv[cond].op; @@ -1217,6 +1146,11 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, tcg_out_opc_branch(s, op, arg1, arg2, 0); } +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, rz), + .out_rr = tgen_brcond, +}; + #define SETCOND_INV TCG_TARGET_NB_REGS #define SETCOND_NEZ (SETCOND_INV << 1) #define SETCOND_FLAGS (SETCOND_INV | SETCOND_NEZ) @@ -1341,6 +1275,24 @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, } } +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_setcond(s, cond, dest, arg1, arg2, false); +} + +static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tcg_out_setcond(s, cond, dest, arg1, arg2, true); +} + +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_setcond, + .out_rri = tgen_setcondi, +}; + static void tcg_out_negsetcond(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg arg1, tcg_target_long arg2, bool c2) { @@ -1379,6 +1331,24 @@ static void tcg_out_negsetcond(TCGContext *s, TCGCond cond, TCGReg ret, } } +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_negsetcond(s, cond, dest, arg1, arg2, false); +} + +static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tcg_out_negsetcond(s, cond, dest, arg1, arg2, true); +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_negsetcond, + .out_rri = tgen_negsetcondi, +}; + static void tcg_out_movcond_zicond(TCGContext *s, TCGReg ret, TCGReg test_ne, int val1, bool c_val1, int val2, bool c_val2) @@ -1476,10 +1446,10 @@ static void tcg_out_movcond_br2(TCGContext *s, TCGCond cond, TCGReg ret, tcg_out_mov(s, TCG_TYPE_REG, ret, tmp); } -static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg cmp1, int cmp2, bool c_cmp2, - TCGReg val1, bool c_val1, - TCGReg val2, bool c_val2) +static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg cmp1, TCGArg cmp2, bool c_cmp2, + TCGArg val1, bool c_val1, + TCGArg val2, bool c_val2) { int tmpflags; TCGReg t; @@ -1506,6 +1476,11 @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, } } +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = C_O1_I4(r, r, rI, rM, rM), + .out = tcg_out_movcond, +}; + static void tcg_out_cltz(TCGContext *s, TCGType type, RISCVInsn insn, TCGReg ret, TCGReg src1, int src2, bool c_src2) { @@ -1517,7 +1492,7 @@ static void tcg_out_cltz(TCGContext *s, TCGType type, RISCVInsn insn, * Note that constraints put 'ret' in a new register, so the * computation above did not clobber either 'src1' or 'src2'. */ - tcg_out_movcond(s, TCG_COND_EQ, ret, src1, 0, true, + tcg_out_movcond(s, type, TCG_COND_EQ, ret, src1, 0, true, src2, c_src2, ret, false); } } @@ -1613,7 +1588,7 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg, tcg_out_call_int(s, arg, false); } -static void tcg_out_mb(TCGContext *s, TCGArg a0) +static void tcg_out_mb(TCGContext *s, unsigned a0) { tcg_insn_unit insn = OPC_FENCE; @@ -1858,22 +1833,31 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val, } } -static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType data_type) +static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data_reg, + TCGReg addr_reg, MemOpIdx oi) { TCGLabelQemuLdst *ldst; TCGReg base; ldst = prepare_host_addr(s, &base, addr_reg, oi, true); - tcg_out_qemu_ld_direct(s, data_reg, base, get_memop(oi), data_type); + tcg_out_qemu_ld_direct(s, data_reg, base, get_memop(oi), type); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = data_reg; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_qemu_ld, +}; + +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + .base.static_constraint = C_NotImplemented, +}; + static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg val, TCGReg base, MemOp opc) { @@ -1898,8 +1882,8 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg val, } } -static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType data_type) +static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data_reg, + TCGReg addr_reg, MemOpIdx oi) { TCGLabelQemuLdst *ldst; TCGReg base; @@ -1908,12 +1892,21 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_out_qemu_st_direct(s, data_reg, base, get_memop(oi)); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = data_reg; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out = tgen_qemu_st, +}; + +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + .base.static_constraint = C_NotImplemented, +}; + static const tcg_insn_unit *tb_ret_addr; static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) @@ -1940,6 +1933,11 @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0); +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { @@ -1957,452 +1955,693 @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, flush_idcache_range(jmp_rx, jmp_rw, 4); } -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) + +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - TCGArg a0 = args[0]; - TCGArg a1 = args[1]; - TCGArg a2 = args[2]; - int c2 = const_args[2]; + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_ADDW : OPC_ADD; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} - switch (opc) { - case INDEX_op_goto_ptr: - tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0); - break; +static void tgen_addi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_ADDIW : OPC_ADDI; + tcg_out_opc_imm(s, insn, a0, a1, a2); +} - case INDEX_op_br: - tcg_out_reloc(s, s->code_ptr, R_RISCV_JAL, arg_label(a0), 0); - tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); - break; +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_add, + .out_rri = tgen_addi, +}; - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - tcg_out_ldst(s, OPC_LBU, a0, a1, a2); - break; - case INDEX_op_ld8s_i32: - case INDEX_op_ld8s_i64: - tcg_out_ldst(s, OPC_LB, a0, a1, a2); - break; - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - tcg_out_ldst(s, OPC_LHU, a0, a1, a2); - break; - case INDEX_op_ld16s_i32: - case INDEX_op_ld16s_i64: - tcg_out_ldst(s, OPC_LH, a0, a1, a2); - break; - case INDEX_op_ld32u_i64: - tcg_out_ldst(s, OPC_LWU, a0, a1, a2); - break; - case INDEX_op_ld_i32: - case INDEX_op_ld32s_i64: - tcg_out_ldst(s, OPC_LW, a0, a1, a2); - break; - case INDEX_op_ld_i64: - tcg_out_ldst(s, OPC_LD, a0, a1, a2); - break; +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - tcg_out_ldst(s, OPC_SB, a0, a1, a2); - break; - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - tcg_out_ldst(s, OPC_SH, a0, a1, a2); - break; - case INDEX_op_st_i32: - case INDEX_op_st32_i64: - tcg_out_ldst(s, OPC_SW, a0, a1, a2); - break; - case INDEX_op_st_i64: - tcg_out_ldst(s, OPC_SD, a0, a1, a2); - break; +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_add_i32: - if (c2) { - tcg_out_opc_imm(s, OPC_ADDIW, a0, a1, a2); - } else { - tcg_out_opc_reg(s, OPC_ADDW, a0, a1, a2); - } - break; - case INDEX_op_add_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_ADDI, a0, a1, a2); - } else { - tcg_out_opc_reg(s, OPC_ADD, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_sub_i32: - if (c2) { - tcg_out_opc_imm(s, OPC_ADDIW, a0, a1, -a2); - } else { - tcg_out_opc_reg(s, OPC_SUBW, a0, a1, a2); - } - break; - case INDEX_op_sub_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_ADDI, a0, a1, -a2); - } else { - tcg_out_opc_reg(s, OPC_SUB, a0, a1, a2); - } - break; +static void tcg_out_set_carry(TCGContext *s) +{ + g_assert_not_reached(); +} - case INDEX_op_and_i32: - case INDEX_op_and_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_ANDI, a0, a1, a2); - } else { - tcg_out_opc_reg(s, OPC_AND, a0, a1, a2); - } - break; +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_AND, a0, a1, a2); +} - case INDEX_op_or_i32: - case INDEX_op_or_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_ORI, a0, a1, a2); - } else { - tcg_out_opc_reg(s, OPC_OR, a0, a1, a2); - } - break; +static void tgen_andi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_opc_imm(s, OPC_ANDI, a0, a1, a2); +} - case INDEX_op_xor_i32: - case INDEX_op_xor_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_XORI, a0, a1, a2); - } else { - tcg_out_opc_reg(s, OPC_XOR, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_and, + .out_rri = tgen_andi, +}; - case INDEX_op_andc_i32: - case INDEX_op_andc_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_ANDI, a0, a1, ~a2); - } else { - tcg_out_opc_reg(s, OPC_ANDN, a0, a1, a2); - } - break; - case INDEX_op_orc_i32: - case INDEX_op_orc_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_ORI, a0, a1, ~a2); - } else { - tcg_out_opc_reg(s, OPC_ORN, a0, a1, a2); - } - break; - case INDEX_op_eqv_i32: - case INDEX_op_eqv_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_XORI, a0, a1, ~a2); - } else { - tcg_out_opc_reg(s, OPC_XNOR, a0, a1, a2); - } - break; +static void tgen_andc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_ANDN, a0, a1, a2); +} - case INDEX_op_not_i32: - case INDEX_op_not_i64: - tcg_out_opc_imm(s, OPC_XORI, a0, a1, -1); - break; +static TCGConstraintSetIndex cset_zbb_rrr(TCGType type, unsigned flags) +{ + return cpuinfo & CPUINFO_ZBB ? C_O1_I2(r, r, r) : C_NotImplemented; +} - case INDEX_op_neg_i32: - tcg_out_opc_reg(s, OPC_SUBW, a0, TCG_REG_ZERO, a1); - break; - case INDEX_op_neg_i64: - tcg_out_opc_reg(s, OPC_SUB, a0, TCG_REG_ZERO, a1); - break; +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_zbb_rrr, + .out_rrr = tgen_andc, +}; - case INDEX_op_mul_i32: - tcg_out_opc_reg(s, OPC_MULW, a0, a1, a2); - break; - case INDEX_op_mul_i64: - tcg_out_opc_reg(s, OPC_MUL, a0, a1, a2); - break; +static void tgen_clz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CLZW : OPC_CLZ; + tcg_out_cltz(s, type, insn, a0, a1, a2, false); +} - case INDEX_op_div_i32: - tcg_out_opc_reg(s, OPC_DIVW, a0, a1, a2); - break; - case INDEX_op_div_i64: - tcg_out_opc_reg(s, OPC_DIV, a0, a1, a2); - break; +static void tgen_clzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CLZW : OPC_CLZ; + tcg_out_cltz(s, type, insn, a0, a1, a2, true); +} - case INDEX_op_divu_i32: - tcg_out_opc_reg(s, OPC_DIVUW, a0, a1, a2); - break; - case INDEX_op_divu_i64: - tcg_out_opc_reg(s, OPC_DIVU, a0, a1, a2); - break; +static TCGConstraintSetIndex cset_clzctz(TCGType type, unsigned flags) +{ + return cpuinfo & CPUINFO_ZBB ? C_N1_I2(r, r, rM) : C_NotImplemented; +} - case INDEX_op_rem_i32: - tcg_out_opc_reg(s, OPC_REMW, a0, a1, a2); - break; - case INDEX_op_rem_i64: - tcg_out_opc_reg(s, OPC_REM, a0, a1, a2); - break; +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_clzctz, + .out_rrr = tgen_clz, + .out_rri = tgen_clzi, +}; - case INDEX_op_remu_i32: - tcg_out_opc_reg(s, OPC_REMUW, a0, a1, a2); - break; - case INDEX_op_remu_i64: - tcg_out_opc_reg(s, OPC_REMU, a0, a1, a2); - break; +static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CPOPW : OPC_CPOP; + tcg_out_opc_imm(s, insn, a0, a1, 0); +} - case INDEX_op_shl_i32: - if (c2) { - tcg_out_opc_imm(s, OPC_SLLIW, a0, a1, a2 & 0x1f); - } else { - tcg_out_opc_reg(s, OPC_SLLW, a0, a1, a2); - } - break; - case INDEX_op_shl_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_SLLI, a0, a1, a2 & 0x3f); - } else { - tcg_out_opc_reg(s, OPC_SLL, a0, a1, a2); - } - break; +static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags) +{ + return cpuinfo & CPUINFO_ZBB ? C_O1_I1(r, r) : C_NotImplemented; +} - case INDEX_op_shr_i32: - if (c2) { - tcg_out_opc_imm(s, OPC_SRLIW, a0, a1, a2 & 0x1f); - } else { - tcg_out_opc_reg(s, OPC_SRLW, a0, a1, a2); - } - break; - case INDEX_op_shr_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_SRLI, a0, a1, a2 & 0x3f); - } else { - tcg_out_opc_reg(s, OPC_SRL, a0, a1, a2); - } - break; +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_ctpop, + .out_rr = tgen_ctpop, +}; - case INDEX_op_sar_i32: - if (c2) { - tcg_out_opc_imm(s, OPC_SRAIW, a0, a1, a2 & 0x1f); - } else { - tcg_out_opc_reg(s, OPC_SRAW, a0, a1, a2); - } - break; - case INDEX_op_sar_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_SRAI, a0, a1, a2 & 0x3f); - } else { - tcg_out_opc_reg(s, OPC_SRA, a0, a1, a2); - } - break; +static void tgen_ctz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CTZW : OPC_CTZ; + tcg_out_cltz(s, type, insn, a0, a1, a2, false); +} - case INDEX_op_rotl_i32: - if (c2) { - tcg_out_opc_imm(s, OPC_RORIW, a0, a1, -a2 & 0x1f); - } else { - tcg_out_opc_reg(s, OPC_ROLW, a0, a1, a2); - } - break; - case INDEX_op_rotl_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_RORI, a0, a1, -a2 & 0x3f); - } else { - tcg_out_opc_reg(s, OPC_ROL, a0, a1, a2); - } - break; +static void tgen_ctzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CTZW : OPC_CTZ; + tcg_out_cltz(s, type, insn, a0, a1, a2, true); +} - case INDEX_op_rotr_i32: - if (c2) { - tcg_out_opc_imm(s, OPC_RORIW, a0, a1, a2 & 0x1f); - } else { - tcg_out_opc_reg(s, OPC_RORW, a0, a1, a2); - } - break; - case INDEX_op_rotr_i64: - if (c2) { - tcg_out_opc_imm(s, OPC_RORI, a0, a1, a2 & 0x3f); - } else { - tcg_out_opc_reg(s, OPC_ROR, a0, a1, a2); - } - break; +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_clzctz, + .out_rrr = tgen_ctz, + .out_rri = tgen_ctzi, +}; - case INDEX_op_bswap64_i64: - tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); - break; - case INDEX_op_bswap32_i32: - a2 = 0; - /* fall through */ - case INDEX_op_bswap32_i64: - tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); - if (a2 & TCG_BSWAP_OZ) { - tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 32); - } else { - tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 32); - } - break; - case INDEX_op_bswap16_i64: - case INDEX_op_bswap16_i32: - tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); - if (a2 & TCG_BSWAP_OZ) { - tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 48); - } else { - tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 48); - } - break; +static void tgen_divs(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_DIVW : OPC_DIV; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} - case INDEX_op_ctpop_i32: - tcg_out_opc_imm(s, OPC_CPOPW, a0, a1, 0); - break; - case INDEX_op_ctpop_i64: - tcg_out_opc_imm(s, OPC_CPOP, a0, a1, 0); - break; +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divs, +}; - case INDEX_op_clz_i32: - tcg_out_cltz(s, TCG_TYPE_I32, OPC_CLZW, a0, a1, a2, c2); - break; - case INDEX_op_clz_i64: - tcg_out_cltz(s, TCG_TYPE_I64, OPC_CLZ, a0, a1, a2, c2); - break; - case INDEX_op_ctz_i32: - tcg_out_cltz(s, TCG_TYPE_I32, OPC_CTZW, a0, a1, a2, c2); - break; - case INDEX_op_ctz_i64: - tcg_out_cltz(s, TCG_TYPE_I64, OPC_CTZ, a0, a1, a2, c2); - break; +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_add2_i32: - tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], - const_args[4], const_args[5], false, true); - break; - case INDEX_op_add2_i64: - tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], - const_args[4], const_args[5], false, false); - break; - case INDEX_op_sub2_i32: - tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], - const_args[4], const_args[5], true, true); - break; - case INDEX_op_sub2_i64: - tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], - const_args[4], const_args[5], true, false); - break; +static void tgen_divu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_DIVUW : OPC_DIVU; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); - break; +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divu, +}; - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - tcg_out_setcond(s, args[3], a0, a1, a2, c2); - break; +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_negsetcond_i32: - case INDEX_op_negsetcond_i64: - tcg_out_negsetcond(s, args[3], a0, a1, a2, c2); - break; +static void tgen_eqv(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_XNOR, a0, a1, a2); +} - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - tcg_out_movcond(s, args[5], a0, a1, a2, c2, - args[3], const_args[3], args[4], const_args[4]); - break; +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_zbb_rrr, + .out_rrr = tgen_eqv, +}; - case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); - break; - case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); - break; +static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_opc_imm(s, OPC_SRAI, a0, a1, 32); +} - case INDEX_op_extrh_i64_i32: - tcg_out_opc_imm(s, OPC_SRAI, a0, a1, 32); - break; +static const TCGOutOpUnary outop_extrh_i64_i32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extrh_i64_i32, +}; - case INDEX_op_mulsh_i32: - case INDEX_op_mulsh_i64: - tcg_out_opc_reg(s, OPC_MULH, a0, a1, a2); - break; +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_MULW : OPC_MUL; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} - case INDEX_op_muluh_i32: - case INDEX_op_muluh_i64: - tcg_out_opc_reg(s, OPC_MULHU, a0, a1, a2); - break; +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_mul, +}; - case INDEX_op_mb: - tcg_out_mb(s, a0); - break; +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_NotImplemented, +}; - case INDEX_op_extract_i64: - if (a2 + args[3] == 32) { - if (a2 == 0) { - tcg_out_ext32u(s, a0, a1); - } else { - tcg_out_opc_imm(s, OPC_SRLIW, a0, a1, a2); - } - break; - } - /* FALLTHRU */ - case INDEX_op_extract_i32: - switch (args[3]) { - case 1: - tcg_out_opc_imm(s, OPC_BEXTI, a0, a1, a2); - break; +static TCGConstraintSetIndex cset_mulh(TCGType type, unsigned flags) +{ + return type == TCG_TYPE_I32 ? C_NotImplemented : C_O1_I2(r, r, r); +} + +static void tgen_mulsh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_MULH, a0, a1, a2); +} + +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mulh, + .out_rrr = tgen_mulsh, +}; + +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_muluh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_MULHU, a0, a1, a2); +} + +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mulh, + .out_rrr = tgen_muluh, +}; + +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_OR, a0, a1, a2); +} + +static void tgen_ori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_opc_imm(s, OPC_ORI, a0, a1, a2); +} + +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_or, + .out_rri = tgen_ori, +}; + +static void tgen_orc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_ORN, a0, a1, a2); +} + +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_zbb_rrr, + .out_rrr = tgen_orc, +}; + +static void tgen_rems(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_REMW : OPC_REM; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_rems, +}; + +static void tgen_remu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_REMUW : OPC_REMU; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_remu, +}; + +static TCGConstraintSetIndex cset_rot(TCGType type, unsigned flags) +{ + return cpuinfo & CPUINFO_ZBB ? C_O1_I2(r, r, ri) : C_NotImplemented; +} + +static void tgen_rotr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_RORW : OPC_ROR; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static void tgen_rotri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_RORIW : OPC_RORI; + unsigned mask = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_opc_imm(s, insn, a0, a1, a2 & mask); +} + +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_rot, + .out_rrr = tgen_rotr, + .out_rri = tgen_rotri, +}; + +static void tgen_rotl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_ROLW : OPC_ROL; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static void tgen_rotli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_rotri(s, type, a0, a1, -a2); +} + +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_rot, + .out_rrr = tgen_rotl, + .out_rri = tgen_rotli, +}; + +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SRAW : OPC_SRA; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static void tgen_sari(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SRAIW : OPC_SRAI; + unsigned mask = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_opc_imm(s, insn, a0, a1, a2 & mask); +} + +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_sar, + .out_rri = tgen_sari, +}; + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SLLW : OPC_SLL; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static void tgen_shli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SLLIW : OPC_SLLI; + unsigned mask = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_opc_imm(s, insn, a0, a1, a2 & mask); +} + +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shl, + .out_rri = tgen_shli, +}; + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SRLW : OPC_SRL; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static void tgen_shri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SRLIW : OPC_SRLI; + unsigned mask = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_opc_imm(s, insn, a0, a1, a2 & mask); +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shr, + .out_rri = tgen_shri, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SUBW : OPC_SUB; + tcg_out_opc_reg(s, insn, a0, a1, a2); +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_sub, +}; + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpAddSubCarry outop_subbi = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_NotImplemented, +}; + +static void tcg_out_set_borrow(TCGContext *s) +{ + g_assert_not_reached(); +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_opc_reg(s, OPC_XOR, a0, a1, a2); +} + +static void tgen_xori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_opc_imm(s, OPC_XORI, a0, a1, a2); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, r, rI), + .out_rrr = tgen_xor, + .out_rri = tgen_xori, +}; + +static TCGConstraintSetIndex cset_bswap(TCGType type, unsigned flags) +{ + return cpuinfo & CPUINFO_ZBB ? C_O1_I1(r, r) : C_NotImplemented; +} + +static void tgen_bswap16(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); + if (flags & TCG_BSWAP_OZ) { + tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 48); + } else { + tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 48); + } +} + +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_bswap, + .out_rr = tgen_bswap16, +}; + +static void tgen_bswap32(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); + if (flags & TCG_BSWAP_OZ) { + tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 32); + } else { + tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 32); + } +} + +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_bswap, + .out_rr = tgen_bswap32, +}; + +static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); +} + +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_bswap, + .out_rr = tgen_bswap64, +}; + +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_sub(s, type, a0, TCG_REG_ZERO, a1); +} + +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_neg, +}; + +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_xori(s, type, a0, a1, -1); +} + +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_not, +}; + +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0) { + switch (len) { case 16: - tcg_debug_assert(a2 == 0); tcg_out_ext16u(s, a0, a1); - break; - default: - g_assert_not_reached(); + return; + case 32: + tcg_out_ext32u(s, a0, a1); + return; } - break; + } + if (ofs + len == 32) { + tgen_shli(s, TCG_TYPE_I32, a0, a1, ofs); + return; + } + if (len == 1) { + tcg_out_opc_imm(s, OPC_BEXTI, a0, a1, ofs); + return; + } + g_assert_not_reached(); +} - case INDEX_op_sextract_i64: - if (a2 + args[3] == 32) { - if (a2 == 0) { - tcg_out_ext32s(s, a0, a1); - } else { - tcg_out_opc_imm(s, OPC_SRAIW, a0, a1, a2); - } - break; - } - /* FALLTHRU */ - case INDEX_op_sextract_i32: - if (a2 == 0 && args[3] == 8) { - tcg_out_ext8s(s, TCG_TYPE_REG, a0, a1); - } else if (a2 == 0 && args[3] == 16) { - tcg_out_ext16s(s, TCG_TYPE_REG, a0, a1); - } else { - g_assert_not_reached(); - } - break; +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extract, +}; - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_mov_i64: - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - default: - g_assert_not_reached(); +static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + if (ofs == 0) { + switch (len) { + case 8: + tcg_out_ext8s(s, type, a0, a1); + return; + case 16: + tcg_out_ext16s(s, type, a0, a1); + return; + case 32: + tcg_out_ext32s(s, a0, a1); + return; + } + } else if (ofs + len == 32) { + tgen_sari(s, TCG_TYPE_I32, a0, a1, ofs); + return; } + g_assert_not_reached(); +} + +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_sextract, +}; + +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LBU, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; + +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LB, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; + +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LHU, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; + +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LH, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; + +static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LWU, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32u, +}; + +static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_LW, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32s, +}; + +static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_SB, data, base, offset); } +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st8_r, +}; + +static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, OPC_SH, data, base, offset); +} + +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st16_r, +}; + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tcg_out_st, +}; + + static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg args[TCG_MAX_OP_ARGS], @@ -2624,142 +2863,6 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) { switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_ld8u_i32: - case INDEX_op_ld8s_i32: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16s_i32: - case INDEX_op_ld_i32: - case INDEX_op_not_i32: - case INDEX_op_neg_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld32u_i64: - case INDEX_op_ld_i64: - case INDEX_op_not_i64: - case INDEX_op_neg_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_ext8s_i32: - case INDEX_op_ext8s_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_extrl_i64_i32: - case INDEX_op_extrh_i64_i32: - case INDEX_op_ext_i32_i64: - case INDEX_op_extract_i32: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i32: - case INDEX_op_sextract_i64: - case INDEX_op_bswap16_i32: - case INDEX_op_bswap32_i32: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: - case INDEX_op_ctpop_i32: - case INDEX_op_ctpop_i64: - return C_O1_I1(r, r); - - case INDEX_op_st8_i32: - case INDEX_op_st16_i32: - case INDEX_op_st_i32: - case INDEX_op_st8_i64: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: - return C_O0_I2(rz, r); - - case INDEX_op_add_i32: - case INDEX_op_and_i32: - case INDEX_op_or_i32: - case INDEX_op_xor_i32: - case INDEX_op_add_i64: - case INDEX_op_and_i64: - case INDEX_op_or_i64: - case INDEX_op_xor_i64: - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - case INDEX_op_negsetcond_i32: - case INDEX_op_negsetcond_i64: - return C_O1_I2(r, r, rI); - - case INDEX_op_andc_i32: - case INDEX_op_andc_i64: - case INDEX_op_orc_i32: - case INDEX_op_orc_i64: - case INDEX_op_eqv_i32: - case INDEX_op_eqv_i64: - return C_O1_I2(r, r, rJ); - - case INDEX_op_sub_i32: - case INDEX_op_sub_i64: - return C_O1_I2(r, rz, rN); - - case INDEX_op_mul_i32: - case INDEX_op_mulsh_i32: - case INDEX_op_muluh_i32: - case INDEX_op_div_i32: - case INDEX_op_divu_i32: - case INDEX_op_rem_i32: - case INDEX_op_remu_i32: - case INDEX_op_mul_i64: - case INDEX_op_mulsh_i64: - case INDEX_op_muluh_i64: - case INDEX_op_div_i64: - case INDEX_op_divu_i64: - case INDEX_op_rem_i64: - case INDEX_op_remu_i64: - return C_O1_I2(r, rz, rz); - - case INDEX_op_shl_i32: - case INDEX_op_shr_i32: - case INDEX_op_sar_i32: - case INDEX_op_rotl_i32: - case INDEX_op_rotr_i32: - case INDEX_op_shl_i64: - case INDEX_op_shr_i64: - case INDEX_op_sar_i64: - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i64: - return C_O1_I2(r, r, ri); - - case INDEX_op_clz_i32: - case INDEX_op_clz_i64: - case INDEX_op_ctz_i32: - case INDEX_op_ctz_i64: - return C_N1_I2(r, r, rM); - - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - return C_O0_I2(rz, rz); - - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - return C_O1_I4(r, r, rI, rM, rM); - - case INDEX_op_add2_i32: - case INDEX_op_add2_i64: - case INDEX_op_sub2_i32: - case INDEX_op_sub2_i64: - return C_O2_I4(r, r, rz, rz, rM, rM); - - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: - return C_O1_I1(r, r); - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: - return C_O0_I2(rz, r); - case INDEX_op_st_vec: return C_O0_I2(v, r); case INDEX_op_dup_vec: diff --git a/tcg/s390x/tcg-target-con-set.h b/tcg/s390x/tcg-target-con-set.h index 370e4b1295..f67fd7898e 100644 --- a/tcg/s390x/tcg-target-con-set.h +++ b/tcg/s390x/tcg-target-con-set.h @@ -22,6 +22,7 @@ C_O1_I1(r, r) C_O1_I1(v, r) C_O1_I1(v, v) C_O1_I1(v, vr) +C_O1_I2(r, 0, r) C_O1_I2(r, 0, ri) C_O1_I2(r, 0, rI) C_O1_I2(r, 0, rJ) @@ -31,20 +32,16 @@ C_O1_I2(r, r, rC) C_O1_I2(r, r, rI) C_O1_I2(r, r, rJ) C_O1_I2(r, r, rK) -C_O1_I2(r, r, rKR) -C_O1_I2(r, r, rNK) C_O1_I2(r, r, rNKR) +C_O1_I2(r, r, rUV) C_O1_I2(r, rZ, r) C_O1_I2(v, v, r) C_O1_I2(v, v, v) C_O1_I3(v, v, v, v) C_O1_I4(v, v, v, vZ, v) C_O1_I4(v, v, v, vZM, v) -C_O1_I4(r, r, ri, rI, r) C_O1_I4(r, r, rC, rI, r) C_O2_I1(o, m, r) C_O2_I2(o, m, 0, r) C_O2_I2(o, m, r, r) C_O2_I3(o, m, 0, 1, r) -C_N1_O1_I4(r, r, 0, 1, ri, r) -C_N1_O1_I4(r, r, 0, 1, rJU, r) diff --git a/tcg/s390x/tcg-target-con-str.h b/tcg/s390x/tcg-target-con-str.h index 3e574e0662..636a38a168 100644 --- a/tcg/s390x/tcg-target-con-str.h +++ b/tcg/s390x/tcg-target-con-str.h @@ -24,4 +24,5 @@ CONST('M', TCG_CT_CONST_M1) CONST('N', TCG_CT_CONST_INV) CONST('R', TCG_CT_CONST_INVRISBG) CONST('U', TCG_CT_CONST_U32) +CONST('V', TCG_CT_CONST_N32) CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/s390x/tcg-target-has.h b/tcg/s390x/tcg-target-has.h index e99e671642..0aeb5ba01a 100644 --- a/tcg/s390x/tcg-target-has.h +++ b/tcg/s390x/tcg-target-has.h @@ -29,65 +29,8 @@ extern uint64_t s390_facilities[3]; ((s390_facilities[FACILITY_##X / 64] >> (63 - FACILITY_##X % 64)) & 1) /* optional instructions */ -#define TCG_TARGET_HAS_div2_i32 1 -#define TCG_TARGET_HAS_rot_i32 1 -#define TCG_TARGET_HAS_ext8s_i32 1 -#define TCG_TARGET_HAS_ext16s_i32 1 -#define TCG_TARGET_HAS_ext8u_i32 1 -#define TCG_TARGET_HAS_ext16u_i32 1 -#define TCG_TARGET_HAS_bswap16_i32 1 -#define TCG_TARGET_HAS_bswap32_i32 1 -#define TCG_TARGET_HAS_not_i32 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_andc_i32 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_orc_i32 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_eqv_i32 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_nand_i32 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_nor_i32 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_clz_i32 0 -#define TCG_TARGET_HAS_ctz_i32 0 -#define TCG_TARGET_HAS_ctpop_i32 1 -#define TCG_TARGET_HAS_extract2_i32 0 -#define TCG_TARGET_HAS_negsetcond_i32 1 -#define TCG_TARGET_HAS_add2_i32 1 -#define TCG_TARGET_HAS_sub2_i32 1 -#define TCG_TARGET_HAS_mulu2_i32 0 -#define TCG_TARGET_HAS_muls2_i32 0 -#define TCG_TARGET_HAS_muluh_i32 0 -#define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_extr_i64_i32 0 -#define TCG_TARGET_HAS_qemu_st8_i32 0 - -#define TCG_TARGET_HAS_div2_i64 1 -#define TCG_TARGET_HAS_rot_i64 1 -#define TCG_TARGET_HAS_ext8s_i64 1 -#define TCG_TARGET_HAS_ext16s_i64 1 -#define TCG_TARGET_HAS_ext32s_i64 1 -#define TCG_TARGET_HAS_ext8u_i64 1 -#define TCG_TARGET_HAS_ext16u_i64 1 -#define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_bswap16_i64 1 -#define TCG_TARGET_HAS_bswap32_i64 1 -#define TCG_TARGET_HAS_bswap64_i64 1 -#define TCG_TARGET_HAS_not_i64 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_andc_i64 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_orc_i64 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_eqv_i64 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_nand_i64 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_nor_i64 HAVE_FACILITY(MISC_INSN_EXT3) -#define TCG_TARGET_HAS_clz_i64 1 -#define TCG_TARGET_HAS_ctz_i64 0 -#define TCG_TARGET_HAS_ctpop_i64 1 -#define TCG_TARGET_HAS_extract2_i64 0 -#define TCG_TARGET_HAS_negsetcond_i64 1 -#define TCG_TARGET_HAS_add2_i64 1 -#define TCG_TARGET_HAS_sub2_i64 1 -#define TCG_TARGET_HAS_mulu2_i64 1 -#define TCG_TARGET_HAS_muls2_i64 HAVE_FACILITY(MISC_INSN_EXT2) -#define TCG_TARGET_HAS_muluh_i64 0 -#define TCG_TARGET_HAS_mulsh_i64 0 - #define TCG_TARGET_HAS_qemu_ldst_i128 1 - #define TCG_TARGET_HAS_tst 1 #define TCG_TARGET_HAS_v64 HAVE_FACILITY(VECTOR) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index b2e1cd60ff..7ca0071f24 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -43,6 +43,7 @@ #define TCG_CT_CONST_INVRISBG (1 << 14) #define TCG_CT_CONST_CMP (1 << 15) #define TCG_CT_CONST_M1 (1 << 16) +#define TCG_CT_CONST_N32 (1 << 17) #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16) #define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32) @@ -134,6 +135,9 @@ typedef enum S390Opcode { RIEc_CLGIJ = 0xec7d, RIEc_CLIJ = 0xec7f, + RIEd_ALHSIK = 0xecda, + RIEd_ALGHSIK = 0xecdb, + RIEf_RISBG = 0xec55, RIEg_LOCGHI = 0xec46, @@ -172,6 +176,8 @@ typedef enum S390Opcode { RRE_SLBGR = 0xb989, RRE_XGR = 0xb982, + RRFa_ALRK = 0xb9fa, + RRFa_ALGRK = 0xb9ea, RRFa_MGRK = 0xb9ec, RRFa_MSRKC = 0xb9fd, RRFa_MSGRKC = 0xb9ed, @@ -613,7 +619,10 @@ static bool tcg_target_const_match(int64_t val, int ct, if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) { return true; } - if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) { + if ((ct & TCG_CT_CONST_U32) && uval <= UINT32_MAX) { + return true; + } + if ((ct & TCG_CT_CONST_N32) && -uval <= UINT32_MAX) { return true; } if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) { @@ -676,8 +685,16 @@ static void tcg_out_insn_RI(TCGContext *s, S390Opcode op, TCGReg r1, int i2) tcg_out32(s, (op << 16) | (r1 << 20) | (i2 & 0xffff)); } +static void tcg_out_insn_RIEd(TCGContext *s, S390Opcode op, + TCGReg r1, TCGReg r3, int i2) +{ + tcg_out16(s, (op & 0xff00) | (r1 << 4) | r3); + tcg_out16(s, i2); + tcg_out16(s, op & 0xff); +} + static void tcg_out_insn_RIEg(TCGContext *s, S390Opcode op, TCGReg r1, - int i2, int m3) + int i2, int m3) { tcg_out16(s, (op & 0xff00) | (r1 << 4) | m3); tcg_out32(s, (i2 << 16) | (op & 0xff)); @@ -951,25 +968,32 @@ static void tcg_out_movi(TCGContext *s, TCGType type, if (pc_off == (int32_t)pc_off) { tcg_out_insn(s, RIL, LARL, ret, pc_off); if (sval & 1) { - tcg_out_insn(s, RI, AGHI, ret, 1); + tcg_out_insn(s, RX, LA, ret, ret, TCG_REG_NONE, 1); } return; } - /* Otherwise, load it by parts. */ - i = is_const_p16((uint32_t)uval); - if (i >= 0) { - tcg_out_insn_RI(s, li_insns[i], ret, uval >> (i * 16)); - } else { - tcg_out_insn(s, RIL, LLILF, ret, uval); - } - uval >>= 32; - i = is_const_p16(uval); - if (i >= 0) { - tcg_out_insn_RI(s, oi_insns[i + 2], ret, uval >> (i * 16)); - } else { - tcg_out_insn(s, RIL, OIHF, ret, uval); + if (!s->carry_live) { + /* Load by parts, at most 2 instructions. */ + i = is_const_p16((uint32_t)uval); + if (i >= 0) { + tcg_out_insn_RI(s, li_insns[i], ret, uval >> (i * 16)); + } else { + tcg_out_insn(s, RIL, LLILF, ret, uval); + } + uval >>= 32; + i = is_const_p16(uval); + if (i >= 0) { + tcg_out_insn_RI(s, oi_insns[i + 2], ret, uval >> (i * 16)); + } else { + tcg_out_insn(s, RIL, OIHF, ret, uval); + } + return; } + + /* Otherwise, stuff it in the constant pool. */ + tcg_out_insn(s, RIL, LGRL, ret, 0); + new_pool_label(s, sval, R_390_PC32DBL, s->code_ptr - 2, 2); } /* Emit a load/store type instruction. Inputs are: @@ -1370,9 +1394,9 @@ static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1, return tgen_cmp2(s, type, c, r1, c2, c2const, need_carry, &inv_cc); } -static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, - TCGReg dest, TCGReg c1, TCGArg c2, - bool c2const, bool neg) +static void tgen_setcond_int(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg c1, TCGArg c2, + bool c2const, bool neg) { int cc; @@ -1464,6 +1488,42 @@ static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, tcg_out_insn(s, RRFc, LOCGR, dest, TCG_TMP0, cc); } +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tgen_setcond_int(s, type, cond, dest, arg1, arg2, false, false); +} + +static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tgen_setcond_int(s, type, cond, dest, arg1, arg2, true, false); +} + +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(r, r, rC), + .out_rrr = tgen_setcond, + .out_rri = tgen_setcondi, +}; + +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tgen_setcond_int(s, type, cond, dest, arg1, arg2, false, true); +} + +static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tgen_setcond_int(s, type, cond, dest, arg1, arg2, true, true); +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(r, r, rC), + .out_rrr = tgen_negsetcond, + .out_rri = tgen_negsetcondi, +}; + static void tgen_movcond_int(TCGContext *s, TCGType type, TCGReg dest, TCGArg v3, int v3const, TCGReg v4, int cc, int inv_cc) @@ -1504,9 +1564,9 @@ static void tgen_movcond_int(TCGContext *s, TCGType type, TCGReg dest, tcg_out_insn(s, RRFc, LOCGR, dest, src, cc); } -static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, TCGReg dest, - TCGReg c1, TCGArg c2, int c2const, - TCGArg v3, int v3const, TCGReg v4) +static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, + TCGReg dest, TCGReg c1, TCGArg c2, bool c2const, + TCGArg v3, bool v3const, TCGArg v4, bool v4const) { int cc, inv_cc; @@ -1514,63 +1574,47 @@ static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, TCGReg dest, tgen_movcond_int(s, type, dest, v3, v3const, v4, cc, inv_cc); } -static void tgen_clz(TCGContext *s, TCGReg dest, TCGReg a1, - TCGArg a2, int a2const) -{ - /* Since this sets both R and R+1, we have no choice but to store the - result into R0, allowing R1 == TCG_TMP0 to be clobbered as well. */ - QEMU_BUILD_BUG_ON(TCG_TMP0 != TCG_REG_R1); - tcg_out_insn(s, RRE, FLOGR, TCG_REG_R0, a1); +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = C_O1_I4(r, r, rC, rI, r), + .out = tgen_movcond, +}; - if (a2const && a2 == 64) { - tcg_out_mov(s, TCG_TYPE_I64, dest, TCG_REG_R0); - return; - } +static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned ofs, unsigned len) +{ + unsigned lsb = (63 - ofs); + unsigned msb = lsb - (len - 1); /* - * Conditions from FLOGR are: - * 2 -> one bit found - * 8 -> no one bit found + * Since we can't support "0Z" as a constraint, we allow a1 in + * any register. Fix things up as if a matching constraint. */ - tgen_movcond_int(s, TCG_TYPE_I64, dest, a2, a2const, TCG_REG_R0, 8, 2); -} - -static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) -{ - /* With MIE3, and bit 0 of m4 set, we get the complete result. */ - if (HAVE_FACILITY(MISC_INSN_EXT3)) { - if (type == TCG_TYPE_I32) { - tcg_out_ext32u(s, dest, src); - src = dest; + if (a0 != a1) { + if (a0 == a2) { + tcg_out_mov(s, type, TCG_TMP0, a2); + a2 = TCG_TMP0; } - tcg_out_insn(s, RRFc, POPCNT, dest, src, 8); - return; - } - - /* Without MIE3, each byte gets the count of bits for the byte. */ - tcg_out_insn(s, RRFc, POPCNT, dest, src, 0); - - /* Multiply to sum each byte at the top of the word. */ - if (type == TCG_TYPE_I32) { - tcg_out_insn(s, RIL, MSFI, dest, 0x01010101); - tcg_out_sh32(s, RS_SRL, dest, TCG_REG_NONE, 24); - } else { - tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 0x0101010101010101ull); - tcg_out_insn(s, RRE, MSGR, dest, TCG_TMP0); - tcg_out_sh64(s, RSY_SRLG, dest, dest, TCG_REG_NONE, 56); + tcg_out_mov(s, type, a0, a1); } + tcg_out_risbg(s, a0, a2, msb, lsb, ofs, false); } -static void tgen_deposit(TCGContext *s, TCGReg dest, TCGReg src, - int ofs, int len, int z) +static void tgen_depositz(TCGContext *s, TCGType type, TCGReg a0, TCGReg a2, + unsigned ofs, unsigned len) { - int lsb = (63 - ofs); - int msb = lsb - (len - 1); - tcg_out_risbg(s, dest, src, msb, lsb, ofs, z); + unsigned lsb = (63 - ofs); + unsigned msb = lsb - (len - 1); + tcg_out_risbg(s, a0, a2, msb, lsb, ofs, true); } -static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src, - int ofs, int len) +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_O1_I2(r, rZ, r), + .out_rrr = tgen_deposit, + .out_rzr = tgen_depositz, +}; + +static void tgen_extract(TCGContext *s, TCGType type, TCGReg dest, + TCGReg src, unsigned ofs, unsigned len) { if (ofs == 0) { switch (len) { @@ -1588,8 +1632,13 @@ static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src, tcg_out_risbg(s, dest, src, 64 - len, 63, 64 - ofs, 1); } -static void tgen_sextract(TCGContext *s, TCGReg dest, TCGReg src, - int ofs, int len) +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extract, +}; + +static void tgen_sextract(TCGContext *s, TCGType type, TCGReg dest, + TCGReg src, unsigned ofs, unsigned len) { if (ofs == 0) { switch (len) { @@ -1607,6 +1656,15 @@ static void tgen_sextract(TCGContext *s, TCGReg dest, TCGReg src, g_assert_not_reached(); } +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_sextract, +}; + +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_NotImplemented, +}; + static void tgen_gotoi(TCGContext *s, int cc, const tcg_insn_unit *dest) { ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1; @@ -1631,6 +1689,11 @@ static void tgen_branch(TCGContext *s, int cc, TCGLabel *l) } } +static void tcg_out_br(TCGContext *s, TCGLabel *l) +{ + tgen_branch(s, S390_CC_ALWAYS, l); +} + static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc, TCGReg r1, TCGReg r2, TCGLabel *l) { @@ -1704,6 +1767,24 @@ static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c, tgen_branch(s, cc, l); } +static void tgen_brcondr(TCGContext *s, TCGType type, TCGCond c, + TCGReg a0, TCGReg a1, TCGLabel *l) +{ + tgen_brcond(s, type, c, a0, a1, false, l); +} + +static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond c, + TCGReg a0, tcg_target_long a1, TCGLabel *l) +{ + tgen_brcond(s, type, c, a0, a1, true, l); +} + +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, rC), + .out_rr = tgen_brcondr, + .out_ri = tgen_brcondi, +}; + static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *dest) { ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1; @@ -2000,8 +2081,8 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, return ldst; } -static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType data_type) +static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data_reg, + TCGReg addr_reg, MemOpIdx oi) { TCGLabelQemuLdst *ldst; HostAddress h; @@ -2010,14 +2091,19 @@ static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, tcg_out_qemu_ld_direct(s, get_memop(oi), data_reg, h); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = data_reg; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } -static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType data_type) +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_qemu_ld, +}; + +static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data_reg, + TCGReg addr_reg, MemOpIdx oi) { TCGLabelQemuLdst *ldst; HostAddress h; @@ -2026,12 +2112,17 @@ static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, tcg_out_qemu_st_direct(s, get_memop(oi), data_reg, h); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = data_reg; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = C_O0_I2(r, r), + .out = tgen_qemu_st, +}; + static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi, TCGReg addr_reg, MemOpIdx oi, bool is_ld) { @@ -2106,6 +2197,28 @@ static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi, } } +static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr_reg, MemOpIdx oi) +{ + tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, true); +} + +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + .base.static_constraint = C_O2_I1(o, m, r), + .out = tgen_qemu_ld2, +}; + +static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr_reg, MemOpIdx oi) +{ + tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, false); +} + +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + .base.static_constraint = C_O0_I3(o, m, r), + .out = tgen_qemu_st2, +}; + static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) { /* Reuse the zeroing that exists for goto_ptr. */ @@ -2132,6 +2245,11 @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0); +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { @@ -2145,660 +2263,903 @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, /* no need to flush icache explicitly */ } -# define OP_32_64(x) \ - case glue(glue(INDEX_op_,x),_i32): \ - case glue(glue(INDEX_op_,x),_i64) -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - S390Opcode op, op2; - TCGArg a0, a1, a2; + if (a0 != a1) { + tcg_out_insn(s, RX, LA, a0, a1, a2, 0); + } else if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RR, AR, a0, a2); + } else { + tcg_out_insn(s, RRE, AGR, a0, a2); + } +} - switch (opc) { - case INDEX_op_goto_ptr: - a0 = args[0]; - tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0); - break; +static void tgen_addi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (a0 == a1) { + if (type == TCG_TYPE_I32) { + if (a2 == (int16_t)a2) { + tcg_out_insn(s, RI, AHI, a0, a2); + } else { + tcg_out_insn(s, RIL, AFI, a0, a2); + } + return; + } + if (a2 == (int16_t)a2) { + tcg_out_insn(s, RI, AGHI, a0, a2); + return; + } + if (a2 == (int32_t)a2) { + tcg_out_insn(s, RIL, AGFI, a0, a2); + return; + } + if (a2 == (uint32_t)a2) { + tcg_out_insn(s, RIL, ALGFI, a0, a2); + return; + } + if (-a2 == (uint32_t)-a2) { + tcg_out_insn(s, RIL, SLGFI, a0, -a2); + return; + } + } + tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2); +} - OP_32_64(ld8u): - /* ??? LLC (RXY format) is only present with the extended-immediate - facility, whereas LLGC is always present. */ - tcg_out_mem(s, 0, RXY_LLGC, args[0], args[1], TCG_REG_NONE, args[2]); - break; +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_add, + .out_rri = tgen_addi, +}; - OP_32_64(ld8s): - /* ??? LB is no smaller than LGB, so no point to using it. */ - tcg_out_mem(s, 0, RXY_LGB, args[0], args[1], TCG_REG_NONE, args[2]); - break; +static void tgen_addco_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type != TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, ALGRK, a0, a1, a2); + } else if (a0 == a1) { + tcg_out_insn(s, RR, ALR, a0, a2); + } else { + tcg_out_insn(s, RRFa, ALRK, a0, a1, a2); + } +} - OP_32_64(ld16u): - /* ??? LLH (RXY format) is only present with the extended-immediate - facility, whereas LLGH is always present. */ - tcg_out_mem(s, 0, RXY_LLGH, args[0], args[1], TCG_REG_NONE, args[2]); - break; +static void tgen_addco_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (a2 == (int16_t)a2) { + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RIEd, ALHSIK, a0, a1, a2); + } else { + tcg_out_insn(s, RIEd, ALGHSIK, a0, a1, a2); + } + return; + } - case INDEX_op_ld16s_i32: - tcg_out_mem(s, RX_LH, RXY_LHY, args[0], args[1], TCG_REG_NONE, args[2]); - break; + tcg_out_mov(s, type, a0, a1); + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RIL, ALFI, a0, a2); + } else if (a2 >= 0) { + tcg_out_insn(s, RIL, ALGFI, a0, a2); + } else { + tcg_out_insn(s, RIL, SLGFI, a0, -a2); + } +} - case INDEX_op_ld_i32: - tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]); - break; +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_O1_I2(r, r, rUV), + .out_rrr = tgen_addco_rrr, + .out_rri = tgen_addco_rri, +}; - OP_32_64(st8): - tcg_out_mem(s, RX_STC, RXY_STCY, args[0], args[1], - TCG_REG_NONE, args[2]); - break; +static void tgen_addcio(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RRE, ALCR, a0, a2); + } else { + tcg_out_insn(s, RRE, ALCGR, a0, a2); + } +} - OP_32_64(st16): - tcg_out_mem(s, RX_STH, RXY_STHY, args[0], args[1], - TCG_REG_NONE, args[2]); - break; +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_O1_I2(r, 0, r), + .out_rrr = tgen_addcio, +}; - case INDEX_op_st_i32: - tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]); - break; +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_O1_I2(r, 0, r), + .out_rrr = tgen_addcio, +}; - case INDEX_op_add_i32: - a0 = args[0], a1 = args[1], a2 = (int32_t)args[2]; - if (const_args[2]) { - do_addi_32: - if (a0 == a1) { - if (a2 == (int16_t)a2) { - tcg_out_insn(s, RI, AHI, a0, a2); - break; - } - tcg_out_insn(s, RIL, AFI, a0, a2); - break; - } - tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2); - } else if (a0 == a1) { - tcg_out_insn(s, RR, AR, a0, a2); - } else { - tcg_out_insn(s, RX, LA, a0, a1, a2, 0); - } - break; - case INDEX_op_sub_i32: - a0 = args[0], a1 = args[1], a2 = (int32_t)args[2]; - if (const_args[2]) { - a2 = -a2; - goto do_addi_32; - } else if (a0 == a1) { - tcg_out_insn(s, RR, SR, a0, a2); - } else { - tcg_out_insn(s, RRFa, SRK, a0, a1, a2); - } - break; +static void tcg_out_set_carry(TCGContext *s) +{ + tcg_out_insn(s, RR, SLR, TCG_REG_R0, TCG_REG_R0); /* cc = 2 */ +} - case INDEX_op_and_i32: - a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I32, a0, a1); - tgen_andi(s, TCG_TYPE_I32, a0, a2); - } else if (a0 == a1) { - tcg_out_insn(s, RR, NR, a0, a2); - } else { - tcg_out_insn(s, RRFa, NRK, a0, a1, a2); - } - break; - case INDEX_op_or_i32: - a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I32, a0, a1); - tgen_ori(s, a0, a2); - } else if (a0 == a1) { - tcg_out_insn(s, RR, OR, a0, a2); - } else { - tcg_out_insn(s, RRFa, ORK, a0, a1, a2); - } - break; - case INDEX_op_xor_i32: - a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I32, a0, a1); - tcg_out_insn(s, RIL, XILF, a0, a2); - } else if (a0 == a1) { - tcg_out_insn(s, RR, XR, args[0], args[2]); - } else { - tcg_out_insn(s, RRFa, XRK, a0, a1, a2); - } - break; +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type != TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, NGRK, a0, a1, a2); + } else if (a0 == a1) { + tcg_out_insn(s, RR, NR, a0, a2); + } else { + tcg_out_insn(s, RRFa, NRK, a0, a1, a2); + } +} - case INDEX_op_andc_i32: - a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I32, a0, a1); - tgen_andi(s, TCG_TYPE_I32, a0, (uint32_t)~a2); - } else { - tcg_out_insn(s, RRFa, NCRK, a0, a1, a2); - } - break; - case INDEX_op_orc_i32: - a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I32, a0, a1); - tgen_ori(s, a0, (uint32_t)~a2); - } else { - tcg_out_insn(s, RRFa, OCRK, a0, a1, a2); - } - break; - case INDEX_op_eqv_i32: - a0 = args[0], a1 = args[1], a2 = (uint32_t)args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I32, a0, a1); - tcg_out_insn(s, RIL, XILF, a0, ~a2); - } else { - tcg_out_insn(s, RRFa, NXRK, a0, a1, a2); +static void tgen_andi_3(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_mov(s, type, a0, a1); + tgen_andi(s, type, a0, a2); +} + +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, r, rNKR), + .out_rrr = tgen_and, + .out_rri = tgen_andi_3, +}; + +static void tgen_andc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, NCRK, a0, a1, a2); + } else { + tcg_out_insn(s, RRFa, NCGRK, a0, a1, a2); + } +} + +static TCGConstraintSetIndex cset_misc3_rrr(TCGType type, unsigned flags) +{ + return HAVE_FACILITY(MISC_INSN_EXT3) ? C_O1_I2(r, r, r) : C_NotImplemented; +} + +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_misc3_rrr, + .out_rrr = tgen_andc, +}; + +static void tgen_clz_int(TCGContext *s, TCGReg dest, TCGReg a1, + TCGArg a2, int a2const) +{ + /* + * Since this sets both R and R+1, we have no choice but to store the + * result into R0, allowing R1 == TCG_TMP0 to be clobbered as well. + */ + QEMU_BUILD_BUG_ON(TCG_TMP0 != TCG_REG_R1); + tcg_out_insn(s, RRE, FLOGR, TCG_REG_R0, a1); + + if (a2const && a2 == 64) { + tcg_out_mov(s, TCG_TYPE_I64, dest, TCG_REG_R0); + return; + } + + /* + * Conditions from FLOGR are: + * 2 -> one bit found + * 8 -> no one bit found + */ + tgen_movcond_int(s, TCG_TYPE_I64, dest, a2, a2const, TCG_REG_R0, 8, 2); +} + +static void tgen_clz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_clz_int(s, a0, a1, a2, false); +} + +static void tgen_clzi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_clz_int(s, a0, a1, a2, true); +} + +static TCGConstraintSetIndex cset_clz(TCGType type, unsigned flags) +{ + return type == TCG_TYPE_I64 ? C_O1_I2(r, r, rI) : C_NotImplemented; +} + +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_clz, + .out_rrr = tgen_clz, + .out_rri = tgen_clzi, +}; + +static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) +{ + /* With MIE3, and bit 0 of m4 set, we get the complete result. */ + if (HAVE_FACILITY(MISC_INSN_EXT3)) { + if (type == TCG_TYPE_I32) { + tcg_out_ext32u(s, dest, src); + src = dest; } - break; - case INDEX_op_nand_i32: - tcg_out_insn(s, RRFa, NNRK, args[0], args[1], args[2]); - break; - case INDEX_op_nor_i32: - tcg_out_insn(s, RRFa, NORK, args[0], args[1], args[2]); - break; + tcg_out_insn(s, RRFc, POPCNT, dest, src, 8); + return; + } - case INDEX_op_neg_i32: - tcg_out_insn(s, RR, LCR, args[0], args[1]); - break; - case INDEX_op_not_i32: - tcg_out_insn(s, RRFa, NORK, args[0], args[1], args[1]); - break; + /* Without MIE3, each byte gets the count of bits for the byte. */ + tcg_out_insn(s, RRFc, POPCNT, dest, src, 0); - case INDEX_op_mul_i32: - a0 = args[0], a1 = args[1], a2 = (int32_t)args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I32, a0, a1); - if (a2 == (int16_t)a2) { - tcg_out_insn(s, RI, MHI, a0, a2); - } else { - tcg_out_insn(s, RIL, MSFI, a0, a2); - } - } else if (a0 == a1) { + /* Multiply to sum each byte at the top of the word. */ + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RIL, MSFI, dest, 0x01010101); + tcg_out_sh32(s, RS_SRL, dest, TCG_REG_NONE, 24); + } else { + tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 0x0101010101010101ull); + tcg_out_insn(s, RRE, MSGR, dest, TCG_TMP0); + tcg_out_sh64(s, RSY_SRLG, dest, dest, TCG_REG_NONE, 56); + } +} + +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_ctpop, +}; + +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divs2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a4) +{ + tcg_debug_assert((a1 & 1) == 0); + tcg_debug_assert(a0 == a1 + 1); + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RR, DR, a1, a4); + } else { + /* + * TODO: Move the sign-extend of the numerator from a2 into a3 + * into the tcg backend, instead of in early expansion. It is + * required for 32-bit DR, but not 64-bit DSGR. + */ + tcg_out_insn(s, RRE, DSGR, a1, a4); + } +} + +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_O2_I3(o, m, 0, 1, r), + .out_rr01r = tgen_divs2, +}; + +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divu2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a4) +{ + tcg_debug_assert((a1 & 1) == 0); + tcg_debug_assert(a0 == a1 + 1); + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RRE, DLR, a1, a4); + } else { + tcg_out_insn(s, RRE, DLGR, a1, a4); + } +} + +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_O2_I3(o, m, 0, 1, r), + .out_rr01r = tgen_divu2, +}; + +static void tgen_eqv(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, NXRK, a0, a1, a2); + } else { + tcg_out_insn(s, RRFa, NXGRK, a0, a1, a2); + } +} + +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_misc3_rrr, + .out_rrr = tgen_eqv, +}; + +static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_sh64(s, RSY_SRLG, a0, a1, TCG_REG_NONE, 32); +} + +static const TCGOutOpUnary outop_extrh_i64_i32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extrh_i64_i32, +}; + +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + if (a0 == a1) { tcg_out_insn(s, RRE, MSR, a0, a2); } else { tcg_out_insn(s, RRFa, MSRKC, a0, a1, a2); } - break; - - case INDEX_op_div2_i32: - tcg_debug_assert(args[0] == args[2]); - tcg_debug_assert(args[1] == args[3]); - tcg_debug_assert((args[1] & 1) == 0); - tcg_debug_assert(args[0] == args[1] + 1); - tcg_out_insn(s, RR, DR, args[1], args[4]); - break; - case INDEX_op_divu2_i32: - tcg_debug_assert(args[0] == args[2]); - tcg_debug_assert(args[1] == args[3]); - tcg_debug_assert((args[1] & 1) == 0); - tcg_debug_assert(args[0] == args[1] + 1); - tcg_out_insn(s, RRE, DLR, args[1], args[4]); - break; - - case INDEX_op_shl_i32: - op = RS_SLL; - op2 = RSY_SLLK; - do_shift32: - a0 = args[0], a1 = args[1], a2 = (int32_t)args[2]; + } else { if (a0 == a1) { - if (const_args[2]) { - tcg_out_sh32(s, op, a0, TCG_REG_NONE, a2); - } else { - tcg_out_sh32(s, op, a0, a2, 0); - } + tcg_out_insn(s, RRE, MSGR, a0, a2); } else { - /* Using tcg_out_sh64 here for the format; it is a 32-bit shift. */ - if (const_args[2]) { - tcg_out_sh64(s, op2, a0, a1, TCG_REG_NONE, a2); - } else { - tcg_out_sh64(s, op2, a0, a1, a2, 0); - } + tcg_out_insn(s, RRFa, MSGRKC, a0, a1, a2); } - break; - case INDEX_op_shr_i32: - op = RS_SRL; - op2 = RSY_SRLK; - goto do_shift32; - case INDEX_op_sar_i32: - op = RS_SRA; - op2 = RSY_SRAK; - goto do_shift32; + } +} - case INDEX_op_rotl_i32: - /* ??? Using tcg_out_sh64 here for the format; it is a 32-bit rol. */ - if (const_args[2]) { - tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_REG_NONE, args[2]); +static void tgen_muli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_mov(s, type, a0, a1); + if (type == TCG_TYPE_I32) { + if (a2 == (int16_t)a2) { + tcg_out_insn(s, RI, MHI, a0, a2); } else { - tcg_out_sh64(s, RSY_RLL, args[0], args[1], args[2], 0); + tcg_out_insn(s, RIL, MSFI, a0, a2); } - break; - case INDEX_op_rotr_i32: - if (const_args[2]) { - tcg_out_sh64(s, RSY_RLL, args[0], args[1], - TCG_REG_NONE, (32 - args[2]) & 31); + } else { + if (a2 == (int16_t)a2) { + tcg_out_insn(s, RI, MGHI, a0, a2); } else { - tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]); - tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_TMP0, 0); + tcg_out_insn(s, RIL, MSGFI, a0, a2); } - break; + } +} + +static TCGConstraintSetIndex cset_mul(TCGType type, unsigned flags) +{ + return (HAVE_FACILITY(MISC_INSN_EXT2) + ? C_O1_I2(r, r, rJ) + : C_O1_I2(r, 0, rJ)); +} + +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mul, + .out_rrr = tgen_mul, + .out_rri = tgen_muli, +}; + +static void tgen_muls2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + tcg_debug_assert((a1 & 1) == 0); + tcg_debug_assert(a0 == a1 + 1); + tcg_out_insn(s, RRFa, MGRK, a1, a2, a3); +} + +static TCGConstraintSetIndex cset_muls2(TCGType type, unsigned flags) +{ + return (type == TCG_TYPE_I64 && HAVE_FACILITY(MISC_INSN_EXT2) + ? C_O2_I2(o, m, r, r) : C_NotImplemented); +} + +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_muls2, + .out_rrrr = tgen_muls2, +}; + +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_mulu2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + tcg_debug_assert(a0 == a2); + tcg_debug_assert((a1 & 1) == 0); + tcg_debug_assert(a0 == a1 + 1); + tcg_out_insn(s, RRE, MLGR, a1, a3); +} + +static TCGConstraintSetIndex cset_mulu2(TCGType type, unsigned flags) +{ + return (type == TCG_TYPE_I64 && HAVE_FACILITY(MISC_INSN_EXT2) + ? C_O2_I2(o, m, 0, r) : C_NotImplemented); +} + +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mulu2, + .out_rrrr = tgen_mulu2, +}; + +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_nand(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, NNRK, a0, a1, a2); + } else { + tcg_out_insn(s, RRFa, NNGRK, a0, a1, a2); + } +} + +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_misc3_rrr, + .out_rrr = tgen_nand, +}; + +static void tgen_nor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, NORK, a0, a1, a2); + } else { + tcg_out_insn(s, RRFa, NOGRK, a0, a1, a2); + } +} + +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_misc3_rrr, + .out_rrr = tgen_nor, +}; - case INDEX_op_bswap16_i32: - a0 = args[0], a1 = args[1], a2 = args[2]; +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type != TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, OGRK, a0, a1, a2); + } else if (a0 == a1) { + tcg_out_insn(s, RR, OR, a0, a2); + } else { + tcg_out_insn(s, RRFa, ORK, a0, a1, a2); + } +} + +static void tgen_ori_3(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_mov(s, type, a0, a1); + tgen_ori(s, a0, type == TCG_TYPE_I32 ? (uint32_t)a2 : a2); +} + +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, r, rK), + .out_rrr = tgen_or, + .out_rri = tgen_ori_3, +}; + +static void tgen_orc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, OCRK, a0, a1, a2); + } else { + tcg_out_insn(s, RRFa, OCGRK, a0, a1, a2); + } +} + +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_misc3_rrr, + .out_rrr = tgen_orc, +}; + +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_rotl_int(TCGContext *s, TCGType type, TCGReg dst, + TCGReg src, TCGReg v, tcg_target_long i) +{ + S390Opcode insn = type == TCG_TYPE_I32 ? RSY_RLL : RSY_RLLG; + tcg_out_sh64(s, insn, dst, src, v, i); +} + +static void tgen_rotl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_rotl_int(s, type, a0, a1, a2, 0); +} + +static void tgen_rotli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_rotl_int(s, type, a0, a1, TCG_REG_NONE, a2); +} + +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_rotl, + .out_rri = tgen_rotli, +}; + +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_sar_int(TCGContext *s, TCGType type, TCGReg dst, + TCGReg src, TCGReg v, tcg_target_long i) +{ + if (type != TCG_TYPE_I32) { + tcg_out_sh64(s, RSY_SRAG, dst, src, v, i); + } else if (dst == src) { + tcg_out_sh32(s, RS_SRA, dst, v, i); + } else { + tcg_out_sh64(s, RSY_SRAK, dst, src, v, i); + } +} + +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_sar_int(s, type, a0, a1, a2, 0); +} + +static void tgen_sari(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_sar_int(s, type, a0, a1, TCG_REG_NONE, a2); +} + +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_sar, + .out_rri = tgen_sari, +}; + +static void tgen_shl_int(TCGContext *s, TCGType type, TCGReg dst, + TCGReg src, TCGReg v, tcg_target_long i) +{ + if (type != TCG_TYPE_I32) { + tcg_out_sh64(s, RSY_SLLG, dst, src, v, i); + } else if (dst == src) { + tcg_out_sh32(s, RS_SLL, dst, v, i); + } else { + tcg_out_sh64(s, RSY_SLLK, dst, src, v, i); + } +} + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_shl_int(s, type, a0, a1, a2, 0); +} + +static void tgen_shli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_shl_int(s, type, a0, a1, TCG_REG_NONE, a2); +} + +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shl, + .out_rri = tgen_shli, +}; + +static void tgen_shr_int(TCGContext *s, TCGType type, TCGReg dst, + TCGReg src, TCGReg v, tcg_target_long i) +{ + if (type != TCG_TYPE_I32) { + tcg_out_sh64(s, RSY_SRLG, dst, src, v, i); + } else if (dst == src) { + tcg_out_sh32(s, RS_SRL, dst, v, i); + } else { + tcg_out_sh64(s, RSY_SRLK, dst, src, v, i); + } +} + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_shr_int(s, type, a0, a1, a2, 0); +} + +static void tgen_shri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_shr_int(s, type, a0, a1, TCG_REG_NONE, a2); +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_O1_I2(r, r, ri), + .out_rrr = tgen_shr, + .out_rri = tgen_shri, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type != TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, SGRK, a0, a1, a2); + } else if (a0 == a1) { + tcg_out_insn(s, RR, SR, a0, a2); + } else { + tcg_out_insn(s, RRFa, SRK, a0, a1, a2); + } +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_sub, +}; + +static void tgen_subbo_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type != TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, SLGRK, a0, a1, a2); + } else if (a0 == a1) { + tcg_out_insn(s, RR, SLR, a0, a2); + } else { + tcg_out_insn(s, RRFa, SLRK, a0, a1, a2); + } +} + +static void tgen_subbo_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_mov(s, type, a0, a1); + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RIL, SLFI, a0, a2); + } else if (a2 >= 0) { + tcg_out_insn(s, RIL, SLGFI, a0, a2); + } else { + tcg_out_insn(s, RIL, ALGFI, a0, -a2); + } +} + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_O1_I2(r, r, rUV), + .out_rrr = tgen_subbo_rrr, + .out_rri = tgen_subbo_rri, +}; + +static void tgen_subbio(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RRE, SLBR, a0, a2); + } else { + tcg_out_insn(s, RRE, SLBGR, a0, a2); + } +} + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_O1_I2(r, 0, r), + .out_rrr = tgen_subbio, +}; + +#define outop_subbi outop_subbio + +static void tcg_out_set_borrow(TCGContext *s) +{ + tcg_out_insn(s, RR, CLR, TCG_REG_R0, TCG_REG_R0); /* cc = 0 */ +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type != TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, XGRK, a0, a1, a2); + } else if (a0 == a1) { + tcg_out_insn(s, RR, XR, a0, a2); + } else { + tcg_out_insn(s, RRFa, XRK, a0, a1, a2); + } +} + +static void tgen_xori_3(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_mov(s, type, a0, a1); + tgen_xori(s, a0, type == TCG_TYPE_I32 ? (uint32_t)a2 : a2); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, r, rK), + .out_rrr = tgen_xor, + .out_rri = tgen_xori_3, +}; + +static void tgen_bswap16(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + if (type == TCG_TYPE_I32) { tcg_out_insn(s, RRE, LRVR, a0, a1); - if (a2 & TCG_BSWAP_OS) { - tcg_out_sh32(s, RS_SRA, a0, TCG_REG_NONE, 16); - } else { - tcg_out_sh32(s, RS_SRL, a0, TCG_REG_NONE, 16); - } - break; - case INDEX_op_bswap16_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; + tcg_out_sh32(s, (flags & TCG_BSWAP_OS ? RS_SRA : RS_SRL), + a0, TCG_REG_NONE, 16); + } else { tcg_out_insn(s, RRE, LRVGR, a0, a1); - if (a2 & TCG_BSWAP_OS) { - tcg_out_sh64(s, RSY_SRAG, a0, a0, TCG_REG_NONE, 48); - } else { - tcg_out_sh64(s, RSY_SRLG, a0, a0, TCG_REG_NONE, 48); - } - break; + tcg_out_sh64(s, (flags & TCG_BSWAP_OS ? RSY_SRAG : RSY_SRLG), + a0, a0, TCG_REG_NONE, 48); + } +} - case INDEX_op_bswap32_i32: - tcg_out_insn(s, RRE, LRVR, args[0], args[1]); - break; - case INDEX_op_bswap32_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - tcg_out_insn(s, RRE, LRVR, a0, a1); - if (a2 & TCG_BSWAP_OS) { - tcg_out_ext32s(s, a0, a0); - } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { - tcg_out_ext32u(s, a0, a0); - } - break; +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap16, +}; - case INDEX_op_add2_i32: - if (const_args[4]) { - tcg_out_insn(s, RIL, ALFI, args[0], args[4]); - } else { - tcg_out_insn(s, RR, ALR, args[0], args[4]); - } - tcg_out_insn(s, RRE, ALCR, args[1], args[5]); - break; - case INDEX_op_sub2_i32: - if (const_args[4]) { - tcg_out_insn(s, RIL, SLFI, args[0], args[4]); - } else { - tcg_out_insn(s, RR, SLR, args[0], args[4]); - } - tcg_out_insn(s, RRE, SLBR, args[1], args[5]); - break; +static void tgen_bswap32(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_insn(s, RRE, LRVR, a0, a1); + if (flags & TCG_BSWAP_OS) { + tcg_out_ext32s(s, a0, a0); + } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { + tcg_out_ext32u(s, a0, a0); + } +} - case INDEX_op_br: - tgen_branch(s, S390_CC_ALWAYS, arg_label(args[0])); - break; +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap32, +}; - case INDEX_op_brcond_i32: - tgen_brcond(s, TCG_TYPE_I32, args[2], args[0], - args[1], const_args[1], arg_label(args[3])); - break; - case INDEX_op_setcond_i32: - tgen_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], - args[2], const_args[2], false); - break; - case INDEX_op_negsetcond_i32: - tgen_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], - args[2], const_args[2], true); - break; - case INDEX_op_movcond_i32: - tgen_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1], - args[2], const_args[2], args[3], const_args[3], args[4]); - break; +static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_insn(s, RRE, LRVGR, a0, a1); +} - case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args[0], args[1], args[2], TCG_TYPE_I32); - break; - case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args[0], args[1], args[2], TCG_TYPE_I64); - break; - case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args[0], args[1], args[2], TCG_TYPE_I32); - break; - case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args[0], args[1], args[2], TCG_TYPE_I64); - break; - case INDEX_op_qemu_ld_i128: - tcg_out_qemu_ldst_i128(s, args[0], args[1], args[2], args[3], true); - break; - case INDEX_op_qemu_st_i128: - tcg_out_qemu_ldst_i128(s, args[0], args[1], args[2], args[3], false); - break; +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap64, +}; - case INDEX_op_ld16s_i64: - tcg_out_mem(s, 0, RXY_LGH, args[0], args[1], TCG_REG_NONE, args[2]); - break; - case INDEX_op_ld32u_i64: - tcg_out_mem(s, 0, RXY_LLGF, args[0], args[1], TCG_REG_NONE, args[2]); - break; - case INDEX_op_ld32s_i64: - tcg_out_mem(s, 0, RXY_LGF, args[0], args[1], TCG_REG_NONE, args[2]); - break; - case INDEX_op_ld_i64: - tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]); - break; +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RR, LCR, a0, a1); + } else { + tcg_out_insn(s, RRE, LCGR, a0, a1); + } +} - case INDEX_op_st32_i64: - tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]); - break; - case INDEX_op_st_i64: - tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]); - break; +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_neg, +}; - case INDEX_op_add_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - do_addi_64: - if (a0 == a1) { - if (a2 == (int16_t)a2) { - tcg_out_insn(s, RI, AGHI, a0, a2); - break; - } - if (a2 == (int32_t)a2) { - tcg_out_insn(s, RIL, AGFI, a0, a2); - break; - } - if (a2 == (uint32_t)a2) { - tcg_out_insn(s, RIL, ALGFI, a0, a2); - break; - } - if (-a2 == (uint32_t)-a2) { - tcg_out_insn(s, RIL, SLGFI, a0, -a2); - break; - } - } - tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2); - } else if (a0 == a1) { - tcg_out_insn(s, RRE, AGR, a0, a2); - } else { - tcg_out_insn(s, RX, LA, a0, a1, a2, 0); - } - break; - case INDEX_op_sub_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - a2 = -a2; - goto do_addi_64; - } else { - tcg_out_insn(s, RRFa, SGRK, a0, a1, a2); - } - break; +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_nor(s, type, a0, a1, a1); +} - case INDEX_op_and_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I64, a0, a1); - tgen_andi(s, TCG_TYPE_I64, args[0], args[2]); - } else { - tcg_out_insn(s, RRFa, NGRK, a0, a1, a2); - } - break; - case INDEX_op_or_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I64, a0, a1); - tgen_ori(s, a0, a2); - } else { - tcg_out_insn(s, RRFa, OGRK, a0, a1, a2); - } - break; - case INDEX_op_xor_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I64, a0, a1); - tgen_xori(s, a0, a2); - } else { - tcg_out_insn(s, RRFa, XGRK, a0, a1, a2); - } - break; +static TCGConstraintSetIndex cset_not(TCGType type, unsigned flags) +{ + return HAVE_FACILITY(MISC_INSN_EXT3) ? C_O1_I1(r, r) : C_NotImplemented; +} - case INDEX_op_andc_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I64, a0, a1); - tgen_andi(s, TCG_TYPE_I64, a0, ~a2); - } else { - tcg_out_insn(s, RRFa, NCGRK, a0, a1, a2); - } - break; - case INDEX_op_orc_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I64, a0, a1); - tgen_ori(s, a0, ~a2); - } else { - tcg_out_insn(s, RRFa, OCGRK, a0, a1, a2); - } - break; - case INDEX_op_eqv_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I64, a0, a1); - tgen_xori(s, a0, ~a2); - } else { - tcg_out_insn(s, RRFa, NXGRK, a0, a1, a2); - } - break; - case INDEX_op_nand_i64: - tcg_out_insn(s, RRFa, NNGRK, args[0], args[1], args[2]); - break; - case INDEX_op_nor_i64: - tcg_out_insn(s, RRFa, NOGRK, args[0], args[1], args[2]); - break; +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_not, + .out_rr = tgen_not, +}; - case INDEX_op_neg_i64: - tcg_out_insn(s, RRE, LCGR, args[0], args[1]); - break; - case INDEX_op_not_i64: - tcg_out_insn(s, RRFa, NOGRK, args[0], args[1], args[1]); - break; - case INDEX_op_bswap64_i64: - tcg_out_insn(s, RRE, LRVGR, args[0], args[1]); - break; +static void tcg_out_mb(TCGContext *s, unsigned a0) +{ + /* + * The host memory model is quite strong, we simply need to + * serialize the instruction stream. + */ + if (a0 & TCG_MO_ST_LD) { + /* fast-bcr-serialization facility (45) is present */ + tcg_out_insn(s, RR, BCR, 14, 0); + } +} - case INDEX_op_mul_i64: - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[2]) { - tcg_out_mov(s, TCG_TYPE_I64, a0, a1); - if (a2 == (int16_t)a2) { - tcg_out_insn(s, RI, MGHI, a0, a2); - } else { - tcg_out_insn(s, RIL, MSGFI, a0, a2); - } - } else if (a0 == a1) { - tcg_out_insn(s, RRE, MSGR, a0, a2); - } else { - tcg_out_insn(s, RRFa, MSGRKC, a0, a1, a2); - } - break; +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem(s, 0, RXY_LLGC, dest, base, TCG_REG_NONE, offset); +} - case INDEX_op_div2_i64: - /* - * ??? We get an unnecessary sign-extension of the dividend - * into op0 with this definition, but as we do in fact always - * produce both quotient and remainder using INDEX_op_div_i64 - * instead requires jumping through even more hoops. - */ - tcg_debug_assert(args[0] == args[2]); - tcg_debug_assert(args[1] == args[3]); - tcg_debug_assert((args[1] & 1) == 0); - tcg_debug_assert(args[0] == args[1] + 1); - tcg_out_insn(s, RRE, DSGR, args[1], args[4]); - break; - case INDEX_op_divu2_i64: - tcg_debug_assert(args[0] == args[2]); - tcg_debug_assert(args[1] == args[3]); - tcg_debug_assert((args[1] & 1) == 0); - tcg_debug_assert(args[0] == args[1] + 1); - tcg_out_insn(s, RRE, DLGR, args[1], args[4]); - break; - case INDEX_op_mulu2_i64: - tcg_debug_assert(args[0] == args[2]); - tcg_debug_assert((args[1] & 1) == 0); - tcg_debug_assert(args[0] == args[1] + 1); - tcg_out_insn(s, RRE, MLGR, args[1], args[3]); - break; - case INDEX_op_muls2_i64: - tcg_debug_assert((args[1] & 1) == 0); - tcg_debug_assert(args[0] == args[1] + 1); - tcg_out_insn(s, RRFa, MGRK, args[1], args[2], args[3]); - break; - - case INDEX_op_shl_i64: - op = RSY_SLLG; - do_shift64: - if (const_args[2]) { - tcg_out_sh64(s, op, args[0], args[1], TCG_REG_NONE, args[2]); - } else { - tcg_out_sh64(s, op, args[0], args[1], args[2], 0); - } - break; - case INDEX_op_shr_i64: - op = RSY_SRLG; - goto do_shift64; - case INDEX_op_sar_i64: - op = RSY_SRAG; - goto do_shift64; +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; - case INDEX_op_rotl_i64: - if (const_args[2]) { - tcg_out_sh64(s, RSY_RLLG, args[0], args[1], - TCG_REG_NONE, args[2]); - } else { - tcg_out_sh64(s, RSY_RLLG, args[0], args[1], args[2], 0); - } - break; - case INDEX_op_rotr_i64: - if (const_args[2]) { - tcg_out_sh64(s, RSY_RLLG, args[0], args[1], - TCG_REG_NONE, (64 - args[2]) & 63); - } else { - /* We can use the smaller 32-bit negate because only the - low 6 bits are examined for the rotate. */ - tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]); - tcg_out_sh64(s, RSY_RLLG, args[0], args[1], TCG_TMP0, 0); - } - break; +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem(s, 0, RXY_LGB, dest, base, TCG_REG_NONE, offset); +} - case INDEX_op_add2_i64: - if (const_args[4]) { - if ((int64_t)args[4] >= 0) { - tcg_out_insn(s, RIL, ALGFI, args[0], args[4]); - } else { - tcg_out_insn(s, RIL, SLGFI, args[0], -args[4]); - } - } else { - tcg_out_insn(s, RRE, ALGR, args[0], args[4]); - } - tcg_out_insn(s, RRE, ALCGR, args[1], args[5]); - break; - case INDEX_op_sub2_i64: - if (const_args[4]) { - if ((int64_t)args[4] >= 0) { - tcg_out_insn(s, RIL, SLGFI, args[0], args[4]); - } else { - tcg_out_insn(s, RIL, ALGFI, args[0], -args[4]); - } - } else { - tcg_out_insn(s, RRE, SLGR, args[0], args[4]); - } - tcg_out_insn(s, RRE, SLBGR, args[1], args[5]); - break; +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; - case INDEX_op_brcond_i64: - tgen_brcond(s, TCG_TYPE_I64, args[2], args[0], - args[1], const_args[1], arg_label(args[3])); - break; - case INDEX_op_setcond_i64: - tgen_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], - args[2], const_args[2], false); - break; - case INDEX_op_negsetcond_i64: - tgen_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], - args[2], const_args[2], true); - break; - case INDEX_op_movcond_i64: - tgen_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1], - args[2], const_args[2], args[3], const_args[3], args[4]); - break; +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem(s, 0, RXY_LLGH, dest, base, TCG_REG_NONE, offset); +} - OP_32_64(deposit): - a0 = args[0], a1 = args[1], a2 = args[2]; - if (const_args[1]) { - tgen_deposit(s, a0, a2, args[3], args[4], 1); - } else { - /* Since we can't support "0Z" as a constraint, we allow a1 in - any register. Fix things up as if a matching constraint. */ - if (a0 != a1) { - if (a0 == a2) { - tcg_out_mov(s, type, TCG_TMP0, a2); - a2 = TCG_TMP0; - } - tcg_out_mov(s, type, a0, a1); - } - tgen_deposit(s, a0, a2, args[3], args[4], 0); - } - break; +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; - OP_32_64(extract): - tgen_extract(s, args[0], args[1], args[2], args[3]); - break; - OP_32_64(sextract): - tgen_sextract(s, args[0], args[1], args[2], args[3]); - break; +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + if (type == TCG_TYPE_I32) { + tcg_out_mem(s, RX_LH, RXY_LHY, dest, base, TCG_REG_NONE, offset); + } else { + tcg_out_mem(s, 0, RXY_LGH, dest, base, TCG_REG_NONE, offset); + } +} - case INDEX_op_clz_i64: - tgen_clz(s, args[0], args[1], args[2], const_args[2]); - break; +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; - case INDEX_op_ctpop_i32: - tgen_ctpop(s, TCG_TYPE_I32, args[0], args[1]); - break; - case INDEX_op_ctpop_i64: - tgen_ctpop(s, TCG_TYPE_I64, args[0], args[1]); - break; +static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem(s, 0, RXY_LLGF, dest, base, TCG_REG_NONE, offset); +} - case INDEX_op_mb: - /* The host memory model is quite strong, we simply need to - serialize the instruction stream. */ - if (args[0] & TCG_MO_ST_LD) { - /* fast-bcr-serialization facility (45) is present */ - tcg_out_insn(s, RR, BCR, 14, 0); - } - break; +static const TCGOutOpLoad outop_ld32u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32u, +}; - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_mov_i64: - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - default: - g_assert_not_reached(); - } +static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem(s, 0, RXY_LGF, dest, base, TCG_REG_NONE, offset); } +static const TCGOutOpLoad outop_ld32s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32s, +}; + +static void tgen_st8(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem(s, RX_STC, RXY_STCY, data, base, TCG_REG_NONE, offset); +} + +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tgen_st8, +}; + +static void tgen_st16(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_mem(s, RX_STH, RXY_STHY, data, base, TCG_REG_NONE, offset); +} + +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tgen_st16, +}; + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tcg_out_st, +}; + + static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg dst, TCGReg src) { @@ -3239,166 +3600,6 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) { switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i32: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i32: - case INDEX_op_ld16s_i64: - case INDEX_op_ld_i32: - case INDEX_op_ld32u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld_i64: - return C_O1_I1(r, r); - - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - case INDEX_op_st_i32: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: - return C_O0_I2(r, r); - - case INDEX_op_add_i32: - case INDEX_op_add_i64: - case INDEX_op_shl_i64: - case INDEX_op_shr_i64: - case INDEX_op_sar_i64: - case INDEX_op_rotl_i32: - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i32: - case INDEX_op_rotr_i64: - return C_O1_I2(r, r, ri); - case INDEX_op_setcond_i32: - case INDEX_op_negsetcond_i32: - case INDEX_op_setcond_i64: - case INDEX_op_negsetcond_i64: - return C_O1_I2(r, r, rC); - - case INDEX_op_clz_i64: - return C_O1_I2(r, r, rI); - - case INDEX_op_sub_i32: - case INDEX_op_sub_i64: - case INDEX_op_and_i32: - case INDEX_op_or_i32: - case INDEX_op_xor_i32: - return C_O1_I2(r, r, ri); - case INDEX_op_and_i64: - return C_O1_I2(r, r, rNKR); - case INDEX_op_or_i64: - case INDEX_op_xor_i64: - return C_O1_I2(r, r, rK); - - case INDEX_op_andc_i32: - case INDEX_op_orc_i32: - case INDEX_op_eqv_i32: - return C_O1_I2(r, r, ri); - case INDEX_op_andc_i64: - return C_O1_I2(r, r, rKR); - case INDEX_op_orc_i64: - case INDEX_op_eqv_i64: - return C_O1_I2(r, r, rNK); - - case INDEX_op_nand_i32: - case INDEX_op_nand_i64: - case INDEX_op_nor_i32: - case INDEX_op_nor_i64: - return C_O1_I2(r, r, r); - - case INDEX_op_mul_i32: - return (HAVE_FACILITY(MISC_INSN_EXT2) - ? C_O1_I2(r, r, ri) - : C_O1_I2(r, 0, ri)); - case INDEX_op_mul_i64: - return (HAVE_FACILITY(MISC_INSN_EXT2) - ? C_O1_I2(r, r, rJ) - : C_O1_I2(r, 0, rJ)); - - case INDEX_op_shl_i32: - case INDEX_op_shr_i32: - case INDEX_op_sar_i32: - return C_O1_I2(r, r, ri); - - case INDEX_op_brcond_i32: - return C_O0_I2(r, ri); - case INDEX_op_brcond_i64: - return C_O0_I2(r, rC); - - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i32: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: - case INDEX_op_neg_i32: - case INDEX_op_neg_i64: - case INDEX_op_not_i32: - case INDEX_op_not_i64: - case INDEX_op_ext8s_i32: - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extract_i32: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i32: - case INDEX_op_sextract_i64: - case INDEX_op_ctpop_i32: - case INDEX_op_ctpop_i64: - return C_O1_I1(r, r); - - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: - return C_O1_I1(r, r); - case INDEX_op_qemu_st_i64: - case INDEX_op_qemu_st_i32: - return C_O0_I2(r, r); - case INDEX_op_qemu_ld_i128: - return C_O2_I1(o, m, r); - case INDEX_op_qemu_st_i128: - return C_O0_I3(o, m, r); - - case INDEX_op_deposit_i32: - case INDEX_op_deposit_i64: - return C_O1_I2(r, rZ, r); - - case INDEX_op_movcond_i32: - return C_O1_I4(r, r, ri, rI, r); - case INDEX_op_movcond_i64: - return C_O1_I4(r, r, rC, rI, r); - - case INDEX_op_div2_i32: - case INDEX_op_div2_i64: - case INDEX_op_divu2_i32: - case INDEX_op_divu2_i64: - return C_O2_I3(o, m, 0, 1, r); - - case INDEX_op_mulu2_i64: - return C_O2_I2(o, m, 0, r); - case INDEX_op_muls2_i64: - return C_O2_I2(o, m, r, r); - - case INDEX_op_add2_i32: - case INDEX_op_sub2_i32: - return C_N1_O1_I4(r, r, 0, 1, ri, r); - - case INDEX_op_add2_i64: - case INDEX_op_sub2_i64: - return C_N1_O1_I4(r, r, 0, 1, rJU, r); - case INDEX_op_st_vec: return C_O0_I2(v, r); case INDEX_op_ld_vec: diff --git a/tcg/sparc64/tcg-target-con-set.h b/tcg/sparc64/tcg-target-con-set.h index 61f9fa3d9f..1a57adc0e8 100644 --- a/tcg/sparc64/tcg-target-con-set.h +++ b/tcg/sparc64/tcg-target-con-set.h @@ -11,10 +11,11 @@ */ C_O0_I1(r) C_O0_I2(rz, r) -C_O0_I2(rz, rJ) +C_O0_I2(r, rJ) C_O1_I1(r, r) C_O1_I2(r, r, r) +C_O1_I2(r, r, rJ) C_O1_I2(r, rz, rJ) -C_O1_I4(r, rz, rJ, rI, 0) -C_O2_I2(r, r, rz, rJ) -C_O2_I4(r, r, rz, rz, rJ, rJ) +C_O1_I2(r, rz, rz) +C_O1_I4(r, r, rJ, rI, 0) +C_O2_I2(r, r, r, r) diff --git a/tcg/sparc64/tcg-target-has.h b/tcg/sparc64/tcg-target-has.h index 2f46df8c61..b29fd177f6 100644 --- a/tcg/sparc64/tcg-target-has.h +++ b/tcg/sparc64/tcg-target-has.h @@ -7,74 +7,9 @@ #ifndef TCG_TARGET_HAS_H #define TCG_TARGET_HAS_H -#if defined(__VIS__) && __VIS__ >= 0x300 -#define use_vis3_instructions 1 -#else -extern bool use_vis3_instructions; -#endif - /* optional instructions */ -#define TCG_TARGET_HAS_div_i32 1 -#define TCG_TARGET_HAS_rem_i32 0 -#define TCG_TARGET_HAS_rot_i32 0 -#define TCG_TARGET_HAS_ext8s_i32 0 -#define TCG_TARGET_HAS_ext16s_i32 0 -#define TCG_TARGET_HAS_ext8u_i32 0 -#define TCG_TARGET_HAS_ext16u_i32 0 -#define TCG_TARGET_HAS_bswap16_i32 0 -#define TCG_TARGET_HAS_bswap32_i32 0 -#define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_andc_i32 1 -#define TCG_TARGET_HAS_orc_i32 1 -#define TCG_TARGET_HAS_eqv_i32 0 -#define TCG_TARGET_HAS_nand_i32 0 -#define TCG_TARGET_HAS_nor_i32 0 -#define TCG_TARGET_HAS_clz_i32 0 -#define TCG_TARGET_HAS_ctz_i32 0 -#define TCG_TARGET_HAS_ctpop_i32 0 -#define TCG_TARGET_HAS_extract2_i32 0 -#define TCG_TARGET_HAS_negsetcond_i32 1 -#define TCG_TARGET_HAS_add2_i32 1 -#define TCG_TARGET_HAS_sub2_i32 1 -#define TCG_TARGET_HAS_mulu2_i32 1 -#define TCG_TARGET_HAS_muls2_i32 1 -#define TCG_TARGET_HAS_muluh_i32 0 -#define TCG_TARGET_HAS_mulsh_i32 0 -#define TCG_TARGET_HAS_qemu_st8_i32 0 - #define TCG_TARGET_HAS_extr_i64_i32 0 -#define TCG_TARGET_HAS_div_i64 1 -#define TCG_TARGET_HAS_rem_i64 0 -#define TCG_TARGET_HAS_rot_i64 0 -#define TCG_TARGET_HAS_ext8s_i64 0 -#define TCG_TARGET_HAS_ext16s_i64 0 -#define TCG_TARGET_HAS_ext32s_i64 1 -#define TCG_TARGET_HAS_ext8u_i64 0 -#define TCG_TARGET_HAS_ext16u_i64 0 -#define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_bswap16_i64 0 -#define TCG_TARGET_HAS_bswap32_i64 0 -#define TCG_TARGET_HAS_bswap64_i64 0 -#define TCG_TARGET_HAS_not_i64 1 -#define TCG_TARGET_HAS_andc_i64 1 -#define TCG_TARGET_HAS_orc_i64 1 -#define TCG_TARGET_HAS_eqv_i64 0 -#define TCG_TARGET_HAS_nand_i64 0 -#define TCG_TARGET_HAS_nor_i64 0 -#define TCG_TARGET_HAS_clz_i64 0 -#define TCG_TARGET_HAS_ctz_i64 0 -#define TCG_TARGET_HAS_ctpop_i64 0 -#define TCG_TARGET_HAS_extract2_i64 0 -#define TCG_TARGET_HAS_negsetcond_i64 1 -#define TCG_TARGET_HAS_add2_i64 1 -#define TCG_TARGET_HAS_sub2_i64 1 -#define TCG_TARGET_HAS_mulu2_i64 0 -#define TCG_TARGET_HAS_muls2_i64 0 -#define TCG_TARGET_HAS_muluh_i64 use_vis3_instructions -#define TCG_TARGET_HAS_mulsh_i64 0 - #define TCG_TARGET_HAS_qemu_ldst_i128 0 - #define TCG_TARGET_HAS_tst 1 #define TCG_TARGET_extract_valid(type, ofs, len) \ diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index 7c722f59a8..9e004fb511 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -199,7 +199,9 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define ARITH_SUB (INSN_OP(2) | INSN_OP3(0x04)) #define ARITH_SUBCC (INSN_OP(2) | INSN_OP3(0x14)) #define ARITH_ADDC (INSN_OP(2) | INSN_OP3(0x08)) +#define ARITH_ADDCCC (INSN_OP(2) | INSN_OP3(0x18)) #define ARITH_SUBC (INSN_OP(2) | INSN_OP3(0x0c)) +#define ARITH_SUBCCC (INSN_OP(2) | INSN_OP3(0x1c)) #define ARITH_UMUL (INSN_OP(2) | INSN_OP3(0x0a)) #define ARITH_SMUL (INSN_OP(2) | INSN_OP3(0x0b)) #define ARITH_UDIV (INSN_OP(2) | INSN_OP3(0x0e)) @@ -208,9 +210,11 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define ARITH_UDIVX (INSN_OP(2) | INSN_OP3(0x0d)) #define ARITH_SDIVX (INSN_OP(2) | INSN_OP3(0x2d)) #define ARITH_MOVCC (INSN_OP(2) | INSN_OP3(0x2c)) +#define ARITH_POPC (INSN_OP(2) | INSN_OP3(0x2e)) #define ARITH_MOVR (INSN_OP(2) | INSN_OP3(0x2f)) #define ARITH_ADDXC (INSN_OP(2) | INSN_OP3(0x36) | INSN_OPF(0x11)) +#define ARITH_ADDXCCC (INSN_OP(2) | INSN_OP3(0x36) | INSN_OPF(0x13)) #define ARITH_UMULXHI (INSN_OP(2) | INSN_OP3(0x36) | INSN_OPF(0x16)) #define SHIFT_SLL (INSN_OP(2) | INSN_OP3(0x25)) @@ -223,6 +227,7 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define RDY (INSN_OP(2) | INSN_OP3(0x28) | INSN_RS1(0)) #define WRY (INSN_OP(2) | INSN_OP3(0x30) | INSN_RD(0)) +#define WRCCR (INSN_OP(2) | INSN_OP3(0x30) | INSN_RD(2)) #define JMPL (INSN_OP(2) | INSN_OP3(0x38)) #define RETURN (INSN_OP(2) | INSN_OP3(0x39)) #define SAVE (INSN_OP(2) | INSN_OP3(0x3c)) @@ -270,8 +275,11 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define STW_LE (STWA | INSN_ASI(ASI_PRIMARY_LITTLE)) #define STX_LE (STXA | INSN_ASI(ASI_PRIMARY_LITTLE)) -#ifndef use_vis3_instructions -bool use_vis3_instructions; +static bool use_popc_instructions; +#if defined(__VIS__) && __VIS__ >= 0x300 +#define use_vis3_instructions 1 +#else +static bool use_vis3_instructions; #endif static bool check_fit_i64(int64_t val, unsigned int bits) @@ -366,7 +374,7 @@ static void tcg_out_arithi(TCGContext *s, TCGReg rd, TCGReg rs1, } static void tcg_out_arithc(TCGContext *s, TCGReg rd, TCGReg rs1, - int32_t val2, int val2const, int op) + int32_t val2, int val2const, int op) { tcg_out32(s, op | INSN_RD(rd) | INSN_RS1(rs1) | (val2const ? INSN_IMM13(val2) : INSN_RS2(val2))); @@ -596,21 +604,6 @@ static void tcg_out_sety(TCGContext *s, TCGReg rs) tcg_out32(s, WRY | INSN_RS1(TCG_REG_G0) | INSN_RS2(rs)); } -static void tcg_out_div32(TCGContext *s, TCGReg rd, TCGReg rs1, - int32_t val2, int val2const, int uns) -{ - /* Load Y with the sign/zero extension of RS1 to 64-bits. */ - if (uns) { - tcg_out_sety(s, TCG_REG_G0); - } else { - tcg_out_arithi(s, TCG_REG_T1, rs1, 31, SHIFT_SRA); - tcg_out_sety(s, TCG_REG_T1); - } - - tcg_out_arithc(s, rd, rs1, val2, val2const, - uns ? ARITH_UDIV : ARITH_SDIV); -} - static const uint8_t tcg_cond_to_bcond[16] = { [TCG_COND_EQ] = COND_E, [TCG_COND_NE] = COND_NE, @@ -652,6 +645,12 @@ static void tcg_out_bpcc(TCGContext *s, int scond, int flags, TCGLabel *l) tcg_out_bpcc0(s, scond, flags, off19); } +static void tcg_out_br(TCGContext *s, TCGLabel *l) +{ + tcg_out_bpcc(s, COND_A, BPCC_PT, l); + tcg_out_nop(s); +} + static void tcg_out_cmp(TCGContext *s, TCGCond cond, TCGReg c1, int32_t c2, int c2const) { @@ -667,11 +666,10 @@ static void tcg_out_brcond_i32(TCGContext *s, TCGCond cond, TCGReg arg1, tcg_out_nop(s); } -static void tcg_out_movcc(TCGContext *s, TCGCond cond, int cc, TCGReg ret, +static void tcg_out_movcc(TCGContext *s, int scond, int cc, TCGReg ret, int32_t v1, int v1const) { - tcg_out32(s, ARITH_MOVCC | cc | INSN_RD(ret) - | INSN_RS1(tcg_cond_to_bcond[cond]) + tcg_out32(s, ARITH_MOVCC | cc | INSN_RD(ret) | INSN_RS1(scond) | (v1const ? INSN_IMM11(v1) : INSN_RS2(v1))); } @@ -680,7 +678,7 @@ static void tcg_out_movcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, int32_t v1, int v1const) { tcg_out_cmp(s, cond, c1, c2, c2const); - tcg_out_movcc(s, cond, MOVCC_ICC, ret, v1, v1const); + tcg_out_movcc(s, tcg_cond_to_bcond[cond], MOVCC_ICC, ret, v1, v1const); } static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond, TCGReg arg1, @@ -724,12 +722,12 @@ static void tcg_out_movcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, tcg_out_movr(s, rcond, ret, c1, v1, v1const); } else { tcg_out_cmp(s, cond, c1, c2, c2const); - tcg_out_movcc(s, cond, MOVCC_XCC, ret, v1, v1const); + tcg_out_movcc(s, tcg_cond_to_bcond[cond], MOVCC_XCC, ret, v1, v1const); } } static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg c1, int32_t c2, int c2const, bool neg) + TCGReg c1, int32_t c2, bool c2const, bool neg) { /* For 32-bit comparisons, we can play games with ADDC/SUBC. */ switch (cond) { @@ -749,7 +747,7 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, } c1 = TCG_REG_G0, c2const = 0; cond = (cond == TCG_COND_EQ ? TCG_COND_GEU : TCG_COND_LTU); - break; + break; case TCG_COND_TSTEQ: case TCG_COND_TSTNE: @@ -758,7 +756,7 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, c1 = TCG_REG_G0; c2 = TCG_REG_T1, c2const = 0; cond = (cond == TCG_COND_TSTEQ ? TCG_COND_GEU : TCG_COND_LTU); - break; + break; case TCG_COND_GTU: case TCG_COND_LEU: @@ -778,7 +776,8 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, default: tcg_out_cmp(s, cond, c1, c2, c2const); tcg_out_movi_s13(s, ret, 0); - tcg_out_movcc(s, cond, MOVCC_ICC, ret, neg ? -1 : 1, 1); + tcg_out_movcc(s, tcg_cond_to_bcond[cond], + MOVCC_ICC, ret, neg ? -1 : 1, 1); return; } @@ -803,7 +802,7 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, } static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg c1, int32_t c2, int c2const, bool neg) + TCGReg c1, int32_t c2, bool c2const, bool neg) { int rcond; @@ -833,78 +832,103 @@ static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, } else { tcg_out_cmp(s, cond, c1, c2, c2const); tcg_out_movi_s13(s, ret, 0); - tcg_out_movcc(s, cond, MOVCC_XCC, ret, neg ? -1 : 1, 1); + tcg_out_movcc(s, tcg_cond_to_bcond[cond], + MOVCC_XCC, ret, neg ? -1 : 1, 1); } } -static void tcg_out_addsub2_i32(TCGContext *s, TCGReg rl, TCGReg rh, - TCGReg al, TCGReg ah, int32_t bl, int blconst, - int32_t bh, int bhconst, int opl, int oph) +static void tcg_out_brcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, TCGArg arg2, bool const_arg2, + TCGLabel *l) { - TCGReg tmp = TCG_REG_T1; - - /* Note that the low parts are fully consumed before tmp is set. */ - if (rl != ah && (bhconst || rl != bh)) { - tmp = rl; + if (type == TCG_TYPE_I32) { + tcg_out_brcond_i32(s, cond, arg1, arg2, const_arg2, l); + } else { + tcg_out_brcond_i64(s, cond, arg1, arg2, const_arg2, l); } +} - tcg_out_arithc(s, tmp, al, bl, blconst, opl); - tcg_out_arithc(s, rh, ah, bh, bhconst, oph); - tcg_out_mov(s, TCG_TYPE_I32, rl, tmp); +static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, TCGReg arg2, TCGLabel *l) +{ + tcg_out_brcond(s, type, cond, arg1, arg2, false, l); } -static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh, - TCGReg al, TCGReg ah, int32_t bl, int blconst, - int32_t bh, int bhconst, bool is_sub) +static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg1, tcg_target_long arg2, TCGLabel *l) { - TCGReg tmp = TCG_REG_T1; + tcg_out_brcond(s, type, cond, arg1, arg2, true, l); +} + +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, rJ), + .out_rr = tgen_brcond, + .out_ri = tgen_brcondi, +}; - /* Note that the low parts are fully consumed before tmp is set. */ - if (rl != ah && (bhconst || rl != bh)) { - tmp = rl; +static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg c1, + TCGArg c2, bool c2const, bool neg) +{ + if (type == TCG_TYPE_I32) { + tcg_out_setcond_i32(s, cond, ret, c1, c2, c2const, neg); + } else { + tcg_out_setcond_i64(s, cond, ret, c1, c2, c2const, neg); } +} - tcg_out_arithc(s, tmp, al, bl, blconst, is_sub ? ARITH_SUBCC : ARITH_ADDCC); +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, false); +} - if (use_vis3_instructions && !is_sub) { - /* Note that ADDXC doesn't accept immediates. */ - if (bhconst && bh != 0) { - tcg_out_movi_s13(s, TCG_REG_T2, bh); - bh = TCG_REG_T2; - } - tcg_out_arith(s, rh, ah, bh, ARITH_ADDXC); - } else if (bh == TCG_REG_G0) { - /* If we have a zero, we can perform the operation in two insns, - with the arithmetic first, and a conditional move into place. */ - if (rh == ah) { - tcg_out_arithi(s, TCG_REG_T2, ah, 1, - is_sub ? ARITH_SUB : ARITH_ADD); - tcg_out_movcc(s, TCG_COND_LTU, MOVCC_XCC, rh, TCG_REG_T2, 0); - } else { - tcg_out_arithi(s, rh, ah, 1, is_sub ? ARITH_SUB : ARITH_ADD); - tcg_out_movcc(s, TCG_COND_GEU, MOVCC_XCC, rh, ah, 0); - } +static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, false); +} + +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_setcond, + .out_rri = tgen_setcondi, +}; + +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, true); +} + +static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, tcg_target_long arg2) +{ + tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, true); +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_negsetcond, + .out_rri = tgen_negsetcondi, +}; + +static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg c1, TCGArg c2, bool c2const, + TCGArg v1, bool v1const, TCGArg v2, bool v2consf) +{ + if (type == TCG_TYPE_I32) { + tcg_out_movcond_i32(s, cond, ret, c1, c2, c2const, v1, v1const); } else { - /* - * Otherwise adjust BH as if there is carry into T2. - * Note that constant BH is constrained to 11 bits for the MOVCC, - * so the adjustment fits 12 bits. - */ - if (bhconst) { - tcg_out_movi_s13(s, TCG_REG_T2, bh + (is_sub ? -1 : 1)); - } else { - tcg_out_arithi(s, TCG_REG_T2, bh, 1, - is_sub ? ARITH_SUB : ARITH_ADD); - } - /* ... smoosh T2 back to original BH if carry is clear ... */ - tcg_out_movcc(s, TCG_COND_GEU, MOVCC_XCC, TCG_REG_T2, bh, bhconst); - /* ... and finally perform the arithmetic with the new operand. */ - tcg_out_arith(s, rh, ah, TCG_REG_T2, is_sub ? ARITH_SUB : ARITH_ADD); + tcg_out_movcond_i64(s, cond, ret, c1, c2, c2const, v1, v1const); } - - tcg_out_mov(s, TCG_TYPE_I64, rl, tmp); } +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = C_O1_I4(r, r, rJ, rI, 0), + .out = tgen_movcond, +}; + static void tcg_out_jmpl_const(TCGContext *s, const tcg_insn_unit *dest, bool in_prologue, bool tail_call) { @@ -935,7 +959,7 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest, tcg_out_nop(s); } -static void tcg_out_mb(TCGContext *s, TCGArg a0) +static void tcg_out_mb(TCGContext *s, unsigned a0) { /* Note that the TCG memory order constants mirror the Sparc MEMBAR. */ tcg_out32(s, MEMBAR | (a0 & TCG_MO_ALL)); @@ -1166,8 +1190,8 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, return ldst; } -static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, - MemOpIdx oi, TCGType data_type) +static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) { static const int ld_opc[(MO_SSIZE | MO_BSWAP) + 1] = { [MO_UB] = LDUB, @@ -1199,14 +1223,23 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, ld_opc[get_memop(oi) & (MO_BSWAP | MO_SSIZE)]); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = data; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } -static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, - MemOpIdx oi, TCGType data_type) +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_qemu_ld, +}; + +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) { static const int st_opc[(MO_SIZE | MO_BSWAP) + 1] = { [MO_UB] = STB, @@ -1229,12 +1262,21 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, st_opc[get_memop(oi) & (MO_BSWAP | MO_SIZE)]); if (ldst) { - ldst->type = data_type; + ldst->type = type; ldst->datalo_reg = data; ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } } +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out = tgen_qemu_st, +}; + +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + .base.static_constraint = C_NotImplemented, +}; + static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) { if (check_fit_ptr(a0, 13)) { @@ -1280,369 +1322,794 @@ static void tcg_out_goto_tb(TCGContext *s, int which) } } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL); + tcg_out_mov_delay(s, TCG_REG_TB, a0); +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { } -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) + +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - TCGArg a0, a1, a2; - int c, c2; + tcg_out_arith(s, a0, a1, a2, ARITH_ADD); +} - /* Hoist the loads of the most common arguments. */ - a0 = args[0]; - a1 = args[1]; - a2 = args[2]; - c2 = const_args[2]; +static void tgen_addi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_arithi(s, a0, a1, a2, ARITH_ADD); +} - switch (opc) { - case INDEX_op_goto_ptr: - tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL); - tcg_out_mov_delay(s, TCG_REG_TB, a0); - break; - case INDEX_op_br: - tcg_out_bpcc(s, COND_A, BPCC_PT, arg_label(a0)); - tcg_out_nop(s); - break; +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_add, + .out_rri = tgen_addi, +}; -#define OP_32_64(x) \ - glue(glue(case INDEX_op_, x), _i32): \ - glue(glue(case INDEX_op_, x), _i64) +static void tgen_addco_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_arith(s, a0, a1, a2, ARITH_ADDCC); +} - OP_32_64(ld8u): - tcg_out_ldst(s, a0, a1, a2, LDUB); - break; - OP_32_64(ld8s): - tcg_out_ldst(s, a0, a1, a2, LDSB); - break; - OP_32_64(ld16u): - tcg_out_ldst(s, a0, a1, a2, LDUH); - break; - OP_32_64(ld16s): - tcg_out_ldst(s, a0, a1, a2, LDSH); - break; - case INDEX_op_ld_i32: - case INDEX_op_ld32u_i64: - tcg_out_ldst(s, a0, a1, a2, LDUW); - break; - OP_32_64(st8): - tcg_out_ldst(s, a0, a1, a2, STB); - break; - OP_32_64(st16): - tcg_out_ldst(s, a0, a1, a2, STH); - break; - case INDEX_op_st_i32: - case INDEX_op_st32_i64: - tcg_out_ldst(s, a0, a1, a2, STW); - break; - OP_32_64(add): - c = ARITH_ADD; - goto gen_arith; - OP_32_64(sub): - c = ARITH_SUB; - goto gen_arith; - OP_32_64(and): - c = ARITH_AND; - goto gen_arith; - OP_32_64(andc): - c = ARITH_ANDN; - goto gen_arith; - OP_32_64(or): - c = ARITH_OR; - goto gen_arith; - OP_32_64(orc): - c = ARITH_ORN; - goto gen_arith; - OP_32_64(xor): - c = ARITH_XOR; - goto gen_arith; - case INDEX_op_shl_i32: - c = SHIFT_SLL; - do_shift32: - /* Limit immediate shift count lest we create an illegal insn. */ - tcg_out_arithc(s, a0, a1, a2 & 31, c2, c); - break; - case INDEX_op_shr_i32: - c = SHIFT_SRL; - goto do_shift32; - case INDEX_op_sar_i32: - c = SHIFT_SRA; - goto do_shift32; - case INDEX_op_mul_i32: - c = ARITH_UMUL; - goto gen_arith; - - OP_32_64(neg): - c = ARITH_SUB; - goto gen_arith1; - OP_32_64(not): - c = ARITH_ORN; - goto gen_arith1; - - case INDEX_op_div_i32: - tcg_out_div32(s, a0, a1, a2, c2, 0); - break; - case INDEX_op_divu_i32: - tcg_out_div32(s, a0, a1, a2, c2, 1); - break; +static void tgen_addco_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_arithi(s, a0, a1, a2, ARITH_ADDCC); +} - case INDEX_op_brcond_i32: - tcg_out_brcond_i32(s, a2, a0, a1, const_args[1], arg_label(args[3])); - break; - case INDEX_op_setcond_i32: - tcg_out_setcond_i32(s, args[3], a0, a1, a2, c2, false); - break; - case INDEX_op_negsetcond_i32: - tcg_out_setcond_i32(s, args[3], a0, a1, a2, c2, true); - break; - case INDEX_op_movcond_i32: - tcg_out_movcond_i32(s, args[5], a0, a1, a2, c2, args[3], const_args[3]); - break; +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_addco_rrr, + .out_rri = tgen_addco_rri, +}; - case INDEX_op_add2_i32: - tcg_out_addsub2_i32(s, args[0], args[1], args[2], args[3], - args[4], const_args[4], args[5], const_args[5], - ARITH_ADDCC, ARITH_ADDC); - break; - case INDEX_op_sub2_i32: - tcg_out_addsub2_i32(s, args[0], args[1], args[2], args[3], - args[4], const_args[4], args[5], const_args[5], - ARITH_SUBCC, ARITH_SUBC); - break; - case INDEX_op_mulu2_i32: - c = ARITH_UMUL; - goto do_mul2; - case INDEX_op_muls2_i32: - c = ARITH_SMUL; - do_mul2: - /* The 32-bit multiply insns produce a full 64-bit result. */ - tcg_out_arithc(s, a0, a2, args[3], const_args[3], c); - tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX); - break; +static void tgen_addci_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_arith(s, a0, a1, a2, ARITH_ADDC); + } else if (use_vis3_instructions) { + tcg_out_arith(s, a0, a1, a2, ARITH_ADDXC); + } else { + tcg_out_arith(s, TCG_REG_T1, a1, a2, ARITH_ADD); /* for CC */ + tcg_out_arithi(s, a0, TCG_REG_T1, 1, ARITH_ADD); /* for CS */ + /* Select the correct result based on actual carry value. */ + tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, TCG_REG_T1, false); + } +} - case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); - break; - case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); - break; - case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); - break; +static void tgen_addci_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_arithi(s, a0, a1, a2, ARITH_ADDC); + return; + } + /* !use_vis3_instructions */ + if (a2 != 0) { + tcg_out_arithi(s, TCG_REG_T1, a1, a2, ARITH_ADD); /* for CC */ + tcg_out_arithi(s, a0, TCG_REG_T1, 1, ARITH_ADD); /* for CS */ + tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, TCG_REG_T1, false); + } else if (a0 == a1) { + tcg_out_arithi(s, TCG_REG_T1, a1, 1, ARITH_ADD); + tcg_out_movcc(s, COND_CS, MOVCC_XCC, a0, TCG_REG_T1, false); + } else { + tcg_out_arithi(s, a0, a1, 1, ARITH_ADD); + tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, a1, false); + } +} - case INDEX_op_ld32s_i64: - tcg_out_ldst(s, a0, a1, a2, LDSW); - break; - case INDEX_op_ld_i64: - tcg_out_ldst(s, a0, a1, a2, LDX); - break; - case INDEX_op_st_i64: - tcg_out_ldst(s, a0, a1, a2, STX); - break; - case INDEX_op_shl_i64: - c = SHIFT_SLLX; - do_shift64: - /* Limit immediate shift count lest we create an illegal insn. */ - tcg_out_arithc(s, a0, a1, a2 & 63, c2, c); - break; - case INDEX_op_shr_i64: - c = SHIFT_SRLX; - goto do_shift64; - case INDEX_op_sar_i64: - c = SHIFT_SRAX; - goto do_shift64; - case INDEX_op_mul_i64: - c = ARITH_MULX; - goto gen_arith; - case INDEX_op_div_i64: - c = ARITH_SDIVX; - goto gen_arith; - case INDEX_op_divu_i64: - c = ARITH_UDIVX; - goto gen_arith; - - case INDEX_op_brcond_i64: - tcg_out_brcond_i64(s, a2, a0, a1, const_args[1], arg_label(args[3])); - break; - case INDEX_op_setcond_i64: - tcg_out_setcond_i64(s, args[3], a0, a1, a2, c2, false); - break; - case INDEX_op_negsetcond_i64: - tcg_out_setcond_i64(s, args[3], a0, a1, a2, c2, true); - break; - case INDEX_op_movcond_i64: - tcg_out_movcond_i64(s, args[5], a0, a1, a2, c2, args[3], const_args[3]); - break; - case INDEX_op_add2_i64: - tcg_out_addsub2_i64(s, args[0], args[1], args[2], args[3], args[4], - const_args[4], args[5], const_args[5], false); - break; - case INDEX_op_sub2_i64: - tcg_out_addsub2_i64(s, args[0], args[1], args[2], args[3], args[4], - const_args[4], args[5], const_args[5], true); - break; - case INDEX_op_muluh_i64: - tcg_out_arith(s, args[0], args[1], args[2], ARITH_UMULXHI); - break; +static TCGConstraintSetIndex cset_addci(TCGType type, unsigned flags) +{ + if (use_vis3_instructions && type == TCG_TYPE_I64) { + /* Note that ADDXC doesn't accept immediates. */ + return C_O1_I2(r, rz, rz); + } + return C_O1_I2(r, rz, rJ); +} - gen_arith: - tcg_out_arithc(s, a0, a1, a2, c2, c); - break; +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addci, + .out_rrr = tgen_addci_rrr, + .out_rri = tgen_addci_rri, +}; - gen_arith1: - tcg_out_arithc(s, a0, TCG_REG_G0, a1, const_args[1], c); - break; +/* Copy %xcc.c to %icc.c */ +static void tcg_out_dup_xcc_c(TCGContext *s) +{ + if (use_vis3_instructions) { + tcg_out_arith(s, TCG_REG_T1, TCG_REG_G0, TCG_REG_G0, ARITH_ADDXC); + } else { + tcg_out_movi_s13(s, TCG_REG_T1, 0); + tcg_out_movcc(s, COND_CS, MOVCC_XCC, TCG_REG_T1, 1, true); + } + /* Write carry-in into %icc via {0,1} + -1. */ + tcg_out_arithi(s, TCG_REG_G0, TCG_REG_T1, -1, ARITH_ADDCC); +} - case INDEX_op_mb: - tcg_out_mb(s, a0); - break; +static void tgen_addcio_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type != TCG_TYPE_I32) { + if (use_vis3_instructions) { + tcg_out_arith(s, a0, a1, a2, ARITH_ADDXCCC); + return; + } + tcg_out_dup_xcc_c(s); + } + tcg_out_arith(s, a0, a1, a2, ARITH_ADDCCC); +} - case INDEX_op_extract_i64: - tcg_debug_assert(a2 + args[3] == 32); - tcg_out_arithi(s, a0, a1, a2, SHIFT_SRL); - break; - case INDEX_op_sextract_i64: - tcg_debug_assert(a2 + args[3] == 32); - tcg_out_arithi(s, a0, a1, a2, SHIFT_SRA); - break; +static void tgen_addcio_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type != TCG_TYPE_I32) { + /* !use_vis3_instructions */ + tcg_out_dup_xcc_c(s); + } + tcg_out_arithi(s, a0, a1, a2, ARITH_ADDCCC); +} - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_mov_i64: - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - default: - g_assert_not_reached(); +static TCGConstraintSetIndex cset_addcio(TCGType type, unsigned flags) +{ + if (use_vis3_instructions && type == TCG_TYPE_I64) { + /* Note that ADDXCCC doesn't accept immediates. */ + return C_O1_I2(r, rz, rz); } + return C_O1_I2(r, rz, rJ); } -static TCGConstraintSetIndex -tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addcio, + .out_rrr = tgen_addcio_rrr, + .out_rri = tgen_addcio_rri, +}; + +static void tcg_out_set_carry(TCGContext *s) +{ + /* 0x11 -> xcc = nzvC, icc = nzvC */ + tcg_out_arithi(s, 0, TCG_REG_G0, 0x11, WRCCR); +} + +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_arith(s, a0, a1, a2, ARITH_AND); +} + +static void tgen_andi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_arithi(s, a0, a1, a2, ARITH_AND); +} + +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_and, + .out_rri = tgen_andi, +}; + +static void tgen_andc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i32: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i32: - case INDEX_op_ld16s_i64: - case INDEX_op_ld_i32: - case INDEX_op_ld32u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld_i64: - case INDEX_op_neg_i32: - case INDEX_op_neg_i64: - case INDEX_op_not_i32: - case INDEX_op_not_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i64: - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: + tcg_out_arith(s, a0, a1, a2, ARITH_ANDN); +} + +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_andc, +}; + +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_arith(s, a0, TCG_REG_G0, a1, ARITH_POPC); +} + +static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags) +{ + if (use_popc_instructions && type == TCG_TYPE_I64) { return C_O1_I1(r, r); + } + return C_NotImplemented; +} - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - case INDEX_op_st_i32: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st_i64: - return C_O0_I2(rz, r); - - case INDEX_op_add_i32: - case INDEX_op_add_i64: - case INDEX_op_mul_i32: - case INDEX_op_mul_i64: - case INDEX_op_div_i32: - case INDEX_op_div_i64: - case INDEX_op_divu_i32: - case INDEX_op_divu_i64: - case INDEX_op_sub_i32: - case INDEX_op_sub_i64: - case INDEX_op_and_i32: - case INDEX_op_and_i64: - case INDEX_op_andc_i32: - case INDEX_op_andc_i64: - case INDEX_op_or_i32: - case INDEX_op_or_i64: - case INDEX_op_orc_i32: - case INDEX_op_orc_i64: - case INDEX_op_xor_i32: - case INDEX_op_xor_i64: - case INDEX_op_shl_i32: - case INDEX_op_shl_i64: - case INDEX_op_shr_i32: - case INDEX_op_shr_i64: - case INDEX_op_sar_i32: - case INDEX_op_sar_i64: - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - case INDEX_op_negsetcond_i32: - case INDEX_op_negsetcond_i64: - return C_O1_I2(r, rz, rJ); - - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - return C_O0_I2(rz, rJ); - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - return C_O1_I4(r, rz, rJ, rI, 0); - case INDEX_op_add2_i32: - case INDEX_op_add2_i64: - case INDEX_op_sub2_i32: - case INDEX_op_sub2_i64: - return C_O2_I4(r, r, rz, rz, rJ, rJ); - case INDEX_op_mulu2_i32: - case INDEX_op_muls2_i32: - return C_O2_I2(r, r, rz, rJ); - case INDEX_op_muluh_i64: - return C_O1_I2(r, r, r); +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_ctpop, + .out_rr = tgen_ctpop, +}; - default: - return C_NotImplemented; +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divs_rJ(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGArg a2, bool c2) +{ + uint32_t insn; + + if (type == TCG_TYPE_I32) { + /* Load Y with the sign extension of a1 to 64-bits. */ + tcg_out_arithi(s, TCG_REG_T1, a1, 31, SHIFT_SRA); + tcg_out_sety(s, TCG_REG_T1); + insn = ARITH_SDIV; + } else { + insn = ARITH_SDIVX; + } + tcg_out_arithc(s, a0, a1, a2, c2, insn); +} + +static void tgen_divs(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_divs_rJ(s, type, a0, a1, a2, false); +} + +static void tgen_divsi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_divs_rJ(s, type, a0, a1, a2, true); +} + +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_divs, + .out_rri = tgen_divsi, +}; + +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divu_rJ(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGArg a2, bool c2) +{ + uint32_t insn; + + if (type == TCG_TYPE_I32) { + /* Load Y with the zero extension to 64-bits. */ + tcg_out_sety(s, TCG_REG_G0); + insn = ARITH_UDIV; + } else { + insn = ARITH_UDIVX; + } + tcg_out_arithc(s, a0, a1, a2, c2, insn); +} + +static void tgen_divu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tgen_divu_rJ(s, type, a0, a1, a2, false); +} + +static void tgen_divui(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tgen_divu_rJ(s, type, a0, a1, a2, true); +} + +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_divu, + .out_rri = tgen_divui, +}; + +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_arithi(s, a0, a1, 32, SHIFT_SRLX); +} + +static const TCGOutOpUnary outop_extrh_i64_i32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extrh_i64_i32, +}; + +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? ARITH_UMUL : ARITH_MULX; + tcg_out_arith(s, a0, a1, a2, insn); +} + +static void tgen_muli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? ARITH_UMUL : ARITH_MULX; + tcg_out_arithi(s, a0, a1, a2, insn); +} + +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_mul, + .out_rri = tgen_muli, +}; + +/* + * The 32-bit multiply insns produce a full 64-bit result. + * Supporting 32-bit mul[us]2 opcodes avoids sign/zero-extensions + * before the actual multiply; we only need extract the high part + * into the separate operand. + */ +static TCGConstraintSetIndex cset_mul2(TCGType type, unsigned flags) +{ + return type == TCG_TYPE_I32 ? C_O2_I2(r, r, r, r) : C_NotImplemented; +} + +static void tgen_muls2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + tcg_out_arith(s, a0, a2, a3, ARITH_SMUL); + tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX); +} + +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mul2, + .out_rrrr = tgen_muls2, +}; + +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_mulu2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + tcg_out_arith(s, a0, a2, a3, ARITH_UMUL); + tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX); +} + +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mul2, + .out_rrrr = tgen_mulu2, +}; + +static void tgen_muluh(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_arith(s, a0, a1, a2, ARITH_UMULXHI); +} + +static TCGConstraintSetIndex cset_muluh(TCGType type, unsigned flags) +{ + return (type == TCG_TYPE_I64 && use_vis3_instructions + ? C_O1_I2(r, r, r) : C_NotImplemented); +} + +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_muluh, + .out_rrr = tgen_muluh, +}; + +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_arith(s, a0, a1, a2, ARITH_OR); +} + +static void tgen_ori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_arithi(s, a0, a1, a2, ARITH_OR); +} + +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_or, + .out_rri = tgen_ori, +}; + +static void tgen_orc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_arith(s, a0, a1, a2, ARITH_ORN); +} + +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_orc, +}; + +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SRA : SHIFT_SRAX; + tcg_out_arith(s, a0, a1, a2, insn); +} + +static void tgen_sari(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SRA : SHIFT_SRAX; + uint32_t mask = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_arithi(s, a0, a1, a2 & mask, insn); +} + +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_sar, + .out_rri = tgen_sari, +}; + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SLL : SHIFT_SLLX; + tcg_out_arith(s, a0, a1, a2, insn); +} + +static void tgen_shli(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SLL : SHIFT_SLLX; + uint32_t mask = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_arithi(s, a0, a1, a2 & mask, insn); +} + +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_shl, + .out_rri = tgen_shli, +}; + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SRL : SHIFT_SRLX; + tcg_out_arith(s, a0, a1, a2, insn); +} + +static void tgen_shri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SRL : SHIFT_SRLX; + uint32_t mask = type == TCG_TYPE_I32 ? 31 : 63; + tcg_out_arithi(s, a0, a1, a2 & mask, insn); +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_shr, + .out_rri = tgen_shri, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_arith(s, a0, a1, a2, ARITH_SUB); +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_sub, +}; + +static void tgen_subbo_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_arith(s, a0, a1, a2, ARITH_SUBCC); +} + +static void tgen_subbo_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_arithi(s, a0, a1, a2, ARITH_SUBCC); +} + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_O1_I2(r, rz, rJ), + .out_rrr = tgen_subbo_rrr, + .out_rri = tgen_subbo_rri, +}; + +static void tgen_subbi_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + /* TODO: OSA 2015 added SUBXC */ + if (type == TCG_TYPE_I32) { + tcg_out_arith(s, a0, a1, a2, ARITH_SUBC); + } else { + tcg_out_arith(s, TCG_REG_T1, a1, a2, ARITH_SUB); /* for CC */ + tcg_out_arithi(s, a0, TCG_REG_T1, 1, ARITH_SUB); /* for CS */ + /* Select the correct result based on actual borrow value. */ + tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, TCG_REG_T1, false); } } +static void tgen_subbi_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type == TCG_TYPE_I32) { + tcg_out_arithi(s, a0, a1, a2, ARITH_SUBC); + } else if (a2 != 0) { + tcg_out_arithi(s, TCG_REG_T1, a1, a2, ARITH_SUB); /* for CC */ + tcg_out_arithi(s, a0, TCG_REG_T1, 1, ARITH_SUB); /* for CS */ + tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, TCG_REG_T1, false); + } else if (a0 == a1) { + tcg_out_arithi(s, TCG_REG_T1, a1, 1, ARITH_SUB); + tcg_out_movcc(s, COND_CS, MOVCC_XCC, a0, TCG_REG_T1, false); + } else { + tcg_out_arithi(s, a0, a1, 1, ARITH_SUB); + tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, a1, false); + } +} + +static const TCGOutOpAddSubCarry outop_subbi = { + .base.static_constraint = C_O1_I2(r, rz, rJ), + .out_rrr = tgen_subbi_rrr, + .out_rri = tgen_subbi_rri, +}; + +static void tgen_subbio_rrr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type != TCG_TYPE_I32) { + /* TODO: OSA 2015 added SUBXCCC */ + tcg_out_dup_xcc_c(s); + } + tcg_out_arith(s, a0, a1, a2, ARITH_SUBCCC); +} + +static void tgen_subbio_rri(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + if (type != TCG_TYPE_I32) { + tcg_out_dup_xcc_c(s); + } + tcg_out_arithi(s, a0, a1, a2, ARITH_SUBCCC); +} + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_O1_I2(r, rz, rJ), + .out_rrr = tgen_subbio_rrr, + .out_rri = tgen_subbio_rri, +}; + +static void tcg_out_set_borrow(TCGContext *s) +{ + tcg_out_set_carry(s); /* borrow == carry */ +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_arith(s, a0, a1, a2, ARITH_XOR); +} + +static void tgen_xori(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2) +{ + tcg_out_arithi(s, a0, a1, a2, ARITH_XOR); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, r, rJ), + .out_rrr = tgen_xor, + .out_rri = tgen_xori, +}; + +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_NotImplemented, +}; + +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_sub(s, type, a0, TCG_REG_G0, a1); +} + +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_neg, +}; + +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tgen_orc(s, type, a0, TCG_REG_G0, a1); +} + +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_not, +}; + +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + tcg_debug_assert(ofs + len == 32); + tcg_out_arithi(s, a0, a1, ofs, SHIFT_SRL); +} + +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extract, +}; + +static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len) +{ + tcg_debug_assert(ofs + len == 32); + tcg_out_arithi(s, a0, a1, ofs, SHIFT_SRA); +} + +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_sextract, +}; + +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, dest, base, offset, LDUB); +} + +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; + +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, dest, base, offset, LDSB); +} + +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; + +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, dest, base, offset, LDUH); +} + +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; + +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, dest, base, offset, LDSH); +} + +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; + +static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, dest, base, offset, LDUW); +} + +static const TCGOutOpLoad outop_ld32u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32u, +}; + +static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, dest, base, offset, LDSW); +} + +static const TCGOutOpLoad outop_ld32s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32s, +}; + +static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, data, base, offset, STB); +} + +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st8_r, +}; + +static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, data, base, offset, STH); +} + +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tgen_st16_r, +}; + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(rz, r), + .out_r = tcg_out_st, +}; + + +static TCGConstraintSetIndex +tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) +{ + return C_NotImplemented; +} + static void tcg_target_init(TCGContext *s) { + unsigned long hwcap = qemu_getauxval(AT_HWCAP); + /* * Only probe for the platform and capabilities if we haven't already * determined maximum values at compile time. */ + use_popc_instructions = (hwcap & HWCAP_SPARC_POPC) != 0; #ifndef use_vis3_instructions - { - unsigned long hwcap = qemu_getauxval(AT_HWCAP); - use_vis3_instructions = (hwcap & HWCAP_SPARC_VIS3) != 0; - } + use_vis3_instructions = (hwcap & HWCAP_SPARC_VIS3) != 0; #endif tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; diff --git a/tcg/tcg-has.h b/tcg/tcg-has.h index 418e4673eb..2fc0e50d20 100644 --- a/tcg/tcg-has.h +++ b/tcg/tcg-has.h @@ -12,53 +12,6 @@ #if TCG_TARGET_REG_BITS == 32 /* Turn some undef macros into false macros. */ #define TCG_TARGET_HAS_extr_i64_i32 0 -#define TCG_TARGET_HAS_div_i64 0 -#define TCG_TARGET_HAS_rem_i64 0 -#define TCG_TARGET_HAS_div2_i64 0 -#define TCG_TARGET_HAS_rot_i64 0 -#define TCG_TARGET_HAS_ext8s_i64 0 -#define TCG_TARGET_HAS_ext16s_i64 0 -#define TCG_TARGET_HAS_ext32s_i64 0 -#define TCG_TARGET_HAS_ext8u_i64 0 -#define TCG_TARGET_HAS_ext16u_i64 0 -#define TCG_TARGET_HAS_ext32u_i64 0 -#define TCG_TARGET_HAS_bswap16_i64 0 -#define TCG_TARGET_HAS_bswap32_i64 0 -#define TCG_TARGET_HAS_bswap64_i64 0 -#define TCG_TARGET_HAS_not_i64 0 -#define TCG_TARGET_HAS_andc_i64 0 -#define TCG_TARGET_HAS_orc_i64 0 -#define TCG_TARGET_HAS_eqv_i64 0 -#define TCG_TARGET_HAS_nand_i64 0 -#define TCG_TARGET_HAS_nor_i64 0 -#define TCG_TARGET_HAS_clz_i64 0 -#define TCG_TARGET_HAS_ctz_i64 0 -#define TCG_TARGET_HAS_ctpop_i64 0 -#define TCG_TARGET_HAS_extract2_i64 0 -#define TCG_TARGET_HAS_negsetcond_i64 0 -#define TCG_TARGET_HAS_add2_i64 0 -#define TCG_TARGET_HAS_sub2_i64 0 -#define TCG_TARGET_HAS_mulu2_i64 0 -#define TCG_TARGET_HAS_muls2_i64 0 -#define TCG_TARGET_HAS_muluh_i64 0 -#define TCG_TARGET_HAS_mulsh_i64 0 -/* Turn some undef macros into true macros. */ -#define TCG_TARGET_HAS_add2_i32 1 -#define TCG_TARGET_HAS_sub2_i32 1 -#endif - -/* Only one of DIV or DIV2 should be defined. */ -#if defined(TCG_TARGET_HAS_div_i32) -#define TCG_TARGET_HAS_div2_i32 0 -#elif defined(TCG_TARGET_HAS_div2_i32) -#define TCG_TARGET_HAS_div_i32 0 -#define TCG_TARGET_HAS_rem_i32 0 -#endif -#if defined(TCG_TARGET_HAS_div_i64) -#define TCG_TARGET_HAS_div2_i64 0 -#elif defined(TCG_TARGET_HAS_div2_i64) -#define TCG_TARGET_HAS_div_i64 0 -#define TCG_TARGET_HAS_rem_i64 0 #endif #if !defined(TCG_TARGET_HAS_v64) \ diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index ff85fb23fa..d6a12afe06 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -107,8 +107,8 @@ void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e); TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, - TCGOpcode opc, unsigned nargs); + TCGOpcode, TCGType, unsigned nargs); TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, - TCGOpcode opc, unsigned nargs); + TCGOpcode, TCGType, unsigned nargs); #endif /* TCG_INTERNAL_H */ diff --git a/tcg/tcg-op-ldst.c b/tcg/tcg-op-ldst.c index 3b073b4ce0..fa9e52277b 100644 --- a/tcg/tcg-op-ldst.c +++ b/tcg/tcg-op-ldst.c @@ -88,24 +88,40 @@ static MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) return op; } -static void gen_ldst(TCGOpcode opc, TCGType type, TCGTemp *vl, TCGTemp *vh, - TCGTemp *addr, MemOpIdx oi) +static void gen_ldst1(TCGOpcode opc, TCGType type, TCGTemp *v, + TCGTemp *addr, MemOpIdx oi) { - if (vh) { - tcg_gen_op4(opc, type, temp_arg(vl), temp_arg(vh), temp_arg(addr), oi); + TCGOp *op = tcg_gen_op3(opc, type, temp_arg(v), temp_arg(addr), oi); + TCGOP_FLAGS(op) = get_memop(oi) & MO_SIZE; +} + +static void gen_ldst2(TCGOpcode opc, TCGType type, TCGTemp *vl, TCGTemp *vh, + TCGTemp *addr, MemOpIdx oi) +{ + TCGOp *op = tcg_gen_op4(opc, type, temp_arg(vl), temp_arg(vh), + temp_arg(addr), oi); + TCGOP_FLAGS(op) = get_memop(oi) & MO_SIZE; +} + +static void gen_ld_i64(TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) +{ + if (TCG_TARGET_REG_BITS == 32) { + gen_ldst2(INDEX_op_qemu_ld2, TCG_TYPE_I64, + tcgv_i32_temp(TCGV_LOW(v)), tcgv_i32_temp(TCGV_HIGH(v)), + addr, oi); } else { - tcg_gen_op3(opc, type, temp_arg(vl), temp_arg(addr), oi); + gen_ldst1(INDEX_op_qemu_ld, TCG_TYPE_I64, tcgv_i64_temp(v), addr, oi); } } -static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) +static void gen_st_i64(TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) { if (TCG_TARGET_REG_BITS == 32) { - TCGTemp *vl = tcgv_i32_temp(TCGV_LOW(v)); - TCGTemp *vh = tcgv_i32_temp(TCGV_HIGH(v)); - gen_ldst(opc, TCG_TYPE_I64, vl, vh, addr, oi); + gen_ldst2(INDEX_op_qemu_st2, TCG_TYPE_I64, + tcgv_i32_temp(TCGV_LOW(v)), tcgv_i32_temp(TCGV_HIGH(v)), + addr, oi); } else { - gen_ldst(opc, TCG_TYPE_I64, tcgv_i64_temp(v), NULL, addr, oi); + gen_ldst1(INDEX_op_qemu_st, TCG_TYPE_I64, tcgv_i64_temp(v), addr, oi); } } @@ -232,8 +248,7 @@ static void tcg_gen_qemu_ld_i32_int(TCGv_i32 val, TCGTemp *addr, } copy_addr = plugin_maybe_preserve_addr(addr); - gen_ldst(INDEX_op_qemu_ld_i32, TCG_TYPE_I32, - tcgv_i32_temp(val), NULL, addr, oi); + gen_ldst1(INDEX_op_qemu_ld, TCG_TYPE_I32, tcgv_i32_temp(val), addr, oi); plugin_gen_mem_callbacks_i32(val, copy_addr, addr, orig_oi, QEMU_PLUGIN_MEM_R); @@ -266,7 +281,6 @@ static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr, { TCGv_i32 swap = NULL; MemOpIdx orig_oi, oi; - TCGOpcode opc; tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); memop = tcg_canonicalize_memop(memop, 0, 1); @@ -289,12 +303,7 @@ static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr, oi = make_memop_idx(memop, idx); } - if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) { - opc = INDEX_op_qemu_st8_i32; - } else { - opc = INDEX_op_qemu_st_i32; - } - gen_ldst(opc, TCG_TYPE_I32, tcgv_i32_temp(val), NULL, addr, oi); + gen_ldst1(INDEX_op_qemu_st, TCG_TYPE_I32, tcgv_i32_temp(val), addr, oi); plugin_gen_mem_callbacks_i32(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); if (swap) { @@ -341,7 +350,7 @@ static void tcg_gen_qemu_ld_i64_int(TCGv_i64 val, TCGTemp *addr, } copy_addr = plugin_maybe_preserve_addr(addr); - gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, oi); + gen_ld_i64(val, addr, oi); plugin_gen_mem_callbacks_i64(val, copy_addr, addr, orig_oi, QEMU_PLUGIN_MEM_R); @@ -408,7 +417,7 @@ static void tcg_gen_qemu_st_i64_int(TCGv_i64 val, TCGTemp *addr, oi = make_memop_idx(memop, idx); } - gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, oi); + gen_st_i64(val, addr, oi); plugin_gen_mem_callbacks_i64(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); if (swap) { @@ -547,8 +556,8 @@ static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, hi = TCGV128_HIGH(val); } - gen_ldst(INDEX_op_qemu_ld_i128, TCG_TYPE_I128, tcgv_i64_temp(lo), - tcgv_i64_temp(hi), addr, oi); + gen_ldst2(INDEX_op_qemu_ld2, TCG_TYPE_I128, tcgv_i64_temp(lo), + tcgv_i64_temp(hi), addr, oi); if (need_bswap) { tcg_gen_bswap64_i64(lo, lo); @@ -576,8 +585,7 @@ static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, y = TCGV128_LOW(val); } - gen_ldst_i64(INDEX_op_qemu_ld_i64, x, addr, - make_memop_idx(mop[0], idx)); + gen_ld_i64(x, addr, make_memop_idx(mop[0], idx)); if (need_bswap) { tcg_gen_bswap64_i64(x, x); @@ -593,8 +601,7 @@ static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, addr_p8 = tcgv_i64_temp(t); } - gen_ldst_i64(INDEX_op_qemu_ld_i64, y, addr_p8, - make_memop_idx(mop[1], idx)); + gen_ld_i64(y, addr_p8, make_memop_idx(mop[1], idx)); tcg_temp_free_internal(addr_p8); if (need_bswap) { @@ -658,8 +665,8 @@ static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, hi = TCGV128_HIGH(val); } - gen_ldst(INDEX_op_qemu_st_i128, TCG_TYPE_I128, - tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi); + gen_ldst2(INDEX_op_qemu_st2, TCG_TYPE_I128, + tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi); if (need_bswap) { tcg_temp_free_i64(lo); @@ -686,8 +693,7 @@ static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, x = b; } - gen_ldst_i64(INDEX_op_qemu_st_i64, x, addr, - make_memop_idx(mop[0], idx)); + gen_st_i64(x, addr, make_memop_idx(mop[0], idx)); if (tcg_ctx->addr_type == TCG_TYPE_I32) { TCGv_i32 t = tcg_temp_ebb_new_i32(); @@ -701,12 +707,10 @@ static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, if (b) { tcg_gen_bswap64_i64(b, y); - gen_ldst_i64(INDEX_op_qemu_st_i64, b, addr_p8, - make_memop_idx(mop[1], idx)); + gen_st_i64(b, addr_p8, make_memop_idx(mop[1], idx)); tcg_temp_free_i64(b); } else { - gen_ldst_i64(INDEX_op_qemu_st_i64, y, addr_p8, - make_memop_idx(mop[1], idx)); + gen_st_i64(y, addr_p8, make_memop_idx(mop[1], idx)); } tcg_temp_free_internal(addr_p8); } else { diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index fec6d678a2..dfa5c38728 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -249,24 +249,6 @@ static void DNI tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, tcgv_i64_arg(a3), a4, a5); } -static void DNI tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGv_i32 a4, - TCGv_i32 a5, TCGv_i32 a6) -{ - tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), - tcgv_i32_arg(a6)); -} - -static void DNI tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGv_i64 a3, TCGv_i64 a4, - TCGv_i64 a5, TCGv_i64 a6) -{ - tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), - tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), - tcgv_i64_arg(a6)); -} - static void DNI tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5, TCGArg a6) @@ -351,7 +333,7 @@ void tcg_gen_discard_i32(TCGv_i32 arg) void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) { if (ret != arg) { - tcg_gen_op2_i32(INDEX_op_mov_i32, ret, arg); + tcg_gen_op2_i32(INDEX_op_mov, ret, arg); } } @@ -362,7 +344,7 @@ void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_add_i32, ret, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_add, ret, arg1, arg2); } void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -377,7 +359,7 @@ void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_sub_i32, ret, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_sub, ret, arg1, arg2); } void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) @@ -396,12 +378,12 @@ void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg) { - tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg); + tcg_gen_op2_i32(INDEX_op_neg, ret, arg); } void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_and_i32, ret, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_and, ret, arg1, arg2); } void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -414,17 +396,19 @@ void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) case -1: tcg_gen_mov_i32(ret, arg1); return; - case 0xff: - /* Don't recurse with tcg_gen_ext8u_i32. */ - if (TCG_TARGET_HAS_ext8u_i32) { - tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); - return; - } - break; - case 0xffff: - if (TCG_TARGET_HAS_ext16u_i32) { - tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); - return; + default: + /* + * Canonicalize on extract, if valid. This aids x86 with its + * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode + * which does not require matching operands. Other backends can + * trivially expand the extract to AND during code generation. + */ + if (!(arg2 & (arg2 + 1))) { + unsigned len = ctz32(~arg2); + if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { + tcg_gen_extract_i32(ret, arg1, 0, len); + return; + } } break; } @@ -434,7 +418,7 @@ void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_or_i32, ret, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_or, ret, arg1, arg2); } void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -451,7 +435,7 @@ void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_xor_i32, ret, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_xor, ret, arg1, arg2); } void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -459,9 +443,10 @@ void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) /* Some cases can be optimized here. */ if (arg2 == 0) { tcg_gen_mov_i32(ret, arg1); - } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { + } else if (arg2 == -1 && + tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { /* Don't recurse with tcg_gen_not_i32. */ - tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); + tcg_gen_op2_i32(INDEX_op_not, ret, arg1); } else { tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); } @@ -469,8 +454,8 @@ void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) { - if (TCG_TARGET_HAS_not_i32) { - tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg); + if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { + tcg_gen_op2_i32(INDEX_op_not, ret, arg); } else { tcg_gen_xori_i32(ret, arg, -1); } @@ -478,7 +463,7 @@ void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_shl_i32, ret, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_shl, ret, arg1, arg2); } void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -493,7 +478,7 @@ void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_shr_i32, ret, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_shr, ret, arg1, arg2); } void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -508,7 +493,7 @@ void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_sar_i32, ret, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_sar, ret, arg1, arg2); } void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -526,7 +511,7 @@ void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) if (cond == TCG_COND_ALWAYS) { tcg_gen_br(l); } else if (cond != TCG_COND_NEVER) { - TCGOp *op = tcg_gen_op4ii_i32(INDEX_op_brcond_i32, + TCGOp *op = tcg_gen_op4ii_i32(INDEX_op_brcond, arg1, arg2, cond, label_arg(l)); add_as_label_use(l, op); } @@ -549,7 +534,7 @@ void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, } else if (cond == TCG_COND_NEVER) { tcg_gen_movi_i32(ret, 0); } else { - tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); + tcg_gen_op4i_i32(INDEX_op_setcond, ret, arg1, arg2, cond); } } @@ -566,11 +551,8 @@ void tcg_gen_negsetcond_i32(TCGCond cond, TCGv_i32 ret, tcg_gen_movi_i32(ret, -1); } else if (cond == TCG_COND_NEVER) { tcg_gen_movi_i32(ret, 0); - } else if (TCG_TARGET_HAS_negsetcond_i32) { - tcg_gen_op4i_i32(INDEX_op_negsetcond_i32, ret, arg1, arg2, cond); } else { - tcg_gen_setcond_i32(cond, ret, arg1, arg2); - tcg_gen_neg_i32(ret, ret); + tcg_gen_op4i_i32(INDEX_op_negsetcond, ret, arg1, arg2, cond); } } @@ -582,7 +564,7 @@ void tcg_gen_negsetcondi_i32(TCGCond cond, TCGv_i32 ret, void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - tcg_gen_op3_i32(INDEX_op_mul_i32, ret, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_mul, ret, arg1, arg2); } void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) @@ -598,12 +580,12 @@ void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_div_i32) { - tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_div2_i32) { + if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_divs, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); tcg_gen_sari_i32(t0, arg1, 31); - tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); + tcg_gen_op5_i32(INDEX_op_divs2, ret, t0, arg1, t0, arg2); tcg_temp_free_i32(t0); } else { gen_helper_div_i32(ret, arg1, arg2); @@ -612,18 +594,18 @@ void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_rem_i32) { - tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_div_i32) { + if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_rems, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); - tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_divs, t0, arg1, arg2); tcg_gen_mul_i32(t0, t0, arg2); tcg_gen_sub_i32(ret, arg1, t0); tcg_temp_free_i32(t0); - } else if (TCG_TARGET_HAS_div2_i32) { + } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); tcg_gen_sari_i32(t0, arg1, 31); - tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); + tcg_gen_op5_i32(INDEX_op_divs2, t0, ret, arg1, t0, arg2); tcg_temp_free_i32(t0); } else { gen_helper_rem_i32(ret, arg1, arg2); @@ -632,12 +614,12 @@ void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_div_i32) { - tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_div2_i32) { + if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_divu, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); TCGv_i32 zero = tcg_constant_i32(0); - tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, zero, arg2); + tcg_gen_op5_i32(INDEX_op_divu2, ret, t0, arg1, zero, arg2); tcg_temp_free_i32(t0); } else { gen_helper_divu_i32(ret, arg1, arg2); @@ -646,18 +628,18 @@ void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_rem_i32) { - tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_div_i32) { + if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_remu, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); - tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_divu, t0, arg1, arg2); tcg_gen_mul_i32(t0, t0, arg2); tcg_gen_sub_i32(ret, arg1, t0); tcg_temp_free_i32(t0); - } else if (TCG_TARGET_HAS_div2_i32) { + } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); TCGv_i32 zero = tcg_constant_i32(0); - tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, zero, arg2); + tcg_gen_op5_i32(INDEX_op_divu2, t0, ret, arg1, zero, arg2); tcg_temp_free_i32(t0); } else { gen_helper_remu_i32(ret, arg1, arg2); @@ -666,8 +648,8 @@ void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_andc_i32) { - tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_andc, ret, arg1, arg2); } else { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); tcg_gen_not_i32(t0, arg2); @@ -678,8 +660,8 @@ void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_eqv_i32) { - tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_eqv, ret, arg1, arg2); } else { tcg_gen_xor_i32(ret, arg1, arg2); tcg_gen_not_i32(ret, ret); @@ -688,8 +670,8 @@ void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_nand_i32) { - tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_nand, ret, arg1, arg2); } else { tcg_gen_and_i32(ret, arg1, arg2); tcg_gen_not_i32(ret, ret); @@ -698,8 +680,8 @@ void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_nor_i32) { - tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_nor, ret, arg1, arg2); } else { tcg_gen_or_i32(ret, arg1, arg2); tcg_gen_not_i32(ret, ret); @@ -708,8 +690,8 @@ void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_orc_i32) { - tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_orc, ret, arg1, arg2); } else { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); tcg_gen_not_i32(t0, arg2); @@ -720,9 +702,9 @@ void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_clz_i32) { - tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_clz_i64) { + if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_clz, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { TCGv_i64 t1 = tcg_temp_ebb_new_i64(); TCGv_i64 t2 = tcg_temp_ebb_new_i64(); tcg_gen_extu_i32_i64(t1, arg1); @@ -745,9 +727,13 @@ void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_ctz_i32) { - tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_ctz_i64) { + TCGv_i32 z, t; + + if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_ctz, ret, arg1, arg2); + return; + } + if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0)) { TCGv_i64 t1 = tcg_temp_ebb_new_i64(); TCGv_i64 t2 = tcg_temp_ebb_new_i64(); tcg_gen_extu_i32_i64(t1, arg1); @@ -756,34 +742,34 @@ void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) tcg_gen_extrl_i64_i32(ret, t1); tcg_temp_free_i64(t1); tcg_temp_free_i64(t2); - } else if (TCG_TARGET_HAS_ctpop_i32 - || TCG_TARGET_HAS_ctpop_i64 - || TCG_TARGET_HAS_clz_i32 - || TCG_TARGET_HAS_clz_i64) { - TCGv_i32 z, t = tcg_temp_ebb_new_i32(); - - if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { - tcg_gen_subi_i32(t, arg1, 1); - tcg_gen_andc_i32(t, t, arg1); - tcg_gen_ctpop_i32(t, t); - } else { - /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ - tcg_gen_neg_i32(t, arg1); - tcg_gen_and_i32(t, t, arg1); - tcg_gen_clzi_i32(t, t, 32); - tcg_gen_xori_i32(t, t, 31); - } - z = tcg_constant_i32(0); - tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); - tcg_temp_free_i32(t); + return; + } + if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_REG, 0)) { + t = tcg_temp_ebb_new_i32(); + tcg_gen_subi_i32(t, arg1, 1); + tcg_gen_andc_i32(t, t, arg1); + tcg_gen_ctpop_i32(t, t); + } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) { + t = tcg_temp_ebb_new_i32(); + tcg_gen_neg_i32(t, arg1); + tcg_gen_and_i32(t, t, arg1); + tcg_gen_clzi_i32(t, t, 32); + tcg_gen_xori_i32(t, t, 31); } else { gen_helper_ctz_i32(ret, arg1, arg2); + return; } + + z = tcg_constant_i32(0); + tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); + tcg_temp_free_i32(t); } void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) { - if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { + if (arg2 == 32 + && !tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0) + && tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_REG, 0)) { /* This equivalence has the advantage of not requiring a fixup. */ TCGv_i32 t = tcg_temp_ebb_new_i32(); tcg_gen_subi_i32(t, arg1, 1); @@ -797,7 +783,7 @@ void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) { - if (TCG_TARGET_HAS_clz_i32) { + if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) { TCGv_i32 t = tcg_temp_ebb_new_i32(); tcg_gen_sari_i32(t, arg, 31); tcg_gen_xor_i32(t, t, arg); @@ -811,9 +797,9 @@ void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) { - if (TCG_TARGET_HAS_ctpop_i32) { - tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); - } else if (TCG_TARGET_HAS_ctpop_i64) { + if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I32, 0)) { + tcg_gen_op2_i32(INDEX_op_ctpop, ret, arg1); + } else if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) { TCGv_i64 t = tcg_temp_ebb_new_i64(); tcg_gen_extu_i32_i64(t, arg1); tcg_gen_ctpop_i64(t, t); @@ -826,15 +812,18 @@ void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_rot_i32) { - tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + tcg_gen_neg_i32(t0, arg2); + tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); + tcg_temp_free_i32(t0); } else { - TCGv_i32 t0, t1; - - t0 = tcg_temp_ebb_new_i32(); - t1 = tcg_temp_ebb_new_i32(); + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + TCGv_i32 t1 = tcg_temp_ebb_new_i32(); tcg_gen_shl_i32(t0, arg1, arg2); - tcg_gen_subfi_i32(t1, 32, arg2); + tcg_gen_neg_i32(t1, arg2); tcg_gen_shr_i32(t1, arg1, t1); tcg_gen_or_i32(ret, t0, t1); tcg_temp_free_i32(t0); @@ -848,12 +837,15 @@ void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) /* some cases can be optimized here */ if (arg2 == 0) { tcg_gen_mov_i32(ret, arg1); - } else if (TCG_TARGET_HAS_rot_i32) { - tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2)); + } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { + TCGv_i32 t0 = tcg_constant_i32(arg2); + tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); + } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { + TCGv_i32 t0 = tcg_constant_i32(32 - arg2); + tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); } else { - TCGv_i32 t0, t1; - t0 = tcg_temp_ebb_new_i32(); - t1 = tcg_temp_ebb_new_i32(); + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + TCGv_i32 t1 = tcg_temp_ebb_new_i32(); tcg_gen_shli_i32(t0, arg1, arg2); tcg_gen_shri_i32(t1, arg1, 32 - arg2); tcg_gen_or_i32(ret, t0, t1); @@ -864,15 +856,18 @@ void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_rot_i32) { - tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { + tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + tcg_gen_neg_i32(t0, arg2); + tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); + tcg_temp_free_i32(t0); } else { - TCGv_i32 t0, t1; - - t0 = tcg_temp_ebb_new_i32(); - t1 = tcg_temp_ebb_new_i32(); + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + TCGv_i32 t1 = tcg_temp_ebb_new_i32(); tcg_gen_shr_i32(t0, arg1, arg2); - tcg_gen_subfi_i32(t1, 32, arg2); + tcg_gen_neg_i32(t1, arg2); tcg_gen_shl_i32(t1, arg1, t1); tcg_gen_or_i32(ret, t0, t1); tcg_temp_free_i32(t0); @@ -883,12 +878,7 @@ void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) { tcg_debug_assert(arg2 >= 0 && arg2 < 32); - /* some cases can be optimized here */ - if (arg2 == 0) { - tcg_gen_mov_i32(ret, arg1); - } else { - tcg_gen_rotli_i32(ret, arg1, 32 - arg2); - } + tcg_gen_rotli_i32(ret, arg1, -arg2 & 31); } void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, @@ -907,13 +897,13 @@ void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, return; } if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { - tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); + tcg_gen_op5ii_i32(INDEX_op_deposit, ret, arg1, arg2, ofs, len); return; } t1 = tcg_temp_ebb_new_i32(); - if (TCG_TARGET_HAS_extract2_i32) { + if (tcg_op_supported(INDEX_op_extract2, TCG_TYPE_I32, 0)) { if (ofs + len == 32) { tcg_gen_shli_i32(t1, arg1, len); tcg_gen_extract2_i32(ret, t1, arg2, len); @@ -953,42 +943,22 @@ void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, tcg_gen_andi_i32(ret, arg, (1u << len) - 1); } else if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { TCGv_i32 zero = tcg_constant_i32(0); - tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); - } else { - /* To help two-operand hosts we prefer to zero-extend first, - which allows ARG to stay live. */ - switch (len) { - case 16: - if (TCG_TARGET_HAS_ext16u_i32) { - tcg_gen_ext16u_i32(ret, arg); - tcg_gen_shli_i32(ret, ret, ofs); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8u_i32) { - tcg_gen_ext8u_i32(ret, arg); - tcg_gen_shli_i32(ret, ret, ofs); - return; - } - break; + tcg_gen_op5ii_i32(INDEX_op_deposit, ret, zero, arg, ofs, len); + } else { + /* + * To help two-operand hosts we prefer to zero-extend first, + * which allows ARG to stay live. + */ + if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { + tcg_gen_extract_i32(ret, arg, 0, len); + tcg_gen_shli_i32(ret, ret, ofs); + return; } /* Otherwise prefer zero-extension over AND for code size. */ - switch (ofs + len) { - case 16: - if (TCG_TARGET_HAS_ext16u_i32) { - tcg_gen_shli_i32(ret, arg, ofs); - tcg_gen_ext16u_i32(ret, ret); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8u_i32) { - tcg_gen_shli_i32(ret, arg, ofs); - tcg_gen_ext8u_i32(ret, ret); - return; - } - break; + if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { + tcg_gen_shli_i32(ret, arg, ofs); + tcg_gen_extract_i32(ret, ret, 0, ofs + len); + return; } tcg_gen_andi_i32(ret, arg, (1u << len) - 1); tcg_gen_shli_i32(ret, ret, ofs); @@ -1008,32 +978,21 @@ void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, tcg_gen_shri_i32(ret, arg, 32 - len); return; } - if (ofs == 0) { - tcg_gen_andi_i32(ret, arg, (1u << len) - 1); - return; - } if (TCG_TARGET_extract_valid(TCG_TYPE_I32, ofs, len)) { - tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); + tcg_gen_op4ii_i32(INDEX_op_extract, ret, arg, ofs, len); + return; + } + if (ofs == 0) { + tcg_gen_andi_i32(ret, arg, (1u << len) - 1); return; } /* Assume that zero-extension, if available, is cheaper than a shift. */ - switch (ofs + len) { - case 16: - if (TCG_TARGET_HAS_ext16u_i32) { - tcg_gen_ext16u_i32(ret, arg); - tcg_gen_shri_i32(ret, ret, ofs); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8u_i32) { - tcg_gen_ext8u_i32(ret, arg); - tcg_gen_shri_i32(ret, ret, ofs); - return; - } - break; + if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { + tcg_gen_op4ii_i32(INDEX_op_extract, ret, arg, 0, ofs + len); + tcg_gen_shri_i32(ret, ret, ofs); + return; } /* ??? Ideally we'd know what values are available for immediate AND. @@ -1064,54 +1023,22 @@ void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, tcg_gen_sari_i32(ret, arg, 32 - len); return; } - if (ofs == 0) { - switch (len) { - case 16: - tcg_gen_ext16s_i32(ret, arg); - return; - case 8: - tcg_gen_ext8s_i32(ret, arg); - return; - } - } if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, ofs, len)) { - tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); + tcg_gen_op4ii_i32(INDEX_op_sextract, ret, arg, ofs, len); return; } /* Assume that sign-extension, if available, is cheaper than a shift. */ - switch (ofs + len) { - case 16: - if (TCG_TARGET_HAS_ext16s_i32) { - tcg_gen_ext16s_i32(ret, arg); - tcg_gen_sari_i32(ret, ret, ofs); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8s_i32) { - tcg_gen_ext8s_i32(ret, arg); - tcg_gen_sari_i32(ret, ret, ofs); - return; - } - break; + if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, ofs + len)) { + tcg_gen_op4ii_i32(INDEX_op_sextract, ret, arg, 0, ofs + len); + tcg_gen_sari_i32(ret, ret, ofs); + return; } - switch (len) { - case 16: - if (TCG_TARGET_HAS_ext16s_i32) { - tcg_gen_shri_i32(ret, arg, ofs); - tcg_gen_ext16s_i32(ret, ret); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8s_i32) { - tcg_gen_shri_i32(ret, arg, ofs); - tcg_gen_ext8s_i32(ret, ret); - return; - } - break; + if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, len)) { + tcg_gen_shri_i32(ret, arg, ofs); + tcg_gen_op4ii_i32(INDEX_op_sextract, ret, ret, 0, len); + return; } tcg_gen_shli_i32(ret, arg, 32 - len - ofs); @@ -1132,8 +1059,8 @@ void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, tcg_gen_mov_i32(ret, ah); } else if (al == ah) { tcg_gen_rotri_i32(ret, al, ofs); - } else if (TCG_TARGET_HAS_extract2_i32) { - tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); + } else if (tcg_op_supported(INDEX_op_extract2, TCG_TYPE_I32, 0)) { + tcg_gen_op4i_i32(INDEX_op_extract2, ret, al, ah, ofs); } else { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); tcg_gen_shri_i32(t0, al, ofs); @@ -1150,52 +1077,89 @@ void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, } else if (cond == TCG_COND_NEVER) { tcg_gen_mov_i32(ret, v2); } else { - tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); + tcg_gen_op6i_i32(INDEX_op_movcond, ret, c1, c2, v1, v2, cond); } } void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) { - if (TCG_TARGET_HAS_add2_i32) { - tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); + if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_I32, 0)) { + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + tcg_gen_op3_i32(INDEX_op_addco, t0, al, bl); + tcg_gen_op3_i32(INDEX_op_addci, rh, ah, bh); + tcg_gen_mov_i32(rl, t0); + tcg_temp_free_i32(t0); } else { - TCGv_i64 t0 = tcg_temp_ebb_new_i64(); - TCGv_i64 t1 = tcg_temp_ebb_new_i64(); - tcg_gen_concat_i32_i64(t0, al, ah); - tcg_gen_concat_i32_i64(t1, bl, bh); - tcg_gen_add_i64(t0, t0, t1); - tcg_gen_extr_i64_i32(rl, rh, t0); - tcg_temp_free_i64(t0); - tcg_temp_free_i64(t1); + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + TCGv_i32 t1 = tcg_temp_ebb_new_i32(); + tcg_gen_add_i32(t0, al, bl); + tcg_gen_setcond_i32(TCG_COND_LTU, t1, t0, al); + tcg_gen_add_i32(rh, ah, bh); + tcg_gen_add_i32(rh, rh, t1); + tcg_gen_mov_i32(rl, t0); + tcg_temp_free_i32(t0); + tcg_temp_free_i32(t1); + } +} + +void tcg_gen_addcio_i32(TCGv_i32 r, TCGv_i32 co, + TCGv_i32 a, TCGv_i32 b, TCGv_i32 ci) +{ + if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_I32, 0)) { + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + TCGv_i32 zero = tcg_constant_i32(0); + TCGv_i32 mone = tcg_constant_i32(-1); + + tcg_gen_op3_i32(INDEX_op_addco, t0, ci, mone); + tcg_gen_op3_i32(INDEX_op_addcio, r, a, b); + tcg_gen_op3_i32(INDEX_op_addci, co, zero, zero); + tcg_temp_free_i32(t0); + } else { + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + TCGv_i32 t1 = tcg_temp_ebb_new_i32(); + + tcg_gen_add_i32(t0, a, b); + tcg_gen_setcond_i32(TCG_COND_LTU, t1, t0, a); + tcg_gen_add_i32(r, t0, ci); + tcg_gen_setcond_i32(TCG_COND_LTU, t0, r, t0); + tcg_gen_or_i32(co, t0, t1); + + tcg_temp_free_i32(t0); + tcg_temp_free_i32(t1); } } void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) { - if (TCG_TARGET_HAS_sub2_i32) { - tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); + if (tcg_op_supported(INDEX_op_subbi, TCG_TYPE_I32, 0)) { + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + tcg_gen_op3_i32(INDEX_op_subbo, t0, al, bl); + tcg_gen_op3_i32(INDEX_op_subbi, rh, ah, bh); + tcg_gen_mov_i32(rl, t0); + tcg_temp_free_i32(t0); } else { - TCGv_i64 t0 = tcg_temp_ebb_new_i64(); - TCGv_i64 t1 = tcg_temp_ebb_new_i64(); - tcg_gen_concat_i32_i64(t0, al, ah); - tcg_gen_concat_i32_i64(t1, bl, bh); - tcg_gen_sub_i64(t0, t0, t1); - tcg_gen_extr_i64_i32(rl, rh, t0); - tcg_temp_free_i64(t0); - tcg_temp_free_i64(t1); + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + TCGv_i32 t1 = tcg_temp_ebb_new_i32(); + tcg_gen_sub_i32(t0, al, bl); + tcg_gen_setcond_i32(TCG_COND_LTU, t1, al, bl); + tcg_gen_sub_i32(rh, ah, bh); + tcg_gen_sub_i32(rh, rh, t1); + tcg_gen_mov_i32(rl, t0); + tcg_temp_free_i32(t0); + tcg_temp_free_i32(t1); } } void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_mulu2_i32) { - tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); - } else if (TCG_TARGET_HAS_muluh_i32) { + if (tcg_op_supported(INDEX_op_mulu2, TCG_TYPE_I32, 0)) { + tcg_gen_op4_i32(INDEX_op_mulu2, rl, rh, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I32, 0)) { TCGv_i32 t = tcg_temp_ebb_new_i32(); - tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); - tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_muluh, rh, arg1, arg2); tcg_gen_mov_i32(rl, t); tcg_temp_free_i32(t); } else if (TCG_TARGET_REG_BITS == 64) { @@ -1208,18 +1172,18 @@ void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) tcg_temp_free_i64(t0); tcg_temp_free_i64(t1); } else { - qemu_build_not_reached(); + g_assert_not_reached(); } } void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) { - if (TCG_TARGET_HAS_muls2_i32) { - tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); - } else if (TCG_TARGET_HAS_mulsh_i32) { + if (tcg_op_supported(INDEX_op_muls2, TCG_TYPE_I32, 0)) { + tcg_gen_op4_i32(INDEX_op_muls2, rl, rh, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I32, 0)) { TCGv_i32 t = tcg_temp_ebb_new_i32(); - tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); - tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); + tcg_gen_op3_i32(INDEX_op_mulsh, rh, arg1, arg2); tcg_gen_mov_i32(rl, t); tcg_temp_free_i32(t); } else if (TCG_TARGET_REG_BITS == 32) { @@ -1281,40 +1245,22 @@ void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) { - if (TCG_TARGET_HAS_ext8s_i32) { - tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); - } else { - tcg_gen_shli_i32(ret, arg, 24); - tcg_gen_sari_i32(ret, ret, 24); - } + tcg_gen_sextract_i32(ret, arg, 0, 8); } void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) { - if (TCG_TARGET_HAS_ext16s_i32) { - tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); - } else { - tcg_gen_shli_i32(ret, arg, 16); - tcg_gen_sari_i32(ret, ret, 16); - } + tcg_gen_sextract_i32(ret, arg, 0, 16); } void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) { - if (TCG_TARGET_HAS_ext8u_i32) { - tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); - } else { - tcg_gen_andi_i32(ret, arg, 0xffu); - } + tcg_gen_extract_i32(ret, arg, 0, 8); } void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) { - if (TCG_TARGET_HAS_ext16u_i32) { - tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); - } else { - tcg_gen_andi_i32(ret, arg, 0xffffu); - } + tcg_gen_extract_i32(ret, arg, 0, 16); } /* @@ -1330,8 +1276,8 @@ void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) /* Only one extension flag may be present. */ tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); - if (TCG_TARGET_HAS_bswap16_i32) { - tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); + if (tcg_op_supported(INDEX_op_bswap16, TCG_TYPE_I32, 0)) { + tcg_gen_op3i_i32(INDEX_op_bswap16, ret, arg, flags); } else { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); TCGv_i32 t1 = tcg_temp_ebb_new_i32(); @@ -1367,8 +1313,8 @@ void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) */ void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) { - if (TCG_TARGET_HAS_bswap32_i32) { - tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); + if (tcg_op_supported(INDEX_op_bswap32, TCG_TYPE_I32, 0)) { + tcg_gen_op3i_i32(INDEX_op_bswap32, ret, arg, 0); } else { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); TCGv_i32 t1 = tcg_temp_ebb_new_i32(); @@ -1433,42 +1379,42 @@ void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) { - tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset); + tcg_gen_ldst_op_i32(INDEX_op_ld8u, ret, arg2, offset); } void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) { - tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset); + tcg_gen_ldst_op_i32(INDEX_op_ld8s, ret, arg2, offset); } void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) { - tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset); + tcg_gen_ldst_op_i32(INDEX_op_ld16u, ret, arg2, offset); } void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) { - tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset); + tcg_gen_ldst_op_i32(INDEX_op_ld16s, ret, arg2, offset); } void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) { - tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset); + tcg_gen_ldst_op_i32(INDEX_op_ld, ret, arg2, offset); } void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) { - tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset); + tcg_gen_ldst_op_i32(INDEX_op_st8, arg1, arg2, offset); } void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) { - tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset); + tcg_gen_ldst_op_i32(INDEX_op_st16, arg1, arg2, offset); } void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) { - tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset); + tcg_gen_ldst_op_i32(INDEX_op_st, arg1, arg2, offset); } @@ -1490,7 +1436,7 @@ void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) return; } if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op2_i64(INDEX_op_mov_i64, ret, arg); + tcg_gen_op2_i64(INDEX_op_mov, ret, arg); } else { TCGTemp *ts = tcgv_i64_temp(arg); @@ -1517,7 +1463,7 @@ void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_ld8u, ret, arg2, offset); } else { tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); tcg_gen_movi_i32(TCGV_HIGH(ret), 0); @@ -1527,7 +1473,7 @@ void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_ld8s, ret, arg2, offset); } else { tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); @@ -1537,7 +1483,7 @@ void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_ld16u, ret, arg2, offset); } else { tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); tcg_gen_movi_i32(TCGV_HIGH(ret), 0); @@ -1547,7 +1493,7 @@ void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_ld16s, ret, arg2, offset); } else { tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); @@ -1557,7 +1503,7 @@ void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_ld32u, ret, arg2, offset); } else { tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); tcg_gen_movi_i32(TCGV_HIGH(ret), 0); @@ -1567,7 +1513,7 @@ void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_ld32s, ret, arg2, offset); } else { tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); @@ -1581,7 +1527,7 @@ void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) * they cannot be the same temporary -- no chance of overlap. */ if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_ld, ret, arg2, offset); } else if (HOST_BIG_ENDIAN) { tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); @@ -1594,7 +1540,7 @@ void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_st8, arg1, arg2, offset); } else { tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); } @@ -1603,7 +1549,7 @@ void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_st16, arg1, arg2, offset); } else { tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); } @@ -1612,7 +1558,7 @@ void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_st32, arg1, arg2, offset); } else { tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); } @@ -1621,7 +1567,7 @@ void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset); + tcg_gen_ldst_op_i64(INDEX_op_st, arg1, arg2, offset); } else if (HOST_BIG_ENDIAN) { tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); @@ -1634,7 +1580,7 @@ void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op3_i64(INDEX_op_add_i64, ret, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_add, ret, arg1, arg2); } else { tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); @@ -1644,7 +1590,7 @@ void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op3_i64(INDEX_op_sub_i64, ret, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_sub, ret, arg1, arg2); } else { tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); @@ -1654,7 +1600,7 @@ void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op3_i64(INDEX_op_and_i64, ret, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_and, ret, arg1, arg2); } else { tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); @@ -1664,7 +1610,7 @@ void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op3_i64(INDEX_op_or_i64, ret, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_or, ret, arg1, arg2); } else { tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); @@ -1674,7 +1620,7 @@ void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op3_i64(INDEX_op_xor_i64, ret, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_xor, ret, arg1, arg2); } else { tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); @@ -1684,7 +1630,7 @@ void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op3_i64(INDEX_op_shl_i64, ret, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_shl, ret, arg1, arg2); } else { gen_helper_shl_i64(ret, arg1, arg2); } @@ -1693,7 +1639,7 @@ void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op3_i64(INDEX_op_shr_i64, ret, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_shr, ret, arg1, arg2); } else { gen_helper_shr_i64(ret, arg1, arg2); } @@ -1702,7 +1648,7 @@ void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op3_i64(INDEX_op_sar_i64, ret, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_sar, ret, arg1, arg2); } else { gen_helper_sar_i64(ret, arg1, arg2); } @@ -1714,7 +1660,7 @@ void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) TCGv_i32 t1; if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op3_i64(INDEX_op_mul_i64, ret, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_mul, ret, arg1, arg2); return; } @@ -1770,7 +1716,7 @@ void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg) { if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg); + tcg_gen_op2_i64(INDEX_op_neg, ret, arg); } else { TCGv_i32 zero = tcg_constant_i32(0); tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), @@ -1794,23 +1740,19 @@ void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) case -1: tcg_gen_mov_i64(ret, arg1); return; - case 0xff: - /* Don't recurse with tcg_gen_ext8u_i64. */ - if (TCG_TARGET_HAS_ext8u_i64) { - tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); - return; - } - break; - case 0xffff: - if (TCG_TARGET_HAS_ext16u_i64) { - tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); - return; - } - break; - case 0xffffffffu: - if (TCG_TARGET_HAS_ext32u_i64) { - tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); - return; + default: + /* + * Canonicalize on extract, if valid. This aids x86 with its + * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode + * which does not require matching operands. Other backends can + * trivially expand the extract to AND during code generation. + */ + if (!(arg2 & (arg2 + 1))) { + unsigned len = ctz64(~arg2); + if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { + tcg_gen_extract_i64(ret, arg1, 0, len); + return; + } } break; } @@ -1845,9 +1787,10 @@ void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) /* Some cases can be optimized here. */ if (arg2 == 0) { tcg_gen_mov_i64(ret, arg1); - } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { + } else if (arg2 == -1 && + tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { /* Don't recurse with tcg_gen_not_i64. */ - tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); + tcg_gen_op2_i64(INDEX_op_not, ret, arg1); } else { tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); } @@ -1875,7 +1818,7 @@ static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, tcg_gen_movi_i32(TCGV_LOW(ret), 0); } } else if (right) { - if (TCG_TARGET_HAS_extract2_i32) { + if (tcg_op_supported(INDEX_op_extract2, TCG_TYPE_I32, 0)) { tcg_gen_extract2_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_HIGH(arg1), c); } else { @@ -1889,7 +1832,7 @@ static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); } } else { - if (TCG_TARGET_HAS_extract2_i32) { + if (tcg_op_supported(INDEX_op_extract2, TCG_TYPE_I32, 0)) { tcg_gen_extract2_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); } else { @@ -1950,7 +1893,7 @@ void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2), cond, label_arg(l)); } else { - op = tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, + op = tcg_gen_op4ii_i64(INDEX_op_brcond, arg1, arg2, cond, label_arg(l)); } add_as_label_use(l, op); @@ -1987,7 +1930,7 @@ void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); tcg_gen_movi_i32(TCGV_HIGH(ret), 0); } else { - tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); + tcg_gen_op4i_i64(INDEX_op_setcond, ret, arg1, arg2, cond); } } } @@ -2023,17 +1966,14 @@ void tcg_gen_negsetcond_i64(TCGCond cond, TCGv_i64 ret, tcg_gen_movi_i64(ret, -1); } else if (cond == TCG_COND_NEVER) { tcg_gen_movi_i64(ret, 0); - } else if (TCG_TARGET_HAS_negsetcond_i64) { - tcg_gen_op4i_i64(INDEX_op_negsetcond_i64, ret, arg1, arg2, cond); - } else if (TCG_TARGET_REG_BITS == 32) { + } else if (TCG_TARGET_REG_BITS == 64) { + tcg_gen_op4i_i64(INDEX_op_negsetcond, ret, arg1, arg2, cond); + } else { tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); tcg_gen_neg_i32(TCGV_LOW(ret), TCGV_LOW(ret)); tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_LOW(ret)); - } else { - tcg_gen_setcond_i64(cond, ret, arg1, arg2); - tcg_gen_neg_i64(ret, ret); } } @@ -2050,12 +1990,12 @@ void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_div_i64) { - tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_div2_i64) { + if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_divs, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); tcg_gen_sari_i64(t0, arg1, 63); - tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); + tcg_gen_op5_i64(INDEX_op_divs2, ret, t0, arg1, t0, arg2); tcg_temp_free_i64(t0); } else { gen_helper_div_i64(ret, arg1, arg2); @@ -2064,18 +2004,18 @@ void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_rem_i64) { - tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_div_i64) { + if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_rems, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); - tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_divs, t0, arg1, arg2); tcg_gen_mul_i64(t0, t0, arg2); tcg_gen_sub_i64(ret, arg1, t0); tcg_temp_free_i64(t0); - } else if (TCG_TARGET_HAS_div2_i64) { + } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); tcg_gen_sari_i64(t0, arg1, 63); - tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); + tcg_gen_op5_i64(INDEX_op_divs2, t0, ret, arg1, t0, arg2); tcg_temp_free_i64(t0); } else { gen_helper_rem_i64(ret, arg1, arg2); @@ -2084,12 +2024,12 @@ void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_div_i64) { - tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_div2_i64) { + if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_divu, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); TCGv_i64 zero = tcg_constant_i64(0); - tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, zero, arg2); + tcg_gen_op5_i64(INDEX_op_divu2, ret, t0, arg1, zero, arg2); tcg_temp_free_i64(t0); } else { gen_helper_divu_i64(ret, arg1, arg2); @@ -2098,18 +2038,18 @@ void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_rem_i64) { - tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_div_i64) { + if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_remu, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); - tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_divu, t0, arg1, arg2); tcg_gen_mul_i64(t0, t0, arg2); tcg_gen_sub_i64(ret, arg1, t0); tcg_temp_free_i64(t0); - } else if (TCG_TARGET_HAS_div2_i64) { + } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); TCGv_i64 zero = tcg_constant_i64(0); - tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, zero, arg2); + tcg_gen_op5_i64(INDEX_op_divu2, t0, ret, arg1, zero, arg2); tcg_temp_free_i64(t0); } else { gen_helper_remu_i64(ret, arg1, arg2); @@ -2118,77 +2058,32 @@ void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) { - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); - tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); - } else if (TCG_TARGET_HAS_ext8s_i64) { - tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); - } else { - tcg_gen_shli_i64(ret, arg, 56); - tcg_gen_sari_i64(ret, ret, 56); - } + tcg_gen_sextract_i64(ret, arg, 0, 8); } void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) { - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); - tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); - } else if (TCG_TARGET_HAS_ext16s_i64) { - tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); - } else { - tcg_gen_shli_i64(ret, arg, 48); - tcg_gen_sari_i64(ret, ret, 48); - } + tcg_gen_sextract_i64(ret, arg, 0, 16); } void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) { - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); - tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); - } else if (TCG_TARGET_HAS_ext32s_i64) { - tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); - } else { - tcg_gen_shli_i64(ret, arg, 32); - tcg_gen_sari_i64(ret, ret, 32); - } + tcg_gen_sextract_i64(ret, arg, 0, 32); } void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) { - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); - tcg_gen_movi_i32(TCGV_HIGH(ret), 0); - } else if (TCG_TARGET_HAS_ext8u_i64) { - tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); - } else { - tcg_gen_andi_i64(ret, arg, 0xffu); - } + tcg_gen_extract_i64(ret, arg, 0, 8); } void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) { - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); - tcg_gen_movi_i32(TCGV_HIGH(ret), 0); - } else if (TCG_TARGET_HAS_ext16u_i64) { - tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); - } else { - tcg_gen_andi_i64(ret, arg, 0xffffu); - } + tcg_gen_extract_i64(ret, arg, 0, 16); } void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) { - if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); - tcg_gen_movi_i32(TCGV_HIGH(ret), 0); - } else if (TCG_TARGET_HAS_ext32u_i64) { - tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); - } else { - tcg_gen_andi_i64(ret, arg, 0xffffffffu); - } + tcg_gen_extract_i64(ret, arg, 0, 32); } /* @@ -2211,8 +2106,8 @@ void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) } else { tcg_gen_movi_i32(TCGV_HIGH(ret), 0); } - } else if (TCG_TARGET_HAS_bswap16_i64) { - tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); + } else if (tcg_op_supported(INDEX_op_bswap16, TCG_TYPE_I64, 0)) { + tcg_gen_op3i_i64(INDEX_op_bswap16, ret, arg, flags); } else { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); TCGv_i64 t1 = tcg_temp_ebb_new_i64(); @@ -2261,8 +2156,8 @@ void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) } else { tcg_gen_movi_i32(TCGV_HIGH(ret), 0); } - } else if (TCG_TARGET_HAS_bswap32_i64) { - tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); + } else if (tcg_op_supported(INDEX_op_bswap32, TCG_TYPE_I64, 0)) { + tcg_gen_op3i_i64(INDEX_op_bswap32, ret, arg, flags); } else { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); TCGv_i64 t1 = tcg_temp_ebb_new_i64(); @@ -2308,8 +2203,8 @@ void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) tcg_gen_mov_i32(TCGV_HIGH(ret), t0); tcg_temp_free_i32(t0); tcg_temp_free_i32(t1); - } else if (TCG_TARGET_HAS_bswap64_i64) { - tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); + } else if (tcg_op_supported(INDEX_op_bswap64, TCG_TYPE_I64, 0)) { + tcg_gen_op3i_i64(INDEX_op_bswap64, ret, arg, 0); } else { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); TCGv_i64 t1 = tcg_temp_ebb_new_i64(); @@ -2380,8 +2275,8 @@ void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) if (TCG_TARGET_REG_BITS == 32) { tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); - } else if (TCG_TARGET_HAS_not_i64) { - tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); + } else if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { + tcg_gen_op2_i64(INDEX_op_not, ret, arg); } else { tcg_gen_xori_i64(ret, arg, -1); } @@ -2392,8 +2287,8 @@ void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) if (TCG_TARGET_REG_BITS == 32) { tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); - } else if (TCG_TARGET_HAS_andc_i64) { - tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_andc, ret, arg1, arg2); } else { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); tcg_gen_not_i64(t0, arg2); @@ -2407,8 +2302,8 @@ void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) if (TCG_TARGET_REG_BITS == 32) { tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); - } else if (TCG_TARGET_HAS_eqv_i64) { - tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_eqv, ret, arg1, arg2); } else { tcg_gen_xor_i64(ret, arg1, arg2); tcg_gen_not_i64(ret, ret); @@ -2420,8 +2315,8 @@ void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) if (TCG_TARGET_REG_BITS == 32) { tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); - } else if (TCG_TARGET_HAS_nand_i64) { - tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_nand, ret, arg1, arg2); } else { tcg_gen_and_i64(ret, arg1, arg2); tcg_gen_not_i64(ret, ret); @@ -2433,8 +2328,8 @@ void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) if (TCG_TARGET_REG_BITS == 32) { tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); - } else if (TCG_TARGET_HAS_nor_i64) { - tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_nor, ret, arg1, arg2); } else { tcg_gen_or_i64(ret, arg1, arg2); tcg_gen_not_i64(ret, ret); @@ -2446,8 +2341,8 @@ void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) if (TCG_TARGET_REG_BITS == 32) { tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); - } else if (TCG_TARGET_HAS_orc_i64) { - tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_orc, ret, arg1, arg2); } else { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); tcg_gen_not_i64(t0, arg2); @@ -2458,8 +2353,8 @@ void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_clz_i64) { - tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_clz, ret, arg1, arg2); } else { gen_helper_clz_i64(ret, arg1, arg2); } @@ -2468,8 +2363,8 @@ void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) { if (TCG_TARGET_REG_BITS == 32 - && TCG_TARGET_HAS_clz_i32 - && arg2 <= 0xffffffffu) { + && arg2 <= 0xffffffffu + && tcg_op_supported(INDEX_op_clz, TCG_TYPE_I32, 0)) { TCGv_i32 t = tcg_temp_ebb_new_i32(); tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); tcg_gen_addi_i32(t, t, 32); @@ -2483,45 +2378,47 @@ void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_ctz_i64) { - tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); - } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { - TCGv_i64 z, t = tcg_temp_ebb_new_i64(); + TCGv_i64 z, t; - if (TCG_TARGET_HAS_ctpop_i64) { - tcg_gen_subi_i64(t, arg1, 1); - tcg_gen_andc_i64(t, t, arg1); - tcg_gen_ctpop_i64(t, t); - } else { - /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ - tcg_gen_neg_i64(t, arg1); - tcg_gen_and_i64(t, t, arg1); - tcg_gen_clzi_i64(t, t, 64); - tcg_gen_xori_i64(t, t, 63); - } - z = tcg_constant_i64(0); - tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); - tcg_temp_free_i64(t); - tcg_temp_free_i64(z); + if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_ctz, ret, arg1, arg2); + return; + } + if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) { + t = tcg_temp_ebb_new_i64(); + tcg_gen_subi_i64(t, arg1, 1); + tcg_gen_andc_i64(t, t, arg1); + tcg_gen_ctpop_i64(t, t); + } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { + t = tcg_temp_ebb_new_i64(); + tcg_gen_neg_i64(t, arg1); + tcg_gen_and_i64(t, t, arg1); + tcg_gen_clzi_i64(t, t, 64); + tcg_gen_xori_i64(t, t, 63); } else { gen_helper_ctz_i64(ret, arg1, arg2); + return; } + + z = tcg_constant_i64(0); + tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); + tcg_temp_free_i64(t); } void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) { if (TCG_TARGET_REG_BITS == 32 - && TCG_TARGET_HAS_ctz_i32 - && arg2 <= 0xffffffffu) { + && arg2 <= 0xffffffffu + && tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0)) { TCGv_i32 t32 = tcg_temp_ebb_new_i32(); tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); tcg_gen_addi_i32(t32, t32, 32); tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); tcg_gen_movi_i32(TCGV_HIGH(ret), 0); tcg_temp_free_i32(t32); - } else if (!TCG_TARGET_HAS_ctz_i64 - && TCG_TARGET_HAS_ctpop_i64 - && arg2 == 64) { + } else if (arg2 == 64 + && !tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0) + && tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) { /* This equivalence has the advantage of not requiring a fixup. */ TCGv_i64 t = tcg_temp_ebb_new_i64(); tcg_gen_subi_i64(t, arg1, 1); @@ -2535,7 +2432,7 @@ void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) { - if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { + if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { TCGv_i64 t = tcg_temp_ebb_new_i64(); tcg_gen_sari_i64(t, arg, 63); tcg_gen_xor_i64(t, t, arg); @@ -2549,28 +2446,37 @@ void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) { - if (TCG_TARGET_HAS_ctpop_i64) { - tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); - } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { - tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); - tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); - tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); - tcg_gen_movi_i32(TCGV_HIGH(ret), 0); + if (TCG_TARGET_REG_BITS == 64) { + if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) { + tcg_gen_op2_i64(INDEX_op_ctpop, ret, arg1); + return; + } } else { - gen_helper_ctpop_i64(ret, arg1); + if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I32, 0)) { + tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); + tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); + tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); + tcg_gen_movi_i32(TCGV_HIGH(ret), 0); + return; + } } + gen_helper_ctpop_i64(ret, arg1); } void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_rot_i64) { - tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + tcg_gen_neg_i64(t0, arg2); + tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); + tcg_temp_free_i64(t0); } else { - TCGv_i64 t0, t1; - t0 = tcg_temp_ebb_new_i64(); - t1 = tcg_temp_ebb_new_i64(); + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + TCGv_i64 t1 = tcg_temp_ebb_new_i64(); tcg_gen_shl_i64(t0, arg1, arg2); - tcg_gen_subfi_i64(t1, 64, arg2); + tcg_gen_neg_i64(t1, arg2); tcg_gen_shr_i64(t1, arg1, t1); tcg_gen_or_i64(ret, t0, t1); tcg_temp_free_i64(t0); @@ -2584,12 +2490,15 @@ void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) /* some cases can be optimized here */ if (arg2 == 0) { tcg_gen_mov_i64(ret, arg1); - } else if (TCG_TARGET_HAS_rot_i64) { - tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2)); + } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { + TCGv_i64 t0 = tcg_constant_i64(arg2); + tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); + } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { + TCGv_i64 t0 = tcg_constant_i64(64 - arg2); + tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); } else { - TCGv_i64 t0, t1; - t0 = tcg_temp_ebb_new_i64(); - t1 = tcg_temp_ebb_new_i64(); + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + TCGv_i64 t1 = tcg_temp_ebb_new_i64(); tcg_gen_shli_i64(t0, arg1, arg2); tcg_gen_shri_i64(t1, arg1, 64 - arg2); tcg_gen_or_i64(ret, t0, t1); @@ -2600,14 +2509,18 @@ void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_rot_i64) { - tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); + if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { + tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + tcg_gen_neg_i64(t0, arg2); + tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); + tcg_temp_free_i64(t0); } else { - TCGv_i64 t0, t1; - t0 = tcg_temp_ebb_new_i64(); - t1 = tcg_temp_ebb_new_i64(); + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + TCGv_i64 t1 = tcg_temp_ebb_new_i64(); tcg_gen_shr_i64(t0, arg1, arg2); - tcg_gen_subfi_i64(t1, 64, arg2); + tcg_gen_neg_i64(t1, arg2); tcg_gen_shl_i64(t1, arg1, t1); tcg_gen_or_i64(ret, t0, t1); tcg_temp_free_i64(t0); @@ -2618,12 +2531,7 @@ void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) { tcg_debug_assert(arg2 >= 0 && arg2 < 64); - /* some cases can be optimized here */ - if (arg2 == 0) { - tcg_gen_mov_i64(ret, arg1); - } else { - tcg_gen_rotli_i64(ret, arg1, 64 - arg2); - } + tcg_gen_rotli_i64(ret, arg1, -arg2 & 63); } void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, @@ -2644,7 +2552,7 @@ void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, if (TCG_TARGET_REG_BITS == 64) { if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { - tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); + tcg_gen_op5ii_i64(INDEX_op_deposit, ret, arg1, arg2, ofs, len); return; } } else { @@ -2664,7 +2572,7 @@ void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, t1 = tcg_temp_ebb_new_i64(); - if (TCG_TARGET_HAS_extract2_i64) { + if (tcg_op_supported(INDEX_op_extract2, TCG_TYPE_I64, 0)) { if (ofs + len == 64) { tcg_gen_shli_i64(t1, arg1, len); tcg_gen_extract2_i64(ret, t1, arg2, len); @@ -2705,7 +2613,7 @@ void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, } else if (TCG_TARGET_REG_BITS == 64 && TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { TCGv_i64 zero = tcg_constant_i64(0); - tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); + tcg_gen_op5ii_i64(INDEX_op_deposit, ret, zero, arg, ofs, len); } else { if (TCG_TARGET_REG_BITS == 32) { if (ofs >= 32) { @@ -2720,54 +2628,20 @@ void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, return; } } - /* To help two-operand hosts we prefer to zero-extend first, - which allows ARG to stay live. */ - switch (len) { - case 32: - if (TCG_TARGET_HAS_ext32u_i64) { - tcg_gen_ext32u_i64(ret, arg); - tcg_gen_shli_i64(ret, ret, ofs); - return; - } - break; - case 16: - if (TCG_TARGET_HAS_ext16u_i64) { - tcg_gen_ext16u_i64(ret, arg); - tcg_gen_shli_i64(ret, ret, ofs); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8u_i64) { - tcg_gen_ext8u_i64(ret, arg); - tcg_gen_shli_i64(ret, ret, ofs); - return; - } - break; + /* + * To help two-operand hosts we prefer to zero-extend first, + * which allows ARG to stay live. + */ + if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { + tcg_gen_extract_i64(ret, arg, 0, len); + tcg_gen_shli_i64(ret, ret, ofs); + return; } /* Otherwise prefer zero-extension over AND for code size. */ - switch (ofs + len) { - case 32: - if (TCG_TARGET_HAS_ext32u_i64) { - tcg_gen_shli_i64(ret, arg, ofs); - tcg_gen_ext32u_i64(ret, ret); - return; - } - break; - case 16: - if (TCG_TARGET_HAS_ext16u_i64) { - tcg_gen_shli_i64(ret, arg, ofs); - tcg_gen_ext16u_i64(ret, ret); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8u_i64) { - tcg_gen_shli_i64(ret, arg, ofs); - tcg_gen_ext8u_i64(ret, ret); - return; - } - break; + if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { + tcg_gen_shli_i64(ret, arg, ofs); + tcg_gen_extract_i64(ret, ret, 0, ofs + len); + return; } tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); tcg_gen_shli_i64(ret, ret, ofs); @@ -2787,10 +2661,6 @@ void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, tcg_gen_shri_i64(ret, arg, 64 - len); return; } - if (ofs == 0) { - tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); - return; - } if (TCG_TARGET_REG_BITS == 32) { /* Look for a 32-bit extract within one of the two words. */ @@ -2804,39 +2674,34 @@ void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, tcg_gen_movi_i32(TCGV_HIGH(ret), 0); return; } - /* The field is split across two words. One double-word - shift is better than two double-word shifts. */ - goto do_shift_and; + + /* The field is split across two words. */ + tcg_gen_extract2_i32(TCGV_LOW(ret), TCGV_LOW(arg), + TCGV_HIGH(arg), ofs); + if (len <= 32) { + tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(ret), 0, len); + tcg_gen_movi_i32(TCGV_HIGH(ret), 0); + } else { + tcg_gen_extract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), + ofs, len - 32); + } + return; } if (TCG_TARGET_extract_valid(TCG_TYPE_I64, ofs, len)) { - tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); + tcg_gen_op4ii_i64(INDEX_op_extract, ret, arg, ofs, len); + return; + } + if (ofs == 0) { + tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); return; } /* Assume that zero-extension, if available, is cheaper than a shift. */ - switch (ofs + len) { - case 32: - if (TCG_TARGET_HAS_ext32u_i64) { - tcg_gen_ext32u_i64(ret, arg); - tcg_gen_shri_i64(ret, ret, ofs); - return; - } - break; - case 16: - if (TCG_TARGET_HAS_ext16u_i64) { - tcg_gen_ext16u_i64(ret, arg); - tcg_gen_shri_i64(ret, ret, ofs); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8u_i64) { - tcg_gen_ext8u_i64(ret, arg); - tcg_gen_shri_i64(ret, ret, ofs); - return; - } - break; + if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { + tcg_gen_op4ii_i64(INDEX_op_extract, ret, arg, 0, ofs + len); + tcg_gen_shri_i64(ret, ret, ofs); + return; } /* ??? Ideally we'd know what values are available for immediate AND. @@ -2844,7 +2709,6 @@ void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, so that we get ext8u, ext16u, and ext32u. */ switch (len) { case 1 ... 8: case 16: case 32: - do_shift_and: tcg_gen_shri_i64(ret, arg, ofs); tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); break; @@ -2868,19 +2732,6 @@ void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, tcg_gen_sari_i64(ret, arg, 64 - len); return; } - if (ofs == 0) { - switch (len) { - case 32: - tcg_gen_ext32s_i64(ret, arg); - return; - case 16: - tcg_gen_ext16s_i64(ret, arg); - return; - case 8: - tcg_gen_ext8s_i64(ret, arg); - return; - } - } if (TCG_TARGET_REG_BITS == 32) { /* Look for a 32-bit extract within one of the two words. */ @@ -2915,57 +2766,22 @@ void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, } if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, ofs, len)) { - tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); + tcg_gen_op4ii_i64(INDEX_op_sextract, ret, arg, ofs, len); return; } /* Assume that sign-extension, if available, is cheaper than a shift. */ - switch (ofs + len) { - case 32: - if (TCG_TARGET_HAS_ext32s_i64) { - tcg_gen_ext32s_i64(ret, arg); - tcg_gen_sari_i64(ret, ret, ofs); - return; - } - break; - case 16: - if (TCG_TARGET_HAS_ext16s_i64) { - tcg_gen_ext16s_i64(ret, arg); - tcg_gen_sari_i64(ret, ret, ofs); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8s_i64) { - tcg_gen_ext8s_i64(ret, arg); - tcg_gen_sari_i64(ret, ret, ofs); - return; - } - break; + if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, ofs + len)) { + tcg_gen_op4ii_i64(INDEX_op_sextract, ret, arg, 0, ofs + len); + tcg_gen_sari_i64(ret, ret, ofs); + return; } - switch (len) { - case 32: - if (TCG_TARGET_HAS_ext32s_i64) { - tcg_gen_shri_i64(ret, arg, ofs); - tcg_gen_ext32s_i64(ret, ret); - return; - } - break; - case 16: - if (TCG_TARGET_HAS_ext16s_i64) { - tcg_gen_shri_i64(ret, arg, ofs); - tcg_gen_ext16s_i64(ret, ret); - return; - } - break; - case 8: - if (TCG_TARGET_HAS_ext8s_i64) { - tcg_gen_shri_i64(ret, arg, ofs); - tcg_gen_ext8s_i64(ret, ret); - return; - } - break; + if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, len)) { + tcg_gen_shri_i64(ret, arg, ofs); + tcg_gen_op4ii_i64(INDEX_op_sextract, ret, ret, 0, len); + return; } + tcg_gen_shli_i64(ret, arg, 64 - len - ofs); tcg_gen_sari_i64(ret, ret, 64 - len); } @@ -2984,8 +2800,8 @@ void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, tcg_gen_mov_i64(ret, ah); } else if (al == ah) { tcg_gen_rotri_i64(ret, al, ofs); - } else if (TCG_TARGET_HAS_extract2_i64) { - tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); + } else if (tcg_op_supported(INDEX_op_extract2, TCG_TYPE_I64, 0)) { + tcg_gen_op4i_i64(INDEX_op_extract2, ret, al, ah, ofs); } else { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); tcg_gen_shri_i64(t0, al, ofs); @@ -3002,7 +2818,7 @@ void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, } else if (cond == TCG_COND_NEVER) { tcg_gen_mov_i64(ret, v2); } else if (TCG_TARGET_REG_BITS == 64) { - tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); + tcg_gen_op6i_i64(INDEX_op_movcond, ret, c1, c2, v1, v2, cond); } else { TCGv_i32 t0 = tcg_temp_ebb_new_i32(); TCGv_i32 zero = tcg_constant_i32(0); @@ -3023,8 +2839,25 @@ void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) { - if (TCG_TARGET_HAS_add2_i64) { - tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); + if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_REG, 0)) { + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + + if (TCG_TARGET_REG_BITS == 32) { + tcg_gen_op3_i32(INDEX_op_addco, TCGV_LOW(t0), + TCGV_LOW(al), TCGV_LOW(bl)); + tcg_gen_op3_i32(INDEX_op_addcio, TCGV_HIGH(t0), + TCGV_HIGH(al), TCGV_HIGH(bl)); + tcg_gen_op3_i32(INDEX_op_addcio, TCGV_LOW(rh), + TCGV_LOW(ah), TCGV_LOW(bh)); + tcg_gen_op3_i32(INDEX_op_addci, TCGV_HIGH(rh), + TCGV_HIGH(ah), TCGV_HIGH(bh)); + } else { + tcg_gen_op3_i64(INDEX_op_addco, t0, al, bl); + tcg_gen_op3_i64(INDEX_op_addci, rh, ah, bh); + } + + tcg_gen_mov_i64(rl, t0); + tcg_temp_free_i64(t0); } else { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); TCGv_i64 t1 = tcg_temp_ebb_new_i64(); @@ -3038,11 +2871,96 @@ void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, } } +void tcg_gen_addcio_i64(TCGv_i64 r, TCGv_i64 co, + TCGv_i64 a, TCGv_i64 b, TCGv_i64 ci) +{ + if (TCG_TARGET_REG_BITS == 64) { + if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_I64, 0)) { + TCGv_i64 discard = tcg_temp_ebb_new_i64(); + TCGv_i64 zero = tcg_constant_i64(0); + TCGv_i64 mone = tcg_constant_i64(-1); + + tcg_gen_op3_i64(INDEX_op_addco, discard, ci, mone); + tcg_gen_op3_i64(INDEX_op_addcio, r, a, b); + tcg_gen_op3_i64(INDEX_op_addci, co, zero, zero); + tcg_temp_free_i64(discard); + } else { + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + TCGv_i64 t1 = tcg_temp_ebb_new_i64(); + + tcg_gen_add_i64(t0, a, b); + tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, a); + tcg_gen_add_i64(r, t0, ci); + tcg_gen_setcond_i64(TCG_COND_LTU, t0, r, t0); + tcg_gen_or_i64(co, t0, t1); + + tcg_temp_free_i64(t0); + tcg_temp_free_i64(t1); + } + } else { + if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_I32, 0)) { + TCGv_i32 discard = tcg_temp_ebb_new_i32(); + TCGv_i32 zero = tcg_constant_i32(0); + TCGv_i32 mone = tcg_constant_i32(-1); + + tcg_gen_op3_i32(INDEX_op_addco, discard, TCGV_LOW(ci), mone); + tcg_gen_op3_i32(INDEX_op_addcio, discard, TCGV_HIGH(ci), mone); + tcg_gen_op3_i32(INDEX_op_addcio, TCGV_LOW(r), + TCGV_LOW(a), TCGV_LOW(b)); + tcg_gen_op3_i32(INDEX_op_addcio, TCGV_HIGH(r), + TCGV_HIGH(a), TCGV_HIGH(b)); + tcg_gen_op3_i32(INDEX_op_addci, TCGV_LOW(co), zero, zero); + tcg_temp_free_i32(discard); + } else { + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + TCGv_i32 c0 = tcg_temp_ebb_new_i32(); + TCGv_i32 c1 = tcg_temp_ebb_new_i32(); + + tcg_gen_or_i32(c1, TCGV_LOW(ci), TCGV_HIGH(ci)); + tcg_gen_setcondi_i32(TCG_COND_NE, c1, c1, 0); + + tcg_gen_add_i32(t0, TCGV_LOW(a), TCGV_LOW(b)); + tcg_gen_setcond_i32(TCG_COND_LTU, c0, t0, TCGV_LOW(a)); + tcg_gen_add_i32(TCGV_LOW(r), t0, c1); + tcg_gen_setcond_i32(TCG_COND_LTU, c1, TCGV_LOW(r), c1); + tcg_gen_or_i32(c1, c1, c0); + + tcg_gen_add_i32(t0, TCGV_HIGH(a), TCGV_HIGH(b)); + tcg_gen_setcond_i32(TCG_COND_LTU, c0, t0, TCGV_HIGH(a)); + tcg_gen_add_i32(TCGV_HIGH(r), t0, c1); + tcg_gen_setcond_i32(TCG_COND_LTU, c1, TCGV_HIGH(r), c1); + tcg_gen_or_i32(TCGV_LOW(co), c0, c1); + + tcg_temp_free_i32(t0); + tcg_temp_free_i32(c0); + tcg_temp_free_i32(c1); + } + tcg_gen_movi_i32(TCGV_HIGH(co), 0); + } +} + void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) { - if (TCG_TARGET_HAS_sub2_i64) { - tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); + if (tcg_op_supported(INDEX_op_subbi, TCG_TYPE_REG, 0)) { + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + + if (TCG_TARGET_REG_BITS == 32) { + tcg_gen_op3_i32(INDEX_op_subbo, TCGV_LOW(t0), + TCGV_LOW(al), TCGV_LOW(bl)); + tcg_gen_op3_i32(INDEX_op_subbio, TCGV_HIGH(t0), + TCGV_HIGH(al), TCGV_HIGH(bl)); + tcg_gen_op3_i32(INDEX_op_subbio, TCGV_LOW(rh), + TCGV_LOW(ah), TCGV_LOW(bh)); + tcg_gen_op3_i32(INDEX_op_subbi, TCGV_HIGH(rh), + TCGV_HIGH(ah), TCGV_HIGH(bh)); + } else { + tcg_gen_op3_i64(INDEX_op_subbo, t0, al, bl); + tcg_gen_op3_i64(INDEX_op_subbi, rh, ah, bh); + } + + tcg_gen_mov_i64(rl, t0); + tcg_temp_free_i64(t0); } else { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); TCGv_i64 t1 = tcg_temp_ebb_new_i64(); @@ -3058,12 +2976,12 @@ void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_mulu2_i64) { - tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); - } else if (TCG_TARGET_HAS_muluh_i64) { + if (tcg_op_supported(INDEX_op_mulu2, TCG_TYPE_I64, 0)) { + tcg_gen_op4_i64(INDEX_op_mulu2, rl, rh, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { TCGv_i64 t = tcg_temp_ebb_new_i64(); - tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); - tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_muluh, rh, arg1, arg2); tcg_gen_mov_i64(rl, t); tcg_temp_free_i64(t); } else { @@ -3077,15 +2995,16 @@ void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) { - if (TCG_TARGET_HAS_muls2_i64) { - tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); - } else if (TCG_TARGET_HAS_mulsh_i64) { + if (tcg_op_supported(INDEX_op_muls2, TCG_TYPE_I64, 0)) { + tcg_gen_op4_i64(INDEX_op_muls2, rl, rh, arg1, arg2); + } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I64, 0)) { TCGv_i64 t = tcg_temp_ebb_new_i64(); - tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); - tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); + tcg_gen_op3_i64(INDEX_op_mulsh, rh, arg1, arg2); tcg_gen_mov_i64(rl, t); tcg_temp_free_i64(t); - } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { + } else if (tcg_op_supported(INDEX_op_mulu2, TCG_TYPE_I64, 0) || + tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { TCGv_i64 t0 = tcg_temp_ebb_new_i64(); TCGv_i64 t1 = tcg_temp_ebb_new_i64(); TCGv_i64 t2 = tcg_temp_ebb_new_i64(); @@ -3164,11 +3083,9 @@ void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) { if (TCG_TARGET_REG_BITS == 32) { tcg_gen_mov_i32(ret, TCGV_LOW(arg)); - } else if (TCG_TARGET_HAS_extr_i64_i32) { + } else { tcg_gen_op2(INDEX_op_extrl_i64_i32, TCG_TYPE_I32, tcgv_i32_arg(ret), tcgv_i64_arg(arg)); - } else { - tcg_gen_mov_i32(ret, (TCGv_i32)arg); } } @@ -3176,14 +3093,9 @@ void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) { if (TCG_TARGET_REG_BITS == 32) { tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); - } else if (TCG_TARGET_HAS_extr_i64_i32) { + } else { tcg_gen_op2(INDEX_op_extrh_i64_i32, TCG_TYPE_I32, tcgv_i32_arg(ret), tcgv_i64_arg(arg)); - } else { - TCGv_i64 t = tcg_temp_ebb_new_i64(); - tcg_gen_shri_i64(t, arg, 32); - tcg_gen_mov_i32(ret, (TCGv_i32)t); - tcg_temp_free_i64(t); } } diff --git a/tcg/tcg.c b/tcg/tcg.c index ec7f6743d7..648333a9fb 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -133,9 +133,11 @@ static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long); static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2); static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); static void tcg_out_goto_tb(TCGContext *s, int which); -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]); +static void tcg_out_goto_ptr(TCGContext *s, TCGReg dest); +static void tcg_out_mb(TCGContext *s, unsigned bar); +static void tcg_out_br(TCGContext *s, TCGLabel *l); +static void tcg_out_set_carry(TCGContext *s); +static void tcg_out_set_borrow(TCGContext *s); #if TCG_TARGET_MAYBE_vec static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg dst, TCGReg src); @@ -861,6 +863,7 @@ static int tcg_out_pool_finalize(TCGContext *s) #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4), typedef enum { + C_Dynamic = -2, C_NotImplemented = -1, #include "tcg-target-con-set.h" } TCGConstraintSetIndex; @@ -954,6 +957,164 @@ static const TCGConstraintSet constraint_sets[] = { #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4) +/* + * TCGOutOp is the base class for a set of structures that describe how + * to generate code for a given TCGOpcode. + * + * @static_constraint: + * C_NotImplemented: The TCGOpcode is not supported by the backend. + * C_Dynamic: Use @dynamic_constraint to select a constraint set + * based on any of @type, @flags, or host isa. + * Otherwise: The register allocation constrains for the TCGOpcode. + * + * Subclasses of TCGOutOp will define a set of output routines that may + * be used. Such routines will often be selected by the set of registers + * and constants that come out of register allocation. The set of + * routines that are provided will guide the set of constraints that are + * legal. In particular, assume that tcg_optimize() has done its job in + * swapping commutative operands and folding operations for which all + * operands are constant. + */ +typedef struct TCGOutOp { + TCGConstraintSetIndex static_constraint; + TCGConstraintSetIndex (*dynamic_constraint)(TCGType type, unsigned flags); +} TCGOutOp; + +typedef struct TCGOutOpAddSubCarry { + TCGOutOp base; + void (*out_rrr)(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2); + void (*out_rri)(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2); + void (*out_rir)(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2); + void (*out_rii)(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, tcg_target_long a2); +} TCGOutOpAddSubCarry; + +typedef struct TCGOutOpBinary { + TCGOutOp base; + void (*out_rrr)(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2); + void (*out_rri)(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, tcg_target_long a2); +} TCGOutOpBinary; + +typedef struct TCGOutOpBrcond { + TCGOutOp base; + void (*out_rr)(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a1, TCGReg a2, TCGLabel *label); + void (*out_ri)(TCGContext *s, TCGType type, TCGCond cond, + TCGReg a1, tcg_target_long a2, TCGLabel *label); +} TCGOutOpBrcond; + +typedef struct TCGOutOpBrcond2 { + TCGOutOp base; + void (*out)(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, + TCGArg bh, bool const_bh, TCGLabel *l); +} TCGOutOpBrcond2; + +typedef struct TCGOutOpBswap { + TCGOutOp base; + void (*out_rr)(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags); +} TCGOutOpBswap; + +typedef struct TCGOutOpDeposit { + TCGOutOp base; + void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned ofs, unsigned len); + void (*out_rri)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + tcg_target_long a2, unsigned ofs, unsigned len); + void (*out_rzr)(TCGContext *s, TCGType type, TCGReg a0, + TCGReg a2, unsigned ofs, unsigned len); +} TCGOutOpDeposit; + +typedef struct TCGOutOpDivRem { + TCGOutOp base; + void (*out_rr01r)(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a4); +} TCGOutOpDivRem; + +typedef struct TCGOutOpExtract { + TCGOutOp base; + void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + unsigned ofs, unsigned len); +} TCGOutOpExtract; + +typedef struct TCGOutOpExtract2 { + TCGOutOp base; + void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned shr); +} TCGOutOpExtract2; + +typedef struct TCGOutOpLoad { + TCGOutOp base; + void (*out)(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, intptr_t offset); +} TCGOutOpLoad; + +typedef struct TCGOutOpMovcond { + TCGOutOp base; + void (*out)(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, + TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf); +} TCGOutOpMovcond; + +typedef struct TCGOutOpMul2 { + TCGOutOp base; + void (*out_rrrr)(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3); +} TCGOutOpMul2; + +typedef struct TCGOutOpQemuLdSt { + TCGOutOp base; + void (*out)(TCGContext *s, TCGType type, TCGReg dest, + TCGReg addr, MemOpIdx oi); +} TCGOutOpQemuLdSt; + +typedef struct TCGOutOpQemuLdSt2 { + TCGOutOp base; + void (*out)(TCGContext *s, TCGType type, TCGReg dlo, TCGReg dhi, + TCGReg addr, MemOpIdx oi); +} TCGOutOpQemuLdSt2; + +typedef struct TCGOutOpUnary { + TCGOutOp base; + void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1); +} TCGOutOpUnary; + +typedef struct TCGOutOpSetcond { + TCGOutOp base; + void (*out_rrr)(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg a1, TCGReg a2); + void (*out_rri)(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg a1, tcg_target_long a2); +} TCGOutOpSetcond; + +typedef struct TCGOutOpSetcond2 { + TCGOutOp base; + void (*out)(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, TCGArg bh, bool const_bh); +} TCGOutOpSetcond2; + +typedef struct TCGOutOpStore { + TCGOutOp base; + void (*out_r)(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, intptr_t offset); + void (*out_i)(TCGContext *s, TCGType type, tcg_target_long data, + TCGReg base, intptr_t offset); +} TCGOutOpStore; + +typedef struct TCGOutOpSubtract { + TCGOutOp base; + void (*out_rrr)(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2); + void (*out_rir)(TCGContext *s, TCGType type, + TCGReg a0, tcg_target_long a1, TCGReg a2); +} TCGOutOpSubtract; + #include "tcg-target.c.inc" #ifndef CONFIG_TCG_INTERPRETER @@ -963,6 +1124,144 @@ QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) - < MIN_TLB_MASK_TABLE_OFS); #endif +#if TCG_TARGET_REG_BITS == 64 +/* + * We require these functions for slow-path function calls. + * Adapt them generically for opcode output. + */ + +static void tgen_exts_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_exts_i32_i64(s, a0, a1); +} + +static const TCGOutOpUnary outop_exts_i32_i64 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_exts_i32_i64, +}; + +static void tgen_extu_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_extu_i32_i64(s, a0, a1); +} + +static const TCGOutOpUnary outop_extu_i32_i64 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extu_i32_i64, +}; + +static void tgen_extrl_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_extrl_i64_i32(s, a0, a1); +} + +static const TCGOutOpUnary outop_extrl_i64_i32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = TCG_TARGET_HAS_extr_i64_i32 ? tgen_extrl_i64_i32 : NULL, +}; +#endif + +static const TCGOutOp outop_goto_ptr = { + .static_constraint = C_O0_I1(r), +}; + +static const TCGOutOpLoad outop_ld = { + .base.static_constraint = C_O1_I1(r, r), + .out = tcg_out_ld, +}; + +/* + * Register V as the TCGOutOp for O. + * This verifies that V is of type T, otherwise give a nice compiler error. + * This prevents trivial mistakes within each arch/tcg-target.c.inc. + */ +#define OUTOP(O, T, V) [O] = _Generic(V, T: &V.base) + +/* Register allocation descriptions for every TCGOpcode. */ +static const TCGOutOp * const all_outop[NB_OPS] = { + OUTOP(INDEX_op_add, TCGOutOpBinary, outop_add), + OUTOP(INDEX_op_addci, TCGOutOpAddSubCarry, outop_addci), + OUTOP(INDEX_op_addcio, TCGOutOpBinary, outop_addcio), + OUTOP(INDEX_op_addco, TCGOutOpBinary, outop_addco), + /* addc1o is implemented with set_carry + addcio */ + OUTOP(INDEX_op_addc1o, TCGOutOpBinary, outop_addcio), + OUTOP(INDEX_op_and, TCGOutOpBinary, outop_and), + OUTOP(INDEX_op_andc, TCGOutOpBinary, outop_andc), + OUTOP(INDEX_op_brcond, TCGOutOpBrcond, outop_brcond), + OUTOP(INDEX_op_bswap16, TCGOutOpBswap, outop_bswap16), + OUTOP(INDEX_op_bswap32, TCGOutOpBswap, outop_bswap32), + OUTOP(INDEX_op_clz, TCGOutOpBinary, outop_clz), + OUTOP(INDEX_op_ctpop, TCGOutOpUnary, outop_ctpop), + OUTOP(INDEX_op_ctz, TCGOutOpBinary, outop_ctz), + OUTOP(INDEX_op_deposit, TCGOutOpDeposit, outop_deposit), + OUTOP(INDEX_op_divs, TCGOutOpBinary, outop_divs), + OUTOP(INDEX_op_divu, TCGOutOpBinary, outop_divu), + OUTOP(INDEX_op_divs2, TCGOutOpDivRem, outop_divs2), + OUTOP(INDEX_op_divu2, TCGOutOpDivRem, outop_divu2), + OUTOP(INDEX_op_eqv, TCGOutOpBinary, outop_eqv), + OUTOP(INDEX_op_extract, TCGOutOpExtract, outop_extract), + OUTOP(INDEX_op_extract2, TCGOutOpExtract2, outop_extract2), + OUTOP(INDEX_op_ld8u, TCGOutOpLoad, outop_ld8u), + OUTOP(INDEX_op_ld8s, TCGOutOpLoad, outop_ld8s), + OUTOP(INDEX_op_ld16u, TCGOutOpLoad, outop_ld16u), + OUTOP(INDEX_op_ld16s, TCGOutOpLoad, outop_ld16s), + OUTOP(INDEX_op_ld, TCGOutOpLoad, outop_ld), + OUTOP(INDEX_op_movcond, TCGOutOpMovcond, outop_movcond), + OUTOP(INDEX_op_mul, TCGOutOpBinary, outop_mul), + OUTOP(INDEX_op_muls2, TCGOutOpMul2, outop_muls2), + OUTOP(INDEX_op_mulsh, TCGOutOpBinary, outop_mulsh), + OUTOP(INDEX_op_mulu2, TCGOutOpMul2, outop_mulu2), + OUTOP(INDEX_op_muluh, TCGOutOpBinary, outop_muluh), + OUTOP(INDEX_op_nand, TCGOutOpBinary, outop_nand), + OUTOP(INDEX_op_neg, TCGOutOpUnary, outop_neg), + OUTOP(INDEX_op_negsetcond, TCGOutOpSetcond, outop_negsetcond), + OUTOP(INDEX_op_nor, TCGOutOpBinary, outop_nor), + OUTOP(INDEX_op_not, TCGOutOpUnary, outop_not), + OUTOP(INDEX_op_or, TCGOutOpBinary, outop_or), + OUTOP(INDEX_op_orc, TCGOutOpBinary, outop_orc), + OUTOP(INDEX_op_qemu_ld, TCGOutOpQemuLdSt, outop_qemu_ld), + OUTOP(INDEX_op_qemu_ld2, TCGOutOpQemuLdSt2, outop_qemu_ld2), + OUTOP(INDEX_op_qemu_st, TCGOutOpQemuLdSt, outop_qemu_st), + OUTOP(INDEX_op_qemu_st2, TCGOutOpQemuLdSt2, outop_qemu_st2), + OUTOP(INDEX_op_rems, TCGOutOpBinary, outop_rems), + OUTOP(INDEX_op_remu, TCGOutOpBinary, outop_remu), + OUTOP(INDEX_op_rotl, TCGOutOpBinary, outop_rotl), + OUTOP(INDEX_op_rotr, TCGOutOpBinary, outop_rotr), + OUTOP(INDEX_op_sar, TCGOutOpBinary, outop_sar), + OUTOP(INDEX_op_setcond, TCGOutOpSetcond, outop_setcond), + OUTOP(INDEX_op_sextract, TCGOutOpExtract, outop_sextract), + OUTOP(INDEX_op_shl, TCGOutOpBinary, outop_shl), + OUTOP(INDEX_op_shr, TCGOutOpBinary, outop_shr), + OUTOP(INDEX_op_st, TCGOutOpStore, outop_st), + OUTOP(INDEX_op_st8, TCGOutOpStore, outop_st8), + OUTOP(INDEX_op_st16, TCGOutOpStore, outop_st16), + OUTOP(INDEX_op_sub, TCGOutOpSubtract, outop_sub), + OUTOP(INDEX_op_subbi, TCGOutOpAddSubCarry, outop_subbi), + OUTOP(INDEX_op_subbio, TCGOutOpAddSubCarry, outop_subbio), + OUTOP(INDEX_op_subbo, TCGOutOpAddSubCarry, outop_subbo), + /* subb1o is implemented with set_borrow + subbio */ + OUTOP(INDEX_op_subb1o, TCGOutOpAddSubCarry, outop_subbio), + OUTOP(INDEX_op_xor, TCGOutOpBinary, outop_xor), + + [INDEX_op_goto_ptr] = &outop_goto_ptr, + +#if TCG_TARGET_REG_BITS == 32 + OUTOP(INDEX_op_brcond2_i32, TCGOutOpBrcond2, outop_brcond2), + OUTOP(INDEX_op_setcond2_i32, TCGOutOpSetcond2, outop_setcond2), +#else + OUTOP(INDEX_op_bswap64, TCGOutOpUnary, outop_bswap64), + OUTOP(INDEX_op_ext_i32_i64, TCGOutOpUnary, outop_exts_i32_i64), + OUTOP(INDEX_op_extu_i32_i64, TCGOutOpUnary, outop_extu_i32_i64), + OUTOP(INDEX_op_extrl_i64_i32, TCGOutOpUnary, outop_extrl_i64_i32), + OUTOP(INDEX_op_extrh_i64_i32, TCGOutOpUnary, outop_extrh_i64_i32), + OUTOP(INDEX_op_ld32u, TCGOutOpLoad, outop_ld32u), + OUTOP(INDEX_op_ld32s, TCGOutOpLoad, outop_ld32s), + OUTOP(INDEX_op_st32, TCGOutOpStore, outop_st), +#endif +}; + +#undef OUTOP + /* * All TCG threads except the parent (i.e. the one that called tcg_context_init * and registered the target's TCG globals) must register with this function @@ -1690,7 +1989,6 @@ void tcg_func_start(TCGContext *s) QSIMPLEQ_INIT(&s->labels); tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); - tcg_debug_assert(s->insn_start_words > 0); } static TCGTemp *tcg_temp_alloc(TCGContext *s) @@ -2146,208 +2444,56 @@ bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags) case INDEX_op_exit_tb: case INDEX_op_goto_tb: case INDEX_op_goto_ptr: - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i64: return true; - case INDEX_op_qemu_st8_i32: - return TCG_TARGET_HAS_qemu_st8_i32; - - case INDEX_op_qemu_ld_i128: - case INDEX_op_qemu_st_i128: - return TCG_TARGET_HAS_qemu_ldst_i128; - - case INDEX_op_mov_i32: - case INDEX_op_setcond_i32: - case INDEX_op_brcond_i32: - case INDEX_op_movcond_i32: - case INDEX_op_ld8u_i32: - case INDEX_op_ld8s_i32: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16s_i32: - case INDEX_op_ld_i32: - case INDEX_op_st8_i32: - case INDEX_op_st16_i32: - case INDEX_op_st_i32: - case INDEX_op_add_i32: - case INDEX_op_sub_i32: - case INDEX_op_neg_i32: - case INDEX_op_mul_i32: - case INDEX_op_and_i32: - case INDEX_op_or_i32: - case INDEX_op_xor_i32: - case INDEX_op_shl_i32: - case INDEX_op_shr_i32: - case INDEX_op_sar_i32: - case INDEX_op_extract_i32: - case INDEX_op_sextract_i32: - case INDEX_op_deposit_i32: + case INDEX_op_qemu_ld: + case INDEX_op_qemu_st: + tcg_debug_assert(type <= TCG_TYPE_REG); return true; - case INDEX_op_negsetcond_i32: - return TCG_TARGET_HAS_negsetcond_i32; - case INDEX_op_div_i32: - case INDEX_op_divu_i32: - return TCG_TARGET_HAS_div_i32; - case INDEX_op_rem_i32: - case INDEX_op_remu_i32: - return TCG_TARGET_HAS_rem_i32; - case INDEX_op_div2_i32: - case INDEX_op_divu2_i32: - return TCG_TARGET_HAS_div2_i32; - case INDEX_op_rotl_i32: - case INDEX_op_rotr_i32: - return TCG_TARGET_HAS_rot_i32; - case INDEX_op_extract2_i32: - return TCG_TARGET_HAS_extract2_i32; - case INDEX_op_add2_i32: - return TCG_TARGET_HAS_add2_i32; - case INDEX_op_sub2_i32: - return TCG_TARGET_HAS_sub2_i32; - case INDEX_op_mulu2_i32: - return TCG_TARGET_HAS_mulu2_i32; - case INDEX_op_muls2_i32: - return TCG_TARGET_HAS_muls2_i32; - case INDEX_op_muluh_i32: - return TCG_TARGET_HAS_muluh_i32; - case INDEX_op_mulsh_i32: - return TCG_TARGET_HAS_mulsh_i32; - case INDEX_op_ext8s_i32: - return TCG_TARGET_HAS_ext8s_i32; - case INDEX_op_ext16s_i32: - return TCG_TARGET_HAS_ext16s_i32; - case INDEX_op_ext8u_i32: - return TCG_TARGET_HAS_ext8u_i32; - case INDEX_op_ext16u_i32: - return TCG_TARGET_HAS_ext16u_i32; - case INDEX_op_bswap16_i32: - return TCG_TARGET_HAS_bswap16_i32; - case INDEX_op_bswap32_i32: - return TCG_TARGET_HAS_bswap32_i32; - case INDEX_op_not_i32: - return TCG_TARGET_HAS_not_i32; - case INDEX_op_andc_i32: - return TCG_TARGET_HAS_andc_i32; - case INDEX_op_orc_i32: - return TCG_TARGET_HAS_orc_i32; - case INDEX_op_eqv_i32: - return TCG_TARGET_HAS_eqv_i32; - case INDEX_op_nand_i32: - return TCG_TARGET_HAS_nand_i32; - case INDEX_op_nor_i32: - return TCG_TARGET_HAS_nor_i32; - case INDEX_op_clz_i32: - return TCG_TARGET_HAS_clz_i32; - case INDEX_op_ctz_i32: - return TCG_TARGET_HAS_ctz_i32; - case INDEX_op_ctpop_i32: - return TCG_TARGET_HAS_ctpop_i32; + case INDEX_op_qemu_ld2: + case INDEX_op_qemu_st2: + if (TCG_TARGET_REG_BITS == 32) { + tcg_debug_assert(type == TCG_TYPE_I64); + return true; + } + tcg_debug_assert(type == TCG_TYPE_I128); + goto do_lookup; + + case INDEX_op_add: + case INDEX_op_and: + case INDEX_op_brcond: + case INDEX_op_deposit: + case INDEX_op_extract: + case INDEX_op_ld8u: + case INDEX_op_ld8s: + case INDEX_op_ld16u: + case INDEX_op_ld16s: + case INDEX_op_ld: + case INDEX_op_mov: + case INDEX_op_movcond: + case INDEX_op_negsetcond: + case INDEX_op_or: + case INDEX_op_setcond: + case INDEX_op_sextract: + case INDEX_op_st8: + case INDEX_op_st16: + case INDEX_op_st: + case INDEX_op_xor: + return has_type; case INDEX_op_brcond2_i32: case INDEX_op_setcond2_i32: return TCG_TARGET_REG_BITS == 32; - case INDEX_op_mov_i64: - case INDEX_op_setcond_i64: - case INDEX_op_brcond_i64: - case INDEX_op_movcond_i64: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i64: - case INDEX_op_ld32u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld_i64: - case INDEX_op_st8_i64: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: - case INDEX_op_add_i64: - case INDEX_op_sub_i64: - case INDEX_op_neg_i64: - case INDEX_op_mul_i64: - case INDEX_op_and_i64: - case INDEX_op_or_i64: - case INDEX_op_xor_i64: - case INDEX_op_shl_i64: - case INDEX_op_shr_i64: - case INDEX_op_sar_i64: + case INDEX_op_ld32u: + case INDEX_op_ld32s: + case INDEX_op_st32: case INDEX_op_ext_i32_i64: case INDEX_op_extu_i32_i64: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i64: - case INDEX_op_deposit_i64: - return TCG_TARGET_REG_BITS == 64; - - case INDEX_op_negsetcond_i64: - return TCG_TARGET_HAS_negsetcond_i64; - case INDEX_op_div_i64: - case INDEX_op_divu_i64: - return TCG_TARGET_HAS_div_i64; - case INDEX_op_rem_i64: - case INDEX_op_remu_i64: - return TCG_TARGET_HAS_rem_i64; - case INDEX_op_div2_i64: - case INDEX_op_divu2_i64: - return TCG_TARGET_HAS_div2_i64; - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i64: - return TCG_TARGET_HAS_rot_i64; - case INDEX_op_extract2_i64: - return TCG_TARGET_HAS_extract2_i64; case INDEX_op_extrl_i64_i32: case INDEX_op_extrh_i64_i32: - return TCG_TARGET_HAS_extr_i64_i32; - case INDEX_op_ext8s_i64: - return TCG_TARGET_HAS_ext8s_i64; - case INDEX_op_ext16s_i64: - return TCG_TARGET_HAS_ext16s_i64; - case INDEX_op_ext32s_i64: - return TCG_TARGET_HAS_ext32s_i64; - case INDEX_op_ext8u_i64: - return TCG_TARGET_HAS_ext8u_i64; - case INDEX_op_ext16u_i64: - return TCG_TARGET_HAS_ext16u_i64; - case INDEX_op_ext32u_i64: - return TCG_TARGET_HAS_ext32u_i64; - case INDEX_op_bswap16_i64: - return TCG_TARGET_HAS_bswap16_i64; - case INDEX_op_bswap32_i64: - return TCG_TARGET_HAS_bswap32_i64; - case INDEX_op_bswap64_i64: - return TCG_TARGET_HAS_bswap64_i64; - case INDEX_op_not_i64: - return TCG_TARGET_HAS_not_i64; - case INDEX_op_andc_i64: - return TCG_TARGET_HAS_andc_i64; - case INDEX_op_orc_i64: - return TCG_TARGET_HAS_orc_i64; - case INDEX_op_eqv_i64: - return TCG_TARGET_HAS_eqv_i64; - case INDEX_op_nand_i64: - return TCG_TARGET_HAS_nand_i64; - case INDEX_op_nor_i64: - return TCG_TARGET_HAS_nor_i64; - case INDEX_op_clz_i64: - return TCG_TARGET_HAS_clz_i64; - case INDEX_op_ctz_i64: - return TCG_TARGET_HAS_ctz_i64; - case INDEX_op_ctpop_i64: - return TCG_TARGET_HAS_ctpop_i64; - case INDEX_op_add2_i64: - return TCG_TARGET_HAS_add2_i64; - case INDEX_op_sub2_i64: - return TCG_TARGET_HAS_sub2_i64; - case INDEX_op_mulu2_i64: - return TCG_TARGET_HAS_mulu2_i64; - case INDEX_op_muls2_i64: - return TCG_TARGET_HAS_muls2_i64; - case INDEX_op_muluh_i64: - return TCG_TARGET_HAS_muluh_i64; - case INDEX_op_mulsh_i64: - return TCG_TARGET_HAS_mulsh_i64; + return TCG_TARGET_REG_BITS == 64; case INDEX_op_mov_vec: case INDEX_op_dup_vec: @@ -2416,8 +2562,33 @@ bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags) return has_type && TCG_TARGET_HAS_cmpsel_vec; default: - tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); + if (op < INDEX_op_last_generic) { + const TCGOutOp *outop; + TCGConstraintSetIndex con_set; + + if (!has_type) { + return false; + } + + do_lookup: + outop = all_outop[op]; + tcg_debug_assert(outop != NULL); + + con_set = outop->static_constraint; + if (con_set == C_Dynamic) { + con_set = outop->dynamic_constraint(type, flags); + } + if (con_set >= 0) { + return true; + } + tcg_debug_assert(con_set == C_NotImplemented); + return false; + } + tcg_debug_assert(op < NB_OPS); return true; + + case INDEX_op_last_generic: + g_assert_not_reached(); } } @@ -2771,7 +2942,7 @@ void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) nb_oargs = 0; col += ne_fprintf(f, "\n ----"); - for (i = 0, k = s->insn_start_words; i < k; ++i) { + for (i = 0, k = INSN_START_WORDS; i < k; ++i) { col += ne_fprintf(f, " %016" PRIx64, tcg_get_insn_start_param(op, i)); } @@ -2808,18 +2979,23 @@ void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) col += ne_fprintf(f, ",%s", t); } } else { - col += ne_fprintf(f, " %s ", def->name); + if (def->flags & TCG_OPF_INT) { + col += ne_fprintf(f, " %s_i%d ", + def->name, + 8 * tcg_type_size(TCGOP_TYPE(op))); + } else if (def->flags & TCG_OPF_VECTOR) { + col += ne_fprintf(f, "%s v%d,e%d,", + def->name, + 8 * tcg_type_size(TCGOP_TYPE(op)), + 8 << TCGOP_VECE(op)); + } else { + col += ne_fprintf(f, " %s ", def->name); + } nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; nb_cargs = def->nb_cargs; - if (def->flags & TCG_OPF_VECTOR) { - col += ne_fprintf(f, "v%d,e%d,", - 8 * tcg_type_size(TCGOP_TYPE(op)), - 8 << TCGOP_VECE(op)); - } - k = 0; for (i = 0; i < nb_oargs; i++) { const char *sep = k ? "," : ""; @@ -2834,16 +3010,12 @@ void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) op->args[k++])); } switch (c) { - case INDEX_op_brcond_i32: - case INDEX_op_setcond_i32: - case INDEX_op_negsetcond_i32: - case INDEX_op_movcond_i32: + case INDEX_op_brcond: + case INDEX_op_setcond: + case INDEX_op_negsetcond: + case INDEX_op_movcond: case INDEX_op_brcond2_i32: case INDEX_op_setcond2_i32: - case INDEX_op_brcond_i64: - case INDEX_op_setcond_i64: - case INDEX_op_negsetcond_i64: - case INDEX_op_movcond_i64: case INDEX_op_cmp_vec: case INDEX_op_cmpsel_vec: if (op->args[k] < ARRAY_SIZE(cond_name) @@ -2854,13 +3026,10 @@ void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) } i = 1; break; - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st8_i32: - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i64: - case INDEX_op_qemu_ld_i128: - case INDEX_op_qemu_st_i128: + case INDEX_op_qemu_ld: + case INDEX_op_qemu_st: + case INDEX_op_qemu_ld2: + case INDEX_op_qemu_st2: { const char *s_al, *s_op, *s_at; MemOpIdx oi = op->args[k++]; @@ -2883,11 +3052,9 @@ void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) i = 1; } break; - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i32: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: + case INDEX_op_bswap16: + case INDEX_op_bswap32: + case INDEX_op_bswap64: { TCGArg flags = op->args[k]; const char *name = NULL; @@ -2928,8 +3095,7 @@ void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) switch (c) { case INDEX_op_set_label: case INDEX_op_br: - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: + case INDEX_op_brcond: case INDEX_op_brcond2_i32: col += ne_fprintf(f, "%s$L%d", k ? "," : "", arg_label(op->args[k])->id); @@ -3335,19 +3501,27 @@ static void process_constraint_sets(void) static const TCGArgConstraint *opcode_args_ct(const TCGOp *op) { - const TCGOpDef *def = &tcg_op_defs[op->opc]; + TCGOpcode opc = op->opc; + TCGType type = TCGOP_TYPE(op); + unsigned flags = TCGOP_FLAGS(op); + const TCGOpDef *def = &tcg_op_defs[opc]; + const TCGOutOp *outop = all_outop[opc]; TCGConstraintSetIndex con_set; -#ifdef CONFIG_DEBUG_TCG - assert(tcg_op_supported(op->opc, TCGOP_TYPE(op), TCGOP_FLAGS(op))); -#endif - if (def->flags & TCG_OPF_NOT_PRESENT) { return empty_cts; } - con_set = tcg_target_op_def(op->opc, TCGOP_TYPE(op), TCGOP_FLAGS(op)); - tcg_debug_assert(con_set >= 0 && con_set < ARRAY_SIZE(constraint_sets)); + if (outop) { + con_set = outop->static_constraint; + if (con_set == C_Dynamic) { + con_set = outop->dynamic_constraint(type, flags); + } + } else { + con_set = tcg_target_op_def(opc, type, flags); + } + tcg_debug_assert(con_set >= 0); + tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); /* The constraint arguments must match TCGOpcode arguments. */ tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs); @@ -3376,8 +3550,7 @@ void tcg_op_remove(TCGContext *s, TCGOp *op) case INDEX_op_br: remove_label_use(op, 0); break; - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: + case INDEX_op_brcond: remove_label_use(op, 3); break; case INDEX_op_brcond2_i32: @@ -3449,21 +3622,21 @@ TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) } TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, - TCGOpcode opc, unsigned nargs) + TCGOpcode opc, TCGType type, unsigned nargs) { TCGOp *new_op = tcg_op_alloc(opc, nargs); - TCGOP_TYPE(new_op) = TCGOP_TYPE(old_op); + TCGOP_TYPE(new_op) = type; QTAILQ_INSERT_BEFORE(old_op, new_op, link); return new_op; } TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, - TCGOpcode opc, unsigned nargs) + TCGOpcode opc, TCGType type, unsigned nargs) { TCGOp *new_op = tcg_op_alloc(opc, nargs); - TCGOP_TYPE(new_op) = TCGOP_TYPE(old_op); + TCGOP_TYPE(new_op) = type; QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); return new_op; } @@ -3478,8 +3651,7 @@ static void move_label_uses(TCGLabel *to, TCGLabel *from) case INDEX_op_br: op->args[0] = label_arg(to); break; - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: + case INDEX_op_brcond: op->args[3] = label_arg(to); break; case INDEX_op_brcond2_i32: @@ -3798,6 +3970,17 @@ liveness_pass_0(TCGContext *s) } } +static void assert_carry_dead(TCGContext *s) +{ + /* + * Carry operations can be separated by a few insns like mov, + * load or store, but they should always be "close", and + * carry-out operations should always be paired with carry-in. + * At various boundaries, carry must have been consumed. + */ + tcg_debug_assert(!s->carry_live); +} + /* Liveness analysis : update the opc_arg_life array to tell if a given input arguments is dead. Instructions updating dead temporaries are removed. */ @@ -3808,28 +3991,28 @@ liveness_pass_1(TCGContext *s) int nb_temps = s->nb_temps; TCGOp *op, *op_prev; TCGRegSet *prefs; - int i; prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); - for (i = 0; i < nb_temps; ++i) { + for (int i = 0; i < nb_temps; ++i) { s->temps[i].state_ptr = prefs + i; } /* ??? Should be redundant with the exit_tb that ends the TB. */ la_func_end(s, nb_globals, nb_temps); + s->carry_live = false; QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { int nb_iargs, nb_oargs; TCGOpcode opc_new, opc_new2; - bool have_opc_new2; TCGLifeData arg_life = 0; TCGTemp *ts; TCGOpcode opc = op->opc; - const TCGOpDef *def = &tcg_op_defs[opc]; + const TCGOpDef *def; const TCGArgConstraint *args_ct; switch (opc) { case INDEX_op_call: + assert_carry_dead(s); { const TCGHelperInfo *info = tcg_call_info(op); int call_flags = tcg_call_flags(op); @@ -3839,7 +4022,7 @@ liveness_pass_1(TCGContext *s) /* pure functions can be removed if their result is unused */ if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { - for (i = 0; i < nb_oargs; i++) { + for (int i = 0; i < nb_oargs; i++) { ts = arg_temp(op->args[i]); if (ts->state != TS_DEAD) { goto do_not_remove_call; @@ -3850,7 +4033,7 @@ liveness_pass_1(TCGContext *s) do_not_remove_call: /* Output args are dead. */ - for (i = 0; i < nb_oargs; i++) { + for (int i = 0; i < nb_oargs; i++) { ts = arg_temp(op->args[i]); if (ts->state & TS_DEAD) { arg_life |= DEAD_ARG << i; @@ -3873,7 +4056,7 @@ liveness_pass_1(TCGContext *s) } /* Record arguments that die in this helper. */ - for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { + for (int i = nb_oargs; i < nb_iargs + nb_oargs; i++) { ts = arg_temp(op->args[i]); if (ts->state & TS_DEAD) { arg_life |= DEAD_ARG << i; @@ -3893,7 +4076,7 @@ liveness_pass_1(TCGContext *s) * order so that if a temp is used more than once, the stack * reset to max happens before the register reset to 0. */ - for (i = nb_iargs - 1; i >= 0; i--) { + for (int i = nb_iargs - 1; i >= 0; i--) { const TCGCallArgumentLoc *loc = &info->in[i]; ts = arg_temp(op->args[nb_oargs + i]); @@ -3921,7 +4104,7 @@ liveness_pass_1(TCGContext *s) * If a temp is used once, this produces a single set bit; * if a temp is used multiple times, this produces a set. */ - for (i = 0; i < nb_iargs; i++) { + for (int i = 0; i < nb_iargs; i++) { const TCGCallArgumentLoc *loc = &info->in[i]; ts = arg_temp(op->args[nb_oargs + i]); @@ -3941,6 +4124,7 @@ liveness_pass_1(TCGContext *s) } break; case INDEX_op_insn_start: + assert_carry_dead(s); break; case INDEX_op_discard: /* mark the temporary as dead */ @@ -3949,62 +4133,15 @@ liveness_pass_1(TCGContext *s) la_reset_pref(ts); break; - case INDEX_op_add2_i32: - opc_new = INDEX_op_add_i32; - goto do_addsub2; - case INDEX_op_sub2_i32: - opc_new = INDEX_op_sub_i32; - goto do_addsub2; - case INDEX_op_add2_i64: - opc_new = INDEX_op_add_i64; - goto do_addsub2; - case INDEX_op_sub2_i64: - opc_new = INDEX_op_sub_i64; - do_addsub2: - nb_iargs = 4; - nb_oargs = 2; - /* Test if the high part of the operation is dead, but not - the low part. The result can be optimized to a simple - add or sub. This happens often for x86_64 guest when the - cpu mode is set to 32 bit. */ - if (arg_temp(op->args[1])->state == TS_DEAD) { - if (arg_temp(op->args[0])->state == TS_DEAD) { - goto do_remove; - } - /* Replace the opcode and adjust the args in place, - leaving 3 unused args at the end. */ - op->opc = opc = opc_new; - op->args[1] = op->args[2]; - op->args[2] = op->args[4]; - /* Fall through and mark the single-word operation live. */ - nb_iargs = 2; - nb_oargs = 1; - } - goto do_not_remove; - - case INDEX_op_mulu2_i32: - opc_new = INDEX_op_mul_i32; - opc_new2 = INDEX_op_muluh_i32; - have_opc_new2 = TCG_TARGET_HAS_muluh_i32; - goto do_mul2; - case INDEX_op_muls2_i32: - opc_new = INDEX_op_mul_i32; - opc_new2 = INDEX_op_mulsh_i32; - have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; - goto do_mul2; - case INDEX_op_mulu2_i64: - opc_new = INDEX_op_mul_i64; - opc_new2 = INDEX_op_muluh_i64; - have_opc_new2 = TCG_TARGET_HAS_muluh_i64; - goto do_mul2; - case INDEX_op_muls2_i64: - opc_new = INDEX_op_mul_i64; - opc_new2 = INDEX_op_mulsh_i64; - have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; + case INDEX_op_muls2: + opc_new = INDEX_op_mul; + opc_new2 = INDEX_op_mulsh; goto do_mul2; + case INDEX_op_mulu2: + opc_new = INDEX_op_mul; + opc_new2 = INDEX_op_muluh; do_mul2: - nb_iargs = 2; - nb_oargs = 2; + assert_carry_dead(s); if (arg_temp(op->args[1])->state == TS_DEAD) { if (arg_temp(op->args[0])->state == TS_DEAD) { /* Both parts of the operation are dead. */ @@ -4014,7 +4151,8 @@ liveness_pass_1(TCGContext *s) op->opc = opc = opc_new; op->args[1] = op->args[2]; op->args[2] = op->args[3]; - } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { + } else if (arg_temp(op->args[0])->state == TS_DEAD && + tcg_op_supported(opc_new2, TCGOP_TYPE(op), 0)) { /* The low part of the operation is dead; generate the high. */ op->opc = opc = opc_new2; op->args[0] = op->args[1]; @@ -4024,19 +4162,94 @@ liveness_pass_1(TCGContext *s) goto do_not_remove; } /* Mark the single-word operation live. */ - nb_oargs = 1; goto do_not_remove; - default: - /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ - nb_iargs = def->nb_iargs; - nb_oargs = def->nb_oargs; + case INDEX_op_addco: + if (s->carry_live) { + goto do_not_remove; + } + op->opc = opc = INDEX_op_add; + goto do_default; + + case INDEX_op_addcio: + if (s->carry_live) { + goto do_not_remove; + } + op->opc = opc = INDEX_op_addci; + goto do_default; - /* Test if the operation can be removed because all - its outputs are dead. We assume that nb_oargs == 0 - implies side effects */ - if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { - for (i = 0; i < nb_oargs; i++) { + case INDEX_op_subbo: + if (s->carry_live) { + goto do_not_remove; + } + /* Lower to sub, but this may also require canonicalization. */ + op->opc = opc = INDEX_op_sub; + ts = arg_temp(op->args[2]); + if (ts->kind == TEMP_CONST) { + ts = tcg_constant_internal(ts->type, -ts->val); + if (ts->state_ptr == NULL) { + tcg_debug_assert(temp_idx(ts) == nb_temps); + nb_temps++; + ts->state_ptr = tcg_malloc(sizeof(TCGRegSet)); + ts->state = TS_DEAD; + la_reset_pref(ts); + } + op->args[2] = temp_arg(ts); + op->opc = opc = INDEX_op_add; + } + goto do_default; + + case INDEX_op_subbio: + if (s->carry_live) { + goto do_not_remove; + } + op->opc = opc = INDEX_op_subbi; + goto do_default; + + case INDEX_op_addc1o: + if (s->carry_live) { + goto do_not_remove; + } + /* Lower to add, add +1. */ + op_prev = tcg_op_insert_before(s, op, INDEX_op_add, + TCGOP_TYPE(op), 3); + op_prev->args[0] = op->args[0]; + op_prev->args[1] = op->args[1]; + op_prev->args[2] = op->args[2]; + op->opc = opc = INDEX_op_add; + op->args[1] = op->args[0]; + ts = arg_temp(op->args[0]); + ts = tcg_constant_internal(ts->type, 1); + op->args[2] = temp_arg(ts); + goto do_default; + + case INDEX_op_subb1o: + if (s->carry_live) { + goto do_not_remove; + } + /* Lower to sub, add -1. */ + op_prev = tcg_op_insert_before(s, op, INDEX_op_sub, + TCGOP_TYPE(op), 3); + op_prev->args[0] = op->args[0]; + op_prev->args[1] = op->args[1]; + op_prev->args[2] = op->args[2]; + op->opc = opc = INDEX_op_add; + op->args[1] = op->args[0]; + ts = arg_temp(op->args[0]); + ts = tcg_constant_internal(ts->type, -1); + op->args[2] = temp_arg(ts); + goto do_default; + + default: + do_default: + /* + * Test if the operation can be removed because all + * its outputs are dead. We assume that nb_oargs == 0 + * implies side effects. + */ + def = &tcg_op_defs[opc]; + if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) { + for (int i = def->nb_oargs - 1; i >= 0; i--) { if (arg_temp(op->args[i])->state != TS_DEAD) { goto do_not_remove; } @@ -4050,7 +4263,11 @@ liveness_pass_1(TCGContext *s) break; do_not_remove: - for (i = 0; i < nb_oargs; i++) { + def = &tcg_op_defs[opc]; + nb_iargs = def->nb_iargs; + nb_oargs = def->nb_oargs; + + for (int i = 0; i < nb_oargs; i++) { ts = arg_temp(op->args[i]); /* Remember the preference of the uses that followed. */ @@ -4071,12 +4288,16 @@ liveness_pass_1(TCGContext *s) /* If end of basic block, update. */ if (def->flags & TCG_OPF_BB_EXIT) { + assert_carry_dead(s); la_func_end(s, nb_globals, nb_temps); } else if (def->flags & TCG_OPF_COND_BRANCH) { + assert_carry_dead(s); la_bb_sync(s, nb_globals, nb_temps); } else if (def->flags & TCG_OPF_BB_END) { + assert_carry_dead(s); la_bb_end(s, nb_globals, nb_temps); } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { + assert_carry_dead(s); la_global_sync(s, nb_globals); if (def->flags & TCG_OPF_CALL_CLOBBER) { la_cross_call(s, nb_temps); @@ -4084,15 +4305,18 @@ liveness_pass_1(TCGContext *s) } /* Record arguments that die in this opcode. */ - for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { + for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { ts = arg_temp(op->args[i]); if (ts->state & TS_DEAD) { arg_life |= DEAD_ARG << i; } } + if (def->flags & TCG_OPF_CARRY_OUT) { + s->carry_live = false; + } /* Input arguments are live for preceding opcodes. */ - for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { + for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { ts = arg_temp(op->args[i]); if (ts->state & TS_DEAD) { /* For operands that were dead, initially allow @@ -4101,11 +4325,13 @@ liveness_pass_1(TCGContext *s) ts->state &= ~TS_DEAD; } } + if (def->flags & TCG_OPF_CARRY_IN) { + s->carry_live = true; + } /* Incorporate constraints for this operand. */ switch (opc) { - case INDEX_op_mov_i32: - case INDEX_op_mov_i64: + case INDEX_op_mov: /* Note that these are TCG_OPF_NOT_PRESENT and do not have proper constraints. That said, special case moves to propagate preferences backward. */ @@ -4117,7 +4343,7 @@ liveness_pass_1(TCGContext *s) default: args_ct = opcode_args_ct(op); - for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { + for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { const TCGArgConstraint *ct = &args_ct[i]; TCGRegSet set, *pset; @@ -4141,6 +4367,7 @@ liveness_pass_1(TCGContext *s) } op->life = arg_life; } + assert_carry_dead(s); } /* Liveness analysis: Convert indirect regs to direct temporaries. */ @@ -4211,10 +4438,8 @@ liveness_pass_2(TCGContext *s) arg_ts = arg_temp(op->args[i]); dir_ts = arg_ts->state_ptr; if (dir_ts && arg_ts->state == TS_DEAD) { - TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 - ? INDEX_op_ld_i32 - : INDEX_op_ld_i64); - TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); + TCGOp *lop = tcg_op_insert_before(s, op, INDEX_op_ld, + arg_ts->type, 3); lop->args[0] = temp_arg(dir_ts); lop->args[1] = temp_arg(arg_ts->mem_base); @@ -4263,7 +4488,7 @@ liveness_pass_2(TCGContext *s) } /* Outputs become available. */ - if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { + if (opc == INDEX_op_mov) { arg_ts = arg_temp(op->args[0]); dir_ts = arg_ts->state_ptr; if (dir_ts) { @@ -4274,10 +4499,8 @@ liveness_pass_2(TCGContext *s) arg_ts->state = 0; if (NEED_SYNC_ARG(0)) { - TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 - ? INDEX_op_st_i32 - : INDEX_op_st_i64); - TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); + TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st, + arg_ts->type, 3); TCGTemp *out_ts = dir_ts; if (IS_DEAD_ARG(0)) { @@ -4310,10 +4533,8 @@ liveness_pass_2(TCGContext *s) /* Sync outputs upon their last write. */ if (NEED_SYNC_ARG(i)) { - TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 - ? INDEX_op_st_i32 - : INDEX_op_st_i64); - TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); + TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st, + arg_ts->type, 3); sop->args[0] = temp_arg(dir_ts); sop->args[1] = temp_arg(arg_ts->mem_base); @@ -4726,9 +4947,8 @@ static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) all globals are stored at their canonical location. */ static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) { - int i; - - for (i = s->nb_globals; i < s->nb_temps; i++) { + assert_carry_dead(s); + for (int i = s->nb_globals; i < s->nb_temps; i++) { TCGTemp *ts = &s->temps[i]; switch (ts->kind) { @@ -4759,6 +4979,7 @@ static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) */ static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) { + assert_carry_dead(s); sync_globals(s, allocated_regs); for (int i = s->nb_globals; i < s->nb_temps; i++) { @@ -5030,6 +5251,10 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) int const_args[TCG_MAX_OP_ARGS]; TCGCond op_cond; + if (def->flags & TCG_OPF_CARRY_IN) { + tcg_debug_assert(s->carry_live); + } + nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; @@ -5042,22 +5267,18 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) o_allocated_regs = s->reserved_regs; switch (op->opc) { - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: + case INDEX_op_brcond: op_cond = op->args[2]; break; - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - case INDEX_op_negsetcond_i32: - case INDEX_op_negsetcond_i64: + case INDEX_op_setcond: + case INDEX_op_negsetcond: case INDEX_op_cmp_vec: op_cond = op->args[3]; break; case INDEX_op_brcond2_i32: op_cond = op->args[4]; break; - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: + case INDEX_op_movcond: case INDEX_op_setcond2_i32: case INDEX_op_cmpsel_vec: op_cond = op->args[5]; @@ -5290,6 +5511,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) tcg_reg_alloc_bb_end(s, i_allocated_regs); } else { if (def->flags & TCG_OPF_CALL_CLOBBER) { + assert_carry_dead(s); /* XXX: permit generic clobber register list ? */ for (i = 0; i < TCG_TARGET_NB_REGS; i++) { if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { @@ -5366,50 +5588,345 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) } /* emit instruction */ + TCGType type = TCGOP_TYPE(op); switch (op->opc) { - case INDEX_op_ext8s_i32: - tcg_out_ext8s(s, TCG_TYPE_I32, new_args[0], new_args[1]); + case INDEX_op_addc1o: + tcg_out_set_carry(s); + /* fall through */ + case INDEX_op_add: + case INDEX_op_addcio: + case INDEX_op_addco: + case INDEX_op_and: + case INDEX_op_andc: + case INDEX_op_clz: + case INDEX_op_ctz: + case INDEX_op_divs: + case INDEX_op_divu: + case INDEX_op_eqv: + case INDEX_op_mul: + case INDEX_op_mulsh: + case INDEX_op_muluh: + case INDEX_op_nand: + case INDEX_op_nor: + case INDEX_op_or: + case INDEX_op_orc: + case INDEX_op_rems: + case INDEX_op_remu: + case INDEX_op_rotl: + case INDEX_op_rotr: + case INDEX_op_sar: + case INDEX_op_shl: + case INDEX_op_shr: + case INDEX_op_xor: + { + const TCGOutOpBinary *out = + container_of(all_outop[op->opc], TCGOutOpBinary, base); + + /* Constants should never appear in the first source operand. */ + tcg_debug_assert(!const_args[1]); + if (const_args[2]) { + out->out_rri(s, type, new_args[0], new_args[1], new_args[2]); + } else { + out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); + } + } break; - case INDEX_op_ext8s_i64: - tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]); + + case INDEX_op_sub: + { + const TCGOutOpSubtract *out = &outop_sub; + + /* + * Constants should never appear in the second source operand. + * These are folded to add with negative constant. + */ + tcg_debug_assert(!const_args[2]); + if (const_args[1]) { + out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); + } else { + out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); + } + } break; - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - tcg_out_ext8u(s, new_args[0], new_args[1]); + + case INDEX_op_subb1o: + tcg_out_set_borrow(s); + /* fall through */ + case INDEX_op_addci: + case INDEX_op_subbi: + case INDEX_op_subbio: + case INDEX_op_subbo: + { + const TCGOutOpAddSubCarry *out = + container_of(all_outop[op->opc], TCGOutOpAddSubCarry, base); + + if (const_args[2]) { + if (const_args[1]) { + out->out_rii(s, type, new_args[0], + new_args[1], new_args[2]); + } else { + out->out_rri(s, type, new_args[0], + new_args[1], new_args[2]); + } + } else if (const_args[1]) { + out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); + } else { + out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); + } + } break; - case INDEX_op_ext16s_i32: - tcg_out_ext16s(s, TCG_TYPE_I32, new_args[0], new_args[1]); + + case INDEX_op_bswap64: + case INDEX_op_ext_i32_i64: + case INDEX_op_extu_i32_i64: + case INDEX_op_extrl_i64_i32: + case INDEX_op_extrh_i64_i32: + assert(TCG_TARGET_REG_BITS == 64); + /* fall through */ + case INDEX_op_ctpop: + case INDEX_op_neg: + case INDEX_op_not: + { + const TCGOutOpUnary *out = + container_of(all_outop[op->opc], TCGOutOpUnary, base); + + /* Constants should have been folded. */ + tcg_debug_assert(!const_args[1]); + out->out_rr(s, type, new_args[0], new_args[1]); + } break; - case INDEX_op_ext16s_i64: - tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]); + + case INDEX_op_bswap16: + case INDEX_op_bswap32: + { + const TCGOutOpBswap *out = + container_of(all_outop[op->opc], TCGOutOpBswap, base); + + tcg_debug_assert(!const_args[1]); + out->out_rr(s, type, new_args[0], new_args[1], new_args[2]); + } break; - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - tcg_out_ext16u(s, new_args[0], new_args[1]); + + case INDEX_op_deposit: + { + const TCGOutOpDeposit *out = &outop_deposit; + + if (const_args[2]) { + tcg_debug_assert(!const_args[1]); + out->out_rri(s, type, new_args[0], new_args[1], + new_args[2], new_args[3], new_args[4]); + } else if (const_args[1]) { + tcg_debug_assert(new_args[1] == 0); + tcg_debug_assert(!const_args[2]); + out->out_rzr(s, type, new_args[0], new_args[2], + new_args[3], new_args[4]); + } else { + out->out_rrr(s, type, new_args[0], new_args[1], + new_args[2], new_args[3], new_args[4]); + } + } break; - case INDEX_op_ext32s_i64: - tcg_out_ext32s(s, new_args[0], new_args[1]); + + case INDEX_op_divs2: + case INDEX_op_divu2: + { + const TCGOutOpDivRem *out = + container_of(all_outop[op->opc], TCGOutOpDivRem, base); + + /* Only used by x86 and s390x, which use matching constraints. */ + tcg_debug_assert(new_args[0] == new_args[2]); + tcg_debug_assert(new_args[1] == new_args[3]); + tcg_debug_assert(!const_args[4]); + out->out_rr01r(s, type, new_args[0], new_args[1], new_args[4]); + } break; - case INDEX_op_ext32u_i64: - tcg_out_ext32u(s, new_args[0], new_args[1]); + + case INDEX_op_extract: + case INDEX_op_sextract: + { + const TCGOutOpExtract *out = + container_of(all_outop[op->opc], TCGOutOpExtract, base); + + tcg_debug_assert(!const_args[1]); + out->out_rr(s, type, new_args[0], new_args[1], + new_args[2], new_args[3]); + } break; - case INDEX_op_ext_i32_i64: - tcg_out_exts_i32_i64(s, new_args[0], new_args[1]); + + case INDEX_op_extract2: + { + const TCGOutOpExtract2 *out = &outop_extract2; + + tcg_debug_assert(!const_args[1]); + tcg_debug_assert(!const_args[2]); + out->out_rrr(s, type, new_args[0], new_args[1], + new_args[2], new_args[3]); + } break; - case INDEX_op_extu_i32_i64: - tcg_out_extu_i32_i64(s, new_args[0], new_args[1]); + + case INDEX_op_ld8u: + case INDEX_op_ld8s: + case INDEX_op_ld16u: + case INDEX_op_ld16s: + case INDEX_op_ld32u: + case INDEX_op_ld32s: + case INDEX_op_ld: + { + const TCGOutOpLoad *out = + container_of(all_outop[op->opc], TCGOutOpLoad, base); + + tcg_debug_assert(!const_args[1]); + out->out(s, type, new_args[0], new_args[1], new_args[2]); + } break; - case INDEX_op_extrl_i64_i32: - tcg_out_extrl_i64_i32(s, new_args[0], new_args[1]); + + case INDEX_op_muls2: + case INDEX_op_mulu2: + { + const TCGOutOpMul2 *out = + container_of(all_outop[op->opc], TCGOutOpMul2, base); + + tcg_debug_assert(!const_args[2]); + tcg_debug_assert(!const_args[3]); + out->out_rrrr(s, type, new_args[0], new_args[1], + new_args[2], new_args[3]); + } break; - default: - if (def->flags & TCG_OPF_VECTOR) { - tcg_out_vec_op(s, op->opc, TCGOP_TYPE(op) - TCG_TYPE_V64, - TCGOP_VECE(op), new_args, const_args); - } else { - tcg_out_op(s, op->opc, TCGOP_TYPE(op), new_args, const_args); + + case INDEX_op_st32: + /* Use tcg_op_st w/ I32. */ + type = TCG_TYPE_I32; + /* fall through */ + case INDEX_op_st: + case INDEX_op_st8: + case INDEX_op_st16: + { + const TCGOutOpStore *out = + container_of(all_outop[op->opc], TCGOutOpStore, base); + + if (const_args[0]) { + out->out_i(s, type, new_args[0], new_args[1], new_args[2]); + } else { + out->out_r(s, type, new_args[0], new_args[1], new_args[2]); + } + } + break; + + case INDEX_op_qemu_ld: + case INDEX_op_qemu_st: + { + const TCGOutOpQemuLdSt *out = + container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base); + + out->out(s, type, new_args[0], new_args[1], new_args[2]); } break; + + case INDEX_op_qemu_ld2: + case INDEX_op_qemu_st2: + { + const TCGOutOpQemuLdSt2 *out = + container_of(all_outop[op->opc], TCGOutOpQemuLdSt2, base); + + out->out(s, type, new_args[0], new_args[1], + new_args[2], new_args[3]); + } + break; + + case INDEX_op_brcond: + { + const TCGOutOpBrcond *out = &outop_brcond; + TCGCond cond = new_args[2]; + TCGLabel *label = arg_label(new_args[3]); + + tcg_debug_assert(!const_args[0]); + if (const_args[1]) { + out->out_ri(s, type, cond, new_args[0], new_args[1], label); + } else { + out->out_rr(s, type, cond, new_args[0], new_args[1], label); + } + } + break; + + case INDEX_op_movcond: + { + const TCGOutOpMovcond *out = &outop_movcond; + TCGCond cond = new_args[5]; + + tcg_debug_assert(!const_args[1]); + out->out(s, type, cond, new_args[0], + new_args[1], new_args[2], const_args[2], + new_args[3], const_args[3], + new_args[4], const_args[4]); + } + break; + + case INDEX_op_setcond: + case INDEX_op_negsetcond: + { + const TCGOutOpSetcond *out = + container_of(all_outop[op->opc], TCGOutOpSetcond, base); + TCGCond cond = new_args[3]; + + tcg_debug_assert(!const_args[1]); + if (const_args[2]) { + out->out_rri(s, type, cond, + new_args[0], new_args[1], new_args[2]); + } else { + out->out_rrr(s, type, cond, + new_args[0], new_args[1], new_args[2]); + } + } + break; + +#if TCG_TARGET_REG_BITS == 32 + case INDEX_op_brcond2_i32: + { + const TCGOutOpBrcond2 *out = &outop_brcond2; + TCGCond cond = new_args[4]; + TCGLabel *label = arg_label(new_args[5]); + + tcg_debug_assert(!const_args[0]); + tcg_debug_assert(!const_args[1]); + out->out(s, cond, new_args[0], new_args[1], + new_args[2], const_args[2], + new_args[3], const_args[3], label); + } + break; + case INDEX_op_setcond2_i32: + { + const TCGOutOpSetcond2 *out = &outop_setcond2; + TCGCond cond = new_args[5]; + + tcg_debug_assert(!const_args[1]); + tcg_debug_assert(!const_args[2]); + out->out(s, cond, new_args[0], new_args[1], new_args[2], + new_args[3], const_args[3], new_args[4], const_args[4]); + } + break; +#else + case INDEX_op_brcond2_i32: + case INDEX_op_setcond2_i32: + g_assert_not_reached(); +#endif + + case INDEX_op_goto_ptr: + tcg_debug_assert(!const_args[0]); + tcg_out_goto_ptr(s, new_args[0]); + break; + + default: + tcg_debug_assert(def->flags & TCG_OPF_VECTOR); + tcg_out_vec_op(s, op->opc, type - TCG_TYPE_V64, + TCGOP_VECE(op), new_args, const_args); + break; + } + + if (def->flags & TCG_OPF_CARRY_IN) { + s->carry_live = false; + } + if (def->flags & TCG_OPF_CARRY_OUT) { + s->carry_live = true; } /* move the outputs in the correct register if needed */ @@ -6317,7 +6834,7 @@ static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) { - int i, start_words, num_insns; + int i, num_insns; TCGOp *op; if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) @@ -6407,19 +6924,34 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) QSIMPLEQ_INIT(&s->ldst_labels); s->pool_labels = NULL; - start_words = s->insn_start_words; s->gen_insn_data = - tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * start_words); + tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * INSN_START_WORDS); tcg_out_tb_start(s); num_insns = -1; + s->carry_live = false; QTAILQ_FOREACH(op, &s->ops, link) { TCGOpcode opc = op->opc; switch (opc) { - case INDEX_op_mov_i32: - case INDEX_op_mov_i64: + case INDEX_op_extrl_i64_i32: + assert(TCG_TARGET_REG_BITS == 64); + /* + * If TCG_TYPE_I32 is represented in some canonical form, + * e.g. zero or sign-extended, then emit as a unary op. + * Otherwise we can treat this as a plain move. + * If the output dies, treat this as a plain move, because + * this will be implemented with a store. + */ + if (TCG_TARGET_HAS_extr_i64_i32) { + TCGLifeData arg_life = op->life; + if (!IS_DEAD_ARG(0)) { + goto do_default; + } + } + /* fall through */ + case INDEX_op_mov: case INDEX_op_mov_vec: tcg_reg_alloc_mov(s, op); break; @@ -6427,6 +6959,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) tcg_reg_alloc_dup(s, op); break; case INDEX_op_insn_start: + assert_carry_dead(s); if (num_insns >= 0) { size_t off = tcg_current_code_size(s); s->gen_insn_end_off[num_insns] = off; @@ -6434,8 +6967,8 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) assert(s->gen_insn_end_off[num_insns] == off); } num_insns++; - for (i = 0; i < start_words; ++i) { - s->gen_insn_data[num_insns * start_words + i] = + for (i = 0; i < INSN_START_WORDS; ++i) { + s->gen_insn_data[num_insns * INSN_START_WORDS + i] = tcg_get_insn_start_param(op, i); } break; @@ -6447,6 +6980,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) tcg_out_label(s, arg_label(op->args[0])); break; case INDEX_op_call: + assert_carry_dead(s); tcg_reg_alloc_call(s, op); break; case INDEX_op_exit_tb: @@ -6455,12 +6989,19 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) case INDEX_op_goto_tb: tcg_out_goto_tb(s, op->args[0]); break; + case INDEX_op_br: + tcg_out_br(s, arg_label(op->args[0])); + break; + case INDEX_op_mb: + tcg_out_mb(s, op->args[0]); + break; case INDEX_op_dup2_vec: if (tcg_reg_alloc_dup2(s, op)) { break; } /* fall through */ default: + do_default: /* Sanity check that we've not introduced any unhandled opcodes. */ tcg_debug_assert(tcg_op_supported(opc, TCGOP_TYPE(op), TCGOP_FLAGS(op))); @@ -6482,6 +7023,8 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) return -2; } } + assert_carry_dead(s); + tcg_debug_assert(num_insns + 1 == s->gen_tb->icount); s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); diff --git a/tcg/tci.c b/tcg/tci.c index d223258efe..700e672616 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -26,6 +26,11 @@ #include <ffi.h> +#define ctpop_tr glue(ctpop, TCG_TARGET_REG_BITS) +#define deposit_tr glue(deposit, TCG_TARGET_REG_BITS) +#define extract_tr glue(extract, TCG_TARGET_REG_BITS) +#define sextract_tr glue(sextract, TCG_TARGET_REG_BITS) + /* * Enable TCI assertions only when debugging TCG (and without NDEBUG defined). * Without assertions, the interpreter runs much faster. @@ -174,17 +179,6 @@ static void tci_args_rrrrrc(uint32_t insn, TCGReg *r0, TCGReg *r1, *c5 = extract32(insn, 28, 4); } -static void tci_args_rrrrrr(uint32_t insn, TCGReg *r0, TCGReg *r1, - TCGReg *r2, TCGReg *r3, TCGReg *r4, TCGReg *r5) -{ - *r0 = extract32(insn, 8, 4); - *r1 = extract32(insn, 12, 4); - *r2 = extract32(insn, 16, 4); - *r3 = extract32(insn, 20, 4); - *r4 = extract32(insn, 24, 4); - *r5 = extract32(insn, 28, 4); -} - static bool tci_compare32(uint32_t u0, uint32_t u1, TCGCond condition) { bool result = false; @@ -331,18 +325,6 @@ static void tci_qemu_st(CPUArchState *env, uint64_t taddr, uint64_t val, } } -#if TCG_TARGET_REG_BITS == 64 -# define CASE_32_64(x) \ - case glue(glue(INDEX_op_, x), _i64): \ - case glue(glue(INDEX_op_, x), _i32): -# define CASE_64(x) \ - case glue(glue(INDEX_op_, x), _i64): -#else -# define CASE_32_64(x) \ - case glue(glue(INDEX_op_, x), _i32): -# define CASE_64(x) -#endif - /* Interpret pseudo code in tb. */ /* * Disable CFI checks. @@ -356,6 +338,7 @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tcg_target_ulong regs[TCG_TARGET_NB_REGS]; uint64_t stack[(TCG_STATIC_CALL_ARGS_SIZE + TCG_STATIC_FRAME_SIZE) / sizeof(uint64_t)]; + bool carry = false; regs[TCG_AREG0] = (tcg_target_ulong)env; regs[TCG_REG_CALL_STACK] = (uintptr_t)stack; @@ -364,13 +347,12 @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, for (;;) { uint32_t insn; TCGOpcode opc; - TCGReg r0, r1, r2, r3, r4, r5; + TCGReg r0, r1, r2, r3, r4; tcg_target_ulong t1; TCGCond condition; uint8_t pos, len; uint32_t tmp32; uint64_t tmp64, taddr; - uint64_t T1, T2; MemOpIdx oi; int32_t ofs; void *ptr; @@ -436,34 +418,25 @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_l(insn, tb_ptr, &ptr); tb_ptr = ptr; continue; - case INDEX_op_setcond_i32: - tci_args_rrrc(insn, &r0, &r1, &r2, &condition); - regs[r0] = tci_compare32(regs[r1], regs[r2], condition); - break; - case INDEX_op_movcond_i32: - tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); - tmp32 = tci_compare32(regs[r1], regs[r2], condition); - regs[r0] = regs[tmp32 ? r3 : r4]; - break; #if TCG_TARGET_REG_BITS == 32 case INDEX_op_setcond2_i32: tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); - T1 = tci_uint64(regs[r2], regs[r1]); - T2 = tci_uint64(regs[r4], regs[r3]); - regs[r0] = tci_compare64(T1, T2, condition); + regs[r0] = tci_compare64(tci_uint64(regs[r2], regs[r1]), + tci_uint64(regs[r4], regs[r3]), + condition); break; #elif TCG_TARGET_REG_BITS == 64 - case INDEX_op_setcond_i64: + case INDEX_op_setcond: tci_args_rrrc(insn, &r0, &r1, &r2, &condition); regs[r0] = tci_compare64(regs[r1], regs[r2], condition); break; - case INDEX_op_movcond_i64: + case INDEX_op_movcond: tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); tmp32 = tci_compare64(regs[r1], regs[r2], condition); regs[r0] = regs[tmp32 ? r3 : r4]; break; #endif - CASE_32_64(mov) + case INDEX_op_mov: tci_args_rr(insn, &r0, &r1); regs[r0] = regs[r1]; break; @@ -475,411 +448,325 @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tci_args_rl(insn, tb_ptr, &r0, &ptr); regs[r0] = *(tcg_target_ulong *)ptr; break; + case INDEX_op_tci_setcarry: + carry = true; + break; /* Load/store operations (32 bit). */ - CASE_32_64(ld8u) + case INDEX_op_ld8u: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(uint8_t *)ptr; break; - CASE_32_64(ld8s) + case INDEX_op_ld8s: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(int8_t *)ptr; break; - CASE_32_64(ld16u) + case INDEX_op_ld16u: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(uint16_t *)ptr; break; - CASE_32_64(ld16s) + case INDEX_op_ld16s: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); regs[r0] = *(int16_t *)ptr; break; - case INDEX_op_ld_i32: - CASE_64(ld32u) + case INDEX_op_ld: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); - regs[r0] = *(uint32_t *)ptr; + regs[r0] = *(tcg_target_ulong *)ptr; break; - CASE_32_64(st8) + case INDEX_op_st8: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); *(uint8_t *)ptr = regs[r0]; break; - CASE_32_64(st16) + case INDEX_op_st16: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); *(uint16_t *)ptr = regs[r0]; break; - case INDEX_op_st_i32: - CASE_64(st32) + case INDEX_op_st: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); - *(uint32_t *)ptr = regs[r0]; + *(tcg_target_ulong *)ptr = regs[r0]; break; /* Arithmetic operations (mixed 32/64 bit). */ - CASE_32_64(add) + case INDEX_op_add: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] + regs[r2]; break; - CASE_32_64(sub) + case INDEX_op_sub: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] - regs[r2]; break; - CASE_32_64(mul) + case INDEX_op_mul: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] * regs[r2]; break; - CASE_32_64(and) + case INDEX_op_and: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] & regs[r2]; break; - CASE_32_64(or) + case INDEX_op_or: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] | regs[r2]; break; - CASE_32_64(xor) + case INDEX_op_xor: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] ^ regs[r2]; break; -#if TCG_TARGET_HAS_andc_i32 || TCG_TARGET_HAS_andc_i64 - CASE_32_64(andc) + case INDEX_op_andc: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] & ~regs[r2]; break; -#endif -#if TCG_TARGET_HAS_orc_i32 || TCG_TARGET_HAS_orc_i64 - CASE_32_64(orc) + case INDEX_op_orc: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] | ~regs[r2]; break; -#endif -#if TCG_TARGET_HAS_eqv_i32 || TCG_TARGET_HAS_eqv_i64 - CASE_32_64(eqv) + case INDEX_op_eqv: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = ~(regs[r1] ^ regs[r2]); break; -#endif -#if TCG_TARGET_HAS_nand_i32 || TCG_TARGET_HAS_nand_i64 - CASE_32_64(nand) + case INDEX_op_nand: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = ~(regs[r1] & regs[r2]); break; -#endif -#if TCG_TARGET_HAS_nor_i32 || TCG_TARGET_HAS_nor_i64 - CASE_32_64(nor) + case INDEX_op_nor: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = ~(regs[r1] | regs[r2]); break; + case INDEX_op_neg: + tci_args_rr(insn, &r0, &r1); + regs[r0] = -regs[r1]; + break; + case INDEX_op_not: + tci_args_rr(insn, &r0, &r1); + regs[r0] = ~regs[r1]; + break; + case INDEX_op_ctpop: + tci_args_rr(insn, &r0, &r1); + regs[r0] = ctpop_tr(regs[r1]); + break; + case INDEX_op_addco: + tci_args_rrr(insn, &r0, &r1, &r2); + t1 = regs[r1] + regs[r2]; + carry = t1 < regs[r1]; + regs[r0] = t1; + break; + case INDEX_op_addci: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] + regs[r2] + carry; + break; + case INDEX_op_addcio: + tci_args_rrr(insn, &r0, &r1, &r2); + if (carry) { + t1 = regs[r1] + regs[r2] + 1; + carry = t1 <= regs[r1]; + } else { + t1 = regs[r1] + regs[r2]; + carry = t1 < regs[r1]; + } + regs[r0] = t1; + break; + case INDEX_op_subbo: + tci_args_rrr(insn, &r0, &r1, &r2); + carry = regs[r1] < regs[r2]; + regs[r0] = regs[r1] - regs[r2]; + break; + case INDEX_op_subbi: + tci_args_rrr(insn, &r0, &r1, &r2); + regs[r0] = regs[r1] - regs[r2] - carry; + break; + case INDEX_op_subbio: + tci_args_rrr(insn, &r0, &r1, &r2); + if (carry) { + carry = regs[r1] <= regs[r2]; + regs[r0] = regs[r1] - regs[r2] - 1; + } else { + carry = regs[r1] < regs[r2]; + regs[r0] = regs[r1] - regs[r2]; + } + break; + case INDEX_op_muls2: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); +#if TCG_TARGET_REG_BITS == 32 + tmp64 = (int64_t)(int32_t)regs[r2] * (int32_t)regs[r3]; + tci_write_reg64(regs, r1, r0, tmp64); +#else + muls64(®s[r0], ®s[r1], regs[r2], regs[r3]); +#endif + break; + case INDEX_op_mulu2: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); +#if TCG_TARGET_REG_BITS == 32 + tmp64 = (uint64_t)(uint32_t)regs[r2] * (uint32_t)regs[r3]; + tci_write_reg64(regs, r1, r0, tmp64); +#else + mulu64(®s[r0], ®s[r1], regs[r2], regs[r3]); #endif + break; /* Arithmetic operations (32 bit). */ - case INDEX_op_div_i32: + case INDEX_op_tci_divs32: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int32_t)regs[r1] / (int32_t)regs[r2]; break; - case INDEX_op_divu_i32: + case INDEX_op_tci_divu32: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint32_t)regs[r1] / (uint32_t)regs[r2]; break; - case INDEX_op_rem_i32: + case INDEX_op_tci_rems32: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int32_t)regs[r1] % (int32_t)regs[r2]; break; - case INDEX_op_remu_i32: + case INDEX_op_tci_remu32: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint32_t)regs[r1] % (uint32_t)regs[r2]; break; -#if TCG_TARGET_HAS_clz_i32 - case INDEX_op_clz_i32: + case INDEX_op_tci_clz32: tci_args_rrr(insn, &r0, &r1, &r2); tmp32 = regs[r1]; regs[r0] = tmp32 ? clz32(tmp32) : regs[r2]; break; -#endif -#if TCG_TARGET_HAS_ctz_i32 - case INDEX_op_ctz_i32: + case INDEX_op_tci_ctz32: tci_args_rrr(insn, &r0, &r1, &r2); tmp32 = regs[r1]; regs[r0] = tmp32 ? ctz32(tmp32) : regs[r2]; break; -#endif -#if TCG_TARGET_HAS_ctpop_i32 - case INDEX_op_ctpop_i32: - tci_args_rr(insn, &r0, &r1); - regs[r0] = ctpop32(regs[r1]); + case INDEX_op_tci_setcond32: + tci_args_rrrc(insn, &r0, &r1, &r2, &condition); + regs[r0] = tci_compare32(regs[r1], regs[r2], condition); + break; + case INDEX_op_tci_movcond32: + tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &condition); + tmp32 = tci_compare32(regs[r1], regs[r2], condition); + regs[r0] = regs[tmp32 ? r3 : r4]; break; -#endif - /* Shift/rotate operations (32 bit). */ + /* Shift/rotate operations. */ - case INDEX_op_shl_i32: + case INDEX_op_shl: tci_args_rrr(insn, &r0, &r1, &r2); - regs[r0] = (uint32_t)regs[r1] << (regs[r2] & 31); + regs[r0] = regs[r1] << (regs[r2] % TCG_TARGET_REG_BITS); break; - case INDEX_op_shr_i32: + case INDEX_op_shr: tci_args_rrr(insn, &r0, &r1, &r2); - regs[r0] = (uint32_t)regs[r1] >> (regs[r2] & 31); + regs[r0] = regs[r1] >> (regs[r2] % TCG_TARGET_REG_BITS); break; - case INDEX_op_sar_i32: + case INDEX_op_sar: tci_args_rrr(insn, &r0, &r1, &r2); - regs[r0] = (int32_t)regs[r1] >> (regs[r2] & 31); + regs[r0] = ((tcg_target_long)regs[r1] + >> (regs[r2] % TCG_TARGET_REG_BITS)); break; -#if TCG_TARGET_HAS_rot_i32 - case INDEX_op_rotl_i32: + case INDEX_op_tci_rotl32: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = rol32(regs[r1], regs[r2] & 31); break; - case INDEX_op_rotr_i32: + case INDEX_op_tci_rotr32: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = ror32(regs[r1], regs[r2] & 31); break; -#endif - case INDEX_op_deposit_i32: + case INDEX_op_deposit: tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); - regs[r0] = deposit32(regs[r1], pos, len, regs[r2]); + regs[r0] = deposit_tr(regs[r1], pos, len, regs[r2]); break; - case INDEX_op_extract_i32: + case INDEX_op_extract: tci_args_rrbb(insn, &r0, &r1, &pos, &len); - regs[r0] = extract32(regs[r1], pos, len); + regs[r0] = extract_tr(regs[r1], pos, len); break; - case INDEX_op_sextract_i32: + case INDEX_op_sextract: tci_args_rrbb(insn, &r0, &r1, &pos, &len); - regs[r0] = sextract32(regs[r1], pos, len); + regs[r0] = sextract_tr(regs[r1], pos, len); break; - case INDEX_op_brcond_i32: + case INDEX_op_brcond: tci_args_rl(insn, tb_ptr, &r0, &ptr); - if ((uint32_t)regs[r0]) { + if (regs[r0]) { tb_ptr = ptr; } break; -#if TCG_TARGET_REG_BITS == 32 || TCG_TARGET_HAS_add2_i32 - case INDEX_op_add2_i32: - tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); - T1 = tci_uint64(regs[r3], regs[r2]); - T2 = tci_uint64(regs[r5], regs[r4]); - tci_write_reg64(regs, r1, r0, T1 + T2); - break; -#endif -#if TCG_TARGET_REG_BITS == 32 || TCG_TARGET_HAS_sub2_i32 - case INDEX_op_sub2_i32: - tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); - T1 = tci_uint64(regs[r3], regs[r2]); - T2 = tci_uint64(regs[r5], regs[r4]); - tci_write_reg64(regs, r1, r0, T1 - T2); - break; -#endif -#if TCG_TARGET_HAS_mulu2_i32 - case INDEX_op_mulu2_i32: - tci_args_rrrr(insn, &r0, &r1, &r2, &r3); - tmp64 = (uint64_t)(uint32_t)regs[r2] * (uint32_t)regs[r3]; - tci_write_reg64(regs, r1, r0, tmp64); - break; -#endif -#if TCG_TARGET_HAS_muls2_i32 - case INDEX_op_muls2_i32: - tci_args_rrrr(insn, &r0, &r1, &r2, &r3); - tmp64 = (int64_t)(int32_t)regs[r2] * (int32_t)regs[r3]; - tci_write_reg64(regs, r1, r0, tmp64); - break; -#endif -#if TCG_TARGET_HAS_ext8s_i32 || TCG_TARGET_HAS_ext8s_i64 - CASE_32_64(ext8s) - tci_args_rr(insn, &r0, &r1); - regs[r0] = (int8_t)regs[r1]; - break; -#endif -#if TCG_TARGET_HAS_ext16s_i32 || TCG_TARGET_HAS_ext16s_i64 || \ - TCG_TARGET_HAS_bswap16_i32 || TCG_TARGET_HAS_bswap16_i64 - CASE_32_64(ext16s) - tci_args_rr(insn, &r0, &r1); - regs[r0] = (int16_t)regs[r1]; - break; -#endif -#if TCG_TARGET_HAS_ext8u_i32 || TCG_TARGET_HAS_ext8u_i64 - CASE_32_64(ext8u) - tci_args_rr(insn, &r0, &r1); - regs[r0] = (uint8_t)regs[r1]; - break; -#endif -#if TCG_TARGET_HAS_ext16u_i32 || TCG_TARGET_HAS_ext16u_i64 - CASE_32_64(ext16u) - tci_args_rr(insn, &r0, &r1); - regs[r0] = (uint16_t)regs[r1]; - break; -#endif -#if TCG_TARGET_HAS_bswap16_i32 || TCG_TARGET_HAS_bswap16_i64 - CASE_32_64(bswap16) + case INDEX_op_bswap16: tci_args_rr(insn, &r0, &r1); regs[r0] = bswap16(regs[r1]); break; -#endif -#if TCG_TARGET_HAS_bswap32_i32 || TCG_TARGET_HAS_bswap32_i64 - CASE_32_64(bswap32) + case INDEX_op_bswap32: tci_args_rr(insn, &r0, &r1); regs[r0] = bswap32(regs[r1]); break; -#endif -#if TCG_TARGET_HAS_not_i32 || TCG_TARGET_HAS_not_i64 - CASE_32_64(not) - tci_args_rr(insn, &r0, &r1); - regs[r0] = ~regs[r1]; - break; -#endif - CASE_32_64(neg) - tci_args_rr(insn, &r0, &r1); - regs[r0] = -regs[r1]; - break; #if TCG_TARGET_REG_BITS == 64 /* Load/store operations (64 bit). */ - case INDEX_op_ld32s_i64: + case INDEX_op_ld32u: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); - regs[r0] = *(int32_t *)ptr; + regs[r0] = *(uint32_t *)ptr; break; - case INDEX_op_ld_i64: + case INDEX_op_ld32s: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); - regs[r0] = *(uint64_t *)ptr; + regs[r0] = *(int32_t *)ptr; break; - case INDEX_op_st_i64: + case INDEX_op_st32: tci_args_rrs(insn, &r0, &r1, &ofs); ptr = (void *)(regs[r1] + ofs); - *(uint64_t *)ptr = regs[r0]; + *(uint32_t *)ptr = regs[r0]; break; /* Arithmetic operations (64 bit). */ - case INDEX_op_div_i64: + case INDEX_op_divs: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int64_t)regs[r1] / (int64_t)regs[r2]; break; - case INDEX_op_divu_i64: + case INDEX_op_divu: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint64_t)regs[r1] / (uint64_t)regs[r2]; break; - case INDEX_op_rem_i64: + case INDEX_op_rems: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (int64_t)regs[r1] % (int64_t)regs[r2]; break; - case INDEX_op_remu_i64: + case INDEX_op_remu: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = (uint64_t)regs[r1] % (uint64_t)regs[r2]; break; -#if TCG_TARGET_HAS_clz_i64 - case INDEX_op_clz_i64: + case INDEX_op_clz: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] ? clz64(regs[r1]) : regs[r2]; break; -#endif -#if TCG_TARGET_HAS_ctz_i64 - case INDEX_op_ctz_i64: + case INDEX_op_ctz: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = regs[r1] ? ctz64(regs[r1]) : regs[r2]; break; -#endif -#if TCG_TARGET_HAS_ctpop_i64 - case INDEX_op_ctpop_i64: - tci_args_rr(insn, &r0, &r1); - regs[r0] = ctpop64(regs[r1]); - break; -#endif -#if TCG_TARGET_HAS_mulu2_i64 - case INDEX_op_mulu2_i64: - tci_args_rrrr(insn, &r0, &r1, &r2, &r3); - mulu64(®s[r0], ®s[r1], regs[r2], regs[r3]); - break; -#endif -#if TCG_TARGET_HAS_muls2_i64 - case INDEX_op_muls2_i64: - tci_args_rrrr(insn, &r0, &r1, &r2, &r3); - muls64(®s[r0], ®s[r1], regs[r2], regs[r3]); - break; -#endif -#if TCG_TARGET_HAS_add2_i64 - case INDEX_op_add2_i64: - tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); - T1 = regs[r2] + regs[r4]; - T2 = regs[r3] + regs[r5] + (T1 < regs[r2]); - regs[r0] = T1; - regs[r1] = T2; - break; -#endif -#if TCG_TARGET_HAS_add2_i64 - case INDEX_op_sub2_i64: - tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); - T1 = regs[r2] - regs[r4]; - T2 = regs[r3] - regs[r5] - (regs[r2] < regs[r4]); - regs[r0] = T1; - regs[r1] = T2; - break; -#endif /* Shift/rotate operations (64 bit). */ - case INDEX_op_shl_i64: - tci_args_rrr(insn, &r0, &r1, &r2); - regs[r0] = regs[r1] << (regs[r2] & 63); - break; - case INDEX_op_shr_i64: - tci_args_rrr(insn, &r0, &r1, &r2); - regs[r0] = regs[r1] >> (regs[r2] & 63); - break; - case INDEX_op_sar_i64: - tci_args_rrr(insn, &r0, &r1, &r2); - regs[r0] = (int64_t)regs[r1] >> (regs[r2] & 63); - break; -#if TCG_TARGET_HAS_rot_i64 - case INDEX_op_rotl_i64: + case INDEX_op_rotl: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = rol64(regs[r1], regs[r2] & 63); break; - case INDEX_op_rotr_i64: + case INDEX_op_rotr: tci_args_rrr(insn, &r0, &r1, &r2); regs[r0] = ror64(regs[r1], regs[r2] & 63); break; -#endif - case INDEX_op_deposit_i64: - tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); - regs[r0] = deposit64(regs[r1], pos, len, regs[r2]); - break; - case INDEX_op_extract_i64: - tci_args_rrbb(insn, &r0, &r1, &pos, &len); - regs[r0] = extract64(regs[r1], pos, len); - break; - case INDEX_op_sextract_i64: - tci_args_rrbb(insn, &r0, &r1, &pos, &len); - regs[r0] = sextract64(regs[r1], pos, len); - break; - case INDEX_op_brcond_i64: - tci_args_rl(insn, tb_ptr, &r0, &ptr); - if (regs[r0]) { - tb_ptr = ptr; - } - break; - case INDEX_op_ext32s_i64: case INDEX_op_ext_i32_i64: tci_args_rr(insn, &r0, &r1); regs[r0] = (int32_t)regs[r1]; break; - case INDEX_op_ext32u_i64: case INDEX_op_extu_i32_i64: tci_args_rr(insn, &r0, &r1); regs[r0] = (uint32_t)regs[r1]; break; -#if TCG_TARGET_HAS_bswap64_i64 - case INDEX_op_bswap64_i64: + case INDEX_op_bswap64: tci_args_rr(insn, &r0, &r1); regs[r0] = bswap64(regs[r1]); break; -#endif #endif /* TCG_TARGET_REG_BITS == 64 */ /* QEMU specific operations. */ @@ -902,46 +789,33 @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, tb_ptr = ptr; break; - case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_ld: tci_args_rrm(insn, &r0, &r1, &oi); taddr = regs[r1]; regs[r0] = tci_qemu_ld(env, taddr, oi, tb_ptr); break; - case INDEX_op_qemu_ld_i64: - if (TCG_TARGET_REG_BITS == 64) { - tci_args_rrm(insn, &r0, &r1, &oi); - taddr = regs[r1]; - } else { - tci_args_rrrr(insn, &r0, &r1, &r2, &r3); - taddr = regs[r2]; - oi = regs[r3]; - } - tmp64 = tci_qemu_ld(env, taddr, oi, tb_ptr); - if (TCG_TARGET_REG_BITS == 32) { - tci_write_reg64(regs, r1, r0, tmp64); - } else { - regs[r0] = tmp64; - } - break; - - case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st: tci_args_rrm(insn, &r0, &r1, &oi); taddr = regs[r1]; tci_qemu_st(env, taddr, regs[r0], oi, tb_ptr); break; - case INDEX_op_qemu_st_i64: - if (TCG_TARGET_REG_BITS == 64) { - tci_args_rrm(insn, &r0, &r1, &oi); - tmp64 = regs[r0]; - taddr = regs[r1]; - } else { - tci_args_rrrr(insn, &r0, &r1, &r2, &r3); - tmp64 = tci_uint64(regs[r1], regs[r0]); - taddr = regs[r2]; - oi = regs[r3]; - } + case INDEX_op_qemu_ld2: + tcg_debug_assert(TCG_TARGET_REG_BITS == 32); + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + taddr = regs[r2]; + oi = regs[r3]; + tmp64 = tci_qemu_ld(env, taddr, oi, tb_ptr); + tci_write_reg64(regs, r1, r0, tmp64); + break; + + case INDEX_op_qemu_st2: + tcg_debug_assert(TCG_TARGET_REG_BITS == 32); + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + tmp64 = tci_uint64(regs[r1], regs[r0]); + taddr = regs[r2]; + oi = regs[r3]; tci_qemu_st(env, taddr, tmp64, oi, tb_ptr); break; @@ -1005,7 +879,7 @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) const char *op_name; uint32_t insn; TCGOpcode op; - TCGReg r0, r1, r2, r3, r4, r5; + TCGReg r0, r1, r2, r3, r4; tcg_target_ulong i1; int32_t s2; TCGCond c; @@ -1040,15 +914,14 @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) info->fprintf_func(info->stream, "%-12s %d, %p", op_name, len, ptr); break; - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: + case INDEX_op_brcond: tci_args_rl(insn, tb_ptr, &r0, &ptr); info->fprintf_func(info->stream, "%-12s %s, 0, ne, %p", op_name, str_r(r0), ptr); break; - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: + case INDEX_op_setcond: + case INDEX_op_tci_setcond32: tci_args_rrrc(insn, &r0, &r1, &r2, &c); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2), str_c(c)); @@ -1066,126 +939,95 @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) op_name, str_r(r0), ptr); break; - case INDEX_op_ld8u_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i32: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i32: - case INDEX_op_ld16s_i64: - case INDEX_op_ld32u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld_i32: - case INDEX_op_ld_i64: - case INDEX_op_st8_i32: - case INDEX_op_st8_i64: - case INDEX_op_st16_i32: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i32: - case INDEX_op_st_i64: + case INDEX_op_tci_setcarry: + info->fprintf_func(info->stream, "%-12s", op_name); + break; + + case INDEX_op_ld8u: + case INDEX_op_ld8s: + case INDEX_op_ld16u: + case INDEX_op_ld16s: + case INDEX_op_ld32u: + case INDEX_op_ld: + case INDEX_op_st8: + case INDEX_op_st16: + case INDEX_op_st32: + case INDEX_op_st: tci_args_rrs(insn, &r0, &r1, &s2); info->fprintf_func(info->stream, "%-12s %s, %s, %d", op_name, str_r(r0), str_r(r1), s2); break; - case INDEX_op_mov_i32: - case INDEX_op_mov_i64: - case INDEX_op_ext8s_i32: - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: + case INDEX_op_bswap16: + case INDEX_op_bswap32: + case INDEX_op_ctpop: + case INDEX_op_mov: + case INDEX_op_neg: + case INDEX_op_not: case INDEX_op_ext_i32_i64: case INDEX_op_extu_i32_i64: - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i32: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: - case INDEX_op_not_i32: - case INDEX_op_not_i64: - case INDEX_op_neg_i32: - case INDEX_op_neg_i64: - case INDEX_op_ctpop_i32: - case INDEX_op_ctpop_i64: + case INDEX_op_bswap64: tci_args_rr(insn, &r0, &r1); info->fprintf_func(info->stream, "%-12s %s, %s", op_name, str_r(r0), str_r(r1)); break; - case INDEX_op_add_i32: - case INDEX_op_add_i64: - case INDEX_op_sub_i32: - case INDEX_op_sub_i64: - case INDEX_op_mul_i32: - case INDEX_op_mul_i64: - case INDEX_op_and_i32: - case INDEX_op_and_i64: - case INDEX_op_or_i32: - case INDEX_op_or_i64: - case INDEX_op_xor_i32: - case INDEX_op_xor_i64: - case INDEX_op_andc_i32: - case INDEX_op_andc_i64: - case INDEX_op_orc_i32: - case INDEX_op_orc_i64: - case INDEX_op_eqv_i32: - case INDEX_op_eqv_i64: - case INDEX_op_nand_i32: - case INDEX_op_nand_i64: - case INDEX_op_nor_i32: - case INDEX_op_nor_i64: - case INDEX_op_div_i32: - case INDEX_op_div_i64: - case INDEX_op_rem_i32: - case INDEX_op_rem_i64: - case INDEX_op_divu_i32: - case INDEX_op_divu_i64: - case INDEX_op_remu_i32: - case INDEX_op_remu_i64: - case INDEX_op_shl_i32: - case INDEX_op_shl_i64: - case INDEX_op_shr_i32: - case INDEX_op_shr_i64: - case INDEX_op_sar_i32: - case INDEX_op_sar_i64: - case INDEX_op_rotl_i32: - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i32: - case INDEX_op_rotr_i64: - case INDEX_op_clz_i32: - case INDEX_op_clz_i64: - case INDEX_op_ctz_i32: - case INDEX_op_ctz_i64: + case INDEX_op_add: + case INDEX_op_addci: + case INDEX_op_addcio: + case INDEX_op_addco: + case INDEX_op_and: + case INDEX_op_andc: + case INDEX_op_clz: + case INDEX_op_ctz: + case INDEX_op_divs: + case INDEX_op_divu: + case INDEX_op_eqv: + case INDEX_op_mul: + case INDEX_op_nand: + case INDEX_op_nor: + case INDEX_op_or: + case INDEX_op_orc: + case INDEX_op_rems: + case INDEX_op_remu: + case INDEX_op_rotl: + case INDEX_op_rotr: + case INDEX_op_sar: + case INDEX_op_shl: + case INDEX_op_shr: + case INDEX_op_sub: + case INDEX_op_subbi: + case INDEX_op_subbio: + case INDEX_op_subbo: + case INDEX_op_xor: + case INDEX_op_tci_ctz32: + case INDEX_op_tci_clz32: + case INDEX_op_tci_divs32: + case INDEX_op_tci_divu32: + case INDEX_op_tci_rems32: + case INDEX_op_tci_remu32: + case INDEX_op_tci_rotl32: + case INDEX_op_tci_rotr32: tci_args_rrr(insn, &r0, &r1, &r2); info->fprintf_func(info->stream, "%-12s %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2)); break; - case INDEX_op_deposit_i32: - case INDEX_op_deposit_i64: + case INDEX_op_deposit: tci_args_rrrbb(insn, &r0, &r1, &r2, &pos, &len); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %d, %d", op_name, str_r(r0), str_r(r1), str_r(r2), pos, len); break; - case INDEX_op_extract_i32: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i32: - case INDEX_op_sextract_i64: + case INDEX_op_extract: + case INDEX_op_sextract: tci_args_rrbb(insn, &r0, &r1, &pos, &len); info->fprintf_func(info->stream, "%-12s %s,%s,%d,%d", op_name, str_r(r0), str_r(r1), pos, len); break; - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: + case INDEX_op_tci_movcond32: + case INDEX_op_movcond: case INDEX_op_setcond2_i32: tci_args_rrrrrc(insn, &r0, &r1, &r2, &r3, &r4, &c); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s, %s, %s", @@ -1193,43 +1035,29 @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) str_r(r3), str_r(r4), str_c(c)); break; - case INDEX_op_mulu2_i32: - case INDEX_op_mulu2_i64: - case INDEX_op_muls2_i32: - case INDEX_op_muls2_i64: + case INDEX_op_muls2: + case INDEX_op_mulu2: tci_args_rrrr(insn, &r0, &r1, &r2, &r3); info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s", op_name, str_r(r0), str_r(r1), str_r(r2), str_r(r3)); break; - case INDEX_op_add2_i32: - case INDEX_op_add2_i64: - case INDEX_op_sub2_i32: - case INDEX_op_sub2_i64: - tci_args_rrrrrr(insn, &r0, &r1, &r2, &r3, &r4, &r5); - info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s, %s, %s", - op_name, str_r(r0), str_r(r1), str_r(r2), - str_r(r3), str_r(r4), str_r(r5)); - break; - - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i64: - if (TCG_TARGET_REG_BITS == 32) { - tci_args_rrrr(insn, &r0, &r1, &r2, &r3); - info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s", - op_name, str_r(r0), str_r(r1), - str_r(r2), str_r(r3)); - break; - } - /* fall through */ - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_ld: + case INDEX_op_qemu_st: tci_args_rrm(insn, &r0, &r1, &oi); info->fprintf_func(info->stream, "%-12s %s, %s, %x", op_name, str_r(r0), str_r(r1), oi); break; + case INDEX_op_qemu_ld2: + case INDEX_op_qemu_st2: + tci_args_rrrr(insn, &r0, &r1, &r2, &r3); + info->fprintf_func(info->stream, "%-12s %s, %s, %s, %s", + op_name, str_r(r0), str_r(r1), + str_r(r2), str_r(r3)); + break; + case 0: /* tcg_out_nop_fill uses zeros */ if (insn == 0) { diff --git a/tcg/tci/tcg-target-has.h b/tcg/tci/tcg-target-has.h index c8785ca8dc..ab07ce1fcb 100644 --- a/tcg/tci/tcg-target-has.h +++ b/tcg/tci/tcg-target-has.h @@ -7,67 +7,8 @@ #ifndef TCG_TARGET_HAS_H #define TCG_TARGET_HAS_H -#define TCG_TARGET_HAS_bswap16_i32 1 -#define TCG_TARGET_HAS_bswap32_i32 1 -#define TCG_TARGET_HAS_div_i32 1 -#define TCG_TARGET_HAS_rem_i32 1 -#define TCG_TARGET_HAS_ext8s_i32 1 -#define TCG_TARGET_HAS_ext16s_i32 1 -#define TCG_TARGET_HAS_ext8u_i32 1 -#define TCG_TARGET_HAS_ext16u_i32 1 -#define TCG_TARGET_HAS_andc_i32 1 -#define TCG_TARGET_HAS_extract2_i32 0 -#define TCG_TARGET_HAS_eqv_i32 1 -#define TCG_TARGET_HAS_nand_i32 1 -#define TCG_TARGET_HAS_nor_i32 1 -#define TCG_TARGET_HAS_clz_i32 1 -#define TCG_TARGET_HAS_ctz_i32 1 -#define TCG_TARGET_HAS_ctpop_i32 1 -#define TCG_TARGET_HAS_not_i32 1 -#define TCG_TARGET_HAS_orc_i32 1 -#define TCG_TARGET_HAS_rot_i32 1 -#define TCG_TARGET_HAS_negsetcond_i32 0 -#define TCG_TARGET_HAS_muls2_i32 1 -#define TCG_TARGET_HAS_muluh_i32 0 -#define TCG_TARGET_HAS_mulsh_i32 0 -#define TCG_TARGET_HAS_qemu_st8_i32 0 - #if TCG_TARGET_REG_BITS == 64 #define TCG_TARGET_HAS_extr_i64_i32 0 -#define TCG_TARGET_HAS_bswap16_i64 1 -#define TCG_TARGET_HAS_bswap32_i64 1 -#define TCG_TARGET_HAS_bswap64_i64 1 -#define TCG_TARGET_HAS_extract2_i64 0 -#define TCG_TARGET_HAS_div_i64 1 -#define TCG_TARGET_HAS_rem_i64 1 -#define TCG_TARGET_HAS_ext8s_i64 1 -#define TCG_TARGET_HAS_ext16s_i64 1 -#define TCG_TARGET_HAS_ext32s_i64 1 -#define TCG_TARGET_HAS_ext8u_i64 1 -#define TCG_TARGET_HAS_ext16u_i64 1 -#define TCG_TARGET_HAS_ext32u_i64 1 -#define TCG_TARGET_HAS_andc_i64 1 -#define TCG_TARGET_HAS_eqv_i64 1 -#define TCG_TARGET_HAS_nand_i64 1 -#define TCG_TARGET_HAS_nor_i64 1 -#define TCG_TARGET_HAS_clz_i64 1 -#define TCG_TARGET_HAS_ctz_i64 1 -#define TCG_TARGET_HAS_ctpop_i64 1 -#define TCG_TARGET_HAS_not_i64 1 -#define TCG_TARGET_HAS_orc_i64 1 -#define TCG_TARGET_HAS_rot_i64 1 -#define TCG_TARGET_HAS_negsetcond_i64 0 -#define TCG_TARGET_HAS_muls2_i64 1 -#define TCG_TARGET_HAS_add2_i32 1 -#define TCG_TARGET_HAS_sub2_i32 1 -#define TCG_TARGET_HAS_mulu2_i32 1 -#define TCG_TARGET_HAS_add2_i64 1 -#define TCG_TARGET_HAS_sub2_i64 1 -#define TCG_TARGET_HAS_mulu2_i64 1 -#define TCG_TARGET_HAS_muluh_i64 0 -#define TCG_TARGET_HAS_mulsh_i64 0 -#else -#define TCG_TARGET_HAS_mulu2_i32 1 #endif /* TCG_TARGET_REG_BITS == 64 */ #define TCG_TARGET_HAS_qemu_ldst_i128 0 diff --git a/tcg/tci/tcg-target-opc.h.inc b/tcg/tci/tcg-target-opc.h.inc index ecc8c4e55e..4eb32ed736 100644 --- a/tcg/tci/tcg-target-opc.h.inc +++ b/tcg/tci/tcg-target-opc.h.inc @@ -2,3 +2,14 @@ /* These opcodes for use between the tci generator and interpreter. */ DEF(tci_movi, 1, 0, 1, TCG_OPF_NOT_PRESENT) DEF(tci_movl, 1, 0, 1, TCG_OPF_NOT_PRESENT) +DEF(tci_setcarry, 0, 0, 0, TCG_OPF_NOT_PRESENT) +DEF(tci_clz32, 1, 2, 0, TCG_OPF_NOT_PRESENT) +DEF(tci_ctz32, 1, 2, 0, TCG_OPF_NOT_PRESENT) +DEF(tci_divs32, 1, 2, 0, TCG_OPF_NOT_PRESENT) +DEF(tci_divu32, 1, 2, 0, TCG_OPF_NOT_PRESENT) +DEF(tci_rems32, 1, 2, 0, TCG_OPF_NOT_PRESENT) +DEF(tci_remu32, 1, 2, 0, TCG_OPF_NOT_PRESENT) +DEF(tci_rotl32, 1, 2, 0, TCG_OPF_NOT_PRESENT) +DEF(tci_rotr32, 1, 2, 0, TCG_OPF_NOT_PRESENT) +DEF(tci_setcond32, 1, 2, 1, TCG_OPF_NOT_PRESENT) +DEF(tci_movcond32, 1, 2, 1, TCG_OPF_NOT_PRESENT) diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index 36e018dd19..35c66a4836 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -39,148 +39,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) { - switch (op) { - case INDEX_op_goto_ptr: - return C_O0_I1(r); - - case INDEX_op_ld8u_i32: - case INDEX_op_ld8s_i32: - case INDEX_op_ld16u_i32: - case INDEX_op_ld16s_i32: - case INDEX_op_ld_i32: - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i64: - case INDEX_op_ld32u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld_i64: - case INDEX_op_not_i32: - case INDEX_op_not_i64: - case INDEX_op_neg_i32: - case INDEX_op_neg_i64: - case INDEX_op_ext8s_i32: - case INDEX_op_ext8s_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_bswap16_i32: - case INDEX_op_bswap16_i64: - case INDEX_op_bswap32_i32: - case INDEX_op_bswap32_i64: - case INDEX_op_bswap64_i64: - case INDEX_op_extract_i32: - case INDEX_op_extract_i64: - case INDEX_op_sextract_i32: - case INDEX_op_sextract_i64: - case INDEX_op_ctpop_i32: - case INDEX_op_ctpop_i64: - return C_O1_I1(r, r); - - case INDEX_op_st8_i32: - case INDEX_op_st16_i32: - case INDEX_op_st_i32: - case INDEX_op_st8_i64: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: - return C_O0_I2(r, r); - - case INDEX_op_div_i32: - case INDEX_op_div_i64: - case INDEX_op_divu_i32: - case INDEX_op_divu_i64: - case INDEX_op_rem_i32: - case INDEX_op_rem_i64: - case INDEX_op_remu_i32: - case INDEX_op_remu_i64: - case INDEX_op_add_i32: - case INDEX_op_add_i64: - case INDEX_op_sub_i32: - case INDEX_op_sub_i64: - case INDEX_op_mul_i32: - case INDEX_op_mul_i64: - case INDEX_op_and_i32: - case INDEX_op_and_i64: - case INDEX_op_andc_i32: - case INDEX_op_andc_i64: - case INDEX_op_eqv_i32: - case INDEX_op_eqv_i64: - case INDEX_op_nand_i32: - case INDEX_op_nand_i64: - case INDEX_op_nor_i32: - case INDEX_op_nor_i64: - case INDEX_op_or_i32: - case INDEX_op_or_i64: - case INDEX_op_orc_i32: - case INDEX_op_orc_i64: - case INDEX_op_xor_i32: - case INDEX_op_xor_i64: - case INDEX_op_shl_i32: - case INDEX_op_shl_i64: - case INDEX_op_shr_i32: - case INDEX_op_shr_i64: - case INDEX_op_sar_i32: - case INDEX_op_sar_i64: - case INDEX_op_rotl_i32: - case INDEX_op_rotl_i64: - case INDEX_op_rotr_i32: - case INDEX_op_rotr_i64: - case INDEX_op_setcond_i32: - case INDEX_op_setcond_i64: - case INDEX_op_deposit_i32: - case INDEX_op_deposit_i64: - case INDEX_op_clz_i32: - case INDEX_op_clz_i64: - case INDEX_op_ctz_i32: - case INDEX_op_ctz_i64: - return C_O1_I2(r, r, r); - - case INDEX_op_brcond_i32: - case INDEX_op_brcond_i64: - return C_O0_I2(r, r); - - case INDEX_op_add2_i32: - case INDEX_op_add2_i64: - case INDEX_op_sub2_i32: - case INDEX_op_sub2_i64: - return C_O2_I4(r, r, r, r, r, r); - -#if TCG_TARGET_REG_BITS == 32 - case INDEX_op_brcond2_i32: - return C_O0_I4(r, r, r, r); -#endif - - case INDEX_op_mulu2_i32: - case INDEX_op_mulu2_i64: - case INDEX_op_muls2_i32: - case INDEX_op_muls2_i64: - return C_O2_I2(r, r, r, r); - - case INDEX_op_movcond_i32: - case INDEX_op_movcond_i64: - case INDEX_op_setcond2_i32: - return C_O1_I4(r, r, r, r, r); - - case INDEX_op_qemu_ld_i32: - return C_O1_I1(r, r); - case INDEX_op_qemu_ld_i64: - return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, r) : C_O2_I1(r, r, r); - case INDEX_op_qemu_st_i32: - return C_O0_I2(r, r); - case INDEX_op_qemu_st_i64: - return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(r, r) : C_O0_I3(r, r, r); - - default: - return C_NotImplemented; - } + return C_NotImplemented; } static const int tcg_target_reg_alloc_order[] = { @@ -443,31 +302,13 @@ static void tcg_out_op_rrrrrc(TCGContext *s, TCGOpcode op, tcg_out32(s, insn); } -static void tcg_out_op_rrrrrr(TCGContext *s, TCGOpcode op, - TCGReg r0, TCGReg r1, TCGReg r2, - TCGReg r3, TCGReg r4, TCGReg r5) -{ - tcg_insn_unit insn = 0; - - insn = deposit32(insn, 0, 8, op); - insn = deposit32(insn, 8, 4, r0); - insn = deposit32(insn, 12, 4, r1); - insn = deposit32(insn, 16, 4, r2); - insn = deposit32(insn, 20, 4, r3); - insn = deposit32(insn, 24, 4, r4); - insn = deposit32(insn, 28, 4, r5); - tcg_out32(s, insn); -} - static void tcg_out_ldst(TCGContext *s, TCGOpcode op, TCGReg val, TCGReg base, intptr_t offset) { stack_bounds_check(base, offset); if (offset != sextract32(offset, 0, 16)) { tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, offset); - tcg_out_op_rrr(s, (TCG_TARGET_REG_BITS == 32 - ? INDEX_op_add_i32 : INDEX_op_add_i64), - TCG_REG_TMP, TCG_REG_TMP, base); + tcg_out_op_rrr(s, INDEX_op_add, TCG_REG_TMP, TCG_REG_TMP, base); base = TCG_REG_TMP; offset = 0; } @@ -477,34 +318,17 @@ static void tcg_out_ldst(TCGContext *s, TCGOpcode op, TCGReg val, static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg val, TCGReg base, intptr_t offset) { - switch (type) { - case TCG_TYPE_I32: - tcg_out_ldst(s, INDEX_op_ld_i32, val, base, offset); - break; -#if TCG_TARGET_REG_BITS == 64 - case TCG_TYPE_I64: - tcg_out_ldst(s, INDEX_op_ld_i64, val, base, offset); - break; -#endif - default: - g_assert_not_reached(); + TCGOpcode op = INDEX_op_ld; + + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { + op = INDEX_op_ld32u; } + tcg_out_ldst(s, op, val, base, offset); } static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) { - switch (type) { - case TCG_TYPE_I32: - tcg_out_op_rr(s, INDEX_op_mov_i32, ret, arg); - break; -#if TCG_TARGET_REG_BITS == 64 - case TCG_TYPE_I64: - tcg_out_op_rr(s, INDEX_op_mov_i64, ret, arg); - break; -#endif - default: - g_assert_not_reached(); - } + tcg_out_op_rr(s, INDEX_op_mov, ret, arg); return true; } @@ -535,76 +359,62 @@ static void tcg_out_movi(TCGContext *s, TCGType type, } } +static void tcg_out_extract(TCGContext *s, TCGType type, TCGReg rd, + TCGReg rs, unsigned pos, unsigned len) +{ + tcg_out_op_rrbb(s, INDEX_op_extract, rd, rs, pos, len); +} + +static const TCGOutOpExtract outop_extract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tcg_out_extract, +}; + +static void tcg_out_sextract(TCGContext *s, TCGType type, TCGReg rd, + TCGReg rs, unsigned pos, unsigned len) +{ + tcg_out_op_rrbb(s, INDEX_op_sextract, rd, rs, pos, len); +} + +static const TCGOutOpExtract outop_sextract = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tcg_out_sextract, +}; + +static const TCGOutOpExtract2 outop_extract2 = { + .base.static_constraint = C_NotImplemented, +}; + static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) { - switch (type) { - case TCG_TYPE_I32: - tcg_debug_assert(TCG_TARGET_HAS_ext8s_i32); - tcg_out_op_rr(s, INDEX_op_ext8s_i32, rd, rs); - break; -#if TCG_TARGET_REG_BITS == 64 - case TCG_TYPE_I64: - tcg_debug_assert(TCG_TARGET_HAS_ext8s_i64); - tcg_out_op_rr(s, INDEX_op_ext8s_i64, rd, rs); - break; -#endif - default: - g_assert_not_reached(); - } + tcg_out_sextract(s, type, rd, rs, 0, 8); } static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs) { - if (TCG_TARGET_REG_BITS == 64) { - tcg_debug_assert(TCG_TARGET_HAS_ext8u_i64); - tcg_out_op_rr(s, INDEX_op_ext8u_i64, rd, rs); - } else { - tcg_debug_assert(TCG_TARGET_HAS_ext8u_i32); - tcg_out_op_rr(s, INDEX_op_ext8u_i32, rd, rs); - } + tcg_out_extract(s, TCG_TYPE_REG, rd, rs, 0, 8); } static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs) { - switch (type) { - case TCG_TYPE_I32: - tcg_debug_assert(TCG_TARGET_HAS_ext16s_i32); - tcg_out_op_rr(s, INDEX_op_ext16s_i32, rd, rs); - break; -#if TCG_TARGET_REG_BITS == 64 - case TCG_TYPE_I64: - tcg_debug_assert(TCG_TARGET_HAS_ext16s_i64); - tcg_out_op_rr(s, INDEX_op_ext16s_i64, rd, rs); - break; -#endif - default: - g_assert_not_reached(); - } + tcg_out_sextract(s, type, rd, rs, 0, 16); } static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs) { - if (TCG_TARGET_REG_BITS == 64) { - tcg_debug_assert(TCG_TARGET_HAS_ext16u_i64); - tcg_out_op_rr(s, INDEX_op_ext16u_i64, rd, rs); - } else { - tcg_debug_assert(TCG_TARGET_HAS_ext16u_i32); - tcg_out_op_rr(s, INDEX_op_ext16u_i32, rd, rs); - } + tcg_out_extract(s, TCG_TYPE_REG, rd, rs, 0, 16); } static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs) { tcg_debug_assert(TCG_TARGET_REG_BITS == 64); - tcg_debug_assert(TCG_TARGET_HAS_ext32s_i64); - tcg_out_op_rr(s, INDEX_op_ext32s_i64, rd, rs); + tcg_out_sextract(s, TCG_TYPE_I64, rd, rs, 0, 32); } static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs) { tcg_debug_assert(TCG_TARGET_REG_BITS == 64); - tcg_debug_assert(TCG_TARGET_HAS_ext32u_i64); - tcg_out_op_rr(s, INDEX_op_ext32u_i64, rd, rs); + tcg_out_extract(s, TCG_TYPE_I64, rd, rs, 0, 32); } static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs) @@ -656,18 +466,6 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func, tcg_out32(s, insn); } -#if TCG_TARGET_REG_BITS == 64 -# define CASE_32_64(x) \ - case glue(glue(INDEX_op_, x), _i64): \ - case glue(glue(INDEX_op_, x), _i32): -# define CASE_64(x) \ - case glue(glue(INDEX_op_, x), _i64): -#else -# define CASE_32_64(x) \ - case glue(glue(INDEX_op_, x), _i32): -# define CASE_64(x) -#endif - static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) { tcg_out_op_p(s, INDEX_op_exit_tb, (void *)arg); @@ -680,197 +478,772 @@ static void tcg_out_goto_tb(TCGContext *s, int which) set_jmp_reset_offset(s, which); } +static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) +{ + tcg_out_op_r(s, INDEX_op_goto_ptr, a0); +} + void tb_target_set_jmp_target(const TranslationBlock *tb, int n, uintptr_t jmp_rx, uintptr_t jmp_rw) { /* Always indirect, nothing to do */ } -static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGType type, - const TCGArg args[TCG_MAX_OP_ARGS], - const int const_args[TCG_MAX_OP_ARGS]) +static void tgen_add(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - TCGOpcode exts; + tcg_out_op_rrr(s, INDEX_op_add, a0, a1, a2); +} - switch (opc) { - case INDEX_op_goto_ptr: - tcg_out_op_r(s, opc, args[0]); - break; +static const TCGOutOpBinary outop_add = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_add, +}; - case INDEX_op_br: - tcg_out_op_l(s, opc, arg_label(args[0])); - break; +static TCGConstraintSetIndex cset_addsubcarry(TCGType type, unsigned flags) +{ + return type == TCG_TYPE_REG ? C_O1_I2(r, r, r) : C_NotImplemented; +} - CASE_32_64(setcond) - tcg_out_op_rrrc(s, opc, args[0], args[1], args[2], args[3]); - break; +static void tgen_addco(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_addco, a0, a1, a2); +} - CASE_32_64(movcond) - case INDEX_op_setcond2_i32: - tcg_out_op_rrrrrc(s, opc, args[0], args[1], args[2], - args[3], args[4], args[5]); - break; +static const TCGOutOpBinary outop_addco = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addsubcarry, + .out_rrr = tgen_addco, +}; - CASE_32_64(ld8u) - CASE_32_64(ld8s) - CASE_32_64(ld16u) - CASE_32_64(ld16s) - case INDEX_op_ld_i32: - CASE_64(ld32u) - CASE_64(ld32s) - CASE_64(ld) - CASE_32_64(st8) - CASE_32_64(st16) - case INDEX_op_st_i32: - CASE_64(st32) - CASE_64(st) - tcg_out_ldst(s, opc, args[0], args[1], args[2]); - break; +static void tgen_addci(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_addci, a0, a1, a2); +} - CASE_32_64(add) - CASE_32_64(sub) - CASE_32_64(mul) - CASE_32_64(and) - CASE_32_64(or) - CASE_32_64(xor) - CASE_32_64(andc) /* Optional (TCG_TARGET_HAS_andc_*). */ - CASE_32_64(orc) /* Optional (TCG_TARGET_HAS_orc_*). */ - CASE_32_64(eqv) /* Optional (TCG_TARGET_HAS_eqv_*). */ - CASE_32_64(nand) /* Optional (TCG_TARGET_HAS_nand_*). */ - CASE_32_64(nor) /* Optional (TCG_TARGET_HAS_nor_*). */ - CASE_32_64(shl) - CASE_32_64(shr) - CASE_32_64(sar) - CASE_32_64(rotl) /* Optional (TCG_TARGET_HAS_rot_*). */ - CASE_32_64(rotr) /* Optional (TCG_TARGET_HAS_rot_*). */ - CASE_32_64(div) /* Optional (TCG_TARGET_HAS_div_*). */ - CASE_32_64(divu) /* Optional (TCG_TARGET_HAS_div_*). */ - CASE_32_64(rem) /* Optional (TCG_TARGET_HAS_div_*). */ - CASE_32_64(remu) /* Optional (TCG_TARGET_HAS_div_*). */ - CASE_32_64(clz) /* Optional (TCG_TARGET_HAS_clz_*). */ - CASE_32_64(ctz) /* Optional (TCG_TARGET_HAS_ctz_*). */ - tcg_out_op_rrr(s, opc, args[0], args[1], args[2]); - break; +static const TCGOutOpAddSubCarry outop_addci = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addsubcarry, + .out_rrr = tgen_addci, +}; - CASE_32_64(deposit) - tcg_out_op_rrrbb(s, opc, args[0], args[1], args[2], args[3], args[4]); - break; +static void tgen_addcio(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_addcio, a0, a1, a2); +} - CASE_32_64(extract) /* Optional (TCG_TARGET_HAS_extract_*). */ - CASE_32_64(sextract) /* Optional (TCG_TARGET_HAS_sextract_*). */ - tcg_out_op_rrbb(s, opc, args[0], args[1], args[2], args[3]); - break; +static const TCGOutOpBinary outop_addcio = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addsubcarry, + .out_rrr = tgen_addcio, +}; - CASE_32_64(brcond) - tcg_out_op_rrrc(s, (opc == INDEX_op_brcond_i32 - ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64), - TCG_REG_TMP, args[0], args[1], args[2]); - tcg_out_op_rl(s, opc, TCG_REG_TMP, arg_label(args[3])); - break; +static void tcg_out_set_carry(TCGContext *s) +{ + tcg_out_op_v(s, INDEX_op_tci_setcarry); +} - CASE_32_64(neg) /* Optional (TCG_TARGET_HAS_neg_*). */ - CASE_32_64(not) /* Optional (TCG_TARGET_HAS_not_*). */ - CASE_32_64(ctpop) /* Optional (TCG_TARGET_HAS_ctpop_*). */ - case INDEX_op_bswap32_i32: /* Optional (TCG_TARGET_HAS_bswap32_i32). */ - case INDEX_op_bswap64_i64: /* Optional (TCG_TARGET_HAS_bswap64_i64). */ - tcg_out_op_rr(s, opc, args[0], args[1]); - break; +static void tgen_and(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_and, a0, a1, a2); +} - case INDEX_op_bswap16_i32: /* Optional (TCG_TARGET_HAS_bswap16_i32). */ - exts = INDEX_op_ext16s_i32; - goto do_bswap; - case INDEX_op_bswap16_i64: /* Optional (TCG_TARGET_HAS_bswap16_i64). */ - exts = INDEX_op_ext16s_i64; - goto do_bswap; - case INDEX_op_bswap32_i64: /* Optional (TCG_TARGET_HAS_bswap32_i64). */ - exts = INDEX_op_ext32s_i64; - do_bswap: - /* The base tci bswaps zero-extend, and ignore high bits. */ - tcg_out_op_rr(s, opc, args[0], args[1]); - if (args[2] & TCG_BSWAP_OS) { - tcg_out_op_rr(s, exts, args[0], args[0]); - } - break; +static const TCGOutOpBinary outop_and = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_and, +}; - CASE_32_64(add2) - CASE_32_64(sub2) - tcg_out_op_rrrrrr(s, opc, args[0], args[1], args[2], - args[3], args[4], args[5]); - break; +static void tgen_andc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_andc, a0, a1, a2); +} -#if TCG_TARGET_REG_BITS == 32 - case INDEX_op_brcond2_i32: - tcg_out_op_rrrrrc(s, INDEX_op_setcond2_i32, TCG_REG_TMP, - args[0], args[1], args[2], args[3], args[4]); - tcg_out_op_rl(s, INDEX_op_brcond_i32, TCG_REG_TMP, arg_label(args[5])); - break; +static const TCGOutOpBinary outop_andc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_andc, +}; + +static void tgen_clz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_clz32 + : INDEX_op_clz); + tcg_out_op_rrr(s, opc, a0, a1, a2); +} + +static const TCGOutOpBinary outop_clz = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_clz, +}; + +static void tgen_ctz(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_ctz32 + : INDEX_op_ctz); + tcg_out_op_rrr(s, opc, a0, a1, a2); +} + +static const TCGOutOpBinary outop_ctz = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_ctz, +}; + +static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, + TCGReg a2, unsigned ofs, unsigned len) +{ + tcg_out_op_rrrbb(s, INDEX_op_deposit, a0, a1, a2, ofs, len); +} + +static const TCGOutOpDeposit outop_deposit = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_deposit, +}; + +static void tgen_divs(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_divs32 + : INDEX_op_divs); + tcg_out_op_rrr(s, opc, a0, a1, a2); +} + +static const TCGOutOpBinary outop_divs = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divs, +}; + +static const TCGOutOpDivRem outop_divs2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_divu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_divu32 + : INDEX_op_divu); + tcg_out_op_rrr(s, opc, a0, a1, a2); +} + +static const TCGOutOpBinary outop_divu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_divu, +}; + +static const TCGOutOpDivRem outop_divu2 = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_eqv(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_eqv, a0, a1, a2); +} + +static const TCGOutOpBinary outop_eqv = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_eqv, +}; + +#if TCG_TARGET_REG_BITS == 64 +static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) +{ + tcg_out_extract(s, TCG_TYPE_I64, a0, a1, 32, 32); +} + +static const TCGOutOpUnary outop_extrh_i64_i32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_extrh_i64_i32, +}; #endif - CASE_32_64(mulu2) - CASE_32_64(muls2) - tcg_out_op_rrrr(s, opc, args[0], args[1], args[2], args[3]); - break; +static void tgen_mul(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_mul, a0, a1, a2); +} - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i64: - if (TCG_TARGET_REG_BITS == 32) { - tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, args[3]); - tcg_out_op_rrrr(s, opc, args[0], args[1], args[2], TCG_REG_TMP); - break; - } - /* fall through */ - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_st_i32: - if (TCG_TARGET_REG_BITS == 64 && s->addr_type == TCG_TYPE_I32) { - tcg_out_ext32u(s, TCG_REG_TMP, args[1]); - tcg_out_op_rrm(s, opc, args[0], TCG_REG_TMP, args[2]); - } else { - tcg_out_op_rrm(s, opc, args[0], args[1], args[2]); - } - break; +static const TCGOutOpBinary outop_mul = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_mul, +}; - case INDEX_op_mb: - tcg_out_op_v(s, opc); - break; +static TCGConstraintSetIndex cset_mul2(TCGType type, unsigned flags) +{ + return type == TCG_TYPE_REG ? C_O2_I2(r, r, r, r) : C_NotImplemented; +} - case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ - case INDEX_op_mov_i64: - case INDEX_op_call: /* Always emitted via tcg_out_call. */ - case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ - case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ - case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ - case INDEX_op_ext8s_i64: - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - default: - g_assert_not_reached(); +static void tgen_muls2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + tcg_out_op_rrrr(s, INDEX_op_muls2, a0, a1, a2, a3); +} + +static const TCGOutOpMul2 outop_muls2 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mul2, + .out_rrrr = tgen_muls2, +}; + +static const TCGOutOpBinary outop_mulsh = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_mulu2(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3) +{ + tcg_out_op_rrrr(s, INDEX_op_mulu2, a0, a1, a2, a3); +} + +static const TCGOutOpMul2 outop_mulu2 = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_mul2, + .out_rrrr = tgen_mulu2, +}; + +static const TCGOutOpBinary outop_muluh = { + .base.static_constraint = C_NotImplemented, +}; + +static void tgen_nand(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_nand, a0, a1, a2); +} + +static const TCGOutOpBinary outop_nand = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_nand, +}; + +static void tgen_nor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_nor, a0, a1, a2); +} + +static const TCGOutOpBinary outop_nor = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_nor, +}; + +static void tgen_or(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_or, a0, a1, a2); +} + +static const TCGOutOpBinary outop_or = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_or, +}; + +static void tgen_orc(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_orc, a0, a1, a2); +} + +static const TCGOutOpBinary outop_orc = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_orc, +}; + +static void tgen_rems(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_rems32 + : INDEX_op_rems); + tcg_out_op_rrr(s, opc, a0, a1, a2); +} + +static const TCGOutOpBinary outop_rems = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_rems, +}; + +static void tgen_remu(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_remu32 + : INDEX_op_remu); + tcg_out_op_rrr(s, opc, a0, a1, a2); +} + +static const TCGOutOpBinary outop_remu = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_remu, +}; + +static void tgen_rotl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_rotl32 + : INDEX_op_rotl); + tcg_out_op_rrr(s, opc, a0, a1, a2); +} + +static const TCGOutOpBinary outop_rotl = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_rotl, +}; + +static void tgen_rotr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_rotr32 + : INDEX_op_rotr); + tcg_out_op_rrr(s, opc, a0, a1, a2); +} + +static const TCGOutOpBinary outop_rotr = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_rotr, +}; + +static void tgen_sar(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type < TCG_TYPE_REG) { + tcg_out_ext32s(s, TCG_REG_TMP, a1); + a1 = TCG_REG_TMP; } + tcg_out_op_rrr(s, INDEX_op_sar, a0, a1, a2); } -static void tcg_out_st(TCGContext *s, TCGType type, TCGReg val, TCGReg base, - intptr_t offset) +static const TCGOutOpBinary outop_sar = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_sar, +}; + +static void tgen_shl(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) { - switch (type) { - case TCG_TYPE_I32: - tcg_out_ldst(s, INDEX_op_st_i32, val, base, offset); - break; + tcg_out_op_rrr(s, INDEX_op_shl, a0, a1, a2); +} + +static const TCGOutOpBinary outop_shl = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_shl, +}; + +static void tgen_shr(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + if (type < TCG_TYPE_REG) { + tcg_out_ext32u(s, TCG_REG_TMP, a1); + a1 = TCG_REG_TMP; + } + tcg_out_op_rrr(s, INDEX_op_shr, a0, a1, a2); +} + +static const TCGOutOpBinary outop_shr = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_shr, +}; + +static void tgen_sub(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_sub, a0, a1, a2); +} + +static const TCGOutOpSubtract outop_sub = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_sub, +}; + +static void tgen_subbo(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_subbo, a0, a1, a2); +} + +static const TCGOutOpAddSubCarry outop_subbo = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addsubcarry, + .out_rrr = tgen_subbo, +}; + +static void tgen_subbi(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_subbi, a0, a1, a2); +} + +static const TCGOutOpAddSubCarry outop_subbi = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addsubcarry, + .out_rrr = tgen_subbi, +}; + +static void tgen_subbio(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_subbio, a0, a1, a2); +} + +static const TCGOutOpAddSubCarry outop_subbio = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_addsubcarry, + .out_rrr = tgen_subbio, +}; + +static void tcg_out_set_borrow(TCGContext *s) +{ + tcg_out_op_v(s, INDEX_op_tci_setcarry); /* borrow == carry */ +} + +static void tgen_xor(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out_op_rrr(s, INDEX_op_xor, a0, a1, a2); +} + +static const TCGOutOpBinary outop_xor = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_xor, +}; + +static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_op_rr(s, INDEX_op_ctpop, a0, a1); +} + +static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags) +{ + return type == TCG_TYPE_REG ? C_O1_I1(r, r) : C_NotImplemented; +} + +static const TCGOutOpUnary outop_ctpop = { + .base.static_constraint = C_Dynamic, + .base.dynamic_constraint = cset_ctpop, + .out_rr = tgen_ctpop, +}; + +static void tgen_bswap16(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_op_rr(s, INDEX_op_bswap16, a0, a1); + if (flags & TCG_BSWAP_OS) { + tcg_out_sextract(s, TCG_TYPE_REG, a0, a0, 0, 16); + } +} + +static const TCGOutOpBswap outop_bswap16 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap16, +}; + +static void tgen_bswap32(TCGContext *s, TCGType type, + TCGReg a0, TCGReg a1, unsigned flags) +{ + tcg_out_op_rr(s, INDEX_op_bswap32, a0, a1); + if (flags & TCG_BSWAP_OS) { + tcg_out_sextract(s, TCG_TYPE_REG, a0, a0, 0, 32); + } +} + +static const TCGOutOpBswap outop_bswap32 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap32, +}; + #if TCG_TARGET_REG_BITS == 64 - case TCG_TYPE_I64: - tcg_out_ldst(s, INDEX_op_st_i64, val, base, offset); - break; +static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_op_rr(s, INDEX_op_bswap64, a0, a1); +} + +static const TCGOutOpUnary outop_bswap64 = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_bswap64, +}; #endif - default: - g_assert_not_reached(); + +static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_op_rr(s, INDEX_op_neg, a0, a1); +} + +static const TCGOutOpUnary outop_neg = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_neg, +}; + +static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) +{ + tcg_out_op_rr(s, INDEX_op_not, a0, a1); +} + +static const TCGOutOpUnary outop_not = { + .base.static_constraint = C_O1_I1(r, r), + .out_rr = tgen_not, +}; + +static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_setcond32 + : INDEX_op_setcond); + tcg_out_op_rrrc(s, opc, dest, arg1, arg2, cond); +} + +static const TCGOutOpSetcond outop_setcond = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_setcond, +}; + +static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg dest, TCGReg arg1, TCGReg arg2) +{ + tgen_setcond(s, type, cond, dest, arg1, arg2); + tgen_neg(s, type, dest, dest); +} + +static const TCGOutOpSetcond outop_negsetcond = { + .base.static_constraint = C_O1_I2(r, r, r), + .out_rrr = tgen_negsetcond, +}; + +static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg arg0, TCGReg arg1, TCGLabel *l) +{ + tgen_setcond(s, type, cond, TCG_REG_TMP, arg0, arg1); + tcg_out_op_rl(s, INDEX_op_brcond, TCG_REG_TMP, l); +} + +static const TCGOutOpBrcond outop_brcond = { + .base.static_constraint = C_O0_I2(r, r), + .out_rr = tgen_brcond, +}; + +static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond, + TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, + TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf) +{ + TCGOpcode opc = (type == TCG_TYPE_I32 + ? INDEX_op_tci_movcond32 + : INDEX_op_movcond); + tcg_out_op_rrrrrc(s, opc, ret, c1, c2, vt, vf, cond); +} + +static const TCGOutOpMovcond outop_movcond = { + .base.static_constraint = C_O1_I4(r, r, r, r, r), + .out = tgen_movcond, +}; + +static void tgen_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, + TCGArg bh, bool const_bh, TCGLabel *l) +{ + tcg_out_op_rrrrrc(s, INDEX_op_setcond2_i32, TCG_REG_TMP, + al, ah, bl, bh, cond); + tcg_out_op_rl(s, INDEX_op_brcond, TCG_REG_TMP, l); +} + +#if TCG_TARGET_REG_BITS != 32 +__attribute__((unused)) +#endif +static const TCGOutOpBrcond2 outop_brcond2 = { + .base.static_constraint = C_O0_I4(r, r, r, r), + .out = tgen_brcond2, +}; + +static void tgen_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg al, TCGReg ah, + TCGArg bl, bool const_bl, + TCGArg bh, bool const_bh) +{ + tcg_out_op_rrrrrc(s, INDEX_op_setcond2_i32, ret, al, ah, bl, bh, cond); +} + +#if TCG_TARGET_REG_BITS != 32 +__attribute__((unused)) +#endif +static const TCGOutOpSetcond2 outop_setcond2 = { + .base.static_constraint = C_O1_I4(r, r, r, r, r), + .out = tgen_setcond2, +}; + +static void tcg_out_mb(TCGContext *s, unsigned a0) +{ + tcg_out_op_v(s, INDEX_op_mb); +} + +static void tcg_out_br(TCGContext *s, TCGLabel *l) +{ + tcg_out_op_l(s, INDEX_op_br, l); +} + +static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, INDEX_op_ld8u, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld8u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8u, +}; + +static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, INDEX_op_ld8s, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld8s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld8s, +}; + +static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, INDEX_op_ld16u, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16u, +}; + +static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, INDEX_op_ld16s, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld16s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld16s, +}; + +#if TCG_TARGET_REG_BITS == 64 +static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, INDEX_op_ld32u, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32u = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32u, +}; + +static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, INDEX_op_ld32s, dest, base, offset); +} + +static const TCGOutOpLoad outop_ld32s = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_ld32s, +}; +#endif + +static void tgen_st8(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, INDEX_op_st8, data, base, offset); +} + +static const TCGOutOpStore outop_st8 = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tgen_st8, +}; + +static void tgen_st16(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, ptrdiff_t offset) +{ + tcg_out_ldst(s, INDEX_op_st16, data, base, offset); +} + +static const TCGOutOpStore outop_st16 = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tgen_st16, +}; + +static const TCGOutOpStore outop_st = { + .base.static_constraint = C_O0_I2(r, r), + .out_r = tcg_out_st, +}; + +static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + tcg_out_op_rrm(s, INDEX_op_qemu_ld, data, addr, oi); +} + +static const TCGOutOpQemuLdSt outop_qemu_ld = { + .base.static_constraint = C_O1_I1(r, r), + .out = tgen_qemu_ld, +}; + +static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) +{ + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, oi); + tcg_out_op_rrrr(s, INDEX_op_qemu_ld2, datalo, datahi, addr, TCG_REG_TMP); +} + +static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { + .base.static_constraint = + TCG_TARGET_REG_BITS == 64 ? C_NotImplemented : C_O2_I1(r, r, r), + .out = + TCG_TARGET_REG_BITS == 64 ? NULL : tgen_qemu_ld2, +}; + +static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data, + TCGReg addr, MemOpIdx oi) +{ + tcg_out_op_rrm(s, INDEX_op_qemu_st, data, addr, oi); +} + +static const TCGOutOpQemuLdSt outop_qemu_st = { + .base.static_constraint = C_O0_I2(r, r), + .out = tgen_qemu_st, +}; + +static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo, + TCGReg datahi, TCGReg addr, MemOpIdx oi) +{ + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, oi); + tcg_out_op_rrrr(s, INDEX_op_qemu_st2, datalo, datahi, addr, TCG_REG_TMP); +} + +static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { + .base.static_constraint = + TCG_TARGET_REG_BITS == 64 ? C_NotImplemented : C_O0_I3(r, r, r), + .out = + TCG_TARGET_REG_BITS == 64 ? NULL : tgen_qemu_st2, +}; + +static void tcg_out_st(TCGContext *s, TCGType type, TCGReg val, TCGReg base, + intptr_t offset) +{ + TCGOpcode op = INDEX_op_st; + + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { + op = INDEX_op_st32; } + tcg_out_ldst(s, op, val, base, offset); } static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker index 0535585428..081f3e00f7 100644 --- a/tests/docker/dockerfiles/debian-amd64-cross.docker +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -50,7 +50,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-web \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker index 6b1e4fc827..91c555a36e 100644 --- a/tests/docker/dockerfiles/debian-arm64-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -50,7 +50,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-web \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker index cf0fe63af9..f0e2efcda0 100644 --- a/tests/docker/dockerfiles/debian-armhf-cross.docker +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -50,7 +50,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-web \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/debian-i686-cross.docker b/tests/docker/dockerfiles/debian-i686-cross.docker index 1c84dfb945..025beb1ce2 100644 --- a/tests/docker/dockerfiles/debian-i686-cross.docker +++ b/tests/docker/dockerfiles/debian-i686-cross.docker @@ -50,7 +50,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-web \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/debian-mips64el-cross.docker b/tests/docker/dockerfiles/debian-mips64el-cross.docker index 257204eae4..4a941dd870 100644 --- a/tests/docker/dockerfiles/debian-mips64el-cross.docker +++ b/tests/docker/dockerfiles/debian-mips64el-cross.docker @@ -50,7 +50,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-web \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/debian-mipsel-cross.docker b/tests/docker/dockerfiles/debian-mipsel-cross.docker index 395c84d65b..4d3e5d711b 100644 --- a/tests/docker/dockerfiles/debian-mipsel-cross.docker +++ b/tests/docker/dockerfiles/debian-mipsel-cross.docker @@ -50,7 +50,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-web \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/debian-ppc64el-cross.docker b/tests/docker/dockerfiles/debian-ppc64el-cross.docker index 1ae227ccde..22b4457ba9 100644 --- a/tests/docker/dockerfiles/debian-ppc64el-cross.docker +++ b/tests/docker/dockerfiles/debian-ppc64el-cross.docker @@ -50,7 +50,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-web \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker index afa81a57ba..13ec52c8ad 100644 --- a/tests/docker/dockerfiles/debian-s390x-cross.docker +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -50,7 +50,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-web \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/debian.docker b/tests/docker/dockerfiles/debian.docker index 5b3bac43cc..0a57c1a1d3 100644 --- a/tests/docker/dockerfiles/debian.docker +++ b/tests/docker/dockerfiles/debian.docker @@ -122,7 +122,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-web \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/emsdk-wasm32-cross.docker b/tests/docker/dockerfiles/emsdk-wasm32-cross.docker new file mode 100644 index 0000000000..60a7d02f56 --- /dev/null +++ b/tests/docker/dockerfiles/emsdk-wasm32-cross.docker @@ -0,0 +1,145 @@ +# syntax = docker/dockerfile:1.5 + +ARG EMSDK_VERSION_QEMU=3.1.50 +ARG ZLIB_VERSION=1.3.1 +ARG GLIB_MINOR_VERSION=2.84 +ARG GLIB_VERSION=${GLIB_MINOR_VERSION}.0 +ARG PIXMAN_VERSION=0.44.2 +ARG FFI_VERSION=v3.4.7 +ARG MESON_VERSION=1.5.0 + +FROM emscripten/emsdk:$EMSDK_VERSION_QEMU AS build-base +ARG MESON_VERSION +ENV TARGET=/builddeps/target +ENV CPATH="$TARGET/include" +ENV PKG_CONFIG_PATH="$TARGET/lib/pkgconfig" +ENV EM_PKG_CONFIG_PATH="$PKG_CONFIG_PATH" +ENV CFLAGS="-O3 -pthread -DWASM_BIGINT" +ENV CXXFLAGS="$CFLAGS" +ENV LDFLAGS="-sWASM_BIGINT -sASYNCIFY=1 -L$TARGET/lib" +RUN apt-get update && apt-get install -y \ + autoconf \ + build-essential \ + libglib2.0-dev \ + libtool \ + pkgconf \ + ninja-build \ + python3-pip +RUN pip3 install meson==${MESON_VERSION} tomli +RUN mkdir /build +WORKDIR /build +RUN mkdir -p $TARGET +RUN <<EOF +cat <<EOT > /cross.meson +[host_machine] +system = 'emscripten' +cpu_family = 'wasm32' +cpu = 'wasm32' +endian = 'little' + +[binaries] +c = 'emcc' +cpp = 'em++' +ar = 'emar' +ranlib = 'emranlib' +pkgconfig = ['pkg-config', '--static'] +EOT +EOF + +FROM build-base AS zlib-dev +ARG ZLIB_VERSION +RUN mkdir -p /zlib +RUN curl -Ls https://zlib.net/zlib-$ZLIB_VERSION.tar.xz | \ + tar xJC /zlib --strip-components=1 +WORKDIR /zlib +RUN emconfigure ./configure --prefix=$TARGET --static +RUN emmake make install -j$(nproc) + +FROM build-base AS libffi-dev +ARG FFI_VERSION +RUN mkdir -p /libffi +RUN git clone https://github.com/libffi/libffi /libffi +WORKDIR /libffi +RUN git checkout $FFI_VERSION +RUN autoreconf -fiv +RUN emconfigure ./configure --host=wasm32-unknown-linux \ + --prefix=$TARGET --enable-static \ + --disable-shared --disable-dependency-tracking \ + --disable-builddir --disable-multi-os-directory \ + --disable-raw-api --disable-docs +RUN emmake make install SUBDIRS='include' -j$(nproc) + +FROM build-base AS pixman-dev +ARG PIXMAN_VERSION +RUN mkdir /pixman/ +RUN git clone https://gitlab.freedesktop.org/pixman/pixman /pixman/ +WORKDIR /pixman +RUN git checkout pixman-$PIXMAN_VERSION +RUN <<EOF +cat <<EOT >> /cross.meson +[built-in options] +c_args = [$(printf "'%s', " $CFLAGS | sed 's/, $//')] +cpp_args = [$(printf "'%s', " $CFLAGS | sed 's/, $//')] +objc_args = [$(printf "'%s', " $CFLAGS | sed 's/, $//')] +c_link_args = [$(printf "'%s', " $LDFLAGS | sed 's/, $//')] +cpp_link_args = [$(printf "'%s', " $LDFLAGS | sed 's/, $//')] +EOT +EOF +RUN meson setup _build --prefix=$TARGET --cross-file=/cross.meson \ + --default-library=static \ + --buildtype=release -Dtests=disabled -Ddemos=disabled +RUN meson install -C _build + +FROM build-base AS glib-dev +ARG GLIB_VERSION +ARG GLIB_MINOR_VERSION +RUN mkdir -p /stub +WORKDIR /stub +RUN <<EOF +cat <<'EOT' > res_query.c +#include <netdb.h> +int res_query(const char *name, int class, + int type, unsigned char *dest, int len) +{ + h_errno = HOST_NOT_FOUND; + return -1; +} +EOT +EOF +RUN emcc ${CFLAGS} -c res_query.c -fPIC -o libresolv.o +RUN ar rcs libresolv.a libresolv.o +RUN mkdir -p $TARGET/lib/ +RUN cp libresolv.a $TARGET/lib/ + +RUN mkdir -p /glib +RUN curl -Lks https://download.gnome.org/sources/glib/${GLIB_MINOR_VERSION}/glib-$GLIB_VERSION.tar.xz | \ + tar xJC /glib --strip-components=1 + +COPY --link --from=zlib-dev /builddeps/ /builddeps/ +COPY --link --from=libffi-dev /builddeps/ /builddeps/ + +WORKDIR /glib +RUN <<EOF +CFLAGS="$CFLAGS -Wno-incompatible-function-pointer-types" ; +cat <<EOT >> /cross.meson +[built-in options] +c_args = [$(printf "'%s', " $CFLAGS | sed 's/, $//')] +cpp_args = [$(printf "'%s', " $CFLAGS | sed 's/, $//')] +objc_args = [$(printf "'%s', " $CFLAGS | sed 's/, $//')] +c_link_args = [$(printf "'%s', " $LDFLAGS | sed 's/, $//')] +cpp_link_args = [$(printf "'%s', " $LDFLAGS | sed 's/, $//')] +EOT +EOF +RUN meson setup _build --prefix=$TARGET --cross-file=/cross.meson \ + --default-library=static --buildtype=release --force-fallback-for=pcre2 \ + -Dselinux=disabled -Dxattr=false -Dlibmount=disabled -Dnls=disabled \ + -Dtests=false -Dglib_debug=disabled -Dglib_assert=false -Dglib_checks=false +# FIXME: emscripten doesn't provide some pthread functions in the final link, +# which isn't detected during meson setup. +RUN sed -i -E "/#define HAVE_POSIX_SPAWN 1/d" ./_build/config.h +RUN sed -i -E "/#define HAVE_PTHREAD_GETNAME_NP 1/d" ./_build/config.h +RUN meson install -C _build + +FROM build-base +COPY --link --from=glib-dev /builddeps/ /builddeps/ +COPY --link --from=pixman-dev /builddeps/ /builddeps/ diff --git a/tests/docker/dockerfiles/python.docker b/tests/docker/dockerfiles/python.docker index 8f0af9ef25..59e70a0248 100644 --- a/tests/docker/dockerfiles/python.docker +++ b/tests/docker/dockerfiles/python.docker @@ -15,7 +15,6 @@ ENV PACKAGES \ python3.11 \ python3.12 \ python3.13 \ - python3.8 \ python3.9 RUN dnf install -y $PACKAGES diff --git a/tests/docker/dockerfiles/ubuntu2204.docker b/tests/docker/dockerfiles/ubuntu2204.docker index 88ce4ef9a9..4a1cf2bdff 100644 --- a/tests/docker/dockerfiles/ubuntu2204.docker +++ b/tests/docker/dockerfiles/ubuntu2204.docker @@ -121,7 +121,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-venv \ python3-yaml \ rpm2cpio \ - rustc \ + rustc-1.77 \ sed \ socat \ sparse \ @@ -150,6 +150,7 @@ ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" ENV NINJA "/usr/bin/ninja" ENV PYTHON "/usr/bin/python3" +ENV RUSTC=/usr/bin/rustc-1.77 ENV CARGO_HOME=/usr/local/cargo ENV PATH=$CARGO_HOME/bin:$PATH RUN DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/functional/meson.build b/tests/functional/meson.build index b317ad42c5..52b4706cfe 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -11,7 +11,9 @@ endif # Timeouts for individual tests that can be slow e.g. with debugging enabled test_timeouts = { - 'aarch64_aspeed' : 600, + 'aarch64_aspeed_ast2700' : 600, + 'aarch64_aspeed_ast2700fc' : 600, + 'aarch64_imx8mp_evk' : 240, 'aarch64_raspi4' : 480, 'aarch64_reverse_debug' : 180, 'aarch64_rme_virt' : 1200, @@ -79,7 +81,9 @@ tests_aarch64_system_quick = [ ] tests_aarch64_system_thorough = [ - 'aarch64_aspeed', + 'aarch64_aspeed_ast2700', + 'aarch64_aspeed_ast2700fc', + 'aarch64_imx8mp_evk', 'aarch64_raspi3', 'aarch64_raspi4', 'aarch64_replay', diff --git a/tests/functional/test_aarch64_aspeed.py b/tests/functional/test_aarch64_aspeed_ast2700.py index c7f3b3b319..d02dc7991c 100755 --- a/tests/functional/test_aarch64_aspeed.py +++ b/tests/functional/test_aarch64_aspeed_ast2700.py @@ -18,22 +18,52 @@ class AST2x00MachineSDK(QemuSystemTest): def do_test_aarch64_aspeed_sdk_start(self, image): self.require_netdev('user') self.vm.set_console() + self.vm.add_args('-device', + 'tmp105,bus=aspeed.i2c.bus.1,address=0x4d,id=tmp-test') self.vm.add_args('-drive', 'file=' + image + ',if=mtd,format=raw', '-net', 'nic', '-net', 'user', '-snapshot') self.vm.launch() + def verify_vbootrom_firmware_flow(self): + wait_for_console_pattern(self, 'Found valid FIT image') + wait_for_console_pattern(self, '[uboot] loading') + wait_for_console_pattern(self, 'done') + wait_for_console_pattern(self, '[fdt] loading') + wait_for_console_pattern(self, 'done') + wait_for_console_pattern(self, '[tee] loading') + wait_for_console_pattern(self, 'done') + wait_for_console_pattern(self, '[atf] loading') + wait_for_console_pattern(self, 'done') + wait_for_console_pattern(self, 'Jumping to BL31 (Trusted Firmware-A)') + + def verify_openbmc_boot_and_login(self, name): wait_for_console_pattern(self, 'U-Boot 2023.10') wait_for_console_pattern(self, '## Loading kernel from FIT Image') wait_for_console_pattern(self, 'Starting kernel ...') - ASSET_SDK_V905_AST2700 = Asset( - 'https://github.com/AspeedTech-BMC/openbmc/releases/download/v09.05/ast2700-a0-default-obmc.tar.gz', - 'cfbbd1cce72f2a3b73b9080c41eecdadebb7077fba4f7806d72ac99f3e84b74a') + wait_for_console_pattern(self, f'{name} login:') + exec_command_and_wait_for_pattern(self, 'root', 'Password:') + exec_command_and_wait_for_pattern(self, '0penBmc', f'root@{name}:~#') - ASSET_SDK_V905_AST2700A1 = Asset( - 'https://github.com/AspeedTech-BMC/openbmc/releases/download/v09.05/ast2700-default-obmc.tar.gz', - 'c1f4496aec06743c812a6e9a1a18d032f34d62f3ddb6956e924fef62aa2046a5') + ASSET_SDK_V906_AST2700 = Asset( + 'https://github.com/AspeedTech-BMC/openbmc/releases/download/v09.06/ast2700-a0-default-obmc.tar.gz', + '7247b6f19dbfb700686f8d9f723ac23f3eb229226c0589cb9b06b80d1b61f3cb') + + ASSET_SDK_V906_AST2700A1 = Asset( + 'https://github.com/AspeedTech-BMC/openbmc/releases/download/v09.06/ast2700-default-obmc.tar.gz', + 'f1d53e0be8a404ecce3e105f72bc50fa4e090ad13160ffa91b10a6e0233a9dc6') + + def do_ast2700_i2c_test(self): + exec_command_and_wait_for_pattern(self, + 'echo lm75 0x4d > /sys/class/i2c-dev/i2c-1/device/new_device ', + 'i2c i2c-1: new_device: Instantiated device lm75 at 0x4d') + exec_command_and_wait_for_pattern(self, + 'cat /sys/bus/i2c/devices/1-004d/hwmon/hwmon*/temp1_input', '0') + self.vm.cmd('qom-set', path='/machine/peripheral/tmp-test', + property='temperature', value=18000) + exec_command_and_wait_for_pattern(self, + 'cat /sys/bus/i2c/devices/1-004d/hwmon/hwmon*/temp1_input', '18000') def start_ast2700_test(self, name): num_cpu = 4 @@ -73,38 +103,38 @@ class AST2x00MachineSDK(QemuSystemTest): f'loader,addr=0x430000000,cpu-num={i}') self.vm.add_args('-smp', str(num_cpu)) - self.vm.add_args('-device', - 'tmp105,bus=aspeed.i2c.bus.1,address=0x4d,id=tmp-test') self.do_test_aarch64_aspeed_sdk_start( self.scratch_file(name, 'image-bmc')) - wait_for_console_pattern(self, f'{name} login:') - - exec_command_and_wait_for_pattern(self, 'root', 'Password:') - exec_command_and_wait_for_pattern(self, '0penBmc', f'root@{name}:~#') - - exec_command_and_wait_for_pattern(self, - 'echo lm75 0x4d > /sys/class/i2c-dev/i2c-1/device/new_device ', - 'i2c i2c-1: new_device: Instantiated device lm75 at 0x4d') - exec_command_and_wait_for_pattern(self, - 'cat /sys/bus/i2c/devices/1-004d/hwmon/hwmon*/temp1_input', '0') - self.vm.cmd('qom-set', path='/machine/peripheral/tmp-test', - property='temperature', value=18000) - exec_command_and_wait_for_pattern(self, - 'cat /sys/bus/i2c/devices/1-004d/hwmon/hwmon*/temp1_input', '18000') + def start_ast2700_test_vbootrom(self, name): + self.vm.add_args('-bios', 'ast27x0_bootrom.bin') + self.do_test_aarch64_aspeed_sdk_start( + self.scratch_file(name, 'image-bmc')) - def test_aarch64_ast2700_evb_sdk_v09_05(self): + def test_aarch64_ast2700_evb_sdk_v09_06(self): self.set_machine('ast2700-evb') - self.archive_extract(self.ASSET_SDK_V905_AST2700) + self.archive_extract(self.ASSET_SDK_V906_AST2700) self.start_ast2700_test('ast2700-a0-default') + self.verify_openbmc_boot_and_login('ast2700-a0-default') + self.do_ast2700_i2c_test() - def test_aarch64_ast2700a1_evb_sdk_v09_05(self): + def test_aarch64_ast2700a1_evb_sdk_v09_06(self): self.set_machine('ast2700a1-evb') - self.archive_extract(self.ASSET_SDK_V905_AST2700A1) + self.archive_extract(self.ASSET_SDK_V906_AST2700A1) self.start_ast2700_test('ast2700-default') + self.verify_openbmc_boot_and_login('ast2700-default') + self.do_ast2700_i2c_test() + + def test_aarch64_ast2700a1_evb_sdk_vbootrom_v09_06(self): + self.set_machine('ast2700a1-evb') + self.archive_extract(self.ASSET_SDK_V906_AST2700A1) + self.start_ast2700_test_vbootrom('ast2700-default') + self.verify_vbootrom_firmware_flow() + self.verify_openbmc_boot_and_login('ast2700-default') + self.do_ast2700_i2c_test() if __name__ == '__main__': QemuSystemTest.main() diff --git a/tests/functional/test_aarch64_aspeed_ast2700fc.py b/tests/functional/test_aarch64_aspeed_ast2700fc.py new file mode 100755 index 0000000000..b85370e182 --- /dev/null +++ b/tests/functional/test_aarch64_aspeed_ast2700fc.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# +# Functional test that boots the ASPEED SoCs with firmware +# +# Copyright (C) 2022 ASPEED Technology Inc +# +# SPDX-License-Identifier: GPL-2.0-or-later + +import os + +from qemu_test import QemuSystemTest, Asset +from qemu_test import wait_for_console_pattern +from qemu_test import exec_command_and_wait_for_pattern + + +class AST2x00MachineSDK(QemuSystemTest): + + def do_test_aarch64_aspeed_sdk_start(self, image): + self.require_netdev('user') + self.vm.set_console() + self.vm.add_args('-device', + 'tmp105,bus=aspeed.i2c.bus.1,address=0x4d,id=tmp-test') + self.vm.add_args('-drive', 'file=' + image + ',if=mtd,format=raw', + '-net', 'nic', '-net', 'user', '-snapshot') + + self.vm.launch() + + def verify_openbmc_boot_and_login(self, name): + wait_for_console_pattern(self, 'U-Boot 2023.10') + wait_for_console_pattern(self, '## Loading kernel from FIT Image') + wait_for_console_pattern(self, 'Starting kernel ...') + + wait_for_console_pattern(self, f'{name} login:') + exec_command_and_wait_for_pattern(self, 'root', 'Password:') + exec_command_and_wait_for_pattern(self, '0penBmc', f'root@{name}:~#') + + ASSET_SDK_V906_AST2700 = Asset( + 'https://github.com/AspeedTech-BMC/openbmc/releases/download/v09.06/ast2700-default-obmc.tar.gz', + 'f1d53e0be8a404ecce3e105f72bc50fa4e090ad13160ffa91b10a6e0233a9dc6') + + def do_ast2700_i2c_test(self): + exec_command_and_wait_for_pattern(self, + 'echo lm75 0x4d > /sys/class/i2c-dev/i2c-1/device/new_device ', + 'i2c i2c-1: new_device: Instantiated device lm75 at 0x4d') + exec_command_and_wait_for_pattern(self, + 'cat /sys/bus/i2c/devices/1-004d/hwmon/hwmon*/temp1_input', '0') + self.vm.cmd('qom-set', path='/machine/peripheral/tmp-test', + property='temperature', value=18000) + exec_command_and_wait_for_pattern(self, + 'cat /sys/bus/i2c/devices/1-004d/hwmon/hwmon*/temp1_input', '18000') + + def do_ast2700fc_ssp_test(self): + self.vm.shutdown() + self.vm.set_console(console_index=1) + self.vm.launch() + + exec_command_and_wait_for_pattern(self, '\012', 'ssp:~$') + exec_command_and_wait_for_pattern(self, 'version', + 'Zephyr version 3.7.1') + exec_command_and_wait_for_pattern(self, 'md 72c02000 1', + '[72c02000] 06010103') + + def do_ast2700fc_tsp_test(self): + self.vm.shutdown() + self.vm.set_console(console_index=2) + self.vm.launch() + + exec_command_and_wait_for_pattern(self, '\012', 'tsp:~$') + exec_command_and_wait_for_pattern(self, 'version', + 'Zephyr version 3.7.1') + exec_command_and_wait_for_pattern(self, 'md 72c02000 1', + '[72c02000] 06010103') + + def start_ast2700fc_test(self, name): + ca35_core = 4 + uboot_size = os.path.getsize(self.scratch_file(name, + 'u-boot-nodtb.bin')) + uboot_dtb_load_addr = hex(0x400000000 + uboot_size) + + load_images_list = [ + { + 'addr': '0x400000000', + 'file': self.scratch_file(name, + 'u-boot-nodtb.bin') + }, + { + 'addr': str(uboot_dtb_load_addr), + 'file': self.scratch_file(name, 'u-boot.dtb') + }, + { + 'addr': '0x430000000', + 'file': self.scratch_file(name, 'bl31.bin') + }, + { + 'addr': '0x430080000', + 'file': self.scratch_file(name, 'optee', + 'tee-raw.bin') + } + ] + + for load_image in load_images_list: + addr = load_image['addr'] + file = load_image['file'] + self.vm.add_args('-device', + f'loader,force-raw=on,addr={addr},file={file}') + + for i in range(ca35_core): + self.vm.add_args('-device', + f'loader,addr=0x430000000,cpu-num={i}') + + load_elf_list = { + 'ssp': self.scratch_file(name, 'zephyr-aspeed-ssp.elf'), + 'tsp': self.scratch_file(name, 'zephyr-aspeed-tsp.elf') + } + + for cpu_num, key in enumerate(load_elf_list, start=4): + file = load_elf_list[key] + self.vm.add_args('-device', + f'loader,file={file},cpu-num={cpu_num}') + + self.do_test_aarch64_aspeed_sdk_start( + self.scratch_file(name, 'image-bmc')) + + def test_aarch64_ast2700fc_sdk_v09_06(self): + self.set_machine('ast2700fc') + + self.archive_extract(self.ASSET_SDK_V906_AST2700) + self.start_ast2700fc_test('ast2700-default') + self.verify_openbmc_boot_and_login('ast2700-default') + self.do_ast2700_i2c_test() + self.do_ast2700fc_ssp_test() + self.do_ast2700fc_tsp_test() + +if __name__ == '__main__': + QemuSystemTest.main() diff --git a/tests/functional/test_aarch64_imx8mp_evk.py b/tests/functional/test_aarch64_imx8mp_evk.py new file mode 100755 index 0000000000..638bf9e131 --- /dev/null +++ b/tests/functional/test_aarch64_imx8mp_evk.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Functional test that boots a Linux kernel and checks the console +# +# SPDX-License-Identifier: GPL-2.0-or-later + +from qemu_test import LinuxKernelTest, Asset + + +class Imx8mpEvkMachine(LinuxKernelTest): + + ASSET_IMAGE = Asset( + ('https://cloud.debian.org/images/cloud/bookworm/20231210-1590/' + 'debian-12-generic-arm64-20231210-1590.tar.xz'), + '7ebf1577b32d5af6204df74b54ca2e4675de9b5a9fa14f3ff70b88eeb7b3b359') + + KERNEL_OFFSET = 0x51000000 + KERNEL_SIZE = 32622528 + INITRD_OFFSET = 0x76000000 + INITRD_SIZE = 30987766 + DTB_OFFSET = 0x64F51000 + DTB_SIZE = 45 * 1024 + + def extract(self, in_path, out_path, offset, size): + try: + with open(in_path, "rb") as source: + source.seek(offset) + data = source.read(size) + with open(out_path, "wb") as target: + target.write(data) + except (IOError, ValueError) as e: + self.log.error(f"Failed to extract {out_path}: {e}") + raise + + def setUp(self): + super().setUp() + + self.image_path = self.scratch_file("disk.raw") + self.kernel_path = self.scratch_file("linux") + self.initrd_path = self.scratch_file("initrd.zstd") + self.dtb_path = self.scratch_file("imx8mp-evk.dtb") + + self.archive_extract(self.ASSET_IMAGE) + self.extract(self.image_path, self.kernel_path, + self.KERNEL_OFFSET, self.KERNEL_SIZE) + self.extract(self.image_path, self.initrd_path, + self.INITRD_OFFSET, self.INITRD_SIZE) + self.extract(self.image_path, self.dtb_path, + self.DTB_OFFSET, self.DTB_SIZE) + + def test_aarch64_imx8mp_evk_usdhc(self): + self.set_machine('imx8mp-evk') + self.vm.set_console(console_index=1) + self.vm.add_args('-m', '2G', + '-smp', '4', + '-kernel', self.kernel_path, + '-initrd', self.initrd_path, + '-dtb', self.dtb_path, + '-append', 'root=/dev/mmcblk2p1', + '-drive', f'file={self.image_path},if=sd,bus=2,' + 'format=raw,id=mmcblk2,snapshot=on') + + self.vm.launch() + self.wait_for_console_pattern('Welcome to ') + +if __name__ == '__main__': + LinuxKernelTest.main() diff --git a/tests/functional/test_arm_aspeed_ast1030.py b/tests/functional/test_arm_aspeed_ast1030.py index d45d9f7c1c..77037f0179 100755 --- a/tests/functional/test_arm_aspeed_ast1030.py +++ b/tests/functional/test_arm_aspeed_ast1030.py @@ -12,17 +12,17 @@ from qemu_test import exec_command_and_wait_for_pattern class AST1030Machine(LinuxKernelTest): - ASSET_ZEPHYR_1_04 = Asset( + ASSET_ZEPHYR_3_00 = Asset( ('https://github.com/AspeedTech-BMC' - '/zephyr/releases/download/v00.01.04/ast1030-evb-demo.zip'), - '4ac6210adcbc61294927918707c6762483fd844dde5e07f3ba834ad1f91434d3') + '/zephyr/releases/download/v00.03.00/ast1030-evb-demo.zip'), + '37fe3ecd4a1b9d620971a15b96492a81093435396eeac69b6f3e384262ff555f') - def test_ast1030_zephyros_1_04(self): + def test_ast1030_zephyros_3_00(self): self.set_machine('ast1030-evb') kernel_name = "ast1030-evb-demo/zephyr.elf" kernel_file = self.archive_extract( - self.ASSET_ZEPHYR_1_04, member=kernel_name) + self.ASSET_ZEPHYR_3_00, member=kernel_name) self.vm.set_console() self.vm.add_args('-kernel', kernel_file, '-nographic') diff --git a/tests/functional/test_arm_aspeed_ast2500.py b/tests/functional/test_arm_aspeed_ast2500.py index a3b44572fc..6923fe8701 100755 --- a/tests/functional/test_arm_aspeed_ast2500.py +++ b/tests/functional/test_arm_aspeed_ast2500.py @@ -37,14 +37,14 @@ class AST2500Machine(AspeedTest): self.do_test_arm_aspeed_buildroot_poweroff() - ASSET_SDK_V806_AST2500 = Asset( - 'https://github.com/AspeedTech-BMC/openbmc/releases/download/v08.06/ast2500-default-obmc.tar.gz', - 'e1755f3cadff69190438c688d52dd0f0d399b70a1e14b1d3d5540fc4851d38ca') + ASSET_SDK_V906_AST2500 = Asset( + 'https://github.com/AspeedTech-BMC/openbmc/releases/download/v09.06/ast2500-default-obmc.tar.gz', + '542db84645b4efd8aed50385d7f4dd1caff379a987032311cfa7b563a3addb2a') def test_arm_ast2500_evb_sdk(self): self.set_machine('ast2500-evb') - self.archive_extract(self.ASSET_SDK_V806_AST2500) + self.archive_extract(self.ASSET_SDK_V906_AST2500) self.do_test_arm_aspeed_sdk_start( self.scratch_file("ast2500-default", "image-bmc")) diff --git a/tests/functional/test_arm_aspeed_ast2600.py b/tests/functional/test_arm_aspeed_ast2600.py index 5ef52f0659..fdae4c939d 100755 --- a/tests/functional/test_arm_aspeed_ast2600.py +++ b/tests/functional/test_arm_aspeed_ast2600.py @@ -97,26 +97,27 @@ class AST2600Machine(AspeedTest): self.do_test_arm_aspeed_buildroot_poweroff() - ASSET_SDK_V806_AST2600_A2 = Asset( - 'https://github.com/AspeedTech-BMC/openbmc/releases/download/v08.06/ast2600-a2-obmc.tar.gz', - '9083506135f622d5e7351fcf7d4e1c7125cee5ba16141220c0ba88931f3681a4') + ASSET_SDK_V906_AST2600 = Asset( + 'https://github.com/AspeedTech-BMC/openbmc/releases/download/v09.06/ast2600-default-obmc.tar.gz', + '768d76e247896ad78c154b9cff4f766da2ce65f217d620b286a4a03a8a4f68f5') def test_arm_ast2600_evb_sdk(self): self.set_machine('ast2600-evb') - self.archive_extract(self.ASSET_SDK_V806_AST2600_A2) + self.archive_extract(self.ASSET_SDK_V906_AST2600) self.vm.add_args('-device', 'tmp105,bus=aspeed.i2c.bus.5,address=0x4d,id=tmp-test') self.vm.add_args('-device', 'ds1338,bus=aspeed.i2c.bus.5,address=0x32') self.do_test_arm_aspeed_sdk_start( - self.scratch_file("ast2600-a2", "image-bmc")) + self.scratch_file("ast2600-default", "image-bmc")) - self.wait_for_console_pattern('ast2600-a2 login:') + self.wait_for_console_pattern('ast2600-default login:') exec_command_and_wait_for_pattern(self, 'root', 'Password:') - exec_command_and_wait_for_pattern(self, '0penBmc', 'root@ast2600-a2:~#') + exec_command_and_wait_for_pattern(self, '0penBmc', + 'root@ast2600-default:~#') exec_command_and_wait_for_pattern(self, 'echo lm75 0x4d > /sys/class/i2c-dev/i2c-5/device/new_device', diff --git a/tests/functional/test_hppa_seabios.py b/tests/functional/test_hppa_seabios.py index a44d1a3eeb..661b2464e1 100755 --- a/tests/functional/test_hppa_seabios.py +++ b/tests/functional/test_hppa_seabios.py @@ -17,9 +17,9 @@ class HppaSeabios(QemuSystemTest): def boot_seabios(self): mach = self.machine bits = self.MACH_BITS[mach] + self.vm.add_args('-no-shutdown') self.vm.set_console() self.vm.launch() - self.machine wait_for_console_pattern(self, f'SeaBIOS PA-RISC {bits}-bit Firmware') wait_for_console_pattern(self, f'Emulated machine: HP {mach} ({bits}-bit') diff --git a/tests/functional/test_ppc64_pseries.py b/tests/functional/test_ppc64_pseries.py index fdc404ed03..67057934e8 100755 --- a/tests/functional/test_ppc64_pseries.py +++ b/tests/functional/test_ppc64_pseries.py @@ -63,6 +63,7 @@ class pseriesMachine(QemuSystemTest): wait_for_console_pattern(self, self.good_message, self.panic_message) def test_ppc64_linux_smt_boot(self): + self.set_machine('pseries') self.vm.add_args('-smp', '4,threads=4') self.do_test_ppc64_linux_boot() console_pattern = 'CPU maps initialized for 4 threads per core' diff --git a/tests/lcitool/mappings.yml b/tests/lcitool/mappings.yml index 74eb13d62b..673baf3936 100644 --- a/tests/lcitool/mappings.yml +++ b/tests/lcitool/mappings.yml @@ -64,6 +64,11 @@ mappings: python3-wheel: OpenSUSELeap15: python311-pip + rust: + Debian12: rustc-web + Ubuntu2204: rustc-1.77 + Ubuntu2404: rustc-1.77 + pypi_mappings: # Request more recent version meson: diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index aa551aca9b..8474ea822f 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -141,7 +141,8 @@ fedora_rustup_nightly_extras = [ 'RUN $CARGO --list\n', ] -ubuntu2204_bindgen_extras = [ +ubuntu2204_rust_extras = [ + "ENV RUSTC=/usr/bin/rustc-1.77\n", "ENV CARGO_HOME=/usr/local/cargo\n", 'ENV PATH=$CARGO_HOME/bin:$PATH\n', "RUN DEBIAN_FRONTEND=noninteractive eatmydata \\\n", @@ -170,7 +171,7 @@ try: generate_dockerfile("fedora", "fedora-40") generate_dockerfile("opensuse-leap", "opensuse-leap-15") generate_dockerfile("ubuntu2204", "ubuntu-2204", - trailer="".join(ubuntu2204_bindgen_extras)) + trailer="".join(ubuntu2204_rust_extras)) # # Non-fatal Rust-enabled build diff --git a/tests/qtest/cpu-plug-test.c b/tests/qtest/cpu-plug-test.c index 6633abfc10..44d704680b 100644 --- a/tests/qtest/cpu-plug-test.c +++ b/tests/qtest/cpu-plug-test.c @@ -156,6 +156,28 @@ static void add_s390x_test_case(const char *mname) g_free(path); } +static void add_loongarch_test_case(const char *mname) +{ + char *path; + PlugTestData *data; + + data = g_new(PlugTestData, 1); + data->machine = g_strdup(mname); + data->cpu_model = "la464"; + data->device_model = g_strdup("la464-loongarch-cpu"); + data->sockets = 1; + data->cores = 3; + data->threads = 1; + data->maxcpus = data->sockets * data->cores * data->threads; + + path = g_strdup_printf("cpu-plug/%s/device-add/%ux%ux%u&maxcpus=%u", + mname, data->sockets, data->cores, + data->threads, data->maxcpus); + qtest_add_data_func_full(path, data, test_plug_with_device_add, + test_data_free); + g_free(path); +} + int main(int argc, char **argv) { const char *arch = qtest_get_arch(); @@ -168,6 +190,8 @@ int main(int argc, char **argv) qtest_cb_for_every_machine(add_pseries_test_case, g_test_quick()); } else if (g_str_equal(arch, "s390x")) { qtest_cb_for_every_machine(add_s390x_test_case, g_test_quick()); + } else if (g_str_equal(arch, "loongarch64")) { + add_loongarch_test_case("virt"); } return g_test_run(); diff --git a/tests/qtest/libqos/virtio-9p-client.c b/tests/qtest/libqos/virtio-9p-client.c index 98b77db51d..6ab4501c6e 100644 --- a/tests/qtest/libqos/virtio-9p-client.c +++ b/tests/qtest/libqos/virtio-9p-client.c @@ -557,6 +557,55 @@ void v9fs_rgetattr(P9Req *req, v9fs_attr *attr) v9fs_req_free(req); } +/* + * size[4] Tsetattr tag[2] fid[4] valid[4] mode[4] uid[4] gid[4] size[8] + * atime_sec[8] atime_nsec[8] mtime_sec[8] mtime_nsec[8] + */ +TSetAttrRes v9fs_tsetattr(TSetAttrOpt opt) +{ + P9Req *req; + uint32_t err; + + g_assert(opt.client); + + req = v9fs_req_init( + opt.client, 4/*fid*/ + 4/*valid*/ + 4/*mode*/ + 4/*uid*/ + 4/*gid*/ + + 8/*size*/ + 8/*atime_sec*/ + 8/*atime_nsec*/ + 8/*mtime_sec*/ + + 8/*mtime_nsec*/, P9_TSETATTR, opt.tag + ); + v9fs_uint32_write(req, opt.fid); + v9fs_uint32_write(req, (uint32_t) opt.attr.valid); + v9fs_uint32_write(req, opt.attr.mode); + v9fs_uint32_write(req, opt.attr.uid); + v9fs_uint32_write(req, opt.attr.gid); + v9fs_uint64_write(req, opt.attr.size); + v9fs_uint64_write(req, opt.attr.atime_sec); + v9fs_uint64_write(req, opt.attr.atime_nsec); + v9fs_uint64_write(req, opt.attr.mtime_sec); + v9fs_uint64_write(req, opt.attr.mtime_nsec); + v9fs_req_send(req); + + if (!opt.requestOnly) { + v9fs_req_wait_for_reply(req, NULL); + if (opt.expectErr) { + v9fs_rlerror(req, &err); + g_assert_cmpint(err, ==, opt.expectErr); + } else { + v9fs_rsetattr(req); + } + req = NULL; /* request was freed */ + } + + return (TSetAttrRes) { .req = req }; +} + +/* size[4] Rsetattr tag[2] */ +void v9fs_rsetattr(P9Req *req) +{ + v9fs_req_recv(req, P9_RSETATTR); + v9fs_req_free(req); +} + /* size[4] Treaddir tag[2] fid[4] offset[8] count[4] */ TReadDirRes v9fs_treaddir(TReadDirOpt opt) { diff --git a/tests/qtest/libqos/virtio-9p-client.h b/tests/qtest/libqos/virtio-9p-client.h index 78228eb97d..e3221a3104 100644 --- a/tests/qtest/libqos/virtio-9p-client.h +++ b/tests/qtest/libqos/virtio-9p-client.h @@ -65,6 +65,16 @@ typedef struct v9fs_attr { #define P9_GETATTR_BASIC 0x000007ffULL /* Mask for fields up to BLOCKS */ #define P9_GETATTR_ALL 0x00003fffULL /* Mask for ALL fields */ +#define P9_SETATTR_MODE 0x00000001UL +#define P9_SETATTR_UID 0x00000002UL +#define P9_SETATTR_GID 0x00000004UL +#define P9_SETATTR_SIZE 0x00000008UL +#define P9_SETATTR_ATIME 0x00000010UL +#define P9_SETATTR_MTIME 0x00000020UL +#define P9_SETATTR_CTIME 0x00000040UL +#define P9_SETATTR_ATIME_SET 0x00000080UL +#define P9_SETATTR_MTIME_SET 0x00000100UL + struct V9fsDirent { v9fs_qid qid; uint64_t offset; @@ -182,6 +192,28 @@ typedef struct TGetAttrRes { P9Req *req; } TGetAttrRes; +/* options for 'Tsetattr' 9p request */ +typedef struct TSetAttrOpt { + /* 9P client being used (mandatory) */ + QVirtio9P *client; + /* user supplied tag number being returned with response (optional) */ + uint16_t tag; + /* file ID of file/dir whose attributes shall be modified (required) */ + uint32_t fid; + /* new attribute values to be set by 9p server */ + v9fs_attr attr; + /* only send Tsetattr request but not wait for a reply? (optional) */ + bool requestOnly; + /* do we expect an Rlerror response, if yes which error code? (optional) */ + uint32_t expectErr; +} TSetAttrOpt; + +/* result of 'Tsetattr' 9p request */ +typedef struct TSetAttrRes { + /* if requestOnly was set: request object for further processing */ + P9Req *req; +} TSetAttrRes; + /* options for 'Treaddir' 9p request */ typedef struct TReadDirOpt { /* 9P client being used (mandatory) */ @@ -470,6 +502,8 @@ TWalkRes v9fs_twalk(TWalkOpt opt); void v9fs_rwalk(P9Req *req, uint16_t *nwqid, v9fs_qid **wqid); TGetAttrRes v9fs_tgetattr(TGetAttrOpt); void v9fs_rgetattr(P9Req *req, v9fs_attr *attr); +TSetAttrRes v9fs_tsetattr(TSetAttrOpt opt); +void v9fs_rsetattr(P9Req *req); TReadDirRes v9fs_treaddir(TReadDirOpt); void v9fs_rreaddir(P9Req *req, uint32_t *count, uint32_t *nentries, struct V9fsDirent **entries); diff --git a/tests/qtest/libqos/virtio.c b/tests/qtest/libqos/virtio.c index 2e7979652f..5a709d0bc5 100644 --- a/tests/qtest/libqos/virtio.c +++ b/tests/qtest/libqos/virtio.c @@ -25,49 +25,63 @@ */ static uint16_t qvirtio_readw(QVirtioDevice *d, QTestState *qts, uint64_t addr) { - uint16_t val = qtest_readw(qts, addr); + uint16_t val; - if (d->features & (1ull << VIRTIO_F_VERSION_1) && qtest_big_endian(qts)) { - val = bswap16(val); + if (d->features & (1ull << VIRTIO_F_VERSION_1)) { + qtest_memread(qts, addr, &val, sizeof(val)); + val = le16_to_cpu(val); + } else { + val = qtest_readw(qts, addr); } + return val; } static uint32_t qvirtio_readl(QVirtioDevice *d, QTestState *qts, uint64_t addr) { - uint32_t val = qtest_readl(qts, addr); + uint32_t val; - if (d->features & (1ull << VIRTIO_F_VERSION_1) && qtest_big_endian(qts)) { - val = bswap32(val); + if (d->features & (1ull << VIRTIO_F_VERSION_1)) { + qtest_memread(qts, addr, &val, sizeof(val)); + val = le32_to_cpu(val); + } else { + val = qtest_readl(qts, addr); } + return val; } static void qvirtio_writew(QVirtioDevice *d, QTestState *qts, uint64_t addr, uint16_t val) { - if (d->features & (1ull << VIRTIO_F_VERSION_1) && qtest_big_endian(qts)) { - val = bswap16(val); + if (d->features & (1ull << VIRTIO_F_VERSION_1)) { + val = cpu_to_le16(val); + qtest_memwrite(qts, addr, &val, sizeof(val)); + } else { + qtest_writew(qts, addr, val); } - qtest_writew(qts, addr, val); } static void qvirtio_writel(QVirtioDevice *d, QTestState *qts, uint64_t addr, uint32_t val) { - if (d->features & (1ull << VIRTIO_F_VERSION_1) && qtest_big_endian(qts)) { - val = bswap32(val); + if (d->features & (1ull << VIRTIO_F_VERSION_1)) { + val = cpu_to_le32(val); + qtest_memwrite(qts, addr, &val, sizeof(val)); + } else { + qtest_writel(qts, addr, val); } - qtest_writel(qts, addr, val); } static void qvirtio_writeq(QVirtioDevice *d, QTestState *qts, uint64_t addr, uint64_t val) { - if (d->features & (1ull << VIRTIO_F_VERSION_1) && qtest_big_endian(qts)) { - val = bswap64(val); + if (d->features & (1ull << VIRTIO_F_VERSION_1)) { + val = cpu_to_le64(val); + qtest_memwrite(qts, addr, &val, sizeof(val)); + } else { + qtest_writeq(qts, addr, val); } - qtest_writeq(qts, addr, val); } uint8_t qvirtio_config_readb(QVirtioDevice *d, uint64_t addr) diff --git a/tests/qtest/libqtest.c b/tests/qtest/libqtest.c index 358580361d..94526b7f9c 100644 --- a/tests/qtest/libqtest.c +++ b/tests/qtest/libqtest.c @@ -574,10 +574,8 @@ void qtest_qmp_handshake(QTestState *s, QList *capabilities) } } -QTestState *qtest_init_with_env_and_capabilities(const char *var, - const char *extra_args, - QList *capabilities, - bool do_connect) +QTestState *qtest_init_ext(const char *var, const char *extra_args, + QList *capabilities, bool do_connect) { QTestState *s = qtest_init_internal(qtest_qemu_binary(var), extra_args, do_connect); @@ -594,15 +592,9 @@ QTestState *qtest_init_with_env_and_capabilities(const char *var, return s; } -QTestState *qtest_init_with_env(const char *var, const char *extra_args, - bool do_connect) -{ - return qtest_init_with_env_and_capabilities(var, extra_args, NULL, true); -} - QTestState *qtest_init(const char *extra_args) { - return qtest_init_with_env(NULL, extra_args, true); + return qtest_init_ext(NULL, extra_args, NULL, true); } QTestState *qtest_vinitf(const char *fmt, va_list ap) @@ -1662,7 +1654,7 @@ static struct MachInfo *qtest_get_machines(const char *var) silence_spawn_log = !g_test_verbose(); - qts = qtest_init_with_env(qemu_var, "-machine none", true); + qts = qtest_init_ext(qemu_var, "-machine none", NULL, true); response = qtest_qmp(qts, "{ 'execute': 'query-machines' }"); g_assert(response); list = qdict_get_qlist(response, "return"); @@ -1717,7 +1709,7 @@ static struct CpuModel *qtest_get_cpu_models(void) silence_spawn_log = !g_test_verbose(); - qts = qtest_init_with_env(NULL, "-machine none", true); + qts = qtest_init_ext(NULL, "-machine none", NULL, true); response = qtest_qmp(qts, "{ 'execute': 'query-cpu-definitions' }"); g_assert(response); list = qdict_get_qlist(response, "return"); diff --git a/tests/qtest/libqtest.h b/tests/qtest/libqtest.h index 930a91dcb7..b3f2e7fbef 100644 --- a/tests/qtest/libqtest.h +++ b/tests/qtest/libqtest.h @@ -57,37 +57,21 @@ QTestState *qtest_vinitf(const char *fmt, va_list ap) G_GNUC_PRINTF(1, 0); QTestState *qtest_init(const char *extra_args); /** - * qtest_init_with_env: - * @var: Environment variable from where to take the QEMU binary - * @extra_args: Other arguments to pass to QEMU. CAUTION: these - * arguments are subject to word splitting and shell evaluation. - * @do_connect: connect to qemu monitor and qtest socket. - * - * Like qtest_init(), but use a different environment variable for the - * QEMU binary. - * - * Returns: #QTestState instance. - */ -QTestState *qtest_init_with_env(const char *var, const char *extra_args, - bool do_connect); - -/** - * qtest_init_with_env_and_capabilities: + * qtest_init_ext: * @var: Environment variable from where to take the QEMU binary * @extra_args: Other arguments to pass to QEMU. CAUTION: these * arguments are subject to word splitting and shell evaluation. * @capabilities: list of QMP capabilities (strings) to enable * @do_connect: connect to qemu monitor and qtest socket. * - * Like qtest_init_with_env(), but enable specified capabilities during - * hadshake. + * Like qtest_init(), but use a different environment variable for the + * QEMU binary, allow specify capabilities and skip connecting + * to QEMU monitor. * * Returns: #QTestState instance. */ -QTestState *qtest_init_with_env_and_capabilities(const char *var, - const char *extra_args, - QList *capabilities, - bool do_connect); +QTestState *qtest_init_ext(const char *var, const char *extra_args, + QList *capabilities, bool do_connect); /** * qtest_init_without_qmp_handshake: @@ -102,7 +86,7 @@ QTestState *qtest_init_without_qmp_handshake(const char *extra_args); * qtest_connect * @s: #QTestState instance to connect * Connect to qemu monitor and qtest socket, after skipping them in - * qtest_init_with_env. Does not handshake with the monitor. + * qtest_init_ext. Does not handshake with the monitor. */ void qtest_connect(QTestState *s); diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build index 3136d15e0f..7daf619845 100644 --- a/tests/qtest/meson.build +++ b/tests/qtest/meson.build @@ -148,7 +148,8 @@ qtests_hppa = \ qtests_loongarch64 = qtests_filter + \ (config_all_devices.has_key('CONFIG_LOONGARCH_VIRT') ? ['numa-test'] : []) + \ - ['boot-serial-test'] + ['boot-serial-test', + 'cpu-plug-test'] qtests_m68k = ['boot-serial-test'] + \ qtests_filter diff --git a/tests/qtest/migration/compression-tests.c b/tests/qtest/migration/compression-tests.c index 8b58401b84..41e79f031b 100644 --- a/tests/qtest/migration/compression-tests.c +++ b/tests/qtest/migration/compression-tests.c @@ -35,6 +35,9 @@ static void test_multifd_tcp_zstd(void) { MigrateCommon args = { .listen_uri = "defer", + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, .start_hook = migrate_hook_start_precopy_tcp_multifd_zstd, }; test_precopy_common(&args); @@ -56,6 +59,9 @@ static void test_multifd_tcp_qatzip(void) { MigrateCommon args = { .listen_uri = "defer", + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, .start_hook = migrate_hook_start_precopy_tcp_multifd_qatzip, }; test_precopy_common(&args); @@ -74,6 +80,9 @@ static void test_multifd_tcp_qpl(void) { MigrateCommon args = { .listen_uri = "defer", + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, .start_hook = migrate_hook_start_precopy_tcp_multifd_qpl, }; test_precopy_common(&args); @@ -92,6 +101,9 @@ static void test_multifd_tcp_uadk(void) { MigrateCommon args = { .listen_uri = "defer", + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, .start_hook = migrate_hook_start_precopy_tcp_multifd_uadk, }; test_precopy_common(&args); @@ -103,10 +115,6 @@ migrate_hook_start_xbzrle(QTestState *from, QTestState *to) { migrate_set_parameter_int(from, "xbzrle-cache-size", 33554432); - - migrate_set_capability(from, "xbzrle", true); - migrate_set_capability(to, "xbzrle", true); - return NULL; } @@ -118,6 +126,9 @@ static void test_precopy_unix_xbzrle(void) .listen_uri = uri, .start_hook = migrate_hook_start_xbzrle, .iterations = 2, + .start = { + .caps[MIGRATION_CAPABILITY_XBZRLE] = true, + }, /* * XBZRLE needs pages to be modified when doing the 2nd+ round * iteration to have real data pushed to the stream. @@ -146,6 +157,9 @@ static void test_multifd_tcp_zlib(void) { MigrateCommon args = { .listen_uri = "defer", + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, .start_hook = migrate_hook_start_precopy_tcp_multifd_zlib, }; test_precopy_common(&args); diff --git a/tests/qtest/migration/cpr-tests.c b/tests/qtest/migration/cpr-tests.c index 4758841824..5536e14610 100644 --- a/tests/qtest/migration/cpr-tests.c +++ b/tests/qtest/migration/cpr-tests.c @@ -24,9 +24,6 @@ static void *migrate_hook_start_mode_reboot(QTestState *from, QTestState *to) migrate_set_parameter_str(from, "mode", "cpr-reboot"); migrate_set_parameter_str(to, "mode", "cpr-reboot"); - migrate_set_capability(from, "x-ignore-shared", true); - migrate_set_capability(to, "x-ignore-shared", true); - return NULL; } @@ -39,6 +36,9 @@ static void test_mode_reboot(void) .connect_uri = uri, .listen_uri = "defer", .start_hook = migrate_hook_start_mode_reboot, + .start = { + .caps[MIGRATION_CAPABILITY_X_IGNORE_SHARED] = true, + }, }; test_file_common(&args, true); diff --git a/tests/qtest/migration/file-tests.c b/tests/qtest/migration/file-tests.c index f260e2871d..4d78ce0855 100644 --- a/tests/qtest/migration/file-tests.c +++ b/tests/qtest/migration/file-tests.c @@ -107,15 +107,6 @@ static void test_precopy_file_offset_bad(void) test_file_common(&args, false); } -static void *migrate_hook_start_mapped_ram(QTestState *from, - QTestState *to) -{ - migrate_set_capability(from, "mapped-ram", true); - migrate_set_capability(to, "mapped-ram", true); - - return NULL; -} - static void test_precopy_file_mapped_ram_live(void) { g_autofree char *uri = g_strdup_printf("file:%s/%s", tmpfs, @@ -123,7 +114,9 @@ static void test_precopy_file_mapped_ram_live(void) MigrateCommon args = { .connect_uri = uri, .listen_uri = "defer", - .start_hook = migrate_hook_start_mapped_ram, + .start = { + .caps[MIGRATION_CAPABILITY_MAPPED_RAM] = true, + }, }; test_file_common(&args, false); @@ -136,26 +129,14 @@ static void test_precopy_file_mapped_ram(void) MigrateCommon args = { .connect_uri = uri, .listen_uri = "defer", - .start_hook = migrate_hook_start_mapped_ram, + .start = { + .caps[MIGRATION_CAPABILITY_MAPPED_RAM] = true, + }, }; test_file_common(&args, true); } -static void *migrate_hook_start_multifd_mapped_ram(QTestState *from, - QTestState *to) -{ - migrate_hook_start_mapped_ram(from, to); - - migrate_set_parameter_int(from, "multifd-channels", 4); - migrate_set_parameter_int(to, "multifd-channels", 4); - - migrate_set_capability(from, "multifd", true); - migrate_set_capability(to, "multifd", true); - - return NULL; -} - static void test_multifd_file_mapped_ram_live(void) { g_autofree char *uri = g_strdup_printf("file:%s/%s", tmpfs, @@ -163,7 +144,10 @@ static void test_multifd_file_mapped_ram_live(void) MigrateCommon args = { .connect_uri = uri, .listen_uri = "defer", - .start_hook = migrate_hook_start_multifd_mapped_ram, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + .caps[MIGRATION_CAPABILITY_MAPPED_RAM] = true, + }, }; test_file_common(&args, false); @@ -176,7 +160,10 @@ static void test_multifd_file_mapped_ram(void) MigrateCommon args = { .connect_uri = uri, .listen_uri = "defer", - .start_hook = migrate_hook_start_multifd_mapped_ram, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + .caps[MIGRATION_CAPABILITY_MAPPED_RAM] = true, + }, }; test_file_common(&args, true); @@ -185,8 +172,6 @@ static void test_multifd_file_mapped_ram(void) static void *migrate_hook_start_multifd_mapped_ram_dio(QTestState *from, QTestState *to) { - migrate_hook_start_multifd_mapped_ram(from, to); - migrate_set_parameter_bool(from, "direct-io", true); migrate_set_parameter_bool(to, "direct-io", true); @@ -201,6 +186,10 @@ static void test_multifd_file_mapped_ram_dio(void) .connect_uri = uri, .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_mapped_ram_dio, + .start = { + .caps[MIGRATION_CAPABILITY_MAPPED_RAM] = true, + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, }; if (!probe_o_direct_support(tmpfs)) { @@ -246,7 +235,6 @@ static void *migrate_hook_start_multifd_mapped_ram_fdset_dio(QTestState *from, fdset_add_fds(from, file, O_WRONLY, 2, true); fdset_add_fds(to, file, O_RDONLY, 2, true); - migrate_hook_start_multifd_mapped_ram(from, to); migrate_set_parameter_bool(from, "direct-io", true); migrate_set_parameter_bool(to, "direct-io", true); @@ -261,8 +249,6 @@ static void *migrate_hook_start_multifd_mapped_ram_fdset(QTestState *from, fdset_add_fds(from, file, O_WRONLY, 2, false); fdset_add_fds(to, file, O_RDONLY, 2, false); - migrate_hook_start_multifd_mapped_ram(from, to); - return NULL; } @@ -275,6 +261,10 @@ static void test_multifd_file_mapped_ram_fdset(void) .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_mapped_ram_fdset, .end_hook = migrate_hook_end_multifd_mapped_ram_fdset, + .start = { + .caps[MIGRATION_CAPABILITY_MAPPED_RAM] = true, + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, }; test_file_common(&args, true); @@ -289,6 +279,10 @@ static void test_multifd_file_mapped_ram_fdset_dio(void) .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_mapped_ram_fdset_dio, .end_hook = migrate_hook_end_multifd_mapped_ram_fdset, + .start = { + .caps[MIGRATION_CAPABILITY_MAPPED_RAM] = true, + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, }; if (!probe_o_direct_support(tmpfs)) { diff --git a/tests/qtest/migration/framework.c b/tests/qtest/migration/framework.c index 10e1d04b58..407c9023c0 100644 --- a/tests/qtest/migration/framework.c +++ b/tests/qtest/migration/framework.c @@ -30,6 +30,7 @@ #define QEMU_VM_FILE_MAGIC 0x5145564d #define QEMU_ENV_SRC "QTEST_QEMU_BINARY_SRC" #define QEMU_ENV_DST "QTEST_QEMU_BINARY_DST" +#define MULTIFD_TEST_CHANNELS 4 unsigned start_address; unsigned end_address; @@ -207,6 +208,51 @@ static QList *migrate_start_get_qmp_capabilities(const MigrateStart *args) return capabilities; } +static void migrate_start_set_capabilities(QTestState *from, QTestState *to, + MigrateStart *args) +{ + /* + * MigrationCapability_lookup and MIGRATION_CAPABILITY_ constants + * are from qapi-types-migration.h. + */ + for (uint8_t i = 0; i < MIGRATION_CAPABILITY__MAX; i++) { + if (!args->caps[i]) { + continue; + } + if (from) { + migrate_set_capability(from, + MigrationCapability_lookup.array[i], true); + } + if (to) { + migrate_set_capability(to, + MigrationCapability_lookup.array[i], true); + } + } + + /* + * Always enable migration events. Libvirt always uses it, let's try + * to mimic as closer as that. + */ + migrate_set_capability(from, "events", true); + if (!args->defer_target_connect) { + migrate_set_capability(to, "events", true); + } + + /* + * Default number of channels should be fine for most + * tests. Individual tests can override by calling + * migrate_set_parameter() directly. + */ + if (args->caps[MIGRATION_CAPABILITY_MULTIFD]) { + migrate_set_parameter_int(from, "multifd-channels", + MULTIFD_TEST_CHANNELS); + migrate_set_parameter_int(to, "multifd-channels", + MULTIFD_TEST_CHANNELS); + } + + return; +} + int migrate_start(QTestState **from, QTestState **to, const char *uri, MigrateStart *args) { @@ -336,8 +382,7 @@ int migrate_start(QTestState **from, QTestState **to, const char *uri, args->opts_source ? args->opts_source : "", ignore_stderr); if (!args->only_target) { - *from = qtest_init_with_env_and_capabilities(QEMU_ENV_SRC, cmd_source, - capabilities, true); + *from = qtest_init_ext(QEMU_ENV_SRC, cmd_source, capabilities, true); qtest_qmp_set_event_callback(*from, migrate_watch_for_events, &src_state); @@ -365,8 +410,8 @@ int migrate_start(QTestState **from, QTestState **to, const char *uri, shmem_opts ? shmem_opts : "", args->opts_target ? args->opts_target : "", ignore_stderr); - *to = qtest_init_with_env_and_capabilities(QEMU_ENV_DST, cmd_target, - capabilities, !args->defer_target_connect); + *to = qtest_init_ext(QEMU_ENV_DST, cmd_target, capabilities, + !args->defer_target_connect); qtest_qmp_set_event_callback(*to, migrate_watch_for_events, &dst_state); @@ -379,14 +424,7 @@ int migrate_start(QTestState **from, QTestState **to, const char *uri, unlink(shmem_path); } - /* - * Always enable migration events. Libvirt always uses it, let's try - * to mimic as closer as that. - */ - migrate_set_capability(*from, "events", true); - if (!args->defer_target_connect) { - migrate_set_capability(*to, "events", true); - } + migrate_start_set_capabilities(*from, *to, args); return 0; } @@ -432,6 +470,10 @@ static int migrate_postcopy_prepare(QTestState **from_ptr, { QTestState *from, *to; + /* set postcopy capabilities */ + args->start.caps[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME] = true; + args->start.caps[MIGRATION_CAPABILITY_POSTCOPY_RAM] = true; + if (migrate_start(&from, &to, "defer", &args->start)) { return -1; } @@ -440,17 +482,7 @@ static int migrate_postcopy_prepare(QTestState **from_ptr, args->postcopy_data = args->start_hook(from, to); } - migrate_set_capability(from, "postcopy-ram", true); - migrate_set_capability(to, "postcopy-ram", true); - migrate_set_capability(to, "postcopy-blocktime", true); - - if (args->postcopy_preempt) { - migrate_set_capability(from, "postcopy-preempt", true); - migrate_set_capability(to, "postcopy-preempt", true); - } - migrate_ensure_non_converge(from); - migrate_prepare_for_dirty_mem(from); qtest_qmp_assert_success(to, "{ 'execute': 'migrate-incoming'," " 'arguments': { " @@ -948,15 +980,9 @@ void *migrate_hook_start_precopy_tcp_multifd_common(QTestState *from, QTestState *to, const char *method) { - migrate_set_parameter_int(from, "multifd-channels", 16); - migrate_set_parameter_int(to, "multifd-channels", 16); - migrate_set_parameter_str(from, "multifd-compression", method); migrate_set_parameter_str(to, "multifd-compression", method); - migrate_set_capability(from, "multifd", true); - migrate_set_capability(to, "multifd", true); - /* Start incoming migration from the 1st socket */ migrate_incoming_qmp(to, "tcp:127.0.0.1:0", NULL, "{}"); diff --git a/tests/qtest/migration/framework.h b/tests/qtest/migration/framework.h index e4a11870f6..01e425e64e 100644 --- a/tests/qtest/migration/framework.h +++ b/tests/qtest/migration/framework.h @@ -12,6 +12,7 @@ #define TEST_FRAMEWORK_H #include "libqtest.h" +#include <qapi/qapi-types-migration.h> #define FILE_TEST_FILENAME "migfile" #define FILE_TEST_OFFSET 0x1000 @@ -120,6 +121,13 @@ typedef struct { /* Do not connect to target monitor and qtest sockets in qtest_init */ bool defer_target_connect; + + /* + * Migration capabilities to be set in both source and + * destination. For unilateral capabilities, use + * migration_set_capabilities(). + */ + bool caps[MIGRATION_CAPABILITY__MAX]; } MigrateStart; typedef enum PostcopyRecoveryFailStage { @@ -207,7 +215,6 @@ typedef struct { /* Postcopy specific fields */ void *postcopy_data; - bool postcopy_preempt; PostcopyRecoveryFailStage postcopy_recovery_fail_stage; } MigrateCommon; diff --git a/tests/qtest/migration/misc-tests.c b/tests/qtest/migration/misc-tests.c index 2e612d9e38..54995256d8 100644 --- a/tests/qtest/migration/misc-tests.c +++ b/tests/qtest/migration/misc-tests.c @@ -98,6 +98,7 @@ static void test_ignore_shared(void) QTestState *from, *to; MigrateStart args = { .use_shmem = true, + .caps[MIGRATION_CAPABILITY_X_IGNORE_SHARED] = true, }; if (migrate_start(&from, &to, uri, &args)) { @@ -107,9 +108,6 @@ static void test_ignore_shared(void) migrate_ensure_non_converge(from); migrate_prepare_for_dirty_mem(from); - migrate_set_capability(from, "x-ignore-shared", true); - migrate_set_capability(to, "x-ignore-shared", true); - /* Wait for the first serial output from the source */ wait_for_serial("src_serial"); diff --git a/tests/qtest/migration/postcopy-tests.c b/tests/qtest/migration/postcopy-tests.c index 982457bed1..483e3ff99f 100644 --- a/tests/qtest/migration/postcopy-tests.c +++ b/tests/qtest/migration/postcopy-tests.c @@ -39,7 +39,9 @@ static void test_postcopy_suspend(void) static void test_postcopy_preempt(void) { MigrateCommon args = { - .postcopy_preempt = true, + .start = { + .caps[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT] = true, + }, }; test_postcopy_common(&args); @@ -73,7 +75,9 @@ static void test_postcopy_recovery_fail_reconnect(void) static void test_postcopy_preempt_recovery(void) { MigrateCommon args = { - .postcopy_preempt = true, + .start = { + .caps[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT] = true, + }, }; test_postcopy_recovery_common(&args); diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c index ba273d10b9..87b0a7e8ef 100644 --- a/tests/qtest/migration/precopy-tests.c +++ b/tests/qtest/migration/precopy-tests.c @@ -99,32 +99,85 @@ static void test_precopy_unix_dirty_ring(void) test_precopy_common(&args); } -static void test_precopy_tcp_plain(void) +#ifdef CONFIG_RDMA + +#define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh" +static int new_rdma_link(char *buffer) { + char cmd[256]; + bool verbose = g_getenv("QTEST_LOG"); + + snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER, + verbose ? "" : "2>/dev/null"); + + FILE *pipe = popen(cmd, "r"); + if (pipe == NULL) { + perror("Failed to run script"); + return -1; + } + + int idx = 0; + while (fgets(buffer + idx, 128 - idx, pipe) != NULL) { + idx += strlen(buffer); + } + + int status = pclose(pipe); + if (status == -1) { + perror("Error reported by pclose()"); + return -1; + } else if (WIFEXITED(status)) { + return WEXITSTATUS(status); + } + + return -1; +} + +static void test_precopy_rdma_plain(void) +{ + char buffer[128] = {}; + + if (new_rdma_link(buffer)) { + g_test_skip("No rdma link available\n" + "# To enable the test:\n" + "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to " + "setup a new rdma/rxe link and rerun the test\n" + "# Optional: run 'scripts/rdma-migration-helper.sh clean' " + "to revert the 'setup'"); + return; + } + + /* + * TODO: query a free port instead of hard code. + * 29200=('R'+'D'+'M'+'A')*100 + **/ + g_autofree char *uri = g_strdup_printf("rdma:%s:29200", buffer); + MigrateCommon args = { - .listen_uri = "tcp:127.0.0.1:0", + .listen_uri = uri, + .connect_uri = uri, }; test_precopy_common(&args); } +#endif -static void *migrate_hook_start_switchover_ack(QTestState *from, QTestState *to) +static void test_precopy_tcp_plain(void) { + MigrateCommon args = { + .listen_uri = "tcp:127.0.0.1:0", + }; - migrate_set_capability(from, "return-path", true); - migrate_set_capability(to, "return-path", true); - - migrate_set_capability(from, "switchover-ack", true); - migrate_set_capability(to, "switchover-ack", true); - - return NULL; + test_precopy_common(&args); } static void test_precopy_tcp_switchover_ack(void) { MigrateCommon args = { .listen_uri = "tcp:127.0.0.1:0", - .start_hook = migrate_hook_start_switchover_ack, + .start = { + .caps[MIGRATION_CAPABILITY_RETURN_PATH] = true, + .caps[MIGRATION_CAPABILITY_SWITCHOVER_ACK] = true, + }, /* * Source VM must be running in order to consider the switchover ACK * when deciding to do switchover or not. @@ -393,6 +446,9 @@ static void test_multifd_tcp_uri_none(void) MigrateCommon args = { .listen_uri = "defer", .start_hook = migrate_hook_start_precopy_tcp_multifd, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, /* * Multifd is more complicated than most of the features, it * directly takes guest page buffers when sending, make sure @@ -408,6 +464,9 @@ static void test_multifd_tcp_zero_page_legacy(void) MigrateCommon args = { .listen_uri = "defer", .start_hook = migrate_hook_start_precopy_tcp_multifd_zero_page_legacy, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, /* * Multifd is more complicated than most of the features, it * directly takes guest page buffers when sending, make sure @@ -423,6 +482,9 @@ static void test_multifd_tcp_no_zero_page(void) MigrateCommon args = { .listen_uri = "defer", .start_hook = migrate_hook_start_precopy_tcp_multifd_no_zero_page, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, /* * Multifd is more complicated than most of the features, it * directly takes guest page buffers when sending, make sure @@ -439,6 +501,9 @@ static void test_multifd_tcp_channels_none(void) .listen_uri = "defer", .start_hook = migrate_hook_start_precopy_tcp_multifd, .live = true, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, .connect_channels = ("[ { 'channel-type': 'main'," " 'addr': { 'transport': 'socket'," " 'type': 'inet'," @@ -1124,6 +1189,10 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env) test_multifd_tcp_uri_none); migration_test_add("/migration/multifd/tcp/plain/cancel", test_multifd_tcp_cancel); +#ifdef CONFIG_RDMA + migration_test_add("/migration/precopy/rdma/plain", + test_precopy_rdma_plain); +#endif } void migration_test_add_precopy(MigrationTestEnv *env) diff --git a/tests/qtest/migration/tls-tests.c b/tests/qtest/migration/tls-tests.c index 2cb4a44bcd..72f44defbb 100644 --- a/tests/qtest/migration/tls-tests.c +++ b/tests/qtest/migration/tls-tests.c @@ -375,9 +375,11 @@ static void test_postcopy_tls_psk(void) static void test_postcopy_preempt_tls_psk(void) { MigrateCommon args = { - .postcopy_preempt = true, .start_hook = migrate_hook_start_tls_psk_match, .end_hook = migrate_hook_end_tls_psk, + .start = { + .caps[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT] = true, + }, }; test_postcopy_common(&args); @@ -397,9 +399,11 @@ static void test_postcopy_recovery_tls_psk(void) static void test_postcopy_preempt_all(void) { MigrateCommon args = { - .postcopy_preempt = true, .start_hook = migrate_hook_start_tls_psk_match, .end_hook = migrate_hook_end_tls_psk, + .start = { + .caps[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT] = true, + }, }; test_postcopy_recovery_common(&args); @@ -631,6 +635,9 @@ static void test_multifd_tcp_tls_psk_match(void) .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_tcp_tls_psk_match, .end_hook = migrate_hook_end_tls_psk, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, }; test_precopy_common(&args); } @@ -640,6 +647,7 @@ static void test_multifd_tcp_tls_psk_mismatch(void) MigrateCommon args = { .start = { .hide_stderr = true, + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, }, .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_tcp_tls_psk_mismatch, @@ -656,6 +664,9 @@ static void test_multifd_tcp_tls_x509_default_host(void) .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_tls_x509_default_host, .end_hook = migrate_hook_end_tls_x509, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, }; test_precopy_common(&args); } @@ -666,6 +677,9 @@ static void test_multifd_tcp_tls_x509_override_host(void) .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_tls_x509_override_host, .end_hook = migrate_hook_end_tls_x509, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, }; test_precopy_common(&args); } @@ -688,6 +702,7 @@ static void test_multifd_tcp_tls_x509_mismatch_host(void) MigrateCommon args = { .start = { .hide_stderr = true, + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, }, .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_tls_x509_mismatch_host, @@ -703,6 +718,9 @@ static void test_multifd_tcp_tls_x509_allow_anon_client(void) .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_tls_x509_allow_anon_client, .end_hook = migrate_hook_end_tls_x509, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, }; test_precopy_common(&args); } @@ -712,6 +730,7 @@ static void test_multifd_tcp_tls_x509_reject_anon_client(void) MigrateCommon args = { .start = { .hide_stderr = true, + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, }, .listen_uri = "defer", .start_hook = migrate_hook_start_multifd_tls_x509_reject_anon_client, diff --git a/tests/qtest/q35-test.c b/tests/qtest/q35-test.c index 75d4078b79..62fff49fc8 100644 --- a/tests/qtest/q35-test.c +++ b/tests/qtest/q35-test.c @@ -246,41 +246,6 @@ static void test_smram_smbase_lock(void) qtest_quit(qts); } -static void test_without_smram_base(void) -{ - QPCIBus *pcibus; - QPCIDevice *pcidev; - QTestState *qts; - int i; - - qts = qtest_init("-M pc-q35-4.1"); - - pcibus = qpci_new_pc(qts, NULL); - g_assert(pcibus != NULL); - - pcidev = qpci_device_find(pcibus, 0); - g_assert(pcidev != NULL); - - /* check that RAM is accessible */ - qtest_writeb(qts, SMBASE, SMRAM_TEST_PATTERN); - g_assert_cmpint(qtest_readb(qts, SMBASE), ==, SMRAM_TEST_PATTERN); - - /* check that writing to 0x9c succeeds */ - for (i = 0; i <= 0xff; i++) { - qpci_config_writeb(pcidev, MCH_HOST_BRIDGE_F_SMBASE, i); - g_assert(qpci_config_readb(pcidev, MCH_HOST_BRIDGE_F_SMBASE) == i); - } - - /* check that RAM is still accessible */ - qtest_writeb(qts, SMBASE, SMRAM_TEST_PATTERN + 1); - g_assert_cmpint(qtest_readb(qts, SMBASE), ==, (SMRAM_TEST_PATTERN + 1)); - - g_free(pcidev); - qpci_free_pc(pcibus); - - qtest_quit(qts); -} - int main(int argc, char **argv) { g_test_init(&argc, &argv, NULL); @@ -293,6 +258,6 @@ int main(int argc, char **argv) qtest_add_data_func("/q35/tseg-size/ext/16mb", &tseg_ext_16mb, test_tseg_size); qtest_add_func("/q35/smram/smbase_lock", test_smram_smbase_lock); - qtest_add_func("/q35/smram/legacy_smbase", test_without_smram_base); + return g_test_run(); } diff --git a/tests/qtest/virtio-9p-test.c b/tests/qtest/virtio-9p-test.c index ab3a12c816..ac38ccf595 100644 --- a/tests/qtest/virtio-9p-test.c +++ b/tests/qtest/virtio-9p-test.c @@ -20,6 +20,7 @@ #define tversion(...) v9fs_tversion((TVersionOpt) __VA_ARGS__) #define tattach(...) v9fs_tattach((TAttachOpt) __VA_ARGS__) #define tgetattr(...) v9fs_tgetattr((TGetAttrOpt) __VA_ARGS__) +#define tsetattr(...) v9fs_tsetattr((TSetAttrOpt) __VA_ARGS__) #define treaddir(...) v9fs_treaddir((TReadDirOpt) __VA_ARGS__) #define tlopen(...) v9fs_tlopen((TLOpenOpt) __VA_ARGS__) #define twrite(...) v9fs_twrite((TWriteOpt) __VA_ARGS__) @@ -735,6 +736,20 @@ static void fs_use_after_unlink(void *obj, void *data, .data = buf }).count; g_assert_cmpint(count, ==, write_count); + + /* truncate file to (arbitrarily chosen) size 2001 */ + tsetattr({ + .client = v9p, .fid = fid_file, .attr = (v9fs_attr) { + .valid = P9_SETATTR_SIZE, + .size = 2001 + } + }); + /* truncate apparently succeeded, let's double-check the size */ + tgetattr({ + .client = v9p, .fid = fid_file, .request_mask = P9_GETATTR_BASIC, + .rgetattr.attr = &attr + }); + g_assert_cmpint(attr.size, ==, 2001); } static void cleanup_9p_local_driver(void *data) diff --git a/tests/tcg/loongarch64/system/kernel.ld b/tests/tcg/loongarch64/system/kernel.ld index f1a7c0168c..56d8588f1a 100644 --- a/tests/tcg/loongarch64/system/kernel.ld +++ b/tests/tcg/loongarch64/system/kernel.ld @@ -3,7 +3,7 @@ ENTRY(_start) SECTIONS { /* Linux kernel legacy start address. */ - . = 0x9000000000200000; + . = 0x200000; _text = .; .text : { *(.text) diff --git a/ui/dbus-display1.xml b/ui/dbus-display1.xml index 72deefa455..4a41a7e0f3 100644 --- a/ui/dbus-display1.xml +++ b/ui/dbus-display1.xml @@ -615,6 +615,51 @@ </interface> <!-- + org.qemu.Display1.Listener.Unix.ScanoutDMABUF2: + + This optional client-side interface can complement + org.qemu.Display1.Listener on ``/org/qemu/Display1/Listener`` for + Unix-specific DMABUF scanout setup which support multi plane. + --> + <?if $(env.HOST_OS) != windows?> + <interface name="org.qemu.Display1.Listener.Unix.ScanoutDMABUF2"> + <!-- + ScanoutDMABUF2: + @dmabuf: DMABUF file descriptor of each plane. + @x: display x offset, in pixels + @y: display y offset, in pixels + @width: display width, in pixels. + @height: display height, in pixels. + @offset: offset of each plane, in bytes. + @stride: stride of each plane, in bytes. + @num_planes: plane number. + @fourcc: DMABUF fourcc. + @backing_width: backing framebuffer width, in pixels + @backing_height: backing framebuffer height, in pixels + @modifier: DMABUF modifier. + @y0_top: whether Y position 0 is the top or not. + + Resize and update the display content with DMABUF. + --> + <method name="ScanoutDMABUF2"> + <arg type="ah" name="dmabuf" direction="in"/> + <arg type="u" name="x" direction="in"/> + <arg type="u" name="y" direction="in"/> + <arg type="u" name="width" direction="in"/> + <arg type="u" name="height" direction="in"/> + <arg type="au" name="offset" direction="in"/> + <arg type="au" name="stride" direction="in"/> + <arg type="u" name="num_planes" direction="in"/> + <arg type="u" name="fourcc" direction="in"/> + <arg type="u" name="backing_width" direction="in"/> + <arg type="u" name="backing_height" direction="in"/> + <arg type="t" name="modifier" direction="in"/> + <arg type="b" name="y0_top" direction="in"/> + </method> + </interface> + <?endif?> + + <!-- org.qemu.Display1.Clipboard: This interface must be implemented by both the client and the server on diff --git a/ui/dbus-listener.c b/ui/dbus-listener.c index 51244c9240..42875b8eed 100644 --- a/ui/dbus-listener.c +++ b/ui/dbus-listener.c @@ -85,6 +85,7 @@ struct _DBusDisplayListener { #endif #else /* !WIN32 */ QemuDBusDisplay1ListenerUnixMap *map_proxy; + QemuDBusDisplay1ListenerUnixScanoutDMABUF2 *scanout_dmabuf_v2_proxy; #endif guint dbus_filter; @@ -288,10 +289,9 @@ static void dbus_call_update_gl(DisplayChangeListener *dcl, } #ifdef CONFIG_GBM -static void dbus_scanout_dmabuf(DisplayChangeListener *dcl, - QemuDmaBuf *dmabuf) +static void dbus_scanout_dmabuf_v1(DBusDisplayListener *ddl, + QemuDmaBuf *dmabuf) { - DBusDisplayListener *ddl = container_of(dcl, DBusDisplayListener, dcl); g_autoptr(GError) err = NULL; g_autoptr(GUnixFDList) fd_list = NULL; int fd; @@ -299,7 +299,7 @@ static void dbus_scanout_dmabuf(DisplayChangeListener *dcl, uint64_t modifier; bool y0_top; - fd = qemu_dmabuf_get_fd(dmabuf); + fd = qemu_dmabuf_get_fds(dmabuf, NULL)[0]; fd_list = g_unix_fd_list_new(); if (g_unix_fd_list_append(fd_list, fd, &err) != 0) { error_report("Failed to setup dmabuf fdlist: %s", err->message); @@ -310,7 +310,7 @@ static void dbus_scanout_dmabuf(DisplayChangeListener *dcl, width = qemu_dmabuf_get_width(dmabuf); height = qemu_dmabuf_get_height(dmabuf); - stride = qemu_dmabuf_get_stride(dmabuf); + stride = qemu_dmabuf_get_strides(dmabuf, NULL)[0]; fourcc = qemu_dmabuf_get_fourcc(dmabuf); modifier = qemu_dmabuf_get_modifier(dmabuf); y0_top = qemu_dmabuf_get_y0_top(dmabuf); @@ -322,6 +322,87 @@ static void dbus_scanout_dmabuf(DisplayChangeListener *dcl, y0_top, G_DBUS_CALL_FLAGS_NONE, -1, fd_list, NULL, NULL, NULL); } + +static void dbus_scanout_dmabuf_v2(DBusDisplayListener *ddl, + QemuDmaBuf *dmabuf) +{ + g_autoptr(GError) err = NULL; + g_autoptr(GUnixFDList) fd_list = NULL; + int i, fd_index[DMABUF_MAX_PLANES], num_fds; + uint32_t x, y, width, height, fourcc, backing_width, backing_height; + GVariant *fd, *offset, *stride, *fd_handles[DMABUF_MAX_PLANES]; + uint64_t modifier; + bool y0_top; + int nfds, noffsets, nstrides; + const int *fds = qemu_dmabuf_get_fds(dmabuf, &nfds); + const uint32_t *offsets = qemu_dmabuf_get_offsets(dmabuf, &noffsets); + const uint32_t *strides = qemu_dmabuf_get_strides(dmabuf, &nstrides); + uint32_t num_planes = qemu_dmabuf_get_num_planes(dmabuf); + + assert(nfds >= num_planes); + assert(noffsets >= num_planes); + assert(nstrides >= num_planes); + + fd_list = g_unix_fd_list_new(); + + for (num_fds = 0; num_fds < num_planes; num_fds++) { + int plane_fd = fds[num_fds]; + + if (plane_fd < 0) { + break; + } + + fd_index[num_fds] = g_unix_fd_list_append(fd_list, plane_fd, &err); + if (fd_index[num_fds] < 0) { + error_report("Failed to setup dmabuf fdlist: %s", err->message); + return; + } + } + + ddl_discard_display_messages(ddl); + + x = qemu_dmabuf_get_x(dmabuf); + y = qemu_dmabuf_get_y(dmabuf); + width = qemu_dmabuf_get_width(dmabuf); + height = qemu_dmabuf_get_height(dmabuf); + fourcc = qemu_dmabuf_get_fourcc(dmabuf); + backing_width = qemu_dmabuf_get_backing_width(dmabuf); + backing_height = qemu_dmabuf_get_backing_height(dmabuf); + modifier = qemu_dmabuf_get_modifier(dmabuf); + y0_top = qemu_dmabuf_get_y0_top(dmabuf); + + offset = g_variant_new_fixed_array(G_VARIANT_TYPE_UINT32, + offsets, num_planes, sizeof(uint32_t)); + stride = g_variant_new_fixed_array(G_VARIANT_TYPE_UINT32, + strides, num_planes, sizeof(uint32_t)); + + for (i = 0; i < num_fds; i++) { + fd_handles[i] = g_variant_new_handle(fd_index[i]); + } + fd = g_variant_new_array(G_VARIANT_TYPE_HANDLE, fd_handles, num_fds); + + qemu_dbus_display1_listener_unix_scanout_dmabuf2_call_scanout_dmabuf2( + ddl->scanout_dmabuf_v2_proxy, fd, x, y, width, height, offset, stride, + num_planes, fourcc, backing_width, backing_height, modifier, y0_top, + G_DBUS_CALL_FLAGS_NONE, -1, fd_list, NULL, NULL, NULL); +} + +static void dbus_scanout_dmabuf(DisplayChangeListener *dcl, + QemuDmaBuf *dmabuf) +{ + DBusDisplayListener *ddl = container_of(dcl, DBusDisplayListener, dcl); + + if (ddl->scanout_dmabuf_v2_proxy) { + dbus_scanout_dmabuf_v2(ddl, dmabuf); + } else { + if (qemu_dmabuf_get_num_planes(dmabuf) > 1) { + g_debug("org.qemu.Display1.Listener.ScanoutDMABUF " + "does not support mutli plane"); + return; + } + dbus_scanout_dmabuf_v1(ddl, dmabuf); + } +} #endif /* GBM */ #endif /* OPENGL */ @@ -504,19 +585,18 @@ static void dbus_scanout_texture(DisplayChangeListener *dcl, backing_width, backing_height, x, y, w, h); #ifdef CONFIG_GBM g_autoptr(QemuDmaBuf) dmabuf = NULL; - int fd; - uint32_t stride, fourcc; + int fd[DMABUF_MAX_PLANES], num_planes; + uint32_t offset[DMABUF_MAX_PLANES], stride[DMABUF_MAX_PLANES], fourcc; uint64_t modifier; assert(tex_id); - fd = egl_get_fd_for_texture(tex_id, (EGLint *)&stride, (EGLint *)&fourcc, - &modifier); - if (fd < 0) { - error_report("%s: failed to get fd for texture", __func__); + if (!egl_dmabuf_export_texture(tex_id, fd, (EGLint *)offset, (EGLint *)stride, + (EGLint *)&fourcc, &num_planes, &modifier)) { + error_report("%s: failed to export dmabuf for texture", __func__); return; } - dmabuf = qemu_dmabuf_new(w, h, stride, x, y, backing_width, - backing_height, fourcc, modifier, fd, + dmabuf = qemu_dmabuf_new(w, h, offset, stride, x, y, backing_width, + backing_height, fourcc, modifier, fd, num_planes, false, backing_y_0_top); dbus_scanout_dmabuf(dcl, dmabuf); @@ -883,6 +963,8 @@ dbus_display_listener_dispose(GObject *object) #ifdef CONFIG_OPENGL egl_fb_destroy(&ddl->fb); #endif +#else /* !WIN32 */ + g_clear_object(&ddl->scanout_dmabuf_v2_proxy); #endif G_OBJECT_CLASS(dbus_display_listener_parent_class)->dispose(object); @@ -1071,6 +1153,26 @@ dbus_display_listener_setup_shared_map(DBusDisplayListener *ddl) #endif } +static void dbus_display_listener_setup_scanout_dmabuf_v2(DBusDisplayListener *ddl) +{ +#ifndef WIN32 + g_autoptr(GError) err = NULL; + + if (!dbus_display_listener_implements( + ddl, "org.qemu.Display1.Listener.Unix.ScanoutDMABUF2")) { + return; + } + ddl->scanout_dmabuf_v2_proxy = + qemu_dbus_display1_listener_unix_scanout_dmabuf2_proxy_new_sync( + ddl->conn, G_DBUS_PROXY_FLAGS_DO_NOT_AUTO_START, NULL, + "/org/qemu/Display1/Listener", NULL, &err); + if (!ddl->scanout_dmabuf_v2_proxy) { + g_debug("Failed to setup Unix scanout dmabuf v2 proxy: %s", err->message); + return; + } +#endif +} + static GDBusMessage * dbus_filter(GDBusConnection *connection, GDBusMessage *message, @@ -1159,6 +1261,7 @@ dbus_display_listener_new(const char *bus_name, dbus_display_listener_setup_shared_map(ddl); trace_dbus_can_share_map(ddl->can_share_map); dbus_display_listener_setup_d3d11(ddl); + dbus_display_listener_setup_scanout_dmabuf_v2(ddl); con = qemu_console_lookup_by_index(dbus_display_console_get_index(console)); assert(con); diff --git a/ui/dmabuf.c b/ui/dmabuf.c index df7a09703f..7433a268f0 100644 --- a/ui/dmabuf.c +++ b/ui/dmabuf.c @@ -11,10 +11,12 @@ #include "ui/dmabuf.h" struct QemuDmaBuf { - int fd; + int fd[DMABUF_MAX_PLANES]; uint32_t width; uint32_t height; - uint32_t stride; + uint32_t offset[DMABUF_MAX_PLANES]; + uint32_t stride[DMABUF_MAX_PLANES]; + uint32_t num_planes; uint32_t fourcc; uint64_t modifier; uint32_t texture; @@ -30,28 +32,33 @@ struct QemuDmaBuf { }; QemuDmaBuf *qemu_dmabuf_new(uint32_t width, uint32_t height, - uint32_t stride, uint32_t x, - uint32_t y, uint32_t backing_width, - uint32_t backing_height, uint32_t fourcc, - uint64_t modifier, int32_t dmabuf_fd, + const uint32_t *offset, const uint32_t *stride, + uint32_t x, uint32_t y, + uint32_t backing_width, uint32_t backing_height, + uint32_t fourcc, uint64_t modifier, + const int32_t *dmabuf_fd, uint32_t num_planes, bool allow_fences, bool y0_top) { QemuDmaBuf *dmabuf; + assert(num_planes > 0 && num_planes <= DMABUF_MAX_PLANES); + dmabuf = g_new0(QemuDmaBuf, 1); dmabuf->width = width; dmabuf->height = height; - dmabuf->stride = stride; + memcpy(dmabuf->offset, offset, num_planes * sizeof(*offset)); + memcpy(dmabuf->stride, stride, num_planes * sizeof(*stride)); dmabuf->x = x; dmabuf->y = y; dmabuf->backing_width = backing_width; dmabuf->backing_height = backing_height; dmabuf->fourcc = fourcc; dmabuf->modifier = modifier; - dmabuf->fd = dmabuf_fd; + memcpy(dmabuf->fd, dmabuf_fd, num_planes * sizeof(*dmabuf_fd)); dmabuf->allow_fences = allow_fences; dmabuf->y0_top = y0_top; dmabuf->fence_fd = -1; + dmabuf->num_planes = num_planes; return dmabuf; } @@ -65,31 +72,40 @@ void qemu_dmabuf_free(QemuDmaBuf *dmabuf) g_free(dmabuf); } -int qemu_dmabuf_get_fd(QemuDmaBuf *dmabuf) +const int *qemu_dmabuf_get_fds(QemuDmaBuf *dmabuf, int *nfds) { assert(dmabuf != NULL); + if (nfds) { + *nfds = ARRAY_SIZE(dmabuf->fd); + } + return dmabuf->fd; } -int qemu_dmabuf_dup_fd(QemuDmaBuf *dmabuf) +void qemu_dmabuf_dup_fds(QemuDmaBuf *dmabuf, int *fds, int nfds) { + int i; + assert(dmabuf != NULL); + assert(nfds >= dmabuf->num_planes); - if (dmabuf->fd >= 0) { - return dup(dmabuf->fd); - } else { - return -1; + for (i = 0; i < dmabuf->num_planes; i++) { + fds[i] = dmabuf->fd[i] >= 0 ? dup(dmabuf->fd[i]) : -1; } } void qemu_dmabuf_close(QemuDmaBuf *dmabuf) { + int i; + assert(dmabuf != NULL); - if (dmabuf->fd >= 0) { - close(dmabuf->fd); - dmabuf->fd = -1; + for (i = 0; i < dmabuf->num_planes; i++) { + if (dmabuf->fd[i] >= 0) { + close(dmabuf->fd[i]); + dmabuf->fd[i] = -1; + } } } @@ -107,13 +123,35 @@ uint32_t qemu_dmabuf_get_height(QemuDmaBuf *dmabuf) return dmabuf->height; } -uint32_t qemu_dmabuf_get_stride(QemuDmaBuf *dmabuf) +const uint32_t *qemu_dmabuf_get_offsets(QemuDmaBuf *dmabuf, int *noffsets) +{ + assert(dmabuf != NULL); + + if (noffsets) { + *noffsets = ARRAY_SIZE(dmabuf->offset); + } + + return dmabuf->offset; +} + +const uint32_t *qemu_dmabuf_get_strides(QemuDmaBuf *dmabuf, int *nstrides) { assert(dmabuf != NULL); + if (nstrides) { + *nstrides = ARRAY_SIZE(dmabuf->stride); + } + return dmabuf->stride; } +uint32_t qemu_dmabuf_get_num_planes(QemuDmaBuf *dmabuf) +{ + assert(dmabuf != NULL); + + return dmabuf->num_planes; +} + uint32_t qemu_dmabuf_get_fourcc(QemuDmaBuf *dmabuf) { assert(dmabuf != NULL); @@ -221,9 +259,3 @@ void qemu_dmabuf_set_draw_submitted(QemuDmaBuf *dmabuf, bool draw_submitted) assert(dmabuf != NULL); dmabuf->draw_submitted = draw_submitted; } - -void qemu_dmabuf_set_fd(QemuDmaBuf *dmabuf, int32_t fd) -{ - assert(dmabuf != NULL); - dmabuf->fd = fd; -} diff --git a/ui/egl-helpers.c b/ui/egl-helpers.c index d591159480..9cda2bbbee 100644 --- a/ui/egl-helpers.c +++ b/ui/egl-helpers.c @@ -23,6 +23,7 @@ #include "system/system.h" #include "qapi/error.h" #include "trace.h" +#include "standard-headers/drm/drm_fourcc.h" EGLDisplay *qemu_egl_display; EGLConfig qemu_egl_config; @@ -257,6 +258,11 @@ int egl_rendernode_init(const char *rendernode, DisplayGLMode mode) error_report("egl: EGL_MESA_image_dma_buf_export not supported"); goto err; } + if (!epoxy_has_egl_extension(qemu_egl_display, + "EGL_EXT_image_dma_buf_import_modifiers")) { + error_report("egl: EGL_EXT_image_dma_buf_import_modifiers not supported"); + goto err; + } qemu_egl_rn_ctx = qemu_egl_init_ctx(); if (!qemu_egl_rn_ctx) { @@ -277,44 +283,86 @@ err: return -1; } -int egl_get_fd_for_texture(uint32_t tex_id, EGLint *stride, EGLint *fourcc, - EGLuint64KHR *modifier) +bool egl_dmabuf_export_texture(uint32_t tex_id, int *fd, EGLint *offset, + EGLint *stride, EGLint *fourcc, int *num_planes, + EGLuint64KHR *modifier) { EGLImageKHR image; - EGLint num_planes, fd; + EGLuint64KHR modifiers[DMABUF_MAX_PLANES]; image = eglCreateImageKHR(qemu_egl_display, eglGetCurrentContext(), EGL_GL_TEXTURE_2D_KHR, (EGLClientBuffer)(unsigned long)tex_id, NULL); if (!image) { - return -1; + return false; } eglExportDMABUFImageQueryMESA(qemu_egl_display, image, fourcc, - &num_planes, modifier); - if (num_planes != 1) { - eglDestroyImageKHR(qemu_egl_display, image); - return -1; - } - eglExportDMABUFImageMESA(qemu_egl_display, image, &fd, stride, NULL); + num_planes, modifiers); + eglExportDMABUFImageMESA(qemu_egl_display, image, fd, stride, offset); eglDestroyImageKHR(qemu_egl_display, image); - return fd; + /* Only first modifier matters. */ + if (modifier) { + *modifier = modifiers[0]; + } + + return true; } void egl_dmabuf_import_texture(QemuDmaBuf *dmabuf) { EGLImageKHR image = EGL_NO_IMAGE_KHR; EGLint attrs[64]; - int i = 0; - uint64_t modifier; + int i = 0, j; + uint64_t modifier = qemu_dmabuf_get_modifier(dmabuf); uint32_t texture = qemu_dmabuf_get_texture(dmabuf); + int nfds, noffsets, nstrides; + const int *fds = qemu_dmabuf_get_fds(dmabuf, &nfds); + const uint32_t *offsets = qemu_dmabuf_get_offsets(dmabuf, &noffsets); + const uint32_t *strides = qemu_dmabuf_get_strides(dmabuf, &nstrides); + uint32_t num_planes = qemu_dmabuf_get_num_planes(dmabuf); + + EGLint fd_attrs[] = { + EGL_DMA_BUF_PLANE0_FD_EXT, + EGL_DMA_BUF_PLANE1_FD_EXT, + EGL_DMA_BUF_PLANE2_FD_EXT, + EGL_DMA_BUF_PLANE3_FD_EXT, + }; + EGLint offset_attrs[] = { + EGL_DMA_BUF_PLANE0_OFFSET_EXT, + EGL_DMA_BUF_PLANE1_OFFSET_EXT, + EGL_DMA_BUF_PLANE2_OFFSET_EXT, + EGL_DMA_BUF_PLANE3_OFFSET_EXT, + }; + EGLint stride_attrs[] = { + EGL_DMA_BUF_PLANE0_PITCH_EXT, + EGL_DMA_BUF_PLANE1_PITCH_EXT, + EGL_DMA_BUF_PLANE2_PITCH_EXT, + EGL_DMA_BUF_PLANE3_PITCH_EXT, + }; + EGLint modifier_lo_attrs[] = { + EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT, + EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT, + EGL_DMA_BUF_PLANE2_MODIFIER_LO_EXT, + EGL_DMA_BUF_PLANE3_MODIFIER_LO_EXT, + }; + EGLint modifier_hi_attrs[] = { + EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT, + EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT, + EGL_DMA_BUF_PLANE2_MODIFIER_HI_EXT, + EGL_DMA_BUF_PLANE3_MODIFIER_HI_EXT, + }; if (texture != 0) { return; } + assert(nfds >= num_planes); + assert(noffsets >= num_planes); + assert(nstrides >= num_planes); + attrs[i++] = EGL_WIDTH; attrs[i++] = qemu_dmabuf_get_backing_width(dmabuf); attrs[i++] = EGL_HEIGHT; @@ -322,21 +370,22 @@ void egl_dmabuf_import_texture(QemuDmaBuf *dmabuf) attrs[i++] = EGL_LINUX_DRM_FOURCC_EXT; attrs[i++] = qemu_dmabuf_get_fourcc(dmabuf); - attrs[i++] = EGL_DMA_BUF_PLANE0_FD_EXT; - attrs[i++] = qemu_dmabuf_get_fd(dmabuf); - attrs[i++] = EGL_DMA_BUF_PLANE0_PITCH_EXT; - attrs[i++] = qemu_dmabuf_get_stride(dmabuf); - attrs[i++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT; - attrs[i++] = 0; -#ifdef EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT - modifier = qemu_dmabuf_get_modifier(dmabuf); - if (modifier) { - attrs[i++] = EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT; - attrs[i++] = (modifier >> 0) & 0xffffffff; - attrs[i++] = EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT; - attrs[i++] = (modifier >> 32) & 0xffffffff; + for (j = 0; j < num_planes; j++) { + attrs[i++] = fd_attrs[j]; + /* fd[1-3] may be -1 if using a joint buffer for all planes */ + attrs[i++] = fds[j] >= 0 ? fds[j] : fds[0]; + attrs[i++] = stride_attrs[j]; + attrs[i++] = strides[j]; + attrs[i++] = offset_attrs[j]; + attrs[i++] = offsets[j]; + if (modifier != DRM_FORMAT_MOD_INVALID) { + attrs[i++] = modifier_lo_attrs[j]; + attrs[i++] = (modifier >> 0) & 0xffffffff; + attrs[i++] = modifier_hi_attrs[j]; + attrs[i++] = (modifier >> 32) & 0xffffffff; + } } -#endif + attrs[i++] = EGL_NONE; image = eglCreateImageKHR(qemu_egl_display, diff --git a/ui/spice-display.c b/ui/spice-display.c index c794ae0649..9c39d2c5c8 100644 --- a/ui/spice-display.c +++ b/ui/spice-display.c @@ -28,6 +28,8 @@ #include "ui/spice-display.h" +#include "standard-headers/drm/drm_fourcc.h" + bool spice_opengl; int qemu_spice_rect_is_empty(const QXLRect* r) @@ -872,23 +874,48 @@ static void spice_gl_update(DisplayChangeListener *dcl, ssd->gl_updates++; } +static void spice_server_gl_scanout(QXLInstance *qxl, + const int *fd, + uint32_t width, uint32_t height, + const uint32_t *offset, + const uint32_t *stride, + uint32_t num_planes, uint32_t format, + uint64_t modifier, int y_0_top) +{ +#ifdef HAVE_SPICE_QXL_GL_SCANOUT2 + spice_qxl_gl_scanout2(qxl, fd, width, height, offset, stride, + num_planes, format, modifier, y_0_top); +#else + if (num_planes <= 1) { + spice_qxl_gl_scanout(qxl, fd[0], width, height, stride[0], format, y_0_top); + } else { + error_report("SPICE server does not support multi plane GL scanout"); + } +#endif +} + static void spice_gl_switch(DisplayChangeListener *dcl, struct DisplaySurface *new_surface) { SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl); - EGLint stride, fourcc; - int fd; if (ssd->ds) { surface_gl_destroy_texture(ssd->gls, ssd->ds); } ssd->ds = new_surface; if (ssd->ds) { + uint32_t offset[DMABUF_MAX_PLANES], stride[DMABUF_MAX_PLANES]; + int fd[DMABUF_MAX_PLANES], num_planes, fourcc; + uint64_t modifier; + surface_gl_create_texture(ssd->gls, ssd->ds); - fd = egl_get_fd_for_texture(ssd->ds->texture, - &stride, &fourcc, - NULL); - if (fd < 0) { + if (!egl_dmabuf_export_texture(ssd->ds->texture, + fd, + (EGLint *)offset, + (EGLint *)stride, + &fourcc, + &num_planes, + &modifier)) { surface_gl_destroy_texture(ssd->gls, ssd->ds); return; } @@ -899,10 +926,11 @@ static void spice_gl_switch(DisplayChangeListener *dcl, fourcc); /* note: spice server will close the fd */ - spice_qxl_gl_scanout(&ssd->qxl, fd, - surface_width(ssd->ds), - surface_height(ssd->ds), - stride, fourcc, false); + spice_server_gl_scanout(&ssd->qxl, fd, + surface_width(ssd->ds), + surface_height(ssd->ds), + offset, stride, num_planes, + fourcc, modifier, false); ssd->have_surface = true; ssd->have_scanout = false; @@ -925,7 +953,8 @@ static void qemu_spice_gl_scanout_disable(DisplayChangeListener *dcl) SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl); trace_qemu_spice_gl_scanout_disable(ssd->qxl.id); - spice_qxl_gl_scanout(&ssd->qxl, -1, 0, 0, 0, 0, false); + spice_server_gl_scanout(&ssd->qxl, NULL, 0, 0, NULL, NULL, 0, DRM_FORMAT_INVALID, + DRM_FORMAT_MOD_INVALID, false); qemu_spice_gl_monitor_config(ssd, 0, 0, 0, 0); ssd->have_surface = false; ssd->have_scanout = false; @@ -941,20 +970,23 @@ static void qemu_spice_gl_scanout_texture(DisplayChangeListener *dcl, void *d3d_tex2d) { SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl); - EGLint stride = 0, fourcc = 0; - int fd = -1; + EGLint offset[DMABUF_MAX_PLANES], stride[DMABUF_MAX_PLANES], fourcc = 0; + int fd[DMABUF_MAX_PLANES], num_planes; + uint64_t modifier; assert(tex_id); - fd = egl_get_fd_for_texture(tex_id, &stride, &fourcc, NULL); - if (fd < 0) { - fprintf(stderr, "%s: failed to get fd for texture\n", __func__); + if (!egl_dmabuf_export_texture(tex_id, fd, offset, stride, &fourcc, + &num_planes, &modifier)) { + fprintf(stderr, "%s: failed to export dmabuf for texture\n", __func__); return; } + trace_qemu_spice_gl_scanout_texture(ssd->qxl.id, w, h, fourcc); /* note: spice server will close the fd */ - spice_qxl_gl_scanout(&ssd->qxl, fd, backing_width, backing_height, - stride, fourcc, y_0_top); + spice_server_gl_scanout(&ssd->qxl, fd, backing_width, backing_height, + (uint32_t *)offset, (uint32_t *)stride, num_planes, + fourcc, modifier, y_0_top); qemu_spice_gl_monitor_config(ssd, x, y, w, h); ssd->have_surface = false; ssd->have_scanout = true; @@ -1025,11 +1057,10 @@ static void qemu_spice_gl_update(DisplayChangeListener *dcl, uint32_t x, uint32_t y, uint32_t w, uint32_t h) { SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl); - EGLint stride = 0, fourcc = 0; + EGLint fourcc = 0; bool render_cursor = false; bool y_0_top = false; /* FIXME */ uint64_t cookie; - int fd; uint32_t width, height, texture; if (!ssd->have_scanout) { @@ -1064,26 +1095,47 @@ static void qemu_spice_gl_update(DisplayChangeListener *dcl, /* dest framebuffer */ if (ssd->blit_fb.width != width || ssd->blit_fb.height != height) { + int fds[DMABUF_MAX_PLANES], num_planes; + uint32_t offsets[DMABUF_MAX_PLANES], strides[DMABUF_MAX_PLANES]; + uint64_t modifier; + trace_qemu_spice_gl_render_dmabuf(ssd->qxl.id, width, height); egl_fb_destroy(&ssd->blit_fb); egl_fb_setup_new_tex(&ssd->blit_fb, width, height); - fd = egl_get_fd_for_texture(ssd->blit_fb.texture, - &stride, &fourcc, NULL); - spice_qxl_gl_scanout(&ssd->qxl, fd, width, height, - stride, fourcc, false); + if (!egl_dmabuf_export_texture(ssd->blit_fb.texture, fds, + (EGLint *)offsets, (EGLint *)strides, + &fourcc, &num_planes, &modifier)) { + fprintf(stderr, + "%s: failed to export dmabuf for texture\n", __func__); + return; + } + + spice_server_gl_scanout(&ssd->qxl, fds, width, height, offsets, strides, + num_planes, fourcc, modifier, false); } } else { - stride = qemu_dmabuf_get_stride(dmabuf); + int fds[DMABUF_MAX_PLANES]; + int noffsets, nstrides; + const uint32_t *offsets = qemu_dmabuf_get_offsets(dmabuf, &noffsets); + const uint32_t *strides = qemu_dmabuf_get_strides(dmabuf, &nstrides); + uint32_t num_planes = qemu_dmabuf_get_num_planes(dmabuf); + + assert(noffsets >= num_planes); + assert(nstrides >= num_planes); + fourcc = qemu_dmabuf_get_fourcc(dmabuf); y_0_top = qemu_dmabuf_get_y0_top(dmabuf); - fd = qemu_dmabuf_dup_fd(dmabuf); + qemu_dmabuf_dup_fds(dmabuf, fds, DMABUF_MAX_PLANES); trace_qemu_spice_gl_forward_dmabuf(ssd->qxl.id, width, height); /* note: spice server will close the fd, so hand over a dup */ - spice_qxl_gl_scanout(&ssd->qxl, fd, width, height, - stride, fourcc, y_0_top); + spice_server_gl_scanout(&ssd->qxl, fds, width, height, + offsets, strides, num_planes, + fourcc, + qemu_dmabuf_get_modifier(dmabuf), + y_0_top); } qemu_spice_gl_monitor_config(ssd, 0, 0, width, height); ssd->guest_dmabuf_refresh = false; diff --git a/util/cacheflush.c b/util/cacheflush.c index 1d12899a39..17c58918de 100644 --- a/util/cacheflush.c +++ b/util/cacheflush.c @@ -229,6 +229,10 @@ static void __attribute__((constructor)) init_cache_info(void) /* Caches are coherent and do not require flushing; symbol inline. */ +#elif defined(EMSCRIPTEN) + +/* Wasm doesn't have executable region of memory. */ + #elif defined(__aarch64__) && !defined(CONFIG_WIN32) /* * For Windows, we use generic implementation of flush_idcache_range, that diff --git a/util/coroutine-wasm.c b/util/coroutine-wasm.c new file mode 100644 index 0000000000..cb1ec92509 --- /dev/null +++ b/util/coroutine-wasm.c @@ -0,0 +1,127 @@ +/* + * emscripten fiber coroutine initialization code + * based on coroutine-ucontext.c + * + * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws> + * Copyright (C) 2011 Kevin Wolf <kwolf@redhat.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.0 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "qemu/osdep.h" +#include "qemu/coroutine_int.h" +#include "qemu/coroutine-tls.h" + +#include <emscripten/fiber.h> + +typedef struct { + Coroutine base; + void *stack; + size_t stack_size; + + void *asyncify_stack; + size_t asyncify_stack_size; + + CoroutineAction action; + + emscripten_fiber_t fiber; +} CoroutineEmscripten; + +/** + * Per-thread coroutine bookkeeping + */ +QEMU_DEFINE_STATIC_CO_TLS(Coroutine *, current); +QEMU_DEFINE_STATIC_CO_TLS(CoroutineEmscripten *, leader); +size_t leader_asyncify_stack_size = COROUTINE_STACK_SIZE; + +static void coroutine_trampoline(void *co_) +{ + Coroutine *co = co_; + + while (true) { + co->entry(co->entry_arg); + qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE); + } +} + +Coroutine *qemu_coroutine_new(void) +{ + CoroutineEmscripten *co; + + co = g_malloc0(sizeof(*co)); + + co->stack_size = COROUTINE_STACK_SIZE; + co->stack = qemu_alloc_stack(&co->stack_size); + + co->asyncify_stack_size = COROUTINE_STACK_SIZE; + co->asyncify_stack = g_malloc0(co->asyncify_stack_size); + emscripten_fiber_init(&co->fiber, coroutine_trampoline, &co->base, + co->stack, co->stack_size, co->asyncify_stack, + co->asyncify_stack_size); + + return &co->base; +} + +void qemu_coroutine_delete(Coroutine *co_) +{ + CoroutineEmscripten *co = DO_UPCAST(CoroutineEmscripten, base, co_); + + qemu_free_stack(co->stack, co->stack_size); + g_free(co->asyncify_stack); + g_free(co); +} + +CoroutineAction qemu_coroutine_switch(Coroutine *from_, Coroutine *to_, + CoroutineAction action) +{ + CoroutineEmscripten *from = DO_UPCAST(CoroutineEmscripten, base, from_); + CoroutineEmscripten *to = DO_UPCAST(CoroutineEmscripten, base, to_); + + set_current(to_); + to->action = action; + emscripten_fiber_swap(&from->fiber, &to->fiber); + return from->action; +} + +Coroutine *qemu_coroutine_self(void) +{ + Coroutine *self = get_current(); + + if (!self) { + CoroutineEmscripten *leaderp = get_leader(); + if (!leaderp) { + leaderp = g_malloc0(sizeof(*leaderp)); + leaderp->asyncify_stack = g_malloc0(leader_asyncify_stack_size); + leaderp->asyncify_stack_size = leader_asyncify_stack_size; + emscripten_fiber_init_from_current_context( + &leaderp->fiber, + leaderp->asyncify_stack, + leaderp->asyncify_stack_size); + leaderp->stack = leaderp->fiber.stack_limit; + leaderp->stack_size = + leaderp->fiber.stack_base - leaderp->fiber.stack_limit; + set_leader(leaderp); + } + self = &leaderp->base; + set_current(self); + } + return self; +} + +bool qemu_in_coroutine(void) +{ + Coroutine *self = get_current(); + + return self && self->caller; +} diff --git a/util/meson.build b/util/meson.build index 780b5977a8..1adff96ebd 100644 --- a/util/meson.build +++ b/util/meson.build @@ -11,7 +11,9 @@ if host_os != 'windows' endif util_ss.add(files('compatfd.c')) util_ss.add(files('event_notifier-posix.c')) - util_ss.add(files('mmap-alloc.c')) + if host_os != 'emscripten' + util_ss.add(files('mmap-alloc.c')) + endif freebsd_dep = [] if host_os == 'freebsd' freebsd_dep = util @@ -131,4 +133,6 @@ elif cpu in ['ppc', 'ppc64'] util_ss.add(files('cpuinfo-ppc.c')) elif cpu in ['riscv32', 'riscv64'] util_ss.add(files('cpuinfo-riscv.c')) +elif cpu == 's390x' + util_ss.add(files('s390x_pci_mmio.c')) endif diff --git a/util/oslib-posix.c b/util/oslib-posix.c index a697c602c6..4ff577e5de 100644 --- a/util/oslib-posix.c +++ b/util/oslib-posix.c @@ -58,6 +58,7 @@ #include <lwp.h> #endif +#include "qemu/memalign.h" #include "qemu/mmap-alloc.h" #define MAX_MEM_PREALLOC_THREAD_COUNT 16 @@ -210,11 +211,21 @@ void *qemu_anon_ram_alloc(size_t size, uint64_t *alignment, bool shared, const uint32_t qemu_map_flags = (shared ? QEMU_MAP_SHARED : 0) | (noreserve ? QEMU_MAP_NORESERVE : 0); size_t align = QEMU_VMALLOC_ALIGN; +#ifndef EMSCRIPTEN void *ptr = qemu_ram_mmap(-1, size, align, qemu_map_flags, 0); if (ptr == MAP_FAILED) { return NULL; } +#else + /* + * qemu_ram_mmap is not implemented for Emscripten. Use qemu_memalign + * for the anonymous allocation. noreserve is ignored as there is no swap + * space on Emscripten, and shared is ignored as there is no other + * processes on Emscripten. + */ + void *ptr = qemu_memalign(align, size); +#endif if (alignment) { *alignment = align; @@ -227,7 +238,16 @@ void *qemu_anon_ram_alloc(size_t size, uint64_t *alignment, bool shared, void qemu_anon_ram_free(void *ptr, size_t size) { trace_qemu_anon_ram_free(ptr, size); +#ifndef EMSCRIPTEN qemu_ram_munmap(-1, ptr, size); +#else + /* + * qemu_ram_munmap is not implemented for Emscripten and qemu_memalign + * was used for the allocation. Use the corresponding freeing function + * here. + */ + qemu_vfree(ptr); +#endif } void qemu_socket_set_block(int fd) @@ -588,7 +608,15 @@ bool qemu_prealloc_mem(int fd, char *area, size_t sz, int max_threads, { static gsize initialized; int ret; +#ifndef EMSCRIPTEN size_t hpagesize = qemu_fd_getpagesize(fd); +#else + /* + * mmap-alloc.c is excluded from Emscripten build, so qemu_fd_getpagesize + * is unavailable. Fallback to the lower level implementation. + */ + size_t hpagesize = qemu_real_host_page_size(); +#endif size_t numpages = DIV_ROUND_UP(sz, hpagesize); bool use_madv_populate_write; struct sigaction act; diff --git a/util/s390x_pci_mmio.c b/util/s390x_pci_mmio.c new file mode 100644 index 0000000000..5ab24fa474 --- /dev/null +++ b/util/s390x_pci_mmio.c @@ -0,0 +1,146 @@ +/* + * s390x PCI MMIO definitions + * + * Copyright 2025 IBM Corp. + * Author(s): Farhan Ali <alifm@linux.ibm.com> + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include <sys/syscall.h> +#include "qemu/s390x_pci_mmio.h" +#include "elf.h" + +union register_pair { + unsigned __int128 pair; + struct { + uint64_t even; + uint64_t odd; + }; +}; + +static bool is_mio_supported; + +static __attribute__((constructor)) void check_is_mio_supported(void) +{ + is_mio_supported = !!(qemu_getauxval(AT_HWCAP) & HWCAP_S390_PCI_MIO); +} + +static uint64_t s390x_pcilgi(const void *ioaddr, size_t len) +{ + union register_pair ioaddr_len = { .even = (uint64_t)ioaddr, + .odd = len }; + uint64_t val; + int cc; + + asm volatile( + /* pcilgi */ + ".insn rre,0xb9d60000,%[val],%[ioaddr_len]\n" + "ipm %[cc]\n" + "srl %[cc],28\n" + : [cc] "=d"(cc), [val] "=d"(val), + [ioaddr_len] "+d"(ioaddr_len.pair) :: "cc"); + + if (cc) { + val = -1ULL; + } + + return val; +} + +static void s390x_pcistgi(void *ioaddr, uint64_t val, size_t len) +{ + union register_pair ioaddr_len = {.even = (uint64_t)ioaddr, .odd = len}; + + asm volatile ( + /* pcistgi */ + ".insn rre,0xb9d40000,%[val],%[ioaddr_len]\n" + : [ioaddr_len] "+d" (ioaddr_len.pair) + : [val] "d" (val) + : "cc", "memory"); +} + +uint8_t s390x_pci_mmio_read_8(const void *ioaddr) +{ + uint8_t val = 0; + + if (is_mio_supported) { + val = s390x_pcilgi(ioaddr, sizeof(val)); + } else { + syscall(__NR_s390_pci_mmio_read, ioaddr, &val, sizeof(val)); + } + return val; +} + +uint16_t s390x_pci_mmio_read_16(const void *ioaddr) +{ + uint16_t val = 0; + + if (is_mio_supported) { + val = s390x_pcilgi(ioaddr, sizeof(val)); + } else { + syscall(__NR_s390_pci_mmio_read, ioaddr, &val, sizeof(val)); + } + return val; +} + +uint32_t s390x_pci_mmio_read_32(const void *ioaddr) +{ + uint32_t val = 0; + + if (is_mio_supported) { + val = s390x_pcilgi(ioaddr, sizeof(val)); + } else { + syscall(__NR_s390_pci_mmio_read, ioaddr, &val, sizeof(val)); + } + return val; +} + +uint64_t s390x_pci_mmio_read_64(const void *ioaddr) +{ + uint64_t val = 0; + + if (is_mio_supported) { + val = s390x_pcilgi(ioaddr, sizeof(val)); + } else { + syscall(__NR_s390_pci_mmio_read, ioaddr, &val, sizeof(val)); + } + return val; +} + +void s390x_pci_mmio_write_8(void *ioaddr, uint8_t val) +{ + if (is_mio_supported) { + s390x_pcistgi(ioaddr, val, sizeof(val)); + } else { + syscall(__NR_s390_pci_mmio_write, ioaddr, &val, sizeof(val)); + } +} + +void s390x_pci_mmio_write_16(void *ioaddr, uint16_t val) +{ + if (is_mio_supported) { + s390x_pcistgi(ioaddr, val, sizeof(val)); + } else { + syscall(__NR_s390_pci_mmio_write, ioaddr, &val, sizeof(val)); + } +} + +void s390x_pci_mmio_write_32(void *ioaddr, uint32_t val) +{ + if (is_mio_supported) { + s390x_pcistgi(ioaddr, val, sizeof(val)); + } else { + syscall(__NR_s390_pci_mmio_write, ioaddr, &val, sizeof(val)); + } +} + +void s390x_pci_mmio_write_64(void *ioaddr, uint64_t val) +{ + if (is_mio_supported) { + s390x_pcistgi(ioaddr, val, sizeof(val)); + } else { + syscall(__NR_s390_pci_mmio_write, ioaddr, &val, sizeof(val)); + } +} |