diff options
Diffstat (limited to 'include/exec')
| -rw-r--r-- | include/exec/cpu-all.h | 3 | ||||
| -rw-r--r-- | include/exec/cpu-defs.h | 50 | ||||
| -rw-r--r-- | include/exec/cpu_ldst.h | 22 | ||||
| -rw-r--r-- | include/exec/exec-all.h | 142 | ||||
| -rw-r--r-- | include/exec/gen-icount.h | 83 | ||||
| -rw-r--r-- | include/exec/helper-gen-common.h | 18 | ||||
| -rw-r--r-- | include/exec/helper-gen.h | 97 | ||||
| -rw-r--r-- | include/exec/helper-gen.h.inc | 102 | ||||
| -rw-r--r-- | include/exec/helper-head.h | 24 | ||||
| -rw-r--r-- | include/exec/helper-info.c.inc | 96 | ||||
| -rw-r--r-- | include/exec/helper-proto-common.h | 18 | ||||
| -rw-r--r-- | include/exec/helper-proto.h | 73 | ||||
| -rw-r--r-- | include/exec/helper-proto.h.inc | 68 | ||||
| -rw-r--r-- | include/exec/helper-tcg.h | 75 | ||||
| -rw-r--r-- | include/exec/plugin-gen.h | 24 | ||||
| -rw-r--r-- | include/exec/poison.h | 1 | ||||
| -rw-r--r-- | include/exec/tlb-common.h | 56 | ||||
| -rw-r--r-- | include/exec/translation-block.h | 149 | ||||
| -rw-r--r-- | include/exec/translator.h | 24 |
19 files changed, 562 insertions, 563 deletions
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index 78d258af44..09bf4c0cc6 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -314,6 +314,9 @@ CPUArchState *cpu_copy(CPUArchState *env); * * Use TARGET_PAGE_BITS_MIN so that these bits are constant * when TARGET_PAGE_BITS_VARY is in effect. + * + * The count, if not the placement of these bits is known + * to tcg/tcg-op-ldst.c, check_max_alignment(). */ /* Zero if TLB entry is valid. */ #define TLB_INVALID_MASK (1 << (TARGET_PAGE_BITS_MIN - 1)) diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index a6e0cf1812..0d418a0384 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -61,16 +61,11 @@ #define NB_MMU_MODES 16 #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG) +#include "exec/tlb-common.h" /* use a fully associative victim tlb of 8 entries */ #define CPU_VTLB_SIZE 8 -#if HOST_LONG_BITS == 32 && TARGET_LONG_BITS == 32 -#define CPU_TLB_ENTRY_BITS 4 -#else -#define CPU_TLB_ENTRY_BITS 5 -#endif - #define CPU_TLB_DYN_MIN_BITS 6 #define CPU_TLB_DYN_DEFAULT_BITS 8 @@ -94,34 +89,6 @@ # endif # endif -/* Minimalized TLB entry for use by TCG fast path. */ -typedef struct CPUTLBEntry { - /* bit TARGET_LONG_BITS to TARGET_PAGE_BITS : virtual address - bit TARGET_PAGE_BITS-1..4 : Nonzero for accesses that should not - go directly to ram. - bit 3 : indicates that the entry is invalid - bit 2..0 : zero - */ - union { - struct { - target_ulong addr_read; - target_ulong addr_write; - target_ulong addr_code; - /* Addend to virtual address to get host address. IO accesses - use the corresponding iotlb value. */ - uintptr_t addend; - }; - /* - * Padding to get a power of two size, as well as index - * access to addr_{read,write,code}. - */ - target_ulong addr_idx[(1 << CPU_TLB_ENTRY_BITS) / TARGET_LONG_SIZE]; - }; -} CPUTLBEntry; - -QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS)); - - #endif /* !CONFIG_USER_ONLY && CONFIG_TCG */ #if !defined(CONFIG_USER_ONLY) @@ -196,17 +163,6 @@ typedef struct CPUTLBDesc { } CPUTLBDesc; /* - * Data elements that are per MMU mode, accessed by the fast path. - * The structure is aligned to aid loading the pair with one insn. - */ -typedef struct CPUTLBDescFast { - /* Contains (n_entries - 1) << CPU_TLB_ENTRY_BITS */ - uintptr_t mask; - /* The array of tlb entries itself. */ - CPUTLBEntry *table; -} CPUTLBDescFast QEMU_ALIGNED(2 * sizeof(void *)); - -/* * Data elements that are shared between all MMU modes. */ typedef struct CPUTLBCommon { @@ -241,10 +197,6 @@ typedef struct CPUTLB { CPUTLBDescFast f[NB_MMU_MODES]; } CPUTLB; -/* This will be used by TCG backends to compute offsets. */ -#define TLB_MASK_TABLE_OFS(IDX) \ - ((int)offsetof(ArchCPU, neg.tlb.f[IDX]) - (int)offsetof(ArchCPU, env)) - #else typedef struct CPUTLB { } CPUTLB; diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index 5939688f69..896f305ff3 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -326,26 +326,32 @@ static inline void clear_helper_retaddr(void) #else -/* Needed for TCG_OVERSIZED_GUEST */ -#include "tcg/tcg.h" +#include "tcg/oversized-guest.h" static inline target_ulong tlb_read_idx(const CPUTLBEntry *entry, MMUAccessType access_type) { /* Do not rearrange the CPUTLBEntry structure members. */ QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_read) != - MMU_DATA_LOAD * TARGET_LONG_SIZE); + MMU_DATA_LOAD * sizeof(uint64_t)); QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_write) != - MMU_DATA_STORE * TARGET_LONG_SIZE); + MMU_DATA_STORE * sizeof(uint64_t)); QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_code) != - MMU_INST_FETCH * TARGET_LONG_SIZE); + MMU_INST_FETCH * sizeof(uint64_t)); - const target_ulong *ptr = &entry->addr_idx[access_type]; -#if TCG_OVERSIZED_GUEST - return *ptr; +#if TARGET_LONG_BITS == 32 + /* Use qatomic_read, in case of addr_write; only care about low bits. */ + const uint32_t *ptr = (uint32_t *)&entry->addr_idx[access_type]; + ptr += HOST_BIG_ENDIAN; + return qatomic_read(ptr); #else + const uint64_t *ptr = &entry->addr_idx[access_type]; +# if TCG_OVERSIZED_GUEST + return *ptr; +# else /* ofs might correspond to .addr_write, so use qatomic_read */ return qatomic_read(ptr); +# endif #endif } diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 3b1b57f6ad..698943d58f 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -24,20 +24,9 @@ #ifdef CONFIG_TCG #include "exec/cpu_ldst.h" #endif -#include "qemu/interval-tree.h" +#include "exec/translation-block.h" #include "qemu/clang-tsa.h" -/* Page tracking code uses ram addresses in system mode, and virtual - addresses in userspace mode. Define tb_page_addr_t to be an appropriate - type. */ -#if defined(CONFIG_USER_ONLY) -typedef abi_ulong tb_page_addr_t; -#define TB_PAGE_ADDR_FMT TARGET_ABI_FMT_lx -#else -typedef ram_addr_t tb_page_addr_t; -#define TB_PAGE_ADDR_FMT RAM_ADDR_FMT -#endif - /** * cpu_unwind_state_data: * @cpu: the cpu context @@ -478,135 +467,6 @@ int probe_access_full(CPUArchState *env, target_ulong addr, int size, CPUTLBEntryFull **pfull, uintptr_t retaddr); #endif -#define CODE_GEN_ALIGN 16 /* must be >= of the size of a icache line */ - -/* Estimated block size for TB allocation. */ -/* ??? The following is based on a 2015 survey of x86_64 host output. - Better would seem to be some sort of dynamically sized TB array, - adapting to the block sizes actually being produced. */ -#if defined(CONFIG_SOFTMMU) -#define CODE_GEN_AVG_BLOCK_SIZE 400 -#else -#define CODE_GEN_AVG_BLOCK_SIZE 150 -#endif - -/* - * Translation Cache-related fields of a TB. - * This struct exists just for convenience; we keep track of TB's in a binary - * search tree, and the only fields needed to compare TB's in the tree are - * @ptr and @size. - * Note: the address of search data can be obtained by adding @size to @ptr. - */ -struct tb_tc { - const void *ptr; /* pointer to the translated code */ - size_t size; -}; - -struct TranslationBlock { - /* - * Guest PC corresponding to this block. This must be the true - * virtual address. Therefore e.g. x86 stores EIP + CS_BASE, and - * targets like Arm, MIPS, HP-PA, which reuse low bits for ISA or - * privilege, must store those bits elsewhere. - * - * If CF_PCREL, the opcodes for the TranslationBlock are written - * such that the TB is associated only with the physical page and - * may be run in any virtual address context. In this case, PC - * must always be taken from ENV in a target-specific manner. - * Unwind information is taken as offsets from the page, to be - * deposited into the "current" PC. - */ - target_ulong pc; - - /* - * Target-specific data associated with the TranslationBlock, e.g.: - * x86: the original user, the Code Segment virtual base, - * arm: an extension of tb->flags, - * s390x: instruction data for EXECUTE, - * sparc: the next pc of the instruction queue (for delay slots). - */ - target_ulong cs_base; - - uint32_t flags; /* flags defining in which context the code was generated */ - uint32_t cflags; /* compile flags */ - -/* Note that TCG_MAX_INSNS is 512; we validate this match elsewhere. */ -#define CF_COUNT_MASK 0x000001ff -#define CF_NO_GOTO_TB 0x00000200 /* Do not chain with goto_tb */ -#define CF_NO_GOTO_PTR 0x00000400 /* Do not chain with goto_ptr */ -#define CF_SINGLE_STEP 0x00000800 /* gdbstub single-step in effect */ -#define CF_LAST_IO 0x00008000 /* Last insn may be an IO access. */ -#define CF_MEMI_ONLY 0x00010000 /* Only instrument memory ops */ -#define CF_USE_ICOUNT 0x00020000 -#define CF_INVALID 0x00040000 /* TB is stale. Set with @jmp_lock held */ -#define CF_PARALLEL 0x00080000 /* Generate code for a parallel context */ -#define CF_NOIRQ 0x00100000 /* Generate an uninterruptible TB */ -#define CF_PCREL 0x00200000 /* Opcodes in TB are PC-relative */ -#define CF_CLUSTER_MASK 0xff000000 /* Top 8 bits are cluster ID */ -#define CF_CLUSTER_SHIFT 24 - - /* - * Above fields used for comparing - */ - - /* size of target code for this block (1 <= size <= TARGET_PAGE_SIZE) */ - uint16_t size; - uint16_t icount; - - struct tb_tc tc; - - /* - * Track tb_page_addr_t intervals that intersect this TB. - * For user-only, the virtual addresses are always contiguous, - * and we use a unified interval tree. For system, we use a - * linked list headed in each PageDesc. Within the list, the lsb - * of the previous pointer tells the index of page_next[], and the - * list is protected by the PageDesc lock(s). - */ -#ifdef CONFIG_USER_ONLY - IntervalTreeNode itree; -#else - uintptr_t page_next[2]; - tb_page_addr_t page_addr[2]; -#endif - - /* jmp_lock placed here to fill a 4-byte hole. Its documentation is below */ - QemuSpin jmp_lock; - - /* The following data are used to directly call another TB from - * the code of this one. This can be done either by emitting direct or - * indirect native jump instructions. These jumps are reset so that the TB - * just continues its execution. The TB can be linked to another one by - * setting one of the jump targets (or patching the jump instruction). Only - * two of such jumps are supported. - */ -#define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */ - uint16_t jmp_reset_offset[2]; /* offset of original jump target */ - uint16_t jmp_insn_offset[2]; /* offset of direct jump insn */ - uintptr_t jmp_target_addr[2]; /* target address */ - - /* - * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps. - * Each TB can have two outgoing jumps, and therefore can participate - * in two lists. The list entries are kept in jmp_list_next[2]. The least - * significant bit (LSB) of the pointers in these lists is used to encode - * which of the two list entries is to be used in the pointed TB. - * - * List traversals are protected by jmp_lock. The destination TB of each - * outgoing jump is kept in jmp_dest[] so that the appropriate jmp_lock - * can be acquired from any origin TB. - * - * jmp_dest[] are tagged pointers as well. The LSB is set when the TB is - * being invalidated, so that no further outgoing jumps from it can be set. - * - * jmp_lock also protects the CF_INVALID cflag; a jump must not be chained - * to a destination TB that has CF_INVALID set. - */ - uintptr_t jmp_list_head; - uintptr_t jmp_list_next[2]; - uintptr_t jmp_dest[2]; -}; - /* Hide the qatomic_read to make code a little easier on the eyes */ static inline uint32_t tb_cflags(const TranslationBlock *tb) { diff --git a/include/exec/gen-icount.h b/include/exec/gen-icount.h deleted file mode 100644 index f6de79a6b4..0000000000 --- a/include/exec/gen-icount.h +++ /dev/null @@ -1,83 +0,0 @@ -#ifndef GEN_ICOUNT_H -#define GEN_ICOUNT_H - -#include "exec/exec-all.h" - -/* Helpers for instruction counting code generation. */ - -static TCGOp *icount_start_insn; - -static inline void gen_io_start(void) -{ - tcg_gen_st_i32(tcg_constant_i32(1), cpu_env, - offsetof(ArchCPU, parent_obj.can_do_io) - - offsetof(ArchCPU, env)); -} - -static inline void gen_tb_start(const TranslationBlock *tb) -{ - TCGv_i32 count = tcg_temp_new_i32(); - - tcg_gen_ld_i32(count, cpu_env, - offsetof(ArchCPU, neg.icount_decr.u32) - - offsetof(ArchCPU, env)); - - if (tb_cflags(tb) & CF_USE_ICOUNT) { - /* - * We emit a sub with a dummy immediate argument. Keep the insn index - * of the sub so that we later (when we know the actual insn count) - * can update the argument with the actual insn count. - */ - tcg_gen_sub_i32(count, count, tcg_constant_i32(0)); - icount_start_insn = tcg_last_op(); - } - - /* - * Emit the check against icount_decr.u32 to see if we should exit - * unless we suppress the check with CF_NOIRQ. If we are using - * icount and have suppressed interruption the higher level code - * should have ensured we don't run more instructions than the - * budget. - */ - if (tb_cflags(tb) & CF_NOIRQ) { - tcg_ctx->exitreq_label = NULL; - } else { - tcg_ctx->exitreq_label = gen_new_label(); - tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label); - } - - if (tb_cflags(tb) & CF_USE_ICOUNT) { - tcg_gen_st16_i32(count, cpu_env, - offsetof(ArchCPU, neg.icount_decr.u16.low) - - offsetof(ArchCPU, env)); - /* - * cpu->can_do_io is cleared automatically here at the beginning of - * each translation block. The cost is minimal and only paid for - * -icount, plus it would be very easy to forget doing it in the - * translator. Doing it here means we don't need a gen_io_end() to - * go with gen_io_start(). - */ - tcg_gen_st_i32(tcg_constant_i32(0), cpu_env, - offsetof(ArchCPU, parent_obj.can_do_io) - - offsetof(ArchCPU, env)); - } -} - -static inline void gen_tb_end(const TranslationBlock *tb, int num_insns) -{ - if (tb_cflags(tb) & CF_USE_ICOUNT) { - /* - * Update the num_insn immediate parameter now that we know - * the actual insn count. - */ - tcg_set_insn_param(icount_start_insn, 2, - tcgv_i32_arg(tcg_constant_i32(num_insns))); - } - - if (tcg_ctx->exitreq_label) { - gen_set_label(tcg_ctx->exitreq_label); - tcg_gen_exit_tb(tb, TB_EXIT_REQUESTED); - } -} - -#endif diff --git a/include/exec/helper-gen-common.h b/include/exec/helper-gen-common.h new file mode 100644 index 0000000000..5d6d78a625 --- /dev/null +++ b/include/exec/helper-gen-common.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands generation functions for tcg opcodes. + */ + +#ifndef HELPER_GEN_COMMON_H +#define HELPER_GEN_COMMON_H + +#define HELPER_H "accel/tcg/tcg-runtime.h" +#include "exec/helper-gen.h.inc" +#undef HELPER_H + +#define HELPER_H "accel/tcg/plugin-helpers.h" +#include "exec/helper-gen.h.inc" +#undef HELPER_H + +#endif /* HELPER_GEN_COMMON_H */ diff --git a/include/exec/helper-gen.h b/include/exec/helper-gen.h index 7b6ca975ef..f7ec155699 100644 --- a/include/exec/helper-gen.h +++ b/include/exec/helper-gen.h @@ -1,95 +1,16 @@ -/* Helper file for declaring TCG helper functions. - This one expands generation functions for tcg opcodes. */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands generation functions for tcg opcodes. + */ #ifndef HELPER_GEN_H #define HELPER_GEN_H -#include "exec/helper-head.h" +#include "exec/helper-gen-common.h" -#define DEF_HELPER_FLAGS_0(name, flags, ret) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl0(ret)) \ -{ \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 0, NULL); \ -} - -#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1)) \ -{ \ - TCGTemp *args[1] = { dh_arg(t1, 1) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 1, args); \ -} - -#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2)) \ -{ \ - TCGTemp *args[2] = { dh_arg(t1, 1), dh_arg(t2, 2) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 2, args); \ -} - -#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3)) \ -{ \ - TCGTemp *args[3] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 3, args); \ -} - -#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), \ - dh_arg_decl(t3, 3), dh_arg_decl(t4, 4)) \ -{ \ - TCGTemp *args[4] = { dh_arg(t1, 1), dh_arg(t2, 2), \ - dh_arg(t3, 3), dh_arg(t4, 4) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 4, args); \ -} - -#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ - dh_arg_decl(t4, 4), dh_arg_decl(t5, 5)) \ -{ \ - TCGTemp *args[5] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ - dh_arg(t4, 4), dh_arg(t5, 5) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 5, args); \ -} - -#define DEF_HELPER_FLAGS_6(name, flags, ret, t1, t2, t3, t4, t5, t6) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ - dh_arg_decl(t4, 4), dh_arg_decl(t5, 5), dh_arg_decl(t6, 6)) \ -{ \ - TCGTemp *args[6] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ - dh_arg(t4, 4), dh_arg(t5, 5), dh_arg(t6, 6) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 6, args); \ -} - -#define DEF_HELPER_FLAGS_7(name, flags, ret, t1, t2, t3, t4, t5, t6, t7)\ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ - dh_arg_decl(t4, 4), dh_arg_decl(t5, 5), dh_arg_decl(t6, 6), \ - dh_arg_decl(t7, 7)) \ -{ \ - TCGTemp *args[7] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ - dh_arg(t4, 4), dh_arg(t5, 5), dh_arg(t6, 6), \ - dh_arg(t7, 7) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 7, args); \ -} - -#include "helper.h" -#include "accel/tcg/tcg-runtime.h" -#include "accel/tcg/plugin-helpers.h" - -#undef DEF_HELPER_FLAGS_0 -#undef DEF_HELPER_FLAGS_1 -#undef DEF_HELPER_FLAGS_2 -#undef DEF_HELPER_FLAGS_3 -#undef DEF_HELPER_FLAGS_4 -#undef DEF_HELPER_FLAGS_5 -#undef DEF_HELPER_FLAGS_6 -#undef DEF_HELPER_FLAGS_7 -#undef GEN_HELPER +#define HELPER_H "helper.h" +#include "exec/helper-gen.h.inc" +#undef HELPER_H #endif /* HELPER_GEN_H */ diff --git a/include/exec/helper-gen.h.inc b/include/exec/helper-gen.h.inc new file mode 100644 index 0000000000..c009641517 --- /dev/null +++ b/include/exec/helper-gen.h.inc @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands generation functions for tcg opcodes. + * Define HELPER_H for the header file to be expanded, + * and static inline to change from global file scope. + */ + +#include "tcg/tcg.h" +#include "tcg/helper-info.h" +#include "exec/helper-head.h" + +#define DEF_HELPER_FLAGS_0(name, flags, ret) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl0(ret)) \ +{ \ + tcg_gen_call0(&glue(helper_info_, name), dh_retvar(ret)); \ +} + +#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1)) \ +{ \ + tcg_gen_call1(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1)); \ +} + +#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2)) \ +{ \ + tcg_gen_call2(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2)); \ +} + +#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3)) \ +{ \ + tcg_gen_call3(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3)); \ +} + +#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), \ + dh_arg_decl(t3, 3), dh_arg_decl(t4, 4)) \ +{ \ + tcg_gen_call4(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), \ + dh_arg(t3, 3), dh_arg(t4, 4)); \ +} + +#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ + dh_arg_decl(t4, 4), dh_arg_decl(t5, 5)) \ +{ \ + tcg_gen_call5(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ + dh_arg(t4, 4), dh_arg(t5, 5)); \ +} + +#define DEF_HELPER_FLAGS_6(name, flags, ret, t1, t2, t3, t4, t5, t6) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ + dh_arg_decl(t4, 4), dh_arg_decl(t5, 5), dh_arg_decl(t6, 6)) \ +{ \ + tcg_gen_call6(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ + dh_arg(t4, 4), dh_arg(t5, 5), dh_arg(t6, 6)); \ +} + +#define DEF_HELPER_FLAGS_7(name, flags, ret, t1, t2, t3, t4, t5, t6, t7)\ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ + dh_arg_decl(t4, 4), dh_arg_decl(t5, 5), dh_arg_decl(t6, 6), \ + dh_arg_decl(t7, 7)) \ +{ \ + tcg_gen_call7(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ + dh_arg(t4, 4), dh_arg(t5, 5), dh_arg(t6, 6), \ + dh_arg(t7, 7)); \ +} + +#include HELPER_H + +#undef DEF_HELPER_FLAGS_0 +#undef DEF_HELPER_FLAGS_1 +#undef DEF_HELPER_FLAGS_2 +#undef DEF_HELPER_FLAGS_3 +#undef DEF_HELPER_FLAGS_4 +#undef DEF_HELPER_FLAGS_5 +#undef DEF_HELPER_FLAGS_6 +#undef DEF_HELPER_FLAGS_7 diff --git a/include/exec/helper-head.h b/include/exec/helper-head.h index f863a6ef5d..28ceab0a46 100644 --- a/include/exec/helper-head.h +++ b/include/exec/helper-head.h @@ -1,18 +1,6 @@ -/* Helper file for declaring TCG helper functions. - Used by other helper files. - - Targets should use DEF_HELPER_N and DEF_HELPER_FLAGS_N to declare helper - functions. Names should be specified without the helper_ prefix, and - the return and argument types specified. 3 basic types are understood - (i32, i64 and ptr). Additional aliases are provided for convenience and - to match the types used by the C helper implementation. - - The target helper.h should be included in all files that use/define - helper functions. THis will ensure that function prototypes are - consistent. In addition it should be included an extra two times for - helper.c, defining: - GEN_HELPER 1 to produce op generation functions (gen_helper_*) - GEN_HELPER 2 to do runtime registration helper functions. +/* + * Helper file for declaring TCG helper functions. + * Used by other helper files. */ #ifndef EXEC_HELPER_HEAD_H @@ -34,6 +22,7 @@ #define dh_alias_f64 i64 #define dh_alias_ptr ptr #define dh_alias_cptr ptr +#define dh_alias_env ptr #define dh_alias_void void #define dh_alias_noreturn noreturn #define dh_alias(t) glue(dh_alias_, t) @@ -49,6 +38,7 @@ #define dh_ctype_f64 float64 #define dh_ctype_ptr void * #define dh_ctype_cptr const void * +#define dh_ctype_env CPUArchState * #define dh_ctype_void void #define dh_ctype_noreturn G_NORETURN void #define dh_ctype(t) dh_ctype_##t @@ -64,9 +54,6 @@ # endif # endif # define dh_ctype_tl target_ulong -# define dh_alias_env ptr -# define dh_ctype_env CPUArchState * -# define dh_typecode_env dh_typecode_ptr #endif /* We can't use glue() here because it falls foul of C preprocessor @@ -108,6 +95,7 @@ #define dh_typecode_f32 dh_typecode_i32 #define dh_typecode_f64 dh_typecode_i64 #define dh_typecode_cptr dh_typecode_ptr +#define dh_typecode_env dh_typecode_ptr #define dh_typecode(t) dh_typecode_##t #define dh_callflag_i32 0 diff --git a/include/exec/helper-info.c.inc b/include/exec/helper-info.c.inc new file mode 100644 index 0000000000..530d2e6d35 --- /dev/null +++ b/include/exec/helper-info.c.inc @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands info structures for tcg helpers. + * Define HELPER_H for the header file to be expanded. + */ + +#include "tcg/tcg.h" +#include "tcg/helper-info.h" +#include "exec/helper-head.h" + +/* + * Need one more level of indirection before stringification + * to get all the macros expanded first. + */ +#define str(s) #s + +#define DEF_HELPER_FLAGS_0(NAME, FLAGS, RET) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) \ + }; + +#define DEF_HELPER_FLAGS_1(NAME, FLAGS, RET, T1) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + }; + +#define DEF_HELPER_FLAGS_2(NAME, FLAGS, RET, T1, T2) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) \ + }; + +#define DEF_HELPER_FLAGS_3(NAME, FLAGS, RET, T1, T2, T3) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + }; + +#define DEF_HELPER_FLAGS_4(NAME, FLAGS, RET, T1, T2, T3, T4) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + | dh_typemask(T4, 4) \ + }; + +#define DEF_HELPER_FLAGS_5(NAME, FLAGS, RET, T1, T2, T3, T4, T5) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + | dh_typemask(T4, 4) | dh_typemask(T5, 5) \ + }; + +#define DEF_HELPER_FLAGS_6(NAME, FLAGS, RET, T1, T2, T3, T4, T5, T6) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + | dh_typemask(T4, 4) | dh_typemask(T5, 5) \ + | dh_typemask(T6, 6) \ + }; + +#define DEF_HELPER_FLAGS_7(NAME, FLAGS, RET, T1, T2, T3, T4, T5, T6, T7) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + | dh_typemask(T4, 4) | dh_typemask(T5, 5) \ + | dh_typemask(T6, 6) | dh_typemask(T7, 7) \ + }; + +#include HELPER_H + +#undef str +#undef DEF_HELPER_FLAGS_0 +#undef DEF_HELPER_FLAGS_1 +#undef DEF_HELPER_FLAGS_2 +#undef DEF_HELPER_FLAGS_3 +#undef DEF_HELPER_FLAGS_4 +#undef DEF_HELPER_FLAGS_5 +#undef DEF_HELPER_FLAGS_6 +#undef DEF_HELPER_FLAGS_7 diff --git a/include/exec/helper-proto-common.h b/include/exec/helper-proto-common.h new file mode 100644 index 0000000000..4d4b022668 --- /dev/null +++ b/include/exec/helper-proto-common.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands prototypes for the helper functions. + */ + +#ifndef HELPER_PROTO_COMMON_H +#define HELPER_PROTO_COMMON_H + +#define HELPER_H "accel/tcg/tcg-runtime.h" +#include "exec/helper-proto.h.inc" +#undef HELPER_H + +#define HELPER_H "accel/tcg/plugin-helpers.h" +#include "exec/helper-proto.h.inc" +#undef HELPER_H + +#endif /* HELPER_PROTO_COMMON_H */ diff --git a/include/exec/helper-proto.h b/include/exec/helper-proto.h index 7a3f04b58c..6935cb4f16 100644 --- a/include/exec/helper-proto.h +++ b/include/exec/helper-proto.h @@ -1,71 +1,16 @@ -/* Helper file for declaring TCG helper functions. - This one expands prototypes for the helper functions. */ - -#ifndef HELPER_PROTO_H -#define HELPER_PROTO_H - -#include "exec/helper-head.h" - +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Work around an issue with --enable-lto, in which GCC's ipa-split pass - * decides to split out the noreturn code paths that raise an exception, - * taking the __builtin_return_address() along into the new function, - * where it no longer computes a value that returns to TCG generated code. - * Despite the name, the noinline attribute affects splitter, so this - * prevents the optimization in question. Given that helpers should not - * otherwise be called directly, this should have any other visible effect. - * - * See https://gitlab.com/qemu-project/qemu/-/issues/1454 + * Helper file for declaring TCG helper functions. + * This one expands prototypes for the helper functions. */ -#define DEF_HELPER_ATTR __attribute__((noinline)) - -#define DEF_HELPER_FLAGS_0(name, flags, ret) \ -dh_ctype(ret) HELPER(name) (void) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2)) DEF_HELPER_ATTR; -#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), \ - dh_ctype(t3)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ - dh_ctype(t4)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ - dh_ctype(t4), dh_ctype(t5)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_6(name, flags, ret, t1, t2, t3, t4, t5, t6) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ - dh_ctype(t4), dh_ctype(t5), \ - dh_ctype(t6)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_7(name, flags, ret, t1, t2, t3, t4, t5, t6, t7) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ - dh_ctype(t4), dh_ctype(t5), dh_ctype(t6), \ - dh_ctype(t7)) DEF_HELPER_ATTR; - -#define IN_HELPER_PROTO - -#include "helper.h" -#include "accel/tcg/tcg-runtime.h" -#include "accel/tcg/plugin-helpers.h" +#ifndef HELPER_PROTO_H +#define HELPER_PROTO_H -#undef IN_HELPER_PROTO +#include "exec/helper-proto-common.h" -#undef DEF_HELPER_FLAGS_0 -#undef DEF_HELPER_FLAGS_1 -#undef DEF_HELPER_FLAGS_2 -#undef DEF_HELPER_FLAGS_3 -#undef DEF_HELPER_FLAGS_4 -#undef DEF_HELPER_FLAGS_5 -#undef DEF_HELPER_FLAGS_6 -#undef DEF_HELPER_FLAGS_7 -#undef DEF_HELPER_ATTR +#define HELPER_H "helper.h" +#include "exec/helper-proto.h.inc" +#undef HELPER_H #endif /* HELPER_PROTO_H */ diff --git a/include/exec/helper-proto.h.inc b/include/exec/helper-proto.h.inc new file mode 100644 index 0000000000..c3aa666929 --- /dev/null +++ b/include/exec/helper-proto.h.inc @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands prototypes for the helper functions. + * Define HELPER_H for the header file to be expanded. + */ + +#include "exec/helper-head.h" + +/* + * Work around an issue with --enable-lto, in which GCC's ipa-split pass + * decides to split out the noreturn code paths that raise an exception, + * taking the __builtin_return_address() along into the new function, + * where it no longer computes a value that returns to TCG generated code. + * Despite the name, the noinline attribute affects splitter, so this + * prevents the optimization in question. Given that helpers should not + * otherwise be called directly, this should not have any other visible effect. + * + * See https://gitlab.com/qemu-project/qemu/-/issues/1454 + */ +#define DEF_HELPER_ATTR __attribute__((noinline)) + +#define DEF_HELPER_FLAGS_0(name, flags, ret) \ +dh_ctype(ret) HELPER(name) (void) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), \ + dh_ctype(t3)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ + dh_ctype(t4)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ + dh_ctype(t4), dh_ctype(t5)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_6(name, flags, ret, t1, t2, t3, t4, t5, t6) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ + dh_ctype(t4), dh_ctype(t5), \ + dh_ctype(t6)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_7(name, flags, ret, t1, t2, t3, t4, t5, t6, t7) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ + dh_ctype(t4), dh_ctype(t5), dh_ctype(t6), \ + dh_ctype(t7)) DEF_HELPER_ATTR; + +#define IN_HELPER_PROTO + +#include HELPER_H + +#undef IN_HELPER_PROTO + +#undef DEF_HELPER_FLAGS_0 +#undef DEF_HELPER_FLAGS_1 +#undef DEF_HELPER_FLAGS_2 +#undef DEF_HELPER_FLAGS_3 +#undef DEF_HELPER_FLAGS_4 +#undef DEF_HELPER_FLAGS_5 +#undef DEF_HELPER_FLAGS_6 +#undef DEF_HELPER_FLAGS_7 +#undef DEF_HELPER_ATTR diff --git a/include/exec/helper-tcg.h b/include/exec/helper-tcg.h deleted file mode 100644 index 3933258f1a..0000000000 --- a/include/exec/helper-tcg.h +++ /dev/null @@ -1,75 +0,0 @@ -/* Helper file for declaring TCG helper functions. - This one defines data structures private to tcg.c. */ - -#ifndef HELPER_TCG_H -#define HELPER_TCG_H - -#include "exec/helper-head.h" - -/* Need one more level of indirection before stringification - to get all the macros expanded first. */ -#define str(s) #s - -#define DEF_HELPER_FLAGS_0(NAME, FLAGS, ret) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) }, - -#define DEF_HELPER_FLAGS_1(NAME, FLAGS, ret, t1) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) }, - -#define DEF_HELPER_FLAGS_2(NAME, FLAGS, ret, t1, t2) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) }, - -#define DEF_HELPER_FLAGS_3(NAME, FLAGS, ret, t1, t2, t3) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) }, - -#define DEF_HELPER_FLAGS_4(NAME, FLAGS, ret, t1, t2, t3, t4) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) }, - -#define DEF_HELPER_FLAGS_5(NAME, FLAGS, ret, t1, t2, t3, t4, t5) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \ - | dh_typemask(t5, 5) }, - -#define DEF_HELPER_FLAGS_6(NAME, FLAGS, ret, t1, t2, t3, t4, t5, t6) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \ - | dh_typemask(t5, 5) | dh_typemask(t6, 6) }, - -#define DEF_HELPER_FLAGS_7(NAME, FLAGS, ret, t1, t2, t3, t4, t5, t6, t7) \ - { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \ - | dh_typemask(t5, 5) | dh_typemask(t6, 6) | dh_typemask(t7, 7) }, - -#include "helper.h" -#include "accel/tcg/tcg-runtime.h" -#include "accel/tcg/plugin-helpers.h" - -#undef str -#undef DEF_HELPER_FLAGS_0 -#undef DEF_HELPER_FLAGS_1 -#undef DEF_HELPER_FLAGS_2 -#undef DEF_HELPER_FLAGS_3 -#undef DEF_HELPER_FLAGS_4 -#undef DEF_HELPER_FLAGS_5 -#undef DEF_HELPER_FLAGS_6 -#undef DEF_HELPER_FLAGS_7 - -#endif /* HELPER_TCG_H */ diff --git a/include/exec/plugin-gen.h b/include/exec/plugin-gen.h index 3af0168e65..52828781bc 100644 --- a/include/exec/plugin-gen.h +++ b/include/exec/plugin-gen.h @@ -12,8 +12,6 @@ #ifndef QEMU_PLUGIN_GEN_H #define QEMU_PLUGIN_GEN_H -#include "exec/cpu_ldst.h" -#include "qemu/plugin.h" #include "tcg/tcg.h" struct DisasContextBase; @@ -29,25 +27,6 @@ void plugin_gen_insn_end(void); void plugin_gen_disable_mem_helpers(void); void plugin_gen_empty_mem_callback(TCGv_i64 addr, uint32_t info); -static inline void plugin_insn_append(abi_ptr pc, const void *from, size_t size) -{ - struct qemu_plugin_insn *insn = tcg_ctx->plugin_insn; - abi_ptr off; - - if (insn == NULL) { - return; - } - off = pc - insn->vaddr; - if (off < insn->data->len) { - g_byte_array_set_size(insn->data, off); - } else if (off > insn->data->len) { - /* we have an unexpected gap */ - g_assert_not_reached(); - } - - insn->data = g_byte_array_append(insn->data, from, size); -} - #else /* !CONFIG_PLUGIN */ static inline bool @@ -72,9 +51,6 @@ static inline void plugin_gen_disable_mem_helpers(void) static inline void plugin_gen_empty_mem_callback(TCGv_i64 addr, uint32_t info) { } -static inline void plugin_insn_append(abi_ptr pc, const void *from, size_t size) -{ } - #endif /* CONFIG_PLUGIN */ #endif /* QEMU_PLUGIN_GEN_H */ diff --git a/include/exec/poison.h b/include/exec/poison.h index 256736e11a..e94ee8dfef 100644 --- a/include/exec/poison.h +++ b/include/exec/poison.h @@ -85,7 +85,6 @@ #pragma GCC poison CONFIG_HVF #pragma GCC poison CONFIG_LINUX_USER #pragma GCC poison CONFIG_KVM -#pragma GCC poison CONFIG_SOFTMMU #pragma GCC poison CONFIG_WHPX #pragma GCC poison CONFIG_XEN diff --git a/include/exec/tlb-common.h b/include/exec/tlb-common.h new file mode 100644 index 0000000000..dc5a5faa0b --- /dev/null +++ b/include/exec/tlb-common.h @@ -0,0 +1,56 @@ +/* + * Common definitions for the softmmu tlb + * + * Copyright (c) 2003 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ +#ifndef EXEC_TLB_COMMON_H +#define EXEC_TLB_COMMON_H 1 + +#define CPU_TLB_ENTRY_BITS 5 + +/* Minimalized TLB entry for use by TCG fast path. */ +typedef union CPUTLBEntry { + struct { + uint64_t addr_read; + uint64_t addr_write; + uint64_t addr_code; + /* + * Addend to virtual address to get host address. IO accesses + * use the corresponding iotlb value. + */ + uintptr_t addend; + }; + /* + * Padding to get a power of two size, as well as index + * access to addr_{read,write,code}. + */ + uint64_t addr_idx[(1 << CPU_TLB_ENTRY_BITS) / sizeof(uint64_t)]; +} CPUTLBEntry; + +QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS)); + +/* + * Data elements that are per MMU mode, accessed by the fast path. + * The structure is aligned to aid loading the pair with one insn. + */ +typedef struct CPUTLBDescFast { + /* Contains (n_entries - 1) << CPU_TLB_ENTRY_BITS */ + uintptr_t mask; + /* The array of tlb entries itself. */ + CPUTLBEntry *table; +} CPUTLBDescFast QEMU_ALIGNED(2 * sizeof(void *)); + +#endif /* EXEC_TLB_COMMON_H */ diff --git a/include/exec/translation-block.h b/include/exec/translation-block.h new file mode 100644 index 0000000000..5119924927 --- /dev/null +++ b/include/exec/translation-block.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Definition of TranslationBlock. + * Copyright (c) 2003 Fabrice Bellard + */ + +#ifndef EXEC_TRANSLATION_BLOCK_H +#define EXEC_TRANSLATION_BLOCK_H + +#include "qemu/atomic.h" +#include "qemu/thread.h" +#include "qemu/interval-tree.h" +#include "exec/cpu-common.h" +#include "exec/target_page.h" + +/* + * Page tracking code uses ram addresses in system mode, and virtual + * addresses in userspace mode. Define tb_page_addr_t to be an + * appropriate type. + */ +#if defined(CONFIG_USER_ONLY) +typedef vaddr tb_page_addr_t; +#define TB_PAGE_ADDR_FMT "%" VADDR_PRIx +#else +typedef ram_addr_t tb_page_addr_t; +#define TB_PAGE_ADDR_FMT RAM_ADDR_FMT +#endif + +/* + * Translation Cache-related fields of a TB. + * This struct exists just for convenience; we keep track of TB's in a binary + * search tree, and the only fields needed to compare TB's in the tree are + * @ptr and @size. + * Note: the address of search data can be obtained by adding @size to @ptr. + */ +struct tb_tc { + const void *ptr; /* pointer to the translated code */ + size_t size; +}; + +struct TranslationBlock { + /* + * Guest PC corresponding to this block. This must be the true + * virtual address. Therefore e.g. x86 stores EIP + CS_BASE, and + * targets like Arm, MIPS, HP-PA, which reuse low bits for ISA or + * privilege, must store those bits elsewhere. + * + * If CF_PCREL, the opcodes for the TranslationBlock are written + * such that the TB is associated only with the physical page and + * may be run in any virtual address context. In this case, PC + * must always be taken from ENV in a target-specific manner. + * Unwind information is taken as offsets from the page, to be + * deposited into the "current" PC. + */ + vaddr pc; + + /* + * Target-specific data associated with the TranslationBlock, e.g.: + * x86: the original user, the Code Segment virtual base, + * arm: an extension of tb->flags, + * s390x: instruction data for EXECUTE, + * sparc: the next pc of the instruction queue (for delay slots). + */ + uint64_t cs_base; + + uint32_t flags; /* flags defining in which context the code was generated */ + uint32_t cflags; /* compile flags */ + +/* Note that TCG_MAX_INSNS is 512; we validate this match elsewhere. */ +#define CF_COUNT_MASK 0x000001ff +#define CF_NO_GOTO_TB 0x00000200 /* Do not chain with goto_tb */ +#define CF_NO_GOTO_PTR 0x00000400 /* Do not chain with goto_ptr */ +#define CF_SINGLE_STEP 0x00000800 /* gdbstub single-step in effect */ +#define CF_LAST_IO 0x00008000 /* Last insn may be an IO access. */ +#define CF_MEMI_ONLY 0x00010000 /* Only instrument memory ops */ +#define CF_USE_ICOUNT 0x00020000 +#define CF_INVALID 0x00040000 /* TB is stale. Set with @jmp_lock held */ +#define CF_PARALLEL 0x00080000 /* Generate code for a parallel context */ +#define CF_NOIRQ 0x00100000 /* Generate an uninterruptible TB */ +#define CF_PCREL 0x00200000 /* Opcodes in TB are PC-relative */ +#define CF_CLUSTER_MASK 0xff000000 /* Top 8 bits are cluster ID */ +#define CF_CLUSTER_SHIFT 24 + + /* + * Above fields used for comparing + */ + + /* size of target code for this block (1 <= size <= TARGET_PAGE_SIZE) */ + uint16_t size; + uint16_t icount; + + struct tb_tc tc; + + /* + * Track tb_page_addr_t intervals that intersect this TB. + * For user-only, the virtual addresses are always contiguous, + * and we use a unified interval tree. For system, we use a + * linked list headed in each PageDesc. Within the list, the lsb + * of the previous pointer tells the index of page_next[], and the + * list is protected by the PageDesc lock(s). + */ +#ifdef CONFIG_USER_ONLY + IntervalTreeNode itree; +#else + uintptr_t page_next[2]; + tb_page_addr_t page_addr[2]; +#endif + + /* jmp_lock placed here to fill a 4-byte hole. Its documentation is below */ + QemuSpin jmp_lock; + + /* The following data are used to directly call another TB from + * the code of this one. This can be done either by emitting direct or + * indirect native jump instructions. These jumps are reset so that the TB + * just continues its execution. The TB can be linked to another one by + * setting one of the jump targets (or patching the jump instruction). Only + * two of such jumps are supported. + */ +#define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */ + uint16_t jmp_reset_offset[2]; /* offset of original jump target */ + uint16_t jmp_insn_offset[2]; /* offset of direct jump insn */ + uintptr_t jmp_target_addr[2]; /* target address */ + + /* + * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps. + * Each TB can have two outgoing jumps, and therefore can participate + * in two lists. The list entries are kept in jmp_list_next[2]. The least + * significant bit (LSB) of the pointers in these lists is used to encode + * which of the two list entries is to be used in the pointed TB. + * + * List traversals are protected by jmp_lock. The destination TB of each + * outgoing jump is kept in jmp_dest[] so that the appropriate jmp_lock + * can be acquired from any origin TB. + * + * jmp_dest[] are tagged pointers as well. The LSB is set when the TB is + * being invalidated, so that no further outgoing jumps from it can be set. + * + * jmp_lock also protects the CF_INVALID cflag; a jump must not be chained + * to a destination TB that has CF_INVALID set. + */ + uintptr_t jmp_list_head; + uintptr_t jmp_list_next[2]; + uintptr_t jmp_dest[2]; +}; + +/* The alignment given to TranslationBlock during allocation. */ +#define CODE_GEN_ALIGN 16 + +#endif /* EXEC_TRANSLATION_BLOCK_H */ diff --git a/include/exec/translator.h b/include/exec/translator.h index 797fef7515..224ae14aa7 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -18,13 +18,8 @@ * member in your target-specific DisasContext. */ - #include "qemu/bswap.h" -#include "exec/exec-all.h" -#include "exec/cpu_ldst.h" -#include "exec/plugin-gen.h" -#include "exec/translate-all.h" -#include "tcg/tcg.h" +#include "exec/cpu_ldst.h" /* for abi_ptr */ /** * gen_intermediate_code @@ -160,6 +155,16 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns, */ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest); +/** + * translator_io_start + * @db: Disassembly context + * + * If icount is enabled, set cpu->can_to_io, adjust db->is_jmp to + * DISAS_TOO_MANY if it is still DISAS_NEXT, and return true. + * Otherwise return false. + */ +bool translator_io_start(DisasContextBase *db); + /* * Translator Load Functions * @@ -219,12 +224,7 @@ translator_ldq_swap(CPUArchState *env, DisasContextBase *db, * re-synthesised for s390x "ex"). It ensures we update other areas of * the translator with details of the executed instruction. */ - -static inline void translator_fake_ldb(uint8_t insn8, abi_ptr pc) -{ - plugin_insn_append(pc, &insn8, sizeof(insn8)); -} - +void translator_fake_ldb(uint8_t insn8, abi_ptr pc); /* * Return whether addr is on the same page as where disassembly started. |