diff options
Diffstat (limited to 'tcg')
| -rw-r--r-- | tcg/aarch64/tcg-target.c.inc | 108 | ||||
| -rw-r--r-- | tcg/arm/tcg-target.c.inc | 355 | ||||
| -rw-r--r-- | tcg/i386/tcg-target.c.inc | 345 | ||||
| -rw-r--r-- | tcg/loongarch64/tcg-target.c.inc | 131 | ||||
| -rw-r--r-- | tcg/mips/tcg-target.c.inc | 186 | ||||
| -rw-r--r-- | tcg/ppc/tcg-target.c.inc | 192 | ||||
| -rw-r--r-- | tcg/riscv/tcg-target-con-set.h | 8 | ||||
| -rw-r--r-- | tcg/riscv/tcg-target.c.inc | 268 | ||||
| -rw-r--r-- | tcg/riscv/tcg-target.h | 22 | ||||
| -rw-r--r-- | tcg/s390x/tcg-target.c.inc | 131 | ||||
| -rw-r--r-- | tcg/sparc64/tcg-target.c.inc | 8 | ||||
| -rw-r--r-- | tcg/tcg-internal.h | 4 | ||||
| -rw-r--r-- | tcg/tcg-ldst.c.inc | 14 | ||||
| -rw-r--r-- | tcg/tcg.c | 58 |
14 files changed, 873 insertions, 957 deletions
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index 4ec3cf3172..d8d464e4a0 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -1587,6 +1587,12 @@ static void tcg_out_adr(TCGContext *s, TCGReg rd, const void *target) tcg_out_insn(s, 3406, ADR, rd, offset); } +typedef struct { + TCGReg base; + TCGReg index; + TCGType index_ext; +} HostAddress; + #ifdef CONFIG_SOFTMMU /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * MemOpIdx oi, uintptr_t ra) @@ -1796,32 +1802,31 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) #endif /* CONFIG_SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext, - TCGReg data_r, TCGReg addr_r, - TCGType otype, TCGReg off_r) + TCGReg data_r, HostAddress h) { switch (memop & MO_SSIZE) { case MO_UB: - tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRB, data_r, h.base, h.index_ext, h.index); break; case MO_SB: tcg_out_ldst_r(s, ext ? I3312_LDRSBX : I3312_LDRSBW, - data_r, addr_r, otype, off_r); + data_r, h.base, h.index_ext, h.index); break; case MO_UW: - tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRH, data_r, h.base, h.index_ext, h.index); break; case MO_SW: tcg_out_ldst_r(s, (ext ? I3312_LDRSHX : I3312_LDRSHW), - data_r, addr_r, otype, off_r); + data_r, h.base, h.index_ext, h.index); break; case MO_UL: - tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRW, data_r, h.base, h.index_ext, h.index); break; case MO_SL: - tcg_out_ldst_r(s, I3312_LDRSWX, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRSWX, data_r, h.base, h.index_ext, h.index); break; case MO_UQ: - tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_LDRX, data_r, h.base, h.index_ext, h.index); break; default: g_assert_not_reached(); @@ -1829,21 +1834,20 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext, } static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop, - TCGReg data_r, TCGReg addr_r, - TCGType otype, TCGReg off_r) + TCGReg data_r, HostAddress h) { switch (memop & MO_SIZE) { case MO_8: - tcg_out_ldst_r(s, I3312_STRB, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRB, data_r, h.base, h.index_ext, h.index); break; case MO_16: - tcg_out_ldst_r(s, I3312_STRH, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRH, data_r, h.base, h.index_ext, h.index); break; case MO_32: - tcg_out_ldst_r(s, I3312_STRW, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRW, data_r, h.base, h.index_ext, h.index); break; case MO_64: - tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, otype, off_r); + tcg_out_ldst_r(s, I3312_STRX, data_r, h.base, h.index_ext, h.index); break; default: g_assert_not_reached(); @@ -1851,22 +1855,28 @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop, } static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi, TCGType ext) + MemOpIdx oi, TCGType data_type) { MemOp memop = get_memop(oi); - const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + HostAddress h; /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((memop & MO_BSWAP) == 0); #ifdef CONFIG_SOFTMMU - unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; - tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 1); - tcg_out_qemu_ld_direct(s, memop, ext, data_reg, - TCG_REG_X1, otype, addr_reg); - add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg, + tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 1); + + h = (HostAddress){ + .base = TCG_REG_X1, + .index = addr_reg, + .index_ext = addr_type + }; + tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h); + + add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ unsigned a_bits = get_alignment_bits(memop); @@ -1874,45 +1884,65 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, tcg_out_test_alignment(s, true, addr_reg, a_bits); } if (USE_GUEST_BASE) { - tcg_out_qemu_ld_direct(s, memop, ext, data_reg, - TCG_REG_GUEST_BASE, otype, addr_reg); + h = (HostAddress){ + .base = TCG_REG_GUEST_BASE, + .index = addr_reg, + .index_ext = addr_type + }; } else { - tcg_out_qemu_ld_direct(s, memop, ext, data_reg, - addr_reg, TCG_TYPE_I64, TCG_REG_XZR); + h = (HostAddress){ + .base = addr_reg, + .index = TCG_REG_XZR, + .index_ext = TCG_TYPE_I64 + }; } + tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h); #endif /* CONFIG_SOFTMMU */ } static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi) + MemOpIdx oi, TCGType data_type) { MemOp memop = get_memop(oi); - const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + HostAddress h; /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((memop & MO_BSWAP) == 0); #ifdef CONFIG_SOFTMMU - unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; - tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 0); - tcg_out_qemu_st_direct(s, memop, data_reg, - TCG_REG_X1, otype, addr_reg); - add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64, - data_reg, addr_reg, s->code_ptr, label_ptr); + tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 0); + + h = (HostAddress){ + .base = TCG_REG_X1, + .index = addr_reg, + .index_ext = addr_type + }; + tcg_out_qemu_st_direct(s, memop, data_reg, h); + + add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #else /* !CONFIG_SOFTMMU */ unsigned a_bits = get_alignment_bits(memop); if (a_bits) { tcg_out_test_alignment(s, false, addr_reg, a_bits); } if (USE_GUEST_BASE) { - tcg_out_qemu_st_direct(s, memop, data_reg, - TCG_REG_GUEST_BASE, otype, addr_reg); + h = (HostAddress){ + .base = TCG_REG_GUEST_BASE, + .index = addr_reg, + .index_ext = addr_type + }; } else { - tcg_out_qemu_st_direct(s, memop, data_reg, - addr_reg, TCG_TYPE_I64, TCG_REG_XZR); + h = (HostAddress){ + .base = addr_reg, + .index = TCG_REG_XZR, + .index_ext = TCG_TYPE_I64 + }; } + tcg_out_qemu_st_direct(s, memop, data_reg, h); #endif /* CONFIG_SOFTMMU */ } @@ -2249,7 +2279,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, REG0(0), a1, a2); + tcg_out_qemu_st(s, REG0(0), a1, a2, ext); break; case INDEX_op_bswap64_i64: diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index 83c818a58b..b6b4ffc546 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -1337,6 +1337,13 @@ static void tcg_out_vldst(TCGContext *s, ARMInsn insn, tcg_out32(s, insn | (rn << 16) | encode_vd(rd) | 0xf); } +typedef struct { + ARMCond cond; + TCGReg base; + int index; + bool index_scratch; +} HostAddress; + #ifdef CONFIG_SOFTMMU /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) @@ -1526,15 +1533,18 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, /* Record the context of a call to the out of line helper code for the slow path for a load or store, so that we can later generate the correct helper code. */ -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, - TCGReg datalo, TCGReg datahi, TCGReg addrlo, - TCGReg addrhi, tcg_insn_unit *raddr, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, + MemOpIdx oi, TCGType type, + TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + tcg_insn_unit *raddr, tcg_insn_unit *label_ptr) { TCGLabelQemuLdst *label = new_ldst_label(s); label->is_ld = is_ld; label->oi = oi; + label->type = type; label->datalo_reg = datalo; label->datahi_reg = datahi; label->addrlo_reg = addrlo; @@ -1693,29 +1703,49 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } #endif /* SOFTMMU */ -static void tcg_out_qemu_ld_index(TCGContext *s, MemOp opc, - TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addend, - bool scratch_addend) +static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, + TCGReg datahi, HostAddress h) { + TCGReg base; + /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & MO_SSIZE) { case MO_UB: - tcg_out_ld8_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld8_12(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld8_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_SB: - tcg_out_ld8s_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld8s_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld8s_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_UW: - tcg_out_ld16u_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld16u_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld16u_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_SW: - tcg_out_ld16s_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld16s_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld16s_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_UL: - tcg_out_ld32_r(s, COND_AL, datalo, addrlo, addend); + if (h.index < 0) { + tcg_out_ld32_12(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_ld32_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_UQ: /* We used pair allocation for datalo, so already should be aligned. */ @@ -1723,182 +1753,112 @@ static void tcg_out_qemu_ld_index(TCGContext *s, MemOp opc, tcg_debug_assert(datahi == datalo + 1); /* LDRD requires alignment; double-check that. */ if (get_alignment_bits(opc) >= MO_64) { + if (h.index < 0) { + tcg_out_ldrd_8(s, h.cond, datalo, h.base, 0); + break; + } /* * Rm (the second address op) must not overlap Rt or Rt + 1. * Since datalo is aligned, we can simplify the test via alignment. * Flip the two address arguments if that works. */ - if ((addend & ~1) != datalo) { - tcg_out_ldrd_r(s, COND_AL, datalo, addrlo, addend); + if ((h.index & ~1) != datalo) { + tcg_out_ldrd_r(s, h.cond, datalo, h.base, h.index); break; } - if ((addrlo & ~1) != datalo) { - tcg_out_ldrd_r(s, COND_AL, datalo, addend, addrlo); + if ((h.base & ~1) != datalo) { + tcg_out_ldrd_r(s, h.cond, datalo, h.index, h.base); break; } } - if (scratch_addend) { - tcg_out_ld32_rwb(s, COND_AL, datalo, addend, addrlo); - tcg_out_ld32_12(s, COND_AL, datahi, addend, 4); - } else { - tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_TMP, - addend, addrlo, SHIFT_IMM_LSL(0)); - tcg_out_ld32_12(s, COND_AL, datalo, TCG_REG_TMP, 0); - tcg_out_ld32_12(s, COND_AL, datahi, TCG_REG_TMP, 4); - } - break; - default: - g_assert_not_reached(); - } -} - -#ifndef CONFIG_SOFTMMU -static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, - TCGReg datahi, TCGReg addrlo) -{ - /* Byte swapping is left to middle-end expansion. */ - tcg_debug_assert((opc & MO_BSWAP) == 0); - - switch (opc & MO_SSIZE) { - case MO_UB: - tcg_out_ld8_12(s, COND_AL, datalo, addrlo, 0); - break; - case MO_SB: - tcg_out_ld8s_8(s, COND_AL, datalo, addrlo, 0); - break; - case MO_UW: - tcg_out_ld16u_8(s, COND_AL, datalo, addrlo, 0); - break; - case MO_SW: - tcg_out_ld16s_8(s, COND_AL, datalo, addrlo, 0); - break; - case MO_UL: - tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0); - break; - case MO_UQ: - /* We used pair allocation for datalo, so already should be aligned. */ - tcg_debug_assert((datalo & 1) == 0); - tcg_debug_assert(datahi == datalo + 1); - /* LDRD requires alignment; double-check that. */ - if (get_alignment_bits(opc) >= MO_64) { - tcg_out_ldrd_8(s, COND_AL, datalo, addrlo, 0); - } else if (datalo == addrlo) { - tcg_out_ld32_12(s, COND_AL, datahi, addrlo, 4); - tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0); + if (h.index < 0) { + base = h.base; + if (datalo == h.base) { + tcg_out_mov_reg(s, h.cond, TCG_REG_TMP, base); + base = TCG_REG_TMP; + } + } else if (h.index_scratch) { + tcg_out_ld32_rwb(s, h.cond, datalo, h.index, h.base); + tcg_out_ld32_12(s, h.cond, datahi, h.index, 4); + break; } else { - tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0); - tcg_out_ld32_12(s, COND_AL, datahi, addrlo, 4); + tcg_out_dat_reg(s, h.cond, ARITH_ADD, TCG_REG_TMP, + h.base, h.index, SHIFT_IMM_LSL(0)); + base = TCG_REG_TMP; } + tcg_out_ld32_12(s, h.cond, datalo, base, 0); + tcg_out_ld32_12(s, h.cond, datahi, base, 4); break; default: g_assert_not_reached(); } } -#endif -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc; -#ifdef CONFIG_SOFTMMU - int mem_index; - TCGReg addend; - tcg_insn_unit *label_ptr; -#else - unsigned a_bits; -#endif - - datalo = *args++; - datahi = (is64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); + MemOp opc = get_memop(oi); + HostAddress h; #ifdef CONFIG_SOFTMMU - mem_index = get_mmuidx(oi); - addend = tcg_out_tlb_read(s, addrlo, addrhi, opc, mem_index, 1); + h.cond = COND_AL; + h.base = addrlo; + h.index_scratch = true; + h.index = tcg_out_tlb_read(s, addrlo, addrhi, opc, get_mmuidx(oi), 1); - /* This a conditional BL only to load a pointer within this opcode into LR - for the slow path. We will not be using the value for a tail call. */ - label_ptr = s->code_ptr; + /* + * This a conditional BL only to load a pointer within this opcode into + * LR for the slow path. We will not be using the value for a tail call. + */ + tcg_insn_unit *label_ptr = s->code_ptr; tcg_out_bl_imm(s, COND_NE, 0); - tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, addend, true); + tcg_out_qemu_ld_direct(s, opc, datalo, datahi, h); - add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); -#else /* !CONFIG_SOFTMMU */ - a_bits = get_alignment_bits(opc); + add_qemu_ldst_label(s, true, oi, data_type, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); +#else + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - if (guest_base) { - tcg_out_qemu_ld_index(s, opc, datalo, datahi, - addrlo, TCG_REG_GUEST_BASE, false); - } else { - tcg_out_qemu_ld_direct(s, opc, datalo, datahi, addrlo); - } + + h.cond = COND_AL; + h.base = addrlo; + h.index = guest_base ? TCG_REG_GUEST_BASE : -1; + h.index_scratch = false; + tcg_out_qemu_ld_direct(s, opc, datalo, datahi, h); #endif } -static void tcg_out_qemu_st_index(TCGContext *s, ARMCond cond, MemOp opc, - TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addend, - bool scratch_addend) +static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, + TCGReg datahi, HostAddress h) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & MO_SIZE) { case MO_8: - tcg_out_st8_r(s, cond, datalo, addrlo, addend); - break; - case MO_16: - tcg_out_st16_r(s, cond, datalo, addrlo, addend); - break; - case MO_32: - tcg_out_st32_r(s, cond, datalo, addrlo, addend); - break; - case MO_64: - /* We used pair allocation for datalo, so already should be aligned. */ - tcg_debug_assert((datalo & 1) == 0); - tcg_debug_assert(datahi == datalo + 1); - /* STRD requires alignment; double-check that. */ - if (get_alignment_bits(opc) >= MO_64) { - tcg_out_strd_r(s, cond, datalo, addrlo, addend); - } else if (scratch_addend) { - tcg_out_st32_rwb(s, cond, datalo, addend, addrlo); - tcg_out_st32_12(s, cond, datahi, addend, 4); + if (h.index < 0) { + tcg_out_st8_12(s, h.cond, datalo, h.base, 0); } else { - tcg_out_dat_reg(s, cond, ARITH_ADD, TCG_REG_TMP, - addend, addrlo, SHIFT_IMM_LSL(0)); - tcg_out_st32_12(s, cond, datalo, TCG_REG_TMP, 0); - tcg_out_st32_12(s, cond, datahi, TCG_REG_TMP, 4); + tcg_out_st8_r(s, h.cond, datalo, h.base, h.index); } break; - default: - g_assert_not_reached(); - } -} - -#ifndef CONFIG_SOFTMMU -static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, - TCGReg datahi, TCGReg addrlo) -{ - /* Byte swapping is left to middle-end expansion. */ - tcg_debug_assert((opc & MO_BSWAP) == 0); - - switch (opc & MO_SIZE) { - case MO_8: - tcg_out_st8_12(s, COND_AL, datalo, addrlo, 0); - break; case MO_16: - tcg_out_st16_8(s, COND_AL, datalo, addrlo, 0); + if (h.index < 0) { + tcg_out_st16_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_st16_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_32: - tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0); + if (h.index < 0) { + tcg_out_st32_12(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_st32_r(s, h.cond, datalo, h.base, h.index); + } break; case MO_64: /* We used pair allocation for datalo, so already should be aligned. */ @@ -1906,62 +1866,59 @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, tcg_debug_assert(datahi == datalo + 1); /* STRD requires alignment; double-check that. */ if (get_alignment_bits(opc) >= MO_64) { - tcg_out_strd_8(s, COND_AL, datalo, addrlo, 0); + if (h.index < 0) { + tcg_out_strd_8(s, h.cond, datalo, h.base, 0); + } else { + tcg_out_strd_r(s, h.cond, datalo, h.base, h.index); + } + } else if (h.index_scratch) { + tcg_out_st32_rwb(s, h.cond, datalo, h.index, h.base); + tcg_out_st32_12(s, h.cond, datahi, h.index, 4); } else { - tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0); - tcg_out_st32_12(s, COND_AL, datahi, addrlo, 4); + tcg_out_dat_reg(s, h.cond, ARITH_ADD, TCG_REG_TMP, + h.base, h.index, SHIFT_IMM_LSL(0)); + tcg_out_st32_12(s, h.cond, datalo, TCG_REG_TMP, 0); + tcg_out_st32_12(s, h.cond, datahi, TCG_REG_TMP, 4); } break; default: g_assert_not_reached(); } } -#endif -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc; -#ifdef CONFIG_SOFTMMU - int mem_index; - TCGReg addend; - tcg_insn_unit *label_ptr; -#else - unsigned a_bits; -#endif - - datalo = *args++; - datahi = (is64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); + MemOp opc = get_memop(oi); + HostAddress h; #ifdef CONFIG_SOFTMMU - mem_index = get_mmuidx(oi); - addend = tcg_out_tlb_read(s, addrlo, addrhi, opc, mem_index, 0); - - tcg_out_qemu_st_index(s, COND_EQ, opc, datalo, datahi, - addrlo, addend, true); + h.cond = COND_EQ; + h.base = addrlo; + h.index_scratch = true; + h.index = tcg_out_tlb_read(s, addrlo, addrhi, opc, get_mmuidx(oi), 0); + tcg_out_qemu_st_direct(s, opc, datalo, datahi, h); /* The conditional call must come last, as we're going to return here. */ - label_ptr = s->code_ptr; + tcg_insn_unit *label_ptr = s->code_ptr; tcg_out_bl_imm(s, COND_NE, 0); - add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); -#else /* !CONFIG_SOFTMMU */ - a_bits = get_alignment_bits(opc); + add_qemu_ldst_label(s, false, oi, data_type, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); +#else + unsigned a_bits = get_alignment_bits(opc); + + h.cond = COND_AL; if (a_bits) { tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); + h.cond = COND_EQ; } - if (guest_base) { - tcg_out_qemu_st_index(s, COND_AL, opc, datalo, datahi, - addrlo, TCG_REG_GUEST_BASE, false); - } else { - tcg_out_qemu_st_direct(s, opc, datalo, datahi, addrlo); - } + + h.base = addrlo; + h.index = guest_base ? TCG_REG_GUEST_BASE : -1; + h.index_scratch = false; + tcg_out_qemu_st_direct(s, opc, datalo, datahi, h); #endif } @@ -2245,16 +2202,40 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, 0); + if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I32); + } else { + tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, 1); + if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_ld(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); + } break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, 0); + if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I32); + } else { + tcg_out_qemu_st(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, 1); + if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_st(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); + } break; case INDEX_op_bswap16_i32: diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index caf91a3151..aae698121a 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -1751,6 +1751,30 @@ static void tcg_out_nopn(TCGContext *s, int n) tcg_out8(s, 0x90); } +/* Test register R vs immediate bits I, setting Z flag for EQ/NE. */ +static void __attribute__((unused)) +tcg_out_testi(TCGContext *s, TCGReg r, uint32_t i) +{ + /* + * This is used for testing alignment, so we can usually use testb. + * For i686, we have to use testl for %esi/%edi. + */ + if (i <= 0xff && (TCG_TARGET_REG_BITS == 64 || r < 4)) { + tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, r); + tcg_out8(s, i); + } else { + tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, r); + tcg_out32(s, i); + } +} + +typedef struct { + TCGReg base; + int index; + int ofs; + int seg; +} HostAddress; + #if defined(CONFIG_SOFTMMU) /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) @@ -1803,8 +1827,6 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, int mem_index, MemOp opc, tcg_insn_unit **label_ptr, int which) { - const TCGReg r0 = TCG_REG_L0; - const TCGReg r1 = TCG_REG_L1; TCGType ttype = TCG_TYPE_I32; TCGType tlbtype = TCG_TYPE_I32; int trexw = 0, hrexw = 0, tlbrexw = 0; @@ -1828,15 +1850,15 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, } } - tcg_out_mov(s, tlbtype, r0, addrlo); - tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0, + tcg_out_mov(s, tlbtype, TCG_REG_L0, addrlo); + tcg_out_shifti(s, SHIFT_SHR + tlbrexw, TCG_REG_L0, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); - tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, r0, TCG_AREG0, + tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, TCG_REG_L0, TCG_AREG0, TLB_MASK_TABLE_OFS(mem_index) + offsetof(CPUTLBDescFast, mask)); - tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r0, TCG_AREG0, + tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, TCG_REG_L0, TCG_AREG0, TLB_MASK_TABLE_OFS(mem_index) + offsetof(CPUTLBDescFast, table)); @@ -1844,19 +1866,21 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, copy the address and mask. For lesser alignments, check that we don't cross pages for the complete access. */ if (a_bits >= s_bits) { - tcg_out_mov(s, ttype, r1, addrlo); + tcg_out_mov(s, ttype, TCG_REG_L1, addrlo); } else { - tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, s_mask - a_mask); + tcg_out_modrm_offset(s, OPC_LEA + trexw, TCG_REG_L1, + addrlo, s_mask - a_mask); } tlb_mask = (target_ulong)TARGET_PAGE_MASK | a_mask; - tgen_arithi(s, ARITH_AND + trexw, r1, tlb_mask, 0); + tgen_arithi(s, ARITH_AND + trexw, TCG_REG_L1, tlb_mask, 0); - /* cmp 0(r0), r1 */ - tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, r1, r0, which); + /* cmp 0(TCG_REG_L0), TCG_REG_L1 */ + tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, + TCG_REG_L1, TCG_REG_L0, which); /* Prepare for both the fast path add of the tlb addend, and the slow path function argument setup. */ - tcg_out_mov(s, ttype, r1, addrlo); + tcg_out_mov(s, ttype, TCG_REG_L1, addrlo); /* jne slow_path */ tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); @@ -1864,8 +1888,8 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, s->code_ptr += 4; if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { - /* cmp 4(r0), addrhi */ - tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, r0, which + 4); + /* cmp 4(TCG_REG_L0), addrhi */ + tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, TCG_REG_L0, which + 4); /* jne slow_path */ tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); @@ -1875,8 +1899,8 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, /* TLB Hit. */ - /* add addend(r0), r1 */ - tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r1, r0, + /* add addend(TCG_REG_L0), TCG_REG_L1 */ + tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, TCG_REG_L1, TCG_REG_L0, offsetof(CPUTLBEntry, addend)); } @@ -1884,8 +1908,8 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, * Record the context of a call to the out of line helper code for the slow path * for a load or store, so that we can later generate the correct helper code */ -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64, - MemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, + TCGType type, MemOpIdx oi, TCGReg datalo, TCGReg datahi, TCGReg addrlo, TCGReg addrhi, tcg_insn_unit *raddr, @@ -1895,7 +1919,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64, label->is_ld = is_ld; label->oi = oi; - label->type = is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32; + label->type = type; label->datalo_reg = datalo; label->datahi_reg = datahi; label->addrlo_reg = addrlo; @@ -2044,18 +2068,7 @@ static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo, unsigned a_mask = (1 << a_bits) - 1; TCGLabelQemuLdst *label; - /* - * We are expecting a_bits to max out at 7, so we can usually use testb. - * For i686, we have to use testl for %esi/%edi. - */ - if (a_mask <= 0xff && (TCG_TARGET_REG_BITS == 64 || addrlo < 4)) { - tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, addrlo); - tcg_out8(s, a_mask); - } else { - tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, addrlo); - tcg_out32(s, a_mask); - } - + tcg_out_testi(s, addrlo, a_mask); /* jne slow_path */ tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); @@ -2113,17 +2126,13 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) return tcg_out_fail_alignment(s, l); } -#if TCG_TARGET_REG_BITS == 32 -# define x86_guest_base_seg 0 -# define x86_guest_base_index -1 -# define x86_guest_base_offset guest_base -#else -static int x86_guest_base_seg; -static int x86_guest_base_index = -1; -static int32_t x86_guest_base_offset; -# if defined(__x86_64__) && defined(__linux__) -# include <asm/prctl.h> -# include <sys/prctl.h> +static HostAddress x86_guest_base = { + .index = -1 +}; + +#if defined(__x86_64__) && defined(__linux__) +# include <asm/prctl.h> +# include <sys/prctl.h> int arch_prctl(int code, unsigned long addr); static inline int setup_guest_base_seg(void) { @@ -2132,8 +2141,9 @@ static inline int setup_guest_base_seg(void) } return 0; } -# elif defined (__FreeBSD__) || defined (__FreeBSD_kernel__) -# include <machine/sysarch.h> +#elif defined(__x86_64__) && \ + (defined (__FreeBSD__) || defined (__FreeBSD_kernel__)) +# include <machine/sysarch.h> static inline int setup_guest_base_seg(void) { if (sysarch(AMD64_SET_GSBASE, &guest_base) == 0) { @@ -2141,22 +2151,19 @@ static inline int setup_guest_base_seg(void) } return 0; } -# else +#else static inline int setup_guest_base_seg(void) { return 0; } -# endif -#endif +#endif /* setup_guest_base_seg */ #endif /* SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg base, int index, intptr_t ofs, - int seg, bool is64, MemOp memop) + HostAddress h, TCGType type, MemOp memop) { - TCGType type = is64 ? TCG_TYPE_I64 : TCG_TYPE_I32; bool use_movbe = false; - int rexw = is64 * P_REXW; + int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW); int movop = OPC_MOVL_GvEv; /* Do big-endian loads with movbe. */ @@ -2168,77 +2175,78 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, switch (memop & MO_SSIZE) { case MO_UB: - tcg_out_modrm_sib_offset(s, OPC_MOVZBL + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVZBL + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; case MO_SB: - tcg_out_modrm_sib_offset(s, OPC_MOVSBL + rexw + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVSBL + rexw + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; case MO_UW: if (use_movbe) { /* There is no extending movbe; only low 16-bits are modified. */ - if (datalo != base && datalo != index) { + if (datalo != h.base && datalo != h.index) { /* XOR breaks dependency chains. */ tgen_arithr(s, ARITH_XOR, datalo, datalo); - tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg, + datalo, h.base, h.index, 0, h.ofs); } else { - tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg, + datalo, h.base, h.index, 0, h.ofs); tcg_out_ext16u(s, datalo, datalo); } } else { - tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVZWL + h.seg, datalo, + h.base, h.index, 0, h.ofs); } break; case MO_SW: if (use_movbe) { - tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg, + datalo, h.base, h.index, 0, h.ofs); tcg_out_ext16s(s, type, datalo, datalo); } else { - tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + h.seg, + datalo, h.base, h.index, 0, h.ofs); } break; case MO_UL: - tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; #if TCG_TARGET_REG_BITS == 64 case MO_SL: if (use_movbe) { - tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + h.seg, datalo, + h.base, h.index, 0, h.ofs); tcg_out_ext32s(s, datalo, datalo); } else { - tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + h.seg, datalo, + h.base, h.index, 0, h.ofs); } break; #endif case MO_UQ: if (TCG_TARGET_REG_BITS == 64) { - tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo, + h.base, h.index, 0, h.ofs); + break; + } + if (use_movbe) { + TCGReg t = datalo; + datalo = datahi; + datahi = t; + } + if (h.base == datalo || h.index == datalo) { + tcg_out_modrm_sib_offset(s, OPC_LEA, datahi, + h.base, h.index, 0, h.ofs); + tcg_out_modrm_offset(s, movop + h.seg, datalo, datahi, 0); + tcg_out_modrm_offset(s, movop + h.seg, datahi, datahi, 4); } else { - if (use_movbe) { - TCGReg t = datalo; - datalo = datahi; - datahi = t; - } - if (base != datalo) { - tcg_out_modrm_sib_offset(s, movop + seg, datalo, - base, index, 0, ofs); - tcg_out_modrm_sib_offset(s, movop + seg, datahi, - base, index, 0, ofs + 4); - } else { - tcg_out_modrm_sib_offset(s, movop + seg, datahi, - base, index, 0, ofs + 4); - tcg_out_modrm_sib_offset(s, movop + seg, datalo, - base, index, 0, ofs); - } + tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, + h.base, h.index, 0, h.ofs); + tcg_out_modrm_sib_offset(s, movop + h.seg, datahi, + h.base, h.index, 0, h.ofs + 4); } break; default: @@ -2246,56 +2254,43 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, } } -/* XXX: qemu_ld and qemu_st could be modified to clobber only EDX and - EAX. It will be useful once fixed registers globals are less - common. */ -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg datalo, datahi, addrlo; - TCGReg addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - int mem_index; - tcg_insn_unit *label_ptr[2]; -#else - unsigned a_bits; -#endif - - datalo = *args++; - datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); + MemOp opc = get_memop(oi); + HostAddress h; #if defined(CONFIG_SOFTMMU) - mem_index = get_mmuidx(oi); + tcg_insn_unit *label_ptr[2]; - tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc, + tcg_out_tlb_load(s, addrlo, addrhi, get_mmuidx(oi), opc, label_ptr, offsetof(CPUTLBEntry, addr_read)); /* TLB Hit. */ - tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, is64, opc); + h.base = TCG_REG_L1; + h.index = -1; + h.ofs = 0; + h.seg = 0; + tcg_out_qemu_ld_direct(s, datalo, datahi, h, data_type, opc); /* Record the current context of a load into ldst label */ - add_qemu_ldst_label(s, true, is64, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, true, data_type, oi, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - tcg_out_qemu_ld_direct(s, datalo, datahi, addrlo, x86_guest_base_index, - x86_guest_base_offset, x86_guest_base_seg, - is64, opc); + h = x86_guest_base; + h.base = addrlo; + tcg_out_qemu_ld_direct(s, datalo, datahi, h, data_type, opc); #endif } static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, - TCGReg base, int index, intptr_t ofs, - int seg, MemOp memop) + HostAddress h, MemOp memop) { bool use_movbe = false; int movop = OPC_MOVL_EvGv; @@ -2314,30 +2309,31 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, case MO_8: /* This is handled with constraints on INDEX_op_qemu_st8_i32. */ tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || datalo < 4); - tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg, - datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + h.seg, + datalo, h.base, h.index, 0, h.ofs); break; case MO_16: - tcg_out_modrm_sib_offset(s, movop + P_DATA16 + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + P_DATA16 + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; case MO_32: - tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, + h.base, h.index, 0, h.ofs); break; case MO_64: if (TCG_TARGET_REG_BITS == 64) { - tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo, - base, index, 0, ofs); + tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo, + h.base, h.index, 0, h.ofs); } else { if (use_movbe) { TCGReg t = datalo; datalo = datahi; datahi = t; } - tcg_out_modrm_sib_offset(s, movop + seg, datalo, - base, index, 0, ofs); - tcg_out_modrm_sib_offset(s, movop + seg, datahi, - base, index, 0, ofs + 4); + tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, + h.base, h.index, 0, h.ofs); + tcg_out_modrm_sib_offset(s, movop + h.seg, datahi, + h.base, h.index, 0, h.ofs + 4); } break; default: @@ -2345,46 +2341,39 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg datalo, datahi, addrlo; - TCGReg addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - int mem_index; - tcg_insn_unit *label_ptr[2]; -#else - unsigned a_bits; -#endif - - datalo = *args++; - datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); + MemOp opc = get_memop(oi); + HostAddress h; #if defined(CONFIG_SOFTMMU) - mem_index = get_mmuidx(oi); + tcg_insn_unit *label_ptr[2]; - tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc, + tcg_out_tlb_load(s, addrlo, addrhi, get_mmuidx(oi), opc, label_ptr, offsetof(CPUTLBEntry, addr_write)); /* TLB Hit. */ - tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, opc); + h.base = TCG_REG_L1; + h.index = -1; + h.ofs = 0; + h.seg = 0; + tcg_out_qemu_st_direct(s, datalo, datahi, h, opc); /* Record the current context of a store into ldst label */ - add_qemu_ldst_label(s, false, is64, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, data_type, oi, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); } - tcg_out_qemu_st_direct(s, datalo, datahi, addrlo, x86_guest_base_index, - x86_guest_base_offset, x86_guest_base_seg, opc); + h = x86_guest_base; + h.base = addrlo; + + tcg_out_qemu_st_direct(s, datalo, datahi, h, opc); #endif } @@ -2673,17 +2662,37 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, 0); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); + } else { + tcg_out_qemu_ld(s, a0, -1, a1, a2, args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, 1); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, a0, a1, a2, -1, args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_ld(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); + } break; case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st8_i32: - tcg_out_qemu_st(s, args, 0); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); + } else { + tcg_out_qemu_st(s, a0, -1, a1, a2, args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, 1); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, a0, a1, a2, -1, args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_st(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); + } break; OP_32_64(mulu2): @@ -4070,18 +4079,18 @@ static void tcg_target_qemu_prologue(TCGContext *s) (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4 + stack_addend); #else -# if !defined(CONFIG_SOFTMMU) && TCG_TARGET_REG_BITS == 64 +# if !defined(CONFIG_SOFTMMU) if (guest_base) { int seg = setup_guest_base_seg(); if (seg != 0) { - x86_guest_base_seg = seg; + x86_guest_base.seg = seg; } else if (guest_base == (int32_t)guest_base) { - x86_guest_base_offset = guest_base; + x86_guest_base.ofs = guest_base; } else { /* Choose R12 because, as a base, it requires a SIB byte. */ - x86_guest_base_index = TCG_REG_R12; - tcg_out_movi(s, TCG_TYPE_PTR, x86_guest_base_index, guest_base); - tcg_regset_set_reg(s->reserved_regs, x86_guest_base_index); + x86_guest_base.index = TCG_REG_R12; + tcg_out_movi(s, TCG_TYPE_PTR, x86_guest_base.index, guest_base); + tcg_regset_set_reg(s->reserved_regs, x86_guest_base.index); } } # endif diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index 0940788c6f..6a87a5e5a3 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -1013,135 +1013,124 @@ static TCGReg tcg_out_zext_addr_if_32_bit(TCGContext *s, return addr; } -static void tcg_out_qemu_ld_indexed(TCGContext *s, TCGReg rd, TCGReg rj, - TCGReg rk, MemOp opc, TCGType type) +typedef struct { + TCGReg base; + TCGReg index; +} HostAddress; + +static void tcg_out_qemu_ld_indexed(TCGContext *s, MemOp opc, TCGType type, + TCGReg rd, HostAddress h) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & MO_SSIZE) { case MO_UB: - tcg_out_opc_ldx_bu(s, rd, rj, rk); + tcg_out_opc_ldx_bu(s, rd, h.base, h.index); break; case MO_SB: - tcg_out_opc_ldx_b(s, rd, rj, rk); + tcg_out_opc_ldx_b(s, rd, h.base, h.index); break; case MO_UW: - tcg_out_opc_ldx_hu(s, rd, rj, rk); + tcg_out_opc_ldx_hu(s, rd, h.base, h.index); break; case MO_SW: - tcg_out_opc_ldx_h(s, rd, rj, rk); + tcg_out_opc_ldx_h(s, rd, h.base, h.index); break; case MO_UL: if (type == TCG_TYPE_I64) { - tcg_out_opc_ldx_wu(s, rd, rj, rk); + tcg_out_opc_ldx_wu(s, rd, h.base, h.index); break; } /* fallthrough */ case MO_SL: - tcg_out_opc_ldx_w(s, rd, rj, rk); + tcg_out_opc_ldx_w(s, rd, h.base, h.index); break; case MO_UQ: - tcg_out_opc_ldx_d(s, rd, rj, rk); + tcg_out_opc_ldx_d(s, rd, h.base, h.index); break; default: g_assert_not_reached(); } } -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, TCGType type) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl; - TCGReg data_regl; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) + MemOp opc = get_memop(oi); + HostAddress h; + +#ifdef CONFIG_SOFTMMU tcg_insn_unit *label_ptr[1]; + + tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 1); + h.index = TCG_REG_TMP2; #else - unsigned a_bits; + unsigned a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, true, addr_reg, a_bits); + } + h.index = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; #endif - TCGReg base; - data_regl = *args++; - addr_regl = *args++; - oi = *args++; - opc = get_memop(oi); + h.base = tcg_out_zext_addr_if_32_bit(s, addr_reg, TCG_REG_TMP0); + tcg_out_qemu_ld_indexed(s, opc, data_type, data_reg, h); -#if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, addr_regl, oi, label_ptr, 1); - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - tcg_out_qemu_ld_indexed(s, data_regl, base, TCG_REG_TMP2, opc, type); - add_qemu_ldst_label(s, 1, oi, type, - data_regl, addr_regl, +#ifdef CONFIG_SOFTMMU + add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); -#else - a_bits = get_alignment_bits(opc); - if (a_bits) { - tcg_out_test_alignment(s, true, addr_regl, a_bits); - } - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - TCGReg guest_base_reg = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; - tcg_out_qemu_ld_indexed(s, data_regl, base, guest_base_reg, opc, type); #endif } -static void tcg_out_qemu_st_indexed(TCGContext *s, TCGReg data, - TCGReg rj, TCGReg rk, MemOp opc) +static void tcg_out_qemu_st_indexed(TCGContext *s, MemOp opc, + TCGReg rd, HostAddress h) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & MO_SIZE) { case MO_8: - tcg_out_opc_stx_b(s, data, rj, rk); + tcg_out_opc_stx_b(s, rd, h.base, h.index); break; case MO_16: - tcg_out_opc_stx_h(s, data, rj, rk); + tcg_out_opc_stx_h(s, rd, h.base, h.index); break; case MO_32: - tcg_out_opc_stx_w(s, data, rj, rk); + tcg_out_opc_stx_w(s, rd, h.base, h.index); break; case MO_64: - tcg_out_opc_stx_d(s, data, rj, rk); + tcg_out_opc_stx_d(s, rd, h.base, h.index); break; default: g_assert_not_reached(); } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, TCGType type) +static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl; - TCGReg data_regl; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) + MemOp opc = get_memop(oi); + HostAddress h; + +#ifdef CONFIG_SOFTMMU tcg_insn_unit *label_ptr[1]; + + tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 0); + h.index = TCG_REG_TMP2; #else - unsigned a_bits; + unsigned a_bits = get_alignment_bits(opc); + if (a_bits) { + tcg_out_test_alignment(s, false, addr_reg, a_bits); + } + h.index = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; #endif - TCGReg base; - data_regl = *args++; - addr_regl = *args++; - oi = *args++; - opc = get_memop(oi); + h.base = tcg_out_zext_addr_if_32_bit(s, addr_reg, TCG_REG_TMP0); + tcg_out_qemu_st_indexed(s, opc, data_reg, h); -#if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, addr_regl, oi, label_ptr, 0); - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - tcg_out_qemu_st_indexed(s, data_regl, base, TCG_REG_TMP2, opc); - add_qemu_ldst_label(s, 0, oi, type, - data_regl, addr_regl, +#ifdef CONFIG_SOFTMMU + add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); -#else - a_bits = get_alignment_bits(opc); - if (a_bits) { - tcg_out_test_alignment(s, false, addr_regl, a_bits); - } - base = tcg_out_zext_addr_if_32_bit(s, addr_regl, TCG_REG_TMP0); - TCGReg guest_base_reg = USE_GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_ZERO; - tcg_out_qemu_st_indexed(s, data_regl, base, guest_base_reg, opc); #endif } @@ -1564,16 +1553,16 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, TCG_TYPE_I32); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, TCG_TYPE_I64); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, TCG_TYPE_I32); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, TCG_TYPE_I64); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index a83ebe8729..ef8350e9cd 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -1479,7 +1479,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) #endif /* SOFTMMU */ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, - TCGReg base, MemOp opc, bool is_64) + TCGReg base, MemOp opc, TCGType type) { switch (opc & (MO_SSIZE | MO_BSWAP)) { case MO_UB: @@ -1503,7 +1503,7 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, tcg_out_opc_imm(s, OPC_LH, lo, base, 0); break; case MO_UL | MO_BSWAP: - if (TCG_TARGET_REG_BITS == 64 && is_64) { + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) { if (use_mips32r2_instructions) { tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); tcg_out_bswap32(s, lo, lo, TCG_BSWAP_IZ | TCG_BSWAP_OZ); @@ -1528,7 +1528,7 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, } break; case MO_UL: - if (TCG_TARGET_REG_BITS == 64 && is_64) { + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) { tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); break; } @@ -1583,7 +1583,7 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, } static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, - TCGReg base, MemOp opc, bool is_64) + TCGReg base, MemOp opc, TCGType type) { const MIPSInsn lw1 = MIPS_BE ? OPC_LWL : OPC_LWR; const MIPSInsn lw2 = MIPS_BE ? OPC_LWR : OPC_LWL; @@ -1623,7 +1623,7 @@ static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, case MO_UL: tcg_out_opc_imm(s, lw1, lo, base, 0); tcg_out_opc_imm(s, lw2, lo, base, 3); - if (TCG_TARGET_REG_BITS == 64 && is_64 && !sgn) { + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64 && !sgn) { tcg_out_ext32u(s, lo, lo); } break; @@ -1634,18 +1634,18 @@ static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, tcg_out_opc_imm(s, lw1, lo, base, 0); tcg_out_opc_imm(s, lw2, lo, base, 3); tcg_out_bswap32(s, lo, lo, - TCG_TARGET_REG_BITS == 64 && is_64 + TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64 ? (sgn ? TCG_BSWAP_OS : TCG_BSWAP_OZ) : 0); } else { const tcg_insn_unit *subr = - (TCG_TARGET_REG_BITS == 64 && is_64 && !sgn + (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64 && !sgn ? bswap32u_addr : bswap32_addr); tcg_out_opc_imm(s, lw1, TCG_TMP0, base, 0); tcg_out_bswap_subr(s, subr); /* delay slot */ tcg_out_opc_imm(s, lw2, TCG_TMP0, base, 3); - tcg_out_mov(s, is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32, lo, TCG_TMP3); + tcg_out_mov(s, type, lo, TCG_TMP3); } break; @@ -1702,68 +1702,59 @@ static void tcg_out_qemu_ld_unalign(TCGContext *s, TCGReg lo, TCGReg hi, } } -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[2]; -#else -#endif - unsigned a_bits, s_bits; - TCGReg base = TCG_REG_A0; - - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - a_bits = get_alignment_bits(opc); - s_bits = opc & MO_SIZE; + MemOp opc = get_memop(oi); + unsigned a_bits = get_alignment_bits(opc); + unsigned s_bits = opc & MO_SIZE; + TCGReg base; /* * R6 removes the left/right instructions but requires the * system to support misaligned memory accesses. */ #if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 1); + tcg_insn_unit *label_ptr[2]; + + base = TCG_REG_A0; + tcg_out_tlb_load(s, base, addrlo, addrhi, oi, label_ptr, 1); if (use_mips32r6_instructions || a_bits >= s_bits) { - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_direct(s, datalo, datahi, base, opc, data_type); } else { - tcg_out_qemu_ld_unalign(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_unalign(s, datalo, datahi, base, opc, data_type); } - add_qemu_ldst_label(s, 1, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, true, oi, data_type, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else + base = addrlo; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, base, addr_regl); - addr_regl = base; + tcg_out_ext32u(s, TCG_REG_A0, base); + base = TCG_REG_A0; } - if (guest_base == 0 && data_regl != addr_regl) { - base = addr_regl; - } else if (guest_base == (int16_t)guest_base) { - tcg_out_opc_imm(s, ALIAS_PADDI, base, addr_regl, guest_base); - } else { - tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_GUEST_BASE_REG, addr_regl); + if (guest_base) { + if (guest_base == (int16_t)guest_base) { + tcg_out_opc_imm(s, ALIAS_PADDI, TCG_REG_A0, base, guest_base); + } else { + tcg_out_opc_reg(s, ALIAS_PADD, TCG_REG_A0, base, + TCG_GUEST_BASE_REG); + } + base = TCG_REG_A0; } if (use_mips32r6_instructions) { if (a_bits) { - tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_direct(s, datalo, datahi, base, opc, data_type); } else { if (a_bits && a_bits != s_bits) { - tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } if (a_bits >= s_bits) { - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_direct(s, datalo, datahi, base, opc, data_type); } else { - tcg_out_qemu_ld_unalign(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_unalign(s, datalo, datahi, base, opc, data_type); } } #endif @@ -1902,67 +1893,60 @@ static void tcg_out_qemu_st_unalign(TCGContext *s, TCGReg lo, TCGReg hi, g_assert_not_reached(); } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) + +static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[2]; -#endif - unsigned a_bits, s_bits; - TCGReg base = TCG_REG_A0; - - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - a_bits = get_alignment_bits(opc); - s_bits = opc & MO_SIZE; + MemOp opc = get_memop(oi); + unsigned a_bits = get_alignment_bits(opc); + unsigned s_bits = opc & MO_SIZE; + TCGReg base; /* * R6 removes the left/right instructions but requires the * system to support misaligned memory accesses. */ #if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 0); + tcg_insn_unit *label_ptr[2]; + + base = TCG_REG_A0; + tcg_out_tlb_load(s, base, addrlo, addrhi, oi, label_ptr, 0); if (use_mips32r6_instructions || a_bits >= s_bits) { - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_direct(s, datalo, datahi, base, opc); } else { - tcg_out_qemu_st_unalign(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_unalign(s, datalo, datahi, base, opc); } - add_qemu_ldst_label(s, 0, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, oi, data_type, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #else + base = addrlo; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, base, addr_regl); - addr_regl = base; + tcg_out_ext32u(s, TCG_REG_A0, base); + base = TCG_REG_A0; } - if (guest_base == 0) { - base = addr_regl; - } else if (guest_base == (int16_t)guest_base) { - tcg_out_opc_imm(s, ALIAS_PADDI, base, addr_regl, guest_base); - } else { - tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_GUEST_BASE_REG, addr_regl); + if (guest_base) { + if (guest_base == (int16_t)guest_base) { + tcg_out_opc_imm(s, ALIAS_PADDI, TCG_REG_A0, base, guest_base); + } else { + tcg_out_opc_reg(s, ALIAS_PADD, TCG_REG_A0, base, + TCG_GUEST_BASE_REG); + } + base = TCG_REG_A0; } if (use_mips32r6_instructions) { if (a_bits) { - tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_direct(s, datalo, datahi, base, opc); } else { if (a_bits && a_bits != s_bits) { - tcg_out_test_alignment(s, true, addr_regl, addr_regh, a_bits); + tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } if (a_bits >= s_bits) { - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_direct(s, datalo, datahi, base, opc); } else { - tcg_out_qemu_st_unalign(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_unalign(s, datalo, datahi, base, opc); } } #endif @@ -2425,16 +2409,36 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, false); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_ld(s, a0, 0, a1, 0, a2, TCG_TYPE_I32); + } else { + tcg_out_qemu_ld(s, a0, 0, a1, a2, args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, true); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_ld(s, a0, 0, a1, 0, a2, TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, a0, a1, a2, 0, args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_ld(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); + } break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, false); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_st(s, a0, 0, a1, 0, a2, TCG_TYPE_I32); + } else { + tcg_out_qemu_st(s, a0, 0, a1, a2, args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, true); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_st(s, a0, 0, a1, 0, a2, TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, a0, a1, a2, 0, args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_st(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); + } break; case INDEX_op_add2_i32: diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 77abb7d20c..cd473deb36 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -2118,7 +2118,8 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, MemOp opc, /* Record the context of a call to the out of line helper code for the slow path for a load or store, so that we can later generate the correct helper code. */ -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, + TCGType type, MemOpIdx oi, TCGReg datalo_reg, TCGReg datahi_reg, TCGReg addrlo_reg, TCGReg addrhi_reg, tcg_insn_unit *raddr, tcg_insn_unit *lptr) @@ -2126,6 +2127,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, TCGLabelQemuLdst *label = new_ldst_label(s); label->is_ld = is_ld; + label->type = type; label->oi = oi; label->datalo_reg = datalo_reg; label->datahi_reg = datahi_reg; @@ -2285,160 +2287,140 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { return tcg_out_fail_alignment(s, l); } - #endif /* SOFTMMU */ -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +typedef struct { + TCGReg base; + TCGReg index; +} HostAddress; + +static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg datalo, datahi, addrlo, rbase; - TCGReg addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc, s_bits; + MemOp opc = get_memop(oi); + MemOp s_bits = opc & MO_SIZE; + HostAddress h; + #ifdef CONFIG_SOFTMMU - int mem_index; tcg_insn_unit *label_ptr; -#else - unsigned a_bits; -#endif - datalo = *args++; - datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - s_bits = opc & MO_SIZE; - -#ifdef CONFIG_SOFTMMU - mem_index = get_mmuidx(oi); - addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true); + h.index = tcg_out_tlb_read(s, opc, addrlo, addrhi, get_mmuidx(oi), true); + h.base = TCG_REG_R3; /* Load a pointer into the current opcode w/conditional branch-link. */ label_ptr = s->code_ptr; tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK); - - rbase = TCG_REG_R3; #else /* !CONFIG_SOFTMMU */ - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); } - rbase = guest_base ? TCG_GUEST_BASE_REG : 0; + h.base = guest_base ? TCG_GUEST_BASE_REG : 0; + h.index = addrlo; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { tcg_out_ext32u(s, TCG_REG_TMP1, addrlo); - addrlo = TCG_REG_TMP1; + h.index = TCG_REG_TMP1; } #endif if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) { if (opc & MO_BSWAP) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo)); - tcg_out32(s, LWBRX | TAB(datahi, rbase, TCG_REG_R0)); - } else if (rbase != 0) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, LWZX | TAB(datahi, rbase, addrlo)); - tcg_out32(s, LWZX | TAB(datalo, rbase, TCG_REG_R0)); - } else if (addrlo == datahi) { - tcg_out32(s, LWZ | TAI(datalo, addrlo, 4)); - tcg_out32(s, LWZ | TAI(datahi, addrlo, 0)); + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, LWBRX | TAB(datalo, h.base, h.index)); + tcg_out32(s, LWBRX | TAB(datahi, h.base, TCG_REG_R0)); + } else if (h.base != 0) { + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, LWZX | TAB(datahi, h.base, h.index)); + tcg_out32(s, LWZX | TAB(datalo, h.base, TCG_REG_R0)); + } else if (h.index == datahi) { + tcg_out32(s, LWZ | TAI(datalo, h.index, 4)); + tcg_out32(s, LWZ | TAI(datahi, h.index, 0)); } else { - tcg_out32(s, LWZ | TAI(datahi, addrlo, 0)); - tcg_out32(s, LWZ | TAI(datalo, addrlo, 4)); + tcg_out32(s, LWZ | TAI(datahi, h.index, 0)); + tcg_out32(s, LWZ | TAI(datalo, h.index, 4)); } } else { uint32_t insn = qemu_ldx_opc[opc & (MO_BSWAP | MO_SSIZE)]; if (!have_isa_2_06 && insn == LDBRX) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo)); - tcg_out32(s, LWBRX | TAB(TCG_REG_R0, rbase, TCG_REG_R0)); + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, LWBRX | TAB(datalo, h.base, h.index)); + tcg_out32(s, LWBRX | TAB(TCG_REG_R0, h.base, TCG_REG_R0)); tcg_out_rld(s, RLDIMI, datalo, TCG_REG_R0, 32, 0); } else if (insn) { - tcg_out32(s, insn | TAB(datalo, rbase, addrlo)); + tcg_out32(s, insn | TAB(datalo, h.base, h.index)); } else { insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)]; - tcg_out32(s, insn | TAB(datalo, rbase, addrlo)); + tcg_out32(s, insn | TAB(datalo, h.base, h.index)); tcg_out_movext(s, TCG_TYPE_REG, datalo, TCG_TYPE_REG, opc & MO_SSIZE, datalo); } } #ifdef CONFIG_SOFTMMU - add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, true, data_type, oi, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #endif } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + MemOpIdx oi, TCGType data_type) { - TCGReg datalo, datahi, addrlo, rbase; - TCGReg addrhi __attribute__((unused)); - MemOpIdx oi; - MemOp opc, s_bits; + MemOp opc = get_memop(oi); + MemOp s_bits = opc & MO_SIZE; + HostAddress h; + #ifdef CONFIG_SOFTMMU - int mem_index; tcg_insn_unit *label_ptr; -#else - unsigned a_bits; -#endif - - datalo = *args++; - datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addrlo = *args++; - addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - s_bits = opc & MO_SIZE; -#ifdef CONFIG_SOFTMMU - mem_index = get_mmuidx(oi); - addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false); + h.index = tcg_out_tlb_read(s, opc, addrlo, addrhi, get_mmuidx(oi), false); + h.base = TCG_REG_R3; /* Load a pointer into the current opcode w/conditional branch-link. */ label_ptr = s->code_ptr; tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK); - - rbase = TCG_REG_R3; #else /* !CONFIG_SOFTMMU */ - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); } - rbase = guest_base ? TCG_GUEST_BASE_REG : 0; + h.base = guest_base ? TCG_GUEST_BASE_REG : 0; + h.index = addrlo; if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { tcg_out_ext32u(s, TCG_REG_TMP1, addrlo); - addrlo = TCG_REG_TMP1; + h.index = TCG_REG_TMP1; } #endif if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) { if (opc & MO_BSWAP) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo)); - tcg_out32(s, STWBRX | SAB(datahi, rbase, TCG_REG_R0)); - } else if (rbase != 0) { - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); - tcg_out32(s, STWX | SAB(datahi, rbase, addrlo)); - tcg_out32(s, STWX | SAB(datalo, rbase, TCG_REG_R0)); + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, STWBRX | SAB(datalo, h.base, h.index)); + tcg_out32(s, STWBRX | SAB(datahi, h.base, TCG_REG_R0)); + } else if (h.base != 0) { + tcg_out32(s, ADDI | TAI(TCG_REG_R0, h.index, 4)); + tcg_out32(s, STWX | SAB(datahi, h.base, h.index)); + tcg_out32(s, STWX | SAB(datalo, h.base, TCG_REG_R0)); } else { - tcg_out32(s, STW | TAI(datahi, addrlo, 0)); - tcg_out32(s, STW | TAI(datalo, addrlo, 4)); + tcg_out32(s, STW | TAI(datahi, h.index, 0)); + tcg_out32(s, STW | TAI(datalo, h.index, 4)); } } else { uint32_t insn = qemu_stx_opc[opc & (MO_BSWAP | MO_SIZE)]; if (!have_isa_2_06 && insn == STDBRX) { - tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo)); - tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, addrlo, 4)); + tcg_out32(s, STWBRX | SAB(datalo, h.base, h.index)); + tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, h.index, 4)); tcg_out_shri64(s, TCG_REG_R0, datalo, 32); - tcg_out32(s, STWBRX | SAB(TCG_REG_R0, rbase, TCG_REG_TMP1)); + tcg_out32(s, STWBRX | SAB(TCG_REG_R0, h.base, TCG_REG_TMP1)); } else { - tcg_out32(s, insn | SAB(datalo, rbase, addrlo)); + tcg_out32(s, insn | SAB(datalo, h.base, h.index)); } } #ifdef CONFIG_SOFTMMU - add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi, - s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, data_type, oi, datalo, datahi, + addrlo, addrhi, s->code_ptr, label_ptr); #endif } @@ -2972,16 +2954,46 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, false); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_ld(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I32); + } else { + tcg_out_qemu_ld(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, true); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_ld(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_ld(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_ld(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); + } break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, false); + if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + tcg_out_qemu_st(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I32); + } else { + tcg_out_qemu_st(s, args[0], -1, args[1], args[2], + args[3], TCG_TYPE_I32); + } break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, true); + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_qemu_st(s, args[0], -1, args[1], -1, + args[2], TCG_TYPE_I64); + } else if (TARGET_LONG_BITS == 32) { + tcg_out_qemu_st(s, args[0], args[1], args[2], -1, + args[3], TCG_TYPE_I64); + } else { + tcg_out_qemu_st(s, args[0], args[1], args[2], args[3], + args[4], TCG_TYPE_I64); + } break; case INDEX_op_setcond_i32: diff --git a/tcg/riscv/tcg-target-con-set.h b/tcg/riscv/tcg-target-con-set.h index cf0ac4d751..d4cff673b0 100644 --- a/tcg/riscv/tcg-target-con-set.h +++ b/tcg/riscv/tcg-target-con-set.h @@ -13,18 +13,10 @@ C_O0_I1(r) C_O0_I2(LZ, L) C_O0_I2(rZ, r) C_O0_I2(rZ, rZ) -C_O0_I3(LZ, L, L) -C_O0_I3(LZ, LZ, L) -C_O0_I4(LZ, LZ, L, L) -C_O0_I4(rZ, rZ, rZ, rZ) C_O1_I1(r, L) C_O1_I1(r, r) -C_O1_I2(r, L, L) C_O1_I2(r, r, ri) C_O1_I2(r, r, rI) C_O1_I2(r, rZ, rN) C_O1_I2(r, rZ, rZ) -C_O1_I4(r, rZ, rZ, rZ, rZ) -C_O2_I1(r, r, L) -C_O2_I2(r, r, L, L) C_O2_I4(r, r, rZ, rZ, rM, rM) diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index 266fe1433d..a4cf60ca75 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -137,15 +137,7 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define SOFTMMU_RESERVE_REGS 0 #endif - -static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) -{ - if (TCG_TARGET_REG_BITS == 32) { - return sextract32(val, pos, len); - } else { - return sextract64(val, pos, len); - } -} +#define sextreg sextract64 /* test if a constant matches the constraint */ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) @@ -235,7 +227,6 @@ typedef enum { OPC_XOR = 0x4033, OPC_XORI = 0x4013, -#if TCG_TARGET_REG_BITS == 64 OPC_ADDIW = 0x1b, OPC_ADDW = 0x3b, OPC_DIVUW = 0x200503b, @@ -250,23 +241,6 @@ typedef enum { OPC_SRLIW = 0x501b, OPC_SRLW = 0x503b, OPC_SUBW = 0x4000003b, -#else - /* Simplify code throughout by defining aliases for RV32. */ - OPC_ADDIW = OPC_ADDI, - OPC_ADDW = OPC_ADD, - OPC_DIVUW = OPC_DIVU, - OPC_DIVW = OPC_DIV, - OPC_MULW = OPC_MUL, - OPC_REMUW = OPC_REMU, - OPC_REMW = OPC_REM, - OPC_SLLIW = OPC_SLLI, - OPC_SLLW = OPC_SLL, - OPC_SRAIW = OPC_SRAI, - OPC_SRAW = OPC_SRA, - OPC_SRLIW = OPC_SRLI, - OPC_SRLW = OPC_SRL, - OPC_SUBW = OPC_SUB, -#endif OPC_FENCE = 0x0000000f, OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */ @@ -500,7 +474,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, tcg_target_long lo, hi, tmp; int shift, ret; - if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { + if (type == TCG_TYPE_I32) { val = (int32_t)val; } @@ -511,7 +485,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, } hi = val - lo; - if (TCG_TARGET_REG_BITS == 32 || val == (int32_t)val) { + if (val == (int32_t)val) { tcg_out_opc_upper(s, OPC_LUI, rd, hi); if (lo != 0) { tcg_out_opc_imm(s, OPC_ADDIW, rd, rd, lo); @@ -519,7 +493,6 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, return; } - /* We can only be here if TCG_TARGET_REG_BITS != 32 */ tmp = tcg_pcrel_diff(s, (void *)val); if (tmp == (int32_t)tmp) { tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); @@ -668,15 +641,15 @@ static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data, static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2) { - bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); - tcg_out_ldst(s, is32bit ? OPC_LW : OPC_LD, arg, arg1, arg2); + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_LW : OPC_LD; + tcg_out_ldst(s, insn, arg, arg1, arg2); } static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2) { - bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); - tcg_out_ldst(s, is32bit ? OPC_SW : OPC_SD, arg, arg1, arg2); + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SW : OPC_SD; + tcg_out_ldst(s, insn, arg, arg1, arg2); } static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, @@ -829,20 +802,6 @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, } } -static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, - TCGReg bl, TCGReg bh, TCGLabel *l) -{ - /* todo */ - g_assert_not_reached(); -} - -static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) -{ - /* todo */ - g_assert_not_reached(); -} - static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; @@ -853,20 +812,18 @@ static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) if (offset == sextreg(offset, 0, 20)) { /* short jump: -2097150 to 2097152 */ tcg_out_opc_jump(s, OPC_JAL, link, offset); - } else if (TCG_TARGET_REG_BITS == 32 || offset == (int32_t)offset) { + } else if (offset == (int32_t)offset) { /* long jump: -2147483646 to 2147483648 */ tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0); tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0); ret = reloc_call(s->code_ptr - 2, arg); tcg_debug_assert(ret == true); - } else if (TCG_TARGET_REG_BITS == 64) { + } else { /* far jump: 64-bit */ tcg_target_long imm = sextreg((tcg_target_long)arg, 0, 12); tcg_target_long base = (tcg_target_long)arg - imm; tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, base); tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, imm); - } else { - g_assert_not_reached(); } } @@ -942,9 +899,6 @@ static void * const qemu_st_helpers[MO_SIZE + 1] = { #endif }; -/* We don't support oversize guests */ -QEMU_BUILD_BUG_ON(TCG_TARGET_REG_BITS < TARGET_LONG_BITS); - /* We expect to use a 12-bit negative offset from ENV. */ QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); @@ -956,8 +910,7 @@ static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) tcg_debug_assert(ok); } -static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, - TCGReg addrh, MemOpIdx oi, +static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, MemOpIdx oi, tcg_insn_unit **label_ptr, bool is_load) { MemOp opc = get_memop(oi); @@ -973,7 +926,7 @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, mask_base, mask_ofs); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, table_base, table_ofs); - tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addrl, + tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addr, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); @@ -992,10 +945,10 @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, /* Clear the non-page, non-alignment bits from the address. */ compare_mask = (tcg_target_long)TARGET_PAGE_MASK | ((1 << a_bits) - 1); if (compare_mask == sextreg(compare_mask, 0, 12)) { - tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addrl, compare_mask); + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr, compare_mask); } else { tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); - tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addrl); + tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addr); } /* Compare masked address with the TLB entry. */ @@ -1003,29 +956,26 @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); /* TLB Hit - translate address using addend. */ - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, TCG_REG_TMP0, addrl); - addrl = TCG_REG_TMP0; + if (TARGET_LONG_BITS == 32) { + tcg_out_ext32u(s, TCG_REG_TMP0, addr); + addr = TCG_REG_TMP0; } - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addrl); + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addr); return TCG_REG_TMP0; } static void add_qemu_ldst_label(TCGContext *s, int is_ld, MemOpIdx oi, - TCGType ext, - TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addrhi, - void *raddr, tcg_insn_unit **label_ptr) + TCGType data_type, TCGReg data_reg, + TCGReg addr_reg, void *raddr, + tcg_insn_unit **label_ptr) { TCGLabelQemuLdst *label = new_ldst_label(s); label->is_ld = is_ld; label->oi = oi; - label->type = ext; - label->datalo_reg = datalo; - label->datahi_reg = datahi; - label->addrlo_reg = addrlo; - label->addrhi_reg = addrhi; + label->type = data_type; + label->datalo_reg = data_reg; + label->addrlo_reg = addr_reg; label->raddr = tcg_splitwx_to_rx(raddr); label->label_ptr[0] = label_ptr[0]; } @@ -1039,11 +989,6 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) TCGReg a2 = tcg_target_call_iarg_regs[2]; TCGReg a3 = tcg_target_call_iarg_regs[3]; - /* We don't support oversize guests */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - g_assert_not_reached(); - } - /* resolve label address */ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; @@ -1073,11 +1018,6 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) TCGReg a3 = tcg_target_call_iarg_regs[3]; TCGReg a4 = tcg_target_call_iarg_regs[4]; - /* We don't support oversize guests */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - g_assert_not_reached(); - } - /* resolve label address */ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { return false; @@ -1146,85 +1086,62 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) #endif /* CONFIG_SOFTMMU */ -static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, - TCGReg base, MemOp opc, bool is_64) +static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val, + TCGReg base, MemOp opc, TCGType type) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & (MO_SSIZE)) { case MO_UB: - tcg_out_opc_imm(s, OPC_LBU, lo, base, 0); + tcg_out_opc_imm(s, OPC_LBU, val, base, 0); break; case MO_SB: - tcg_out_opc_imm(s, OPC_LB, lo, base, 0); + tcg_out_opc_imm(s, OPC_LB, val, base, 0); break; case MO_UW: - tcg_out_opc_imm(s, OPC_LHU, lo, base, 0); + tcg_out_opc_imm(s, OPC_LHU, val, base, 0); break; case MO_SW: - tcg_out_opc_imm(s, OPC_LH, lo, base, 0); + tcg_out_opc_imm(s, OPC_LH, val, base, 0); break; case MO_UL: - if (TCG_TARGET_REG_BITS == 64 && is_64) { - tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); + if (type == TCG_TYPE_I64) { + tcg_out_opc_imm(s, OPC_LWU, val, base, 0); break; } /* FALLTHRU */ case MO_SL: - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); + tcg_out_opc_imm(s, OPC_LW, val, base, 0); break; case MO_UQ: - /* Prefer to load from offset 0 first, but allow for overlap. */ - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_opc_imm(s, OPC_LD, lo, base, 0); - } else if (lo != base) { - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); - tcg_out_opc_imm(s, OPC_LW, hi, base, 4); - } else { - tcg_out_opc_imm(s, OPC_LW, hi, base, 4); - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); - } + tcg_out_opc_imm(s, OPC_LD, val, base, 0); break; default: g_assert_not_reached(); } } -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[1]; -#else - unsigned a_bits; -#endif + MemOp opc = get_memop(oi); TCGReg base; - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - #if defined(CONFIG_SOFTMMU) - base = tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 1); - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); - add_qemu_ldst_label(s, 1, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, + tcg_insn_unit *label_ptr[1]; + + base = tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 1); + tcg_out_qemu_ld_direct(s, data_reg, base, opc, data_type); + add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { - tcg_out_test_alignment(s, true, addr_regl, a_bits); + tcg_out_test_alignment(s, true, addr_reg, a_bits); } - base = addr_regl; - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + base = addr_reg; + if (TARGET_LONG_BITS == 32) { tcg_out_ext32u(s, TCG_REG_TMP0, base); base = TCG_REG_TMP0; } @@ -1232,11 +1149,11 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); base = TCG_REG_TMP0; } - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + tcg_out_qemu_ld_direct(s, data_reg, base, opc, data_type); #endif } -static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, +static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg val, TCGReg base, MemOp opc) { /* Byte swapping is left to middle-end expansion. */ @@ -1244,61 +1161,42 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, switch (opc & (MO_SSIZE)) { case MO_8: - tcg_out_opc_store(s, OPC_SB, base, lo, 0); + tcg_out_opc_store(s, OPC_SB, base, val, 0); break; case MO_16: - tcg_out_opc_store(s, OPC_SH, base, lo, 0); + tcg_out_opc_store(s, OPC_SH, base, val, 0); break; case MO_32: - tcg_out_opc_store(s, OPC_SW, base, lo, 0); + tcg_out_opc_store(s, OPC_SW, base, val, 0); break; case MO_64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_opc_store(s, OPC_SD, base, lo, 0); - } else { - tcg_out_opc_store(s, OPC_SW, base, lo, 0); - tcg_out_opc_store(s, OPC_SW, base, hi, 4); - } + tcg_out_opc_store(s, OPC_SD, base, val, 0); break; default: g_assert_not_reached(); } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[1]; -#else - unsigned a_bits; -#endif + MemOp opc = get_memop(oi); TCGReg base; - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - #if defined(CONFIG_SOFTMMU) - base = tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 0); - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); - add_qemu_ldst_label(s, 0, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, + tcg_insn_unit *label_ptr[1]; + + base = tcg_out_tlb_load(s, addr_reg, oi, label_ptr, 0); + tcg_out_qemu_st_direct(s, data_reg, base, opc); + add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, s->code_ptr, label_ptr); #else - a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); if (a_bits) { - tcg_out_test_alignment(s, false, addr_regl, a_bits); + tcg_out_test_alignment(s, false, addr_reg, a_bits); } - base = addr_regl; - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + base = addr_reg; + if (TARGET_LONG_BITS == 32) { tcg_out_ext32u(s, TCG_REG_TMP0, base); base = TCG_REG_TMP0; } @@ -1306,7 +1204,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); base = TCG_REG_TMP0; } - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + tcg_out_qemu_st_direct(s, data_reg, base, opc); #endif } @@ -1585,29 +1483,23 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_brcond_i64: tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); break; - case INDEX_op_brcond2_i32: - tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], arg_label(args[5])); - break; case INDEX_op_setcond_i32: case INDEX_op_setcond_i64: tcg_out_setcond(s, args[3], a0, a1, a2); break; - case INDEX_op_setcond2_i32: - tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]); - break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, false); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, true); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, false); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, true); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_extrh_i64_i32: @@ -1748,26 +1640,12 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_sub2_i64: return C_O2_I4(r, r, rZ, rZ, rM, rM); - case INDEX_op_brcond2_i32: - return C_O0_I4(rZ, rZ, rZ, rZ); - - case INDEX_op_setcond2_i32: - return C_O1_I4(r, rZ, rZ, rZ, rZ); - case INDEX_op_qemu_ld_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); - case INDEX_op_qemu_st_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O0_I2(LZ, L) : C_O0_I3(LZ, L, L)); case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L) - : C_O2_I2(r, r, L, L)); + return C_O1_I1(r, L); + case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(LZ, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(LZ, LZ, L) - : C_O0_I4(LZ, LZ, L, L)); + return C_O0_I2(LZ, L); default: g_assert_not_reached(); @@ -1843,9 +1721,7 @@ static void tcg_target_qemu_prologue(TCGContext *s) static void tcg_target_init(TCGContext *s) { tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; - if (TCG_TARGET_REG_BITS == 64) { - tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; - } + tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; tcg_target_call_clobber_regs = -1u; tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S0); diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index 0deb33701f..dddf2486c1 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -25,11 +25,14 @@ #ifndef RISCV_TCG_TARGET_H #define RISCV_TCG_TARGET_H -#if __riscv_xlen == 32 -# define TCG_TARGET_REG_BITS 32 -#elif __riscv_xlen == 64 -# define TCG_TARGET_REG_BITS 64 +/* + * We don't support oversize guests. + * Since we will only build tcg once, this in turn requires a 64-bit host. + */ +#if __riscv_xlen != 64 +#error "unsupported code generation mode" #endif +#define TCG_TARGET_REG_BITS 64 #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 20 @@ -83,13 +86,8 @@ typedef enum { #define TCG_TARGET_STACK_ALIGN 16 #define TCG_TARGET_CALL_STACK_OFFSET 0 #define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL -#if TCG_TARGET_REG_BITS == 32 -#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN -#define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_EVEN -#else #define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL #define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_NORMAL -#endif #define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL /* optional instructions */ @@ -106,8 +104,8 @@ typedef enum { #define TCG_TARGET_HAS_sub2_i32 1 #define TCG_TARGET_HAS_mulu2_i32 0 #define TCG_TARGET_HAS_muls2_i32 0 -#define TCG_TARGET_HAS_muluh_i32 (TCG_TARGET_REG_BITS == 32) -#define TCG_TARGET_HAS_mulsh_i32 (TCG_TARGET_REG_BITS == 32) +#define TCG_TARGET_HAS_muluh_i32 0 +#define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_ext8s_i32 1 #define TCG_TARGET_HAS_ext16s_i32 1 #define TCG_TARGET_HAS_ext8u_i32 1 @@ -128,7 +126,6 @@ typedef enum { #define TCG_TARGET_HAS_setcond2 1 #define TCG_TARGET_HAS_qemu_st8_i32 0 -#if TCG_TARGET_REG_BITS == 64 #define TCG_TARGET_HAS_movcond_i64 0 #define TCG_TARGET_HAS_div_i64 1 #define TCG_TARGET_HAS_rem_i64 1 @@ -165,7 +162,6 @@ typedef enum { #define TCG_TARGET_HAS_muls2_i64 0 #define TCG_TARGET_HAS_muluh_i64 1 #define TCG_TARGET_HAS_mulsh_i64 1 -#endif #define TCG_TARGET_DEFAULT_MO (0) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index b399798664..da7ee5b085 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -1606,58 +1606,64 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest, tcg_out_call_int(s, dest); } +typedef struct { + TCGReg base; + TCGReg index; + int disp; +} HostAddress; + static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data, - TCGReg base, TCGReg index, int disp) + HostAddress h) { switch (opc & (MO_SSIZE | MO_BSWAP)) { case MO_UB: - tcg_out_insn(s, RXY, LLGC, data, base, index, disp); + tcg_out_insn(s, RXY, LLGC, data, h.base, h.index, h.disp); break; case MO_SB: - tcg_out_insn(s, RXY, LGB, data, base, index, disp); + tcg_out_insn(s, RXY, LGB, data, h.base, h.index, h.disp); break; case MO_UW | MO_BSWAP: /* swapped unsigned halfword load with upper bits zeroed */ - tcg_out_insn(s, RXY, LRVH, data, base, index, disp); + tcg_out_insn(s, RXY, LRVH, data, h.base, h.index, h.disp); tcg_out_ext16u(s, data, data); break; case MO_UW: - tcg_out_insn(s, RXY, LLGH, data, base, index, disp); + tcg_out_insn(s, RXY, LLGH, data, h.base, h.index, h.disp); break; case MO_SW | MO_BSWAP: /* swapped sign-extended halfword load */ - tcg_out_insn(s, RXY, LRVH, data, base, index, disp); + tcg_out_insn(s, RXY, LRVH, data, h.base, h.index, h.disp); tcg_out_ext16s(s, TCG_TYPE_REG, data, data); break; case MO_SW: - tcg_out_insn(s, RXY, LGH, data, base, index, disp); + tcg_out_insn(s, RXY, LGH, data, h.base, h.index, h.disp); break; case MO_UL | MO_BSWAP: /* swapped unsigned int load with upper bits zeroed */ - tcg_out_insn(s, RXY, LRV, data, base, index, disp); + tcg_out_insn(s, RXY, LRV, data, h.base, h.index, h.disp); tcg_out_ext32u(s, data, data); break; case MO_UL: - tcg_out_insn(s, RXY, LLGF, data, base, index, disp); + tcg_out_insn(s, RXY, LLGF, data, h.base, h.index, h.disp); break; case MO_SL | MO_BSWAP: /* swapped sign-extended int load */ - tcg_out_insn(s, RXY, LRV, data, base, index, disp); + tcg_out_insn(s, RXY, LRV, data, h.base, h.index, h.disp); tcg_out_ext32s(s, data, data); break; case MO_SL: - tcg_out_insn(s, RXY, LGF, data, base, index, disp); + tcg_out_insn(s, RXY, LGF, data, h.base, h.index, h.disp); break; case MO_UQ | MO_BSWAP: - tcg_out_insn(s, RXY, LRVG, data, base, index, disp); + tcg_out_insn(s, RXY, LRVG, data, h.base, h.index, h.disp); break; case MO_UQ: - tcg_out_insn(s, RXY, LG, data, base, index, disp); + tcg_out_insn(s, RXY, LG, data, h.base, h.index, h.disp); break; default: @@ -1666,44 +1672,44 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data, } static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data, - TCGReg base, TCGReg index, int disp) + HostAddress h) { switch (opc & (MO_SIZE | MO_BSWAP)) { case MO_UB: - if (disp >= 0 && disp < 0x1000) { - tcg_out_insn(s, RX, STC, data, base, index, disp); + if (h.disp >= 0 && h.disp < 0x1000) { + tcg_out_insn(s, RX, STC, data, h.base, h.index, h.disp); } else { - tcg_out_insn(s, RXY, STCY, data, base, index, disp); + tcg_out_insn(s, RXY, STCY, data, h.base, h.index, h.disp); } break; case MO_UW | MO_BSWAP: - tcg_out_insn(s, RXY, STRVH, data, base, index, disp); + tcg_out_insn(s, RXY, STRVH, data, h.base, h.index, h.disp); break; case MO_UW: - if (disp >= 0 && disp < 0x1000) { - tcg_out_insn(s, RX, STH, data, base, index, disp); + if (h.disp >= 0 && h.disp < 0x1000) { + tcg_out_insn(s, RX, STH, data, h.base, h.index, h.disp); } else { - tcg_out_insn(s, RXY, STHY, data, base, index, disp); + tcg_out_insn(s, RXY, STHY, data, h.base, h.index, h.disp); } break; case MO_UL | MO_BSWAP: - tcg_out_insn(s, RXY, STRV, data, base, index, disp); + tcg_out_insn(s, RXY, STRV, data, h.base, h.index, h.disp); break; case MO_UL: - if (disp >= 0 && disp < 0x1000) { - tcg_out_insn(s, RX, ST, data, base, index, disp); + if (h.disp >= 0 && h.disp < 0x1000) { + tcg_out_insn(s, RX, ST, data, h.base, h.index, h.disp); } else { - tcg_out_insn(s, RXY, STY, data, base, index, disp); + tcg_out_insn(s, RXY, STY, data, h.base, h.index, h.disp); } break; case MO_UQ | MO_BSWAP: - tcg_out_insn(s, RXY, STRVG, data, base, index, disp); + tcg_out_insn(s, RXY, STRVG, data, h.base, h.index, h.disp); break; case MO_UQ: - tcg_out_insn(s, RXY, STG, data, base, index, disp); + tcg_out_insn(s, RXY, STG, data, h.base, h.index, h.disp); break; default: @@ -1770,13 +1776,14 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc, } static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, - TCGReg data, TCGReg addr, + TCGType type, TCGReg data, TCGReg addr, tcg_insn_unit *raddr, tcg_insn_unit *label_ptr) { TCGLabelQemuLdst *label = new_ldst_label(s); label->is_ld = is_ld; label->oi = oi; + label->type = type; label->datalo_reg = data; label->addrlo_reg = addr; label->raddr = tcg_splitwx_to_rx(raddr); @@ -1882,82 +1889,89 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) return tcg_out_fail_alignment(s, l); } -static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg, - TCGReg *index_reg, tcg_target_long *disp) +static HostAddress tcg_prepare_user_ldst(TCGContext *s, TCGReg addr_reg) { + TCGReg index; + int disp; + if (TARGET_LONG_BITS == 32) { - tcg_out_ext32u(s, TCG_TMP0, *addr_reg); - *addr_reg = TCG_TMP0; + tcg_out_ext32u(s, TCG_TMP0, addr_reg); + addr_reg = TCG_TMP0; } if (guest_base < 0x80000) { - *index_reg = TCG_REG_NONE; - *disp = guest_base; + index = TCG_REG_NONE; + disp = guest_base; } else { - *index_reg = TCG_GUEST_BASE_REG; - *disp = 0; + index = TCG_GUEST_BASE_REG; + disp = 0; } + return (HostAddress){ .base = addr_reg, .index = index, .disp = disp }; } #endif /* CONFIG_SOFTMMU */ static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi) + MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); + HostAddress h; + #ifdef CONFIG_SOFTMMU unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; - TCGReg base_reg; - base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1); + h.base = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1); + h.index = TCG_REG_R2; + h.disp = 0; tcg_out16(s, RI_BRC | (S390_CC_NE << 4)); label_ptr = s->code_ptr; s->code_ptr += 1; - tcg_out_qemu_ld_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0); + tcg_out_qemu_ld_direct(s, opc, data_reg, h); - add_qemu_ldst_label(s, 1, oi, data_reg, addr_reg, s->code_ptr, label_ptr); + add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #else - TCGReg index_reg; - tcg_target_long disp; unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, true, addr_reg, a_bits); } - tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp); - tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, index_reg, disp); + h = tcg_prepare_user_ldst(s, addr_reg); + tcg_out_qemu_ld_direct(s, opc, data_reg, h); #endif } static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, - MemOpIdx oi) + MemOpIdx oi, TCGType data_type) { MemOp opc = get_memop(oi); + HostAddress h; + #ifdef CONFIG_SOFTMMU unsigned mem_index = get_mmuidx(oi); tcg_insn_unit *label_ptr; - TCGReg base_reg; - base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0); + h.base = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0); + h.index = TCG_REG_R2; + h.disp = 0; tcg_out16(s, RI_BRC | (S390_CC_NE << 4)); label_ptr = s->code_ptr; s->code_ptr += 1; - tcg_out_qemu_st_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0); + tcg_out_qemu_st_direct(s, opc, data_reg, h); - add_qemu_ldst_label(s, 0, oi, data_reg, addr_reg, s->code_ptr, label_ptr); + add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg, + s->code_ptr, label_ptr); #else - TCGReg index_reg; - tcg_target_long disp; unsigned a_bits = get_alignment_bits(opc); if (a_bits) { tcg_out_test_alignment(s, false, addr_reg, a_bits); } - tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp); - tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, index_reg, disp); + h = tcg_prepare_user_ldst(s, addr_reg); + tcg_out_qemu_st_direct(s, opc, data_reg, h); #endif } @@ -2307,13 +2321,16 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - /* ??? Technically we can use a non-extending instruction. */ + tcg_out_qemu_ld(s, args[0], args[1], args[2], TCG_TYPE_I32); + break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args[0], args[1], args[2]); + tcg_out_qemu_ld(s, args[0], args[1], args[2], TCG_TYPE_I64); break; case INDEX_op_qemu_st_i32: + tcg_out_qemu_st(s, args[0], args[1], args[2], TCG_TYPE_I32); + break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args[0], args[1], args[2]); + tcg_out_qemu_st(s, args[0], args[1], args[2], TCG_TYPE_I64); break; case INDEX_op_ld16s_i64: diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index 4f477d539c..7e6466d3b6 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -1178,7 +1178,7 @@ static const int qemu_st_opc[(MO_SIZE | MO_BSWAP) + 1] = { }; static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, - MemOpIdx oi, bool is_64) + MemOpIdx oi, TCGType data_type) { MemOp memop = get_memop(oi); tcg_insn_unit *label_ptr; @@ -1220,7 +1220,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_O2, oi); /* We let the helper sign-extend SB and SW, but leave SL for here. */ - if (is_64 && (memop & MO_SSIZE) == MO_SL) { + if ((memop & MO_SSIZE) == MO_SL) { tcg_out_ext32s(s, data, TCG_REG_O0); } else { tcg_out_mov(s, TCG_TYPE_REG, data, TCG_REG_O0); @@ -1636,10 +1636,10 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, a0, a1, a2, false); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, a0, a1, a2, true); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_qemu_st_i32: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index e542a4e9b7..0f1ba01a9a 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -58,10 +58,6 @@ typedef struct TCGCallArgumentLoc { unsigned tmp_subindex : 2; } TCGCallArgumentLoc; -/* Avoid "unsigned < 0 is always false" Werror, when iarg_regs is empty. */ -#define REG_P(L) \ - ((int)(L)->arg_slot < (int)ARRAY_SIZE(tcg_target_call_iarg_regs)) - typedef struct TCGHelperInfo { void *func; const char *name; diff --git a/tcg/tcg-ldst.c.inc b/tcg/tcg-ldst.c.inc index 403cbb0f06..ffada04af0 100644 --- a/tcg/tcg-ldst.c.inc +++ b/tcg/tcg-ldst.c.inc @@ -20,20 +20,6 @@ * THE SOFTWARE. */ -typedef struct TCGLabelQemuLdst { - bool is_ld; /* qemu_ld: true, qemu_st: false */ - MemOpIdx oi; - TCGType type; /* result type of a load */ - TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */ - TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */ - TCGReg datalo_reg; /* reg index for low word to be loaded or stored */ - TCGReg datahi_reg; /* reg index for high word to be loaded or stored */ - const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */ - tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */ - QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next; -} TCGLabelQemuLdst; - - /* * Generate TB finalization at the end of block */ diff --git a/tcg/tcg.c b/tcg/tcg.c index cfd3262a4a..057423c121 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -94,6 +94,19 @@ typedef struct QEMU_PACKED { DebugFrameFDEHeader fde; } DebugFrameHeader; +typedef struct TCGLabelQemuLdst { + bool is_ld; /* qemu_ld: true, qemu_st: false */ + MemOpIdx oi; + TCGType type; /* result type of a load */ + TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */ + TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */ + TCGReg datalo_reg; /* reg index for low word to be loaded or stored */ + TCGReg datahi_reg; /* reg index for high word to be loaded or stored */ + const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */ + tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */ + QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next; +} TCGLabelQemuLdst; + static void tcg_register_jit_int(const void *buf, size_t size, const void *debug_frame, size_t debug_frame_size) @@ -793,6 +806,25 @@ static void init_ffi_layouts(void) } #endif /* CONFIG_TCG_INTERPRETER */ +static inline bool arg_slot_reg_p(unsigned arg_slot) +{ + /* + * Split the sizeof away from the comparison to avoid Werror from + * "unsigned < 0 is always false", when iarg_regs is empty. + */ + unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs); + return arg_slot < nreg; +} + +static inline int arg_slot_stk_ofs(unsigned arg_slot) +{ + unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); + unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); + + tcg_debug_assert(stk_slot < max); + return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long); +} + typedef struct TCGCumulativeArgs { int arg_idx; /* tcg_gen_callN args[] */ int info_in_idx; /* TCGHelperInfo in[] */ @@ -1032,6 +1064,7 @@ static void init_call_layout(TCGHelperInfo *info) } } assert(ref_base + cum.ref_slot <= max_stk_slots); + ref_base += max_reg_slots; if (ref_base != 0) { for (int i = cum.info_in_idx - 1; i >= 0; --i) { @@ -3218,7 +3251,7 @@ liveness_pass_1(TCGContext *s) case TCG_CALL_ARG_NORMAL: case TCG_CALL_ARG_EXTEND_U: case TCG_CALL_ARG_EXTEND_S: - if (REG_P(loc)) { + if (arg_slot_reg_p(loc->arg_slot)) { *la_temp_pref(ts) = 0; break; } @@ -3245,7 +3278,7 @@ liveness_pass_1(TCGContext *s) case TCG_CALL_ARG_NORMAL: case TCG_CALL_ARG_EXTEND_U: case TCG_CALL_ARG_EXTEND_S: - if (REG_P(loc)) { + if (arg_slot_reg_p(loc->arg_slot)) { tcg_regset_set_reg(*la_temp_pref(ts), tcg_target_call_iarg_regs[loc->arg_slot]); } @@ -4803,7 +4836,7 @@ static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, } } -static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, +static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts, TCGRegSet allocated_regs) { /* @@ -4813,30 +4846,27 @@ static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, */ temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, - TCG_TARGET_CALL_STACK_OFFSET + - stk_slot * sizeof(tcg_target_long)); + arg_slot_stk_ofs(arg_slot)); } static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, TCGTemp *ts, TCGRegSet *allocated_regs) { - if (REG_P(l)) { + if (arg_slot_reg_p(l->arg_slot)) { TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; load_arg_reg(s, reg, ts, *allocated_regs); tcg_regset_set_reg(*allocated_regs, reg); } else { - load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs), - ts, *allocated_regs); + load_arg_stk(s, l->arg_slot, ts, *allocated_regs); } } -static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base, +static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base, intptr_t ref_off, TCGRegSet *allocated_regs) { TCGReg reg; - int stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); - if (stk_slot < 0) { + if (arg_slot_reg_p(arg_slot)) { reg = tcg_target_call_iarg_regs[arg_slot]; tcg_reg_free(s, reg, *allocated_regs); tcg_out_addi_ptr(s, reg, ref_base, ref_off); @@ -4846,8 +4876,7 @@ static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base, *allocated_regs, 0, false); tcg_out_addi_ptr(s, reg, ref_base, ref_off); tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK, - TCG_TARGET_CALL_STACK_OFFSET - + stk_slot * sizeof(tcg_target_long)); + arg_slot_stk_ofs(arg_slot)); } } @@ -4877,8 +4906,7 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) case TCG_CALL_ARG_BY_REF: load_arg_stk(s, loc->ref_slot, ts, allocated_regs); load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, - TCG_TARGET_CALL_STACK_OFFSET - + loc->ref_slot * sizeof(tcg_target_long), + arg_slot_stk_ofs(loc->ref_slot), &allocated_regs); break; case TCG_CALL_ARG_BY_REF_N: |