diff options
Diffstat (limited to 'tcg/riscv/tcg-target.c.inc')
| -rw-r--r-- | tcg/riscv/tcg-target.c.inc | 575 |
1 files changed, 197 insertions, 378 deletions
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index 558de127ef..d12b824d8c 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -125,27 +125,9 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define TCG_CT_CONST_N12 0x400 #define TCG_CT_CONST_M12 0x800 -#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) -/* - * For softmmu, we need to avoid conflicts with the first 5 - * argument registers to call the helper. Some of these are - * also used for the tlb lookup. - */ -#ifdef CONFIG_SOFTMMU -#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_A0, 5) -#else -#define SOFTMMU_RESERVE_REGS 0 -#endif - +#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) -static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) -{ - if (TCG_TARGET_REG_BITS == 32) { - return sextract32(val, pos, len); - } else { - return sextract64(val, pos, len); - } -} +#define sextreg sextract64 /* test if a constant matches the constraint */ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) @@ -235,7 +217,6 @@ typedef enum { OPC_XOR = 0x4033, OPC_XORI = 0x4013, -#if TCG_TARGET_REG_BITS == 64 OPC_ADDIW = 0x1b, OPC_ADDW = 0x3b, OPC_DIVUW = 0x200503b, @@ -250,23 +231,6 @@ typedef enum { OPC_SRLIW = 0x501b, OPC_SRLW = 0x503b, OPC_SUBW = 0x4000003b, -#else - /* Simplify code throughout by defining aliases for RV32. */ - OPC_ADDIW = OPC_ADDI, - OPC_ADDW = OPC_ADD, - OPC_DIVUW = OPC_DIVU, - OPC_DIVW = OPC_DIV, - OPC_MULW = OPC_MUL, - OPC_REMUW = OPC_REMU, - OPC_REMW = OPC_REM, - OPC_SLLIW = OPC_SLLI, - OPC_SLLW = OPC_SLL, - OPC_SRAIW = OPC_SRAI, - OPC_SRAW = OPC_SRA, - OPC_SRLIW = OPC_SRLI, - OPC_SRLW = OPC_SRL, - OPC_SUBW = OPC_SUB, -#endif OPC_FENCE = 0x0000000f, OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */ @@ -500,7 +464,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, tcg_target_long lo, hi, tmp; int shift, ret; - if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { + if (type == TCG_TYPE_I32) { val = (int32_t)val; } @@ -511,7 +475,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, } hi = val - lo; - if (TCG_TARGET_REG_BITS == 32 || val == (int32_t)val) { + if (val == (int32_t)val) { tcg_out_opc_upper(s, OPC_LUI, rd, hi); if (lo != 0) { tcg_out_opc_imm(s, OPC_ADDIW, rd, rd, lo); @@ -519,7 +483,6 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, return; } - /* We can only be here if TCG_TARGET_REG_BITS != 32 */ tmp = tcg_pcrel_diff(s, (void *)val); if (tmp == (int32_t)tmp) { tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); @@ -561,6 +524,11 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, tcg_out_opc_imm(s, OPC_LD, rd, rd, 0); } +static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2) +{ + return false; +} + static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, tcg_target_long imm) { @@ -585,13 +553,13 @@ static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg) tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32); } -static void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg) +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) { tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24); tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24); } -static void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg) +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) { tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16); @@ -602,6 +570,23 @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg) tcg_out_opc_imm(s, OPC_ADDIW, ret, arg, 0); } +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) +{ + if (ret != arg) { + tcg_out_ext32s(s, ret, arg); + } +} + +static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_ext32u(s, ret, arg); +} + +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_ext32s(s, ret, arg); +} + static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data, TCGReg addr, intptr_t offset) { @@ -646,15 +631,15 @@ static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data, static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2) { - bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); - tcg_out_ldst(s, is32bit ? OPC_LW : OPC_LD, arg, arg1, arg2); + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_LW : OPC_LD; + tcg_out_ldst(s, insn, arg, arg1, arg2); } static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2) { - bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); - tcg_out_ldst(s, is32bit ? OPC_SW : OPC_SD, arg, arg1, arg2); + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SW : OPC_SD; + tcg_out_ldst(s, insn, arg, arg1, arg2); } static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, @@ -807,20 +792,6 @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, } } -static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, - TCGReg bl, TCGReg bh, TCGLabel *l) -{ - /* todo */ - g_assert_not_reached(); -} - -static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) -{ - /* todo */ - g_assert_not_reached(); -} - static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) { TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; @@ -831,20 +802,18 @@ static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) if (offset == sextreg(offset, 0, 20)) { /* short jump: -2097150 to 2097152 */ tcg_out_opc_jump(s, OPC_JAL, link, offset); - } else if (TCG_TARGET_REG_BITS == 32 || offset == (int32_t)offset) { + } else if (offset == (int32_t)offset) { /* long jump: -2147483646 to 2147483648 */ tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0); tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0); ret = reloc_call(s->code_ptr - 2, arg); tcg_debug_assert(ret == true); - } else if (TCG_TARGET_REG_BITS == 64) { + } else { /* far jump: 64-bit */ tcg_target_long imm = sextreg((tcg_target_long)arg, 0, 12); tcg_target_long base = (tcg_target_long)arg - imm; tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, base); tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, imm); - } else { - g_assert_not_reached(); } } @@ -920,13 +889,6 @@ static void * const qemu_st_helpers[MO_SIZE + 1] = { #endif }; -/* We don't support oversize guests */ -QEMU_BUILD_BUG_ON(TCG_TARGET_REG_BITS < TARGET_LONG_BITS); - -/* We expect to use a 12-bit negative offset from ENV. */ -QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); -QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); - static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) { tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); @@ -934,93 +896,14 @@ static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) tcg_debug_assert(ok); } -static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, - TCGReg addrh, MemOpIdx oi, - tcg_insn_unit **label_ptr, bool is_load) -{ - MemOp opc = get_memop(oi); - unsigned s_bits = opc & MO_SIZE; - unsigned a_bits = get_alignment_bits(opc); - tcg_target_long compare_mask; - int mem_index = get_mmuidx(oi); - int fast_ofs = TLB_MASK_TABLE_OFS(mem_index); - int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); - int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); - TCGReg mask_base = TCG_AREG0, table_base = TCG_AREG0; - - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, mask_base, mask_ofs); - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, table_base, table_ofs); - - tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addrl, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); - tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); - - /* Load the tlb comparator and the addend. */ - tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP0, TCG_REG_TMP2, - is_load ? offsetof(CPUTLBEntry, addr_read) - : offsetof(CPUTLBEntry, addr_write)); - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2, - offsetof(CPUTLBEntry, addend)); - - /* We don't support unaligned accesses. */ - if (a_bits < s_bits) { - a_bits = s_bits; - } - /* Clear the non-page, non-alignment bits from the address. */ - compare_mask = (tcg_target_long)TARGET_PAGE_MASK | ((1 << a_bits) - 1); - if (compare_mask == sextreg(compare_mask, 0, 12)) { - tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addrl, compare_mask); - } else { - tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); - tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addrl); - } - - /* Compare masked address with the TLB entry. */ - label_ptr[0] = s->code_ptr; - tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); - - /* TLB Hit - translate address using addend. */ - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, TCG_REG_TMP0, addrl); - addrl = TCG_REG_TMP0; - } - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addrl); - return TCG_REG_TMP0; -} - -static void add_qemu_ldst_label(TCGContext *s, int is_ld, MemOpIdx oi, - TCGType ext, - TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addrhi, - void *raddr, tcg_insn_unit **label_ptr) -{ - TCGLabelQemuLdst *label = new_ldst_label(s); - - label->is_ld = is_ld; - label->oi = oi; - label->type = ext; - label->datalo_reg = datalo; - label->datahi_reg = datahi; - label->addrlo_reg = addrlo; - label->addrhi_reg = addrhi; - label->raddr = tcg_splitwx_to_rx(raddr); - label->label_ptr[0] = label_ptr[0]; -} +/* We have three temps, we might as well expose them. */ +static const TCGLdstHelperParam ldst_helper_param = { + .ntmp = 3, .tmp = { TCG_REG_TMP0, TCG_REG_TMP1, TCG_REG_TMP2 } +}; static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { - MemOpIdx oi = l->oi; - MemOp opc = get_memop(oi); - TCGReg a0 = tcg_target_call_iarg_regs[0]; - TCGReg a1 = tcg_target_call_iarg_regs[1]; - TCGReg a2 = tcg_target_call_iarg_regs[2]; - TCGReg a3 = tcg_target_call_iarg_regs[3]; - - /* We don't support oversize guests */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - g_assert_not_reached(); - } + MemOp opc = get_memop(l->oi); /* resolve label address */ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { @@ -1028,13 +911,9 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } /* call load helper */ - tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0); - tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg); - tcg_out_movi(s, TCG_TYPE_PTR, a2, oi); - tcg_out_movi(s, TCG_TYPE_PTR, a3, (tcg_target_long)l->raddr); - + tcg_out_ld_helper_args(s, l, &ldst_helper_param); tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SSIZE], false); - tcg_out_mov(s, (opc & MO_SIZE) == MO_64, l->datalo_reg, a0); + tcg_out_ld_helper_ret(s, l, true, &ldst_helper_param); tcg_out_goto(s, l->raddr); return true; @@ -1042,19 +921,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { - MemOpIdx oi = l->oi; - MemOp opc = get_memop(oi); - MemOp s_bits = opc & MO_SIZE; - TCGReg a0 = tcg_target_call_iarg_regs[0]; - TCGReg a1 = tcg_target_call_iarg_regs[1]; - TCGReg a2 = tcg_target_call_iarg_regs[2]; - TCGReg a3 = tcg_target_call_iarg_regs[3]; - TCGReg a4 = tcg_target_call_iarg_regs[4]; - - /* We don't support oversize guests */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - g_assert_not_reached(); - } + MemOp opc = get_memop(l->oi); /* resolve label address */ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { @@ -1062,48 +929,13 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } /* call store helper */ - tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0); - tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg); - tcg_out_mov(s, TCG_TYPE_PTR, a2, l->datalo_reg); - switch (s_bits) { - case MO_8: - tcg_out_ext8u(s, a2, a2); - break; - case MO_16: - tcg_out_ext16u(s, a2, a2); - break; - default: - break; - } - tcg_out_movi(s, TCG_TYPE_PTR, a3, oi); - tcg_out_movi(s, TCG_TYPE_PTR, a4, (tcg_target_long)l->raddr); - + tcg_out_st_helper_args(s, l, &ldst_helper_param); tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE], false); tcg_out_goto(s, l->raddr); return true; } #else - -static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addr_reg, - unsigned a_bits) -{ - unsigned a_mask = (1 << a_bits) - 1; - TCGLabelQemuLdst *l = new_ldst_label(s); - - l->is_ld = is_ld; - l->addrlo_reg = addr_reg; - - /* We are expecting a_bits to max out at 7, so we can always use andi. */ - tcg_debug_assert(a_bits < 12); - tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, a_mask); - - l->label_ptr[0] = s->code_ptr; - tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP1, TCG_REG_ZERO, 0); - - l->raddr = tcg_splitwx_to_rx(s->code_ptr); -} - static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) { /* resolve label address */ @@ -1130,100 +962,161 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { return tcg_out_fail_alignment(s, l); } - #endif /* CONFIG_SOFTMMU */ -static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, - TCGReg base, MemOp opc, bool is_64) +/* + * For softmmu, perform the TLB load and compare. + * For useronly, perform any required alignment tests. + * In both cases, return a TCGLabelQemuLdst structure if the slow path + * is required and fill in @h with the host address for the fast path. + */ +static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, + TCGReg addr_reg, MemOpIdx oi, + bool is_ld) +{ + TCGLabelQemuLdst *ldst = NULL; + MemOp opc = get_memop(oi); + unsigned a_bits = get_alignment_bits(opc); + unsigned a_mask = (1u << a_bits) - 1; + +#ifdef CONFIG_SOFTMMU + unsigned s_bits = opc & MO_SIZE; + int mem_index = get_mmuidx(oi); + int fast_ofs = TLB_MASK_TABLE_OFS(mem_index); + int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); + int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); + TCGReg mask_base = TCG_AREG0, table_base = TCG_AREG0; + tcg_target_long compare_mask; + + ldst = new_ldst_label(s); + ldst->is_ld = is_ld; + ldst->oi = oi; + ldst->addrlo_reg = addr_reg; + + QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); + QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, mask_base, mask_ofs); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, table_base, table_ofs); + + tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addr_reg, + TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); + + /* Load the tlb comparator and the addend. */ + tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP0, TCG_REG_TMP2, + is_ld ? offsetof(CPUTLBEntry, addr_read) + : offsetof(CPUTLBEntry, addr_write)); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2, + offsetof(CPUTLBEntry, addend)); + + /* We don't support unaligned accesses. */ + if (a_bits < s_bits) { + a_bits = s_bits; + } + /* Clear the non-page, non-alignment bits from the address. */ + compare_mask = (tcg_target_long)TARGET_PAGE_MASK | a_mask; + if (compare_mask == sextreg(compare_mask, 0, 12)) { + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, compare_mask); + } else { + tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); + tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addr_reg); + } + + /* Compare masked address with the TLB entry. */ + ldst->label_ptr[0] = s->code_ptr; + tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); + + /* TLB Hit - translate address using addend. */ + if (TARGET_LONG_BITS == 32) { + tcg_out_ext32u(s, TCG_REG_TMP0, addr_reg); + addr_reg = TCG_REG_TMP0; + } + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addr_reg); + *pbase = TCG_REG_TMP0; +#else + if (a_mask) { + ldst = new_ldst_label(s); + ldst->is_ld = is_ld; + ldst->oi = oi; + ldst->addrlo_reg = addr_reg; + + /* We are expecting a_bits max 7, so we can always use andi. */ + tcg_debug_assert(a_bits < 12); + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, a_mask); + + ldst->label_ptr[0] = s->code_ptr; + tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP1, TCG_REG_ZERO, 0); + } + + TCGReg base = addr_reg; + if (TARGET_LONG_BITS == 32) { + tcg_out_ext32u(s, TCG_REG_TMP0, base); + base = TCG_REG_TMP0; + } + if (guest_base != 0) { + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); + base = TCG_REG_TMP0; + } + *pbase = base; +#endif + + return ldst; +} + +static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val, + TCGReg base, MemOp opc, TCGType type) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & (MO_SSIZE)) { case MO_UB: - tcg_out_opc_imm(s, OPC_LBU, lo, base, 0); + tcg_out_opc_imm(s, OPC_LBU, val, base, 0); break; case MO_SB: - tcg_out_opc_imm(s, OPC_LB, lo, base, 0); + tcg_out_opc_imm(s, OPC_LB, val, base, 0); break; case MO_UW: - tcg_out_opc_imm(s, OPC_LHU, lo, base, 0); + tcg_out_opc_imm(s, OPC_LHU, val, base, 0); break; case MO_SW: - tcg_out_opc_imm(s, OPC_LH, lo, base, 0); + tcg_out_opc_imm(s, OPC_LH, val, base, 0); break; case MO_UL: - if (TCG_TARGET_REG_BITS == 64 && is_64) { - tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); + if (type == TCG_TYPE_I64) { + tcg_out_opc_imm(s, OPC_LWU, val, base, 0); break; } /* FALLTHRU */ case MO_SL: - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); + tcg_out_opc_imm(s, OPC_LW, val, base, 0); break; case MO_UQ: - /* Prefer to load from offset 0 first, but allow for overlap. */ - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_opc_imm(s, OPC_LD, lo, base, 0); - } else if (lo != base) { - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); - tcg_out_opc_imm(s, OPC_LW, hi, base, 4); - } else { - tcg_out_opc_imm(s, OPC_LW, hi, base, 4); - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); - } + tcg_out_opc_imm(s, OPC_LD, val, base, 0); break; default: g_assert_not_reached(); } } -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[1]; -#else - unsigned a_bits; -#endif + TCGLabelQemuLdst *ldst; TCGReg base; - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); + ldst = prepare_host_addr(s, &base, addr_reg, oi, true); + tcg_out_qemu_ld_direct(s, data_reg, base, get_memop(oi), data_type); -#if defined(CONFIG_SOFTMMU) - base = tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 1); - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); - add_qemu_ldst_label(s, 1, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); -#else - a_bits = get_alignment_bits(opc); - if (a_bits) { - tcg_out_test_alignment(s, true, addr_regl, a_bits); + if (ldst) { + ldst->type = data_type; + ldst->datalo_reg = data_reg; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } - base = addr_regl; - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, TCG_REG_TMP0, base); - base = TCG_REG_TMP0; - } - if (guest_base != 0) { - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); - base = TCG_REG_TMP0; - } - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); -#endif } -static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, +static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg val, TCGReg base, MemOp opc) { /* Byte swapping is left to middle-end expansion. */ @@ -1231,70 +1124,36 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, switch (opc & (MO_SSIZE)) { case MO_8: - tcg_out_opc_store(s, OPC_SB, base, lo, 0); + tcg_out_opc_store(s, OPC_SB, base, val, 0); break; case MO_16: - tcg_out_opc_store(s, OPC_SH, base, lo, 0); + tcg_out_opc_store(s, OPC_SH, base, val, 0); break; case MO_32: - tcg_out_opc_store(s, OPC_SW, base, lo, 0); + tcg_out_opc_store(s, OPC_SW, base, val, 0); break; case MO_64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_opc_store(s, OPC_SD, base, lo, 0); - } else { - tcg_out_opc_store(s, OPC_SW, base, lo, 0); - tcg_out_opc_store(s, OPC_SW, base, hi, 4); - } + tcg_out_opc_store(s, OPC_SD, base, val, 0); break; default: g_assert_not_reached(); } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - MemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[1]; -#else - unsigned a_bits; -#endif + TCGLabelQemuLdst *ldst; TCGReg base; - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); + ldst = prepare_host_addr(s, &base, addr_reg, oi, false); + tcg_out_qemu_st_direct(s, data_reg, base, get_memop(oi)); -#if defined(CONFIG_SOFTMMU) - base = tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 0); - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); - add_qemu_ldst_label(s, 0, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); -#else - a_bits = get_alignment_bits(opc); - if (a_bits) { - tcg_out_test_alignment(s, false, addr_regl, a_bits); + if (ldst) { + ldst->type = data_type; + ldst->datalo_reg = data_reg; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } - base = addr_regl; - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, TCG_REG_TMP0, base); - base = TCG_REG_TMP0; - } - if (guest_base != 0) { - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); - base = TCG_REG_TMP0; - } - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); -#endif } static const tcg_insn_unit *tb_ret_addr; @@ -1572,60 +1431,23 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_brcond_i64: tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); break; - case INDEX_op_brcond2_i32: - tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], arg_label(args[5])); - break; case INDEX_op_setcond_i32: case INDEX_op_setcond_i64: tcg_out_setcond(s, args[3], a0, a1, a2); break; - case INDEX_op_setcond2_i32: - tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]); - break; case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, false); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, true); + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, false); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); break; case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, true); - break; - - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - tcg_out_ext8u(s, a0, a1); - break; - - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - tcg_out_ext16u(s, a0, a1); - break; - - case INDEX_op_ext32u_i64: - case INDEX_op_extu_i32_i64: - tcg_out_ext32u(s, a0, a1); - break; - - case INDEX_op_ext8s_i32: - case INDEX_op_ext8s_i64: - tcg_out_ext8s(s, a0, a1); - break; - - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - tcg_out_ext16s(s, a0, a1); - break; - - case INDEX_op_ext32s_i64: - case INDEX_op_extrl_i64_i32: - case INDEX_op_ext_i32_i64: - tcg_out_ext32s(s, a0, a1); + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_extrh_i64_i32: @@ -1651,6 +1473,19 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_call: /* Always emitted via tcg_out_call. */ case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ + case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ + case INDEX_op_ext8s_i64: + case INDEX_op_ext8u_i32: + case INDEX_op_ext8u_i64: + case INDEX_op_ext16s_i32: + case INDEX_op_ext16s_i64: + case INDEX_op_ext16u_i32: + case INDEX_op_ext16u_i64: + case INDEX_op_ext32s_i64: + case INDEX_op_ext32u_i64: + case INDEX_op_ext_i32_i64: + case INDEX_op_extu_i32_i64: + case INDEX_op_extrl_i64_i32: default: g_assert_not_reached(); } @@ -1753,26 +1588,12 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_sub2_i64: return C_O2_I4(r, r, rZ, rZ, rM, rM); - case INDEX_op_brcond2_i32: - return C_O0_I4(rZ, rZ, rZ, rZ); - - case INDEX_op_setcond2_i32: - return C_O1_I4(r, rZ, rZ, rZ, rZ); - case INDEX_op_qemu_ld_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); - case INDEX_op_qemu_st_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O0_I2(LZ, L) : C_O0_I3(LZ, L, L)); case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L) - : C_O2_I2(r, r, L, L)); + return C_O1_I1(r, r); + case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(LZ, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(LZ, LZ, L) - : C_O0_I4(LZ, LZ, L, L)); + return C_O0_I2(rZ, r); default: g_assert_not_reached(); @@ -1848,9 +1669,7 @@ static void tcg_target_qemu_prologue(TCGContext *s) static void tcg_target_init(TCGContext *s) { tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; - if (TCG_TARGET_REG_BITS == 64) { - tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; - } + tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; tcg_target_call_clobber_regs = -1u; tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S0); |