diff options
| -rw-r--r-- | accel/tcg/translate-all.c | 2 | ||||
| -rw-r--r-- | tcg/aarch64/tcg-target.inc.c | 79 | ||||
| -rw-r--r-- | tcg/mips/tcg-target.inc.c | 8 | ||||
| -rw-r--r-- | util/cacheinfo.c | 2 |
4 files changed, 70 insertions, 21 deletions
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index dfb9f0de46..0caf80db75 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -504,7 +504,7 @@ static inline PageDesc *page_find(tb_page_addr_t index) #elif defined(__powerpc__) # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) #elif defined(__aarch64__) -# define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) +# define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) #elif defined(__s390x__) /* We have a +- 4GB range on the branches; leave some slop. */ # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) diff --git a/tcg/aarch64/tcg-target.inc.c b/tcg/aarch64/tcg-target.inc.c index 1fa3bccc89..04bc369a92 100644 --- a/tcg/aarch64/tcg-target.inc.c +++ b/tcg/aarch64/tcg-target.inc.c @@ -269,6 +269,8 @@ typedef enum { I3207_BLR = 0xd63f0000, I3207_RET = 0xd65f0000, + /* Load literal for loading the address at pc-relative offset */ + I3305_LDR = 0x58000000, /* Load/store register. Described here as 3.3.12, but the helper that emits them can transform to 3.3.10 or 3.3.13. */ I3312_STRB = 0x38000000 | LDST_ST << 22 | MO_8 << 30, @@ -372,6 +374,7 @@ typedef enum { I3510_EON = 0x4a200000, I3510_ANDS = 0x6a000000, + NOP = 0xd503201f, /* System instructions. */ DMB_ISH = 0xd50338bf, DMB_LD = 0x00000100, @@ -388,6 +391,11 @@ static inline uint32_t tcg_in32(TCGContext *s) #define tcg_out_insn(S, FMT, OP, ...) \ glue(tcg_out_insn_,FMT)(S, glue(glue(glue(I,FMT),_),OP), ## __VA_ARGS__) +static void tcg_out_insn_3305(TCGContext *s, AArch64Insn insn, int imm19, TCGReg rt) +{ + tcg_out32(s, insn | (imm19 & 0x7ffff) << 5 | rt); +} + static void tcg_out_insn_3201(TCGContext *s, AArch64Insn insn, TCGType ext, TCGReg rt, int imm19) { @@ -819,6 +827,17 @@ static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target) tcg_out_insn(s, 3206, B, offset); } +static inline void tcg_out_goto_long(TCGContext *s, tcg_insn_unit *target) +{ + ptrdiff_t offset = target - s->code_ptr; + if (offset == sextract64(offset, 0, 26)) { + tcg_out_insn(s, 3206, BL, offset); + } else { + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target); + tcg_out_insn(s, 3207, BR, TCG_REG_TMP); + } +} + static inline void tcg_out_goto_noaddr(TCGContext *s) { /* We pay attention here to not modify the branch target by reading from @@ -852,15 +871,35 @@ static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target) } } +#ifdef USE_DIRECT_JUMP + void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr) { - tcg_insn_unit *code_ptr = (tcg_insn_unit *)jmp_addr; - tcg_insn_unit *target = (tcg_insn_unit *)addr; + tcg_insn_unit i1, i2; + TCGType rt = TCG_TYPE_I64; + TCGReg rd = TCG_REG_TMP; + uint64_t pair; - reloc_pc26_atomic(code_ptr, target); - flush_icache_range(jmp_addr, jmp_addr + 4); + ptrdiff_t offset = addr - jmp_addr; + + if (offset == sextract64(offset, 0, 26)) { + i1 = I3206_B | ((offset >> 2) & 0x3ffffff); + i2 = NOP; + } else { + offset = (addr >> 12) - (jmp_addr >> 12); + + /* patch ADRP */ + i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd; + /* patch ADDI */ + i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd; + } + pair = (uint64_t)i2 << 32 | i1; + atomic_set((uint64_t *)jmp_addr, pair); + flush_icache_range(jmp_addr, jmp_addr + 8); } +#endif + static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l) { if (!l->has_value) { @@ -1364,23 +1403,33 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_exit_tb: /* Reuse the zeroing that exists for goto_ptr. */ if (a0 == 0) { - tcg_out_goto(s, s->code_gen_epilogue); + tcg_out_goto_long(s, s->code_gen_epilogue); } else { tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0); - tcg_out_goto(s, tb_ret_addr); + tcg_out_goto_long(s, tb_ret_addr); } break; case INDEX_op_goto_tb: -#ifndef USE_DIRECT_JUMP -#error "USE_DIRECT_JUMP required for aarch64" -#endif - /* consistency for USE_DIRECT_JUMP */ - tcg_debug_assert(s->tb_jmp_insn_offset != NULL); - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); - /* actual branch destination will be patched by - aarch64_tb_set_jmp_target later, beware retranslation. */ - tcg_out_goto_noaddr(s); + if (s->tb_jmp_insn_offset != NULL) { + /* USE_DIRECT_JUMP */ + /* Ensure that ADRP+ADD are 8-byte aligned so that an atomic + write can be used to patch the target address. */ + if ((uintptr_t)s->code_ptr & 7) { + tcg_out32(s, NOP); + } + s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); + /* actual branch destination will be patched by + aarch64_tb_set_jmp_target later. */ + tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0); + tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0); + } else { + /* !USE_DIRECT_JUMP */ + tcg_debug_assert(s->tb_jmp_target_addr != NULL); + intptr_t offset = tcg_pcrel_diff(s, (s->tb_jmp_target_addr + a0)) >> 2; + tcg_out_insn(s, 3305, LDR, offset, TCG_REG_TMP); + } + tcg_out_insn(s, 3207, BR, TCG_REG_TMP); s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s); break; diff --git a/tcg/mips/tcg-target.inc.c b/tcg/mips/tcg-target.inc.c index 8cff9a6bf9..85756b81d5 100644 --- a/tcg/mips/tcg-target.inc.c +++ b/tcg/mips/tcg-target.inc.c @@ -1547,8 +1547,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) } else if (guest_base == (int16_t)guest_base) { tcg_out_opc_imm(s, ALIAS_PADDI, base, addr_regl, guest_base); } else { - tcg_out_movi(s, TCG_TYPE_PTR, base, guest_base); - tcg_out_opc_reg(s, ALIAS_PADD, base, base, addr_regl); + tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, guest_base); + tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_TMP0, addr_regl); } tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); #endif @@ -1652,8 +1652,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) } else if (guest_base == (int16_t)guest_base) { tcg_out_opc_imm(s, ALIAS_PADDI, base, addr_regl, guest_base); } else { - tcg_out_movi(s, TCG_TYPE_PTR, base, guest_base); - tcg_out_opc_reg(s, ALIAS_PADD, base, base, addr_regl); + tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, guest_base); + tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_TMP0, addr_regl); } tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); #endif diff --git a/util/cacheinfo.c b/util/cacheinfo.c index f987522df4..6253049533 100644 --- a/util/cacheinfo.c +++ b/util/cacheinfo.c @@ -112,7 +112,7 @@ static void sys_cache_info(int *isize, int *dsize) static void arch_cache_info(int *isize, int *dsize) { if (*isize == 0 || *dsize == 0) { - unsigned ctr; + unsigned long ctr; /* The real cache geometry is in CCSIDR_EL1/CLIDR_EL1/CSSELR_EL1, but (at least under Linux) these are marked protected by the |