diff options
| -rw-r--r-- | target-arm/helper-a64.c | 113 | ||||
| -rw-r--r-- | target-arm/helper-a64.h | 2 | ||||
| -rw-r--r-- | target-arm/translate-a64.c | 106 |
3 files changed, 163 insertions, 58 deletions
diff --git a/target-arm/helper-a64.c b/target-arm/helper-a64.c index 41e48a41b4..98b97df461 100644 --- a/target-arm/helper-a64.c +++ b/target-arm/helper-a64.c @@ -27,6 +27,10 @@ #include "qemu/bitops.h" #include "internals.h" #include "qemu/crc32c.h" +#include "exec/exec-all.h" +#include "exec/cpu_ldst.h" +#include "qemu/int128.h" +#include "tcg.h" #include <zlib.h> /* For crc32 */ /* C2.4.7 Multiply and divide */ @@ -444,3 +448,112 @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes) /* Linux crc32c converts the output to one's complement. */ return crc32c(acc, buf, bytes) ^ 0xffffffff; } + +/* Returns 0 on success; 1 otherwise. */ +uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr, + uint64_t new_lo, uint64_t new_hi) +{ + uintptr_t ra = GETPC(); + Int128 oldv, cmpv, newv; + bool success; + + cmpv = int128_make128(env->exclusive_val, env->exclusive_high); + newv = int128_make128(new_lo, new_hi); + + if (parallel_cpus) { +#ifndef CONFIG_ATOMIC128 + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); + oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra); + success = int128_eq(oldv, cmpv); +#endif + } else { + uint64_t o0, o1; + +#ifdef CONFIG_USER_ONLY + /* ??? Enforce alignment. */ + uint64_t *haddr = g2h(addr); + o0 = ldq_le_p(haddr + 0); + o1 = ldq_le_p(haddr + 1); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + stq_le_p(haddr + 0, int128_getlo(newv)); + stq_le_p(haddr + 1, int128_gethi(newv)); + } +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); + TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx); + + o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra); + o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra); + helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra); + } +#endif + } + + return !success; +} + +uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr, + uint64_t new_lo, uint64_t new_hi) +{ + uintptr_t ra = GETPC(); + Int128 oldv, cmpv, newv; + bool success; + + cmpv = int128_make128(env->exclusive_val, env->exclusive_high); + newv = int128_make128(new_lo, new_hi); + + if (parallel_cpus) { +#ifndef CONFIG_ATOMIC128 + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); + oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra); + success = int128_eq(oldv, cmpv); +#endif + } else { + uint64_t o0, o1; + +#ifdef CONFIG_USER_ONLY + /* ??? Enforce alignment. */ + uint64_t *haddr = g2h(addr); + o1 = ldq_be_p(haddr + 0); + o0 = ldq_be_p(haddr + 1); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + stq_be_p(haddr + 0, int128_gethi(newv)); + stq_be_p(haddr + 1, int128_getlo(newv)); + } +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); + TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx); + + o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra); + o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra); + oldv = int128_make128(o0, o1); + + success = int128_eq(oldv, cmpv); + if (success) { + helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra); + helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra); + } +#endif + } + + return !success; +} diff --git a/target-arm/helper-a64.h b/target-arm/helper-a64.h index 1d3d10fffb..dd32000e63 100644 --- a/target-arm/helper-a64.h +++ b/target-arm/helper-a64.h @@ -46,3 +46,5 @@ DEF_HELPER_FLAGS_2(frecpx_f32, TCG_CALL_NO_RWG, f32, f32, ptr) DEF_HELPER_FLAGS_2(fcvtx_f64_to_f32, TCG_CALL_NO_RWG, f32, f64, env) DEF_HELPER_FLAGS_3(crc32_64, TCG_CALL_NO_RWG_SE, i64, i64, i64, i32) DEF_HELPER_FLAGS_3(crc32c_64, TCG_CALL_NO_RWG_SE, i64, i64, i64, i32) +DEF_HELPER_FLAGS_4(paired_cmpxchg64_le, TCG_CALL_NO_WG, i64, env, i64, i64, i64) +DEF_HELPER_FLAGS_4(paired_cmpxchg64_be, TCG_CALL_NO_WG, i64, env, i64, i64, i64) diff --git a/target-arm/translate-a64.c b/target-arm/translate-a64.c index 96c222722e..ded924a0a9 100644 --- a/target-arm/translate-a64.c +++ b/target-arm/translate-a64.c @@ -1839,37 +1839,41 @@ static void disas_b_exc_sys(DisasContext *s, uint32_t insn) } } -/* - * Load/Store exclusive instructions are implemented by remembering - * the value/address loaded, and seeing if these are the same - * when the store is performed. This is not actually the architecturally - * mandated semantics, but it works for typical guest code sequences - * and avoids having to monitor regular stores. - * - * In system emulation mode only one CPU will be running at once, so - * this sequence is effectively atomic. In user emulation mode we - * throw an exception and handle the atomic operation elsewhere. - */ static void gen_load_exclusive(DisasContext *s, int rt, int rt2, TCGv_i64 addr, int size, bool is_pair) { TCGv_i64 tmp = tcg_temp_new_i64(); - TCGMemOp memop = s->be_data + size; + TCGMemOp be = s->be_data; g_assert(size <= 3); - tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), memop); - if (is_pair) { - TCGv_i64 addr2 = tcg_temp_new_i64(); TCGv_i64 hitmp = tcg_temp_new_i64(); - g_assert(size >= 2); - tcg_gen_addi_i64(addr2, addr, 1 << size); - tcg_gen_qemu_ld_i64(hitmp, addr2, get_mem_index(s), memop); - tcg_temp_free_i64(addr2); + if (size == 3) { + TCGv_i64 addr2 = tcg_temp_new_i64(); + + tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), + MO_64 | MO_ALIGN_16 | be); + tcg_gen_addi_i64(addr2, addr, 8); + tcg_gen_qemu_ld_i64(hitmp, addr2, get_mem_index(s), + MO_64 | MO_ALIGN | be); + tcg_temp_free_i64(addr2); + } else { + g_assert(size == 2); + tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), + MO_64 | MO_ALIGN | be); + if (be == MO_LE) { + tcg_gen_extr32_i64(tmp, hitmp, tmp); + } else { + tcg_gen_extr32_i64(hitmp, tmp, tmp); + } + } + tcg_gen_mov_i64(cpu_exclusive_high, hitmp); tcg_gen_mov_i64(cpu_reg(s, rt2), hitmp); tcg_temp_free_i64(hitmp); + } else { + tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), size | MO_ALIGN | be); } tcg_gen_mov_i64(cpu_exclusive_val, tmp); @@ -1879,16 +1883,6 @@ static void gen_load_exclusive(DisasContext *s, int rt, int rt2, tcg_gen_mov_i64(cpu_exclusive_addr, addr); } -#ifdef CONFIG_USER_ONLY -static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2, - TCGv_i64 addr, int size, int is_pair) -{ - tcg_gen_mov_i64(cpu_exclusive_test, addr); - tcg_gen_movi_i32(cpu_exclusive_info, - size | is_pair << 2 | (rd << 4) | (rt << 9) | (rt2 << 14)); - gen_exception_internal_insn(s, 4, EXCP_STREX); -} -#else static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2, TCGv_i64 inaddr, int size, int is_pair) { @@ -1916,46 +1910,42 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2, tcg_gen_brcond_i64(TCG_COND_NE, addr, cpu_exclusive_addr, fail_label); tmp = tcg_temp_new_i64(); - tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), s->be_data + size); - tcg_gen_brcond_i64(TCG_COND_NE, tmp, cpu_exclusive_val, fail_label); - tcg_temp_free_i64(tmp); - - if (is_pair) { - TCGv_i64 addrhi = tcg_temp_new_i64(); - TCGv_i64 tmphi = tcg_temp_new_i64(); - - tcg_gen_addi_i64(addrhi, addr, 1 << size); - tcg_gen_qemu_ld_i64(tmphi, addrhi, get_mem_index(s), - s->be_data + size); - tcg_gen_brcond_i64(TCG_COND_NE, tmphi, cpu_exclusive_high, fail_label); - - tcg_temp_free_i64(tmphi); - tcg_temp_free_i64(addrhi); - } - - /* We seem to still have the exclusive monitor, so do the store */ - tcg_gen_qemu_st_i64(cpu_reg(s, rt), addr, get_mem_index(s), - s->be_data + size); if (is_pair) { - TCGv_i64 addrhi = tcg_temp_new_i64(); - - tcg_gen_addi_i64(addrhi, addr, 1 << size); - tcg_gen_qemu_st_i64(cpu_reg(s, rt2), addrhi, - get_mem_index(s), s->be_data + size); - tcg_temp_free_i64(addrhi); + if (size == 2) { + TCGv_i64 val = tcg_temp_new_i64(); + tcg_gen_concat32_i64(tmp, cpu_reg(s, rt), cpu_reg(s, rt2)); + tcg_gen_concat32_i64(val, cpu_exclusive_val, cpu_exclusive_high); + tcg_gen_atomic_cmpxchg_i64(tmp, addr, val, tmp, + get_mem_index(s), + size | MO_ALIGN | s->be_data); + tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, val); + tcg_temp_free_i64(val); + } else if (s->be_data == MO_LE) { + gen_helper_paired_cmpxchg64_le(tmp, cpu_env, addr, cpu_reg(s, rt), + cpu_reg(s, rt2)); + } else { + gen_helper_paired_cmpxchg64_be(tmp, cpu_env, addr, cpu_reg(s, rt), + cpu_reg(s, rt2)); + } + } else { + TCGv_i64 val = cpu_reg(s, rt); + tcg_gen_atomic_cmpxchg_i64(tmp, addr, cpu_exclusive_val, val, + get_mem_index(s), + size | MO_ALIGN | s->be_data); + tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val); } tcg_temp_free_i64(addr); - tcg_gen_movi_i64(cpu_reg(s, rd), 0); + tcg_gen_mov_i64(cpu_reg(s, rd), tmp); + tcg_temp_free_i64(tmp); tcg_gen_br(done_label); + gen_set_label(fail_label); tcg_gen_movi_i64(cpu_reg(s, rd), 1); gen_set_label(done_label); tcg_gen_movi_i64(cpu_exclusive_addr, -1); - } -#endif /* Update the Sixty-Four bit (SF) registersize. This logic is derived * from the ARMv8 specs for LDR (Shared decode for all encodings). |