diff options
Diffstat (limited to 'accel/tcg/atomic_template.h')
| -rw-r--r-- | accel/tcg/atomic_template.h | 141 |
1 files changed, 62 insertions, 79 deletions
diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index afa8a9daf3..d89af4cc1e 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -28,8 +28,8 @@ # define SHIFT 4 #elif DATA_SIZE == 8 # define SUFFIX q -# define DATA_TYPE uint64_t -# define SDATA_TYPE int64_t +# define DATA_TYPE aligned_uint64_t +# define SDATA_TYPE aligned_int64_t # define BSWAP bswap64 # define SHIFT 3 #elif DATA_SIZE == 4 @@ -71,15 +71,14 @@ #endif ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, - ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) + ABI_TYPE cmpv, ABI_TYPE newv, + TCGMemOpIdx oi, uintptr_t retaddr) { - ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, + PAGE_READ | PAGE_WRITE, retaddr); DATA_TYPE ret; - uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, - ATOMIC_MMU_IDX); + uint16_t info = atomic_trace_rmw_pre(env, addr, oi); - atomic_trace_rmw_pre(env, addr, info); #if DATA_SIZE == 16 ret = atomic16_cmpxchg(haddr, cmpv, newv); #else @@ -92,45 +91,41 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, #if DATA_SIZE >= 16 #if HAVE_ATOMIC128 -ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) +ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, + TCGMemOpIdx oi, uintptr_t retaddr) { - ATOMIC_MMU_DECLS; - DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP_R; - uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, - ATOMIC_MMU_IDX); + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, + PAGE_READ, retaddr); + DATA_TYPE val; + uint16_t info = atomic_trace_ld_pre(env, addr, oi); - atomic_trace_ld_pre(env, addr, info); val = atomic16_read(haddr); ATOMIC_MMU_CLEANUP; atomic_trace_ld_post(env, addr, info); return val; } -void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, - ABI_TYPE val EXTRA_ARGS) +void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, + TCGMemOpIdx oi, uintptr_t retaddr) { - ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_W; - uint16_t info = trace_mem_build_info(SHIFT, false, 0, true, - ATOMIC_MMU_IDX); + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, + PAGE_WRITE, retaddr); + uint16_t info = atomic_trace_st_pre(env, addr, oi); - atomic_trace_st_pre(env, addr, info); atomic16_set(haddr, val); ATOMIC_MMU_CLEANUP; atomic_trace_st_post(env, addr, info); } #endif #else -ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, - ABI_TYPE val EXTRA_ARGS) +ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, + TCGMemOpIdx oi, uintptr_t retaddr) { - ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, + PAGE_READ | PAGE_WRITE, retaddr); DATA_TYPE ret; - uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, - ATOMIC_MMU_IDX); + uint16_t info = atomic_trace_rmw_pre(env, addr, oi); - atomic_trace_rmw_pre(env, addr, info); ret = qatomic_xchg__nocheck(haddr, val); ATOMIC_MMU_CLEANUP; atomic_trace_rmw_post(env, addr, info); @@ -139,14 +134,12 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, #define GEN_ATOMIC_HELPER(X) \ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ - ABI_TYPE val EXTRA_ARGS) \ + ABI_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) \ { \ - ATOMIC_MMU_DECLS; \ - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ + PAGE_READ | PAGE_WRITE, retaddr); \ DATA_TYPE ret; \ - uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ - ATOMIC_MMU_IDX); \ - atomic_trace_rmw_pre(env, addr, info); \ + uint16_t info = atomic_trace_rmw_pre(env, addr, oi); \ ret = qatomic_##X(haddr, val); \ ATOMIC_MMU_CLEANUP; \ atomic_trace_rmw_post(env, addr, info); \ @@ -164,7 +157,8 @@ GEN_ATOMIC_HELPER(xor_fetch) #undef GEN_ATOMIC_HELPER -/* These helpers are, as a whole, full barriers. Within the helper, +/* + * These helpers are, as a whole, full barriers. Within the helper, * the leading barrier is explicit and the trailing barrier is within * cmpxchg primitive. * @@ -173,14 +167,12 @@ GEN_ATOMIC_HELPER(xor_fetch) */ #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ - ABI_TYPE xval EXTRA_ARGS) \ + ABI_TYPE xval, TCGMemOpIdx oi, uintptr_t retaddr) \ { \ - ATOMIC_MMU_DECLS; \ - XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ + XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ + PAGE_READ | PAGE_WRITE, retaddr); \ XDATA_TYPE cmp, old, new, val = xval; \ - uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ - ATOMIC_MMU_IDX); \ - atomic_trace_rmw_pre(env, addr, info); \ + uint16_t info = atomic_trace_rmw_pre(env, addr, oi); \ smp_mb(); \ cmp = qatomic_read__nocheck(haddr); \ do { \ @@ -218,15 +210,14 @@ GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) #endif ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, - ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) + ABI_TYPE cmpv, ABI_TYPE newv, + TCGMemOpIdx oi, uintptr_t retaddr) { - ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, + PAGE_READ | PAGE_WRITE, retaddr); DATA_TYPE ret; - uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, - ATOMIC_MMU_IDX); + uint16_t info = atomic_trace_rmw_pre(env, addr, oi); - atomic_trace_rmw_pre(env, addr, info); #if DATA_SIZE == 16 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); #else @@ -239,30 +230,28 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, #if DATA_SIZE >= 16 #if HAVE_ATOMIC128 -ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) +ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, + TCGMemOpIdx oi, uintptr_t retaddr) { - ATOMIC_MMU_DECLS; - DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP_R; - uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, - ATOMIC_MMU_IDX); + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, + PAGE_READ, retaddr); + DATA_TYPE val; + uint16_t info = atomic_trace_ld_pre(env, addr, oi); - atomic_trace_ld_pre(env, addr, info); val = atomic16_read(haddr); ATOMIC_MMU_CLEANUP; atomic_trace_ld_post(env, addr, info); return BSWAP(val); } -void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, - ABI_TYPE val EXTRA_ARGS) +void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, + TCGMemOpIdx oi, uintptr_t retaddr) { - ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_W; - uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true, - ATOMIC_MMU_IDX); + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, + PAGE_WRITE, retaddr); + uint16_t info = atomic_trace_st_pre(env, addr, oi); val = BSWAP(val); - atomic_trace_st_pre(env, addr, info); val = BSWAP(val); atomic16_set(haddr, val); ATOMIC_MMU_CLEANUP; @@ -270,16 +259,14 @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, } #endif #else -ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, - ABI_TYPE val EXTRA_ARGS) +ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, + TCGMemOpIdx oi, uintptr_t retaddr) { - ATOMIC_MMU_DECLS; - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, + PAGE_READ | PAGE_WRITE, retaddr); ABI_TYPE ret; - uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, - ATOMIC_MMU_IDX); + uint16_t info = atomic_trace_rmw_pre(env, addr, oi); - atomic_trace_rmw_pre(env, addr, info); ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); ATOMIC_MMU_CLEANUP; atomic_trace_rmw_post(env, addr, info); @@ -288,14 +275,12 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, #define GEN_ATOMIC_HELPER(X) \ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ - ABI_TYPE val EXTRA_ARGS) \ + ABI_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) \ { \ - ATOMIC_MMU_DECLS; \ - DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ + PAGE_READ | PAGE_WRITE, retaddr); \ DATA_TYPE ret; \ - uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ - false, ATOMIC_MMU_IDX); \ - atomic_trace_rmw_pre(env, addr, info); \ + uint16_t info = atomic_trace_rmw_pre(env, addr, oi); \ ret = qatomic_##X(haddr, BSWAP(val)); \ ATOMIC_MMU_CLEANUP; \ atomic_trace_rmw_post(env, addr, info); \ @@ -320,14 +305,12 @@ GEN_ATOMIC_HELPER(xor_fetch) */ #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ - ABI_TYPE xval EXTRA_ARGS) \ + ABI_TYPE xval, TCGMemOpIdx oi, uintptr_t retaddr) \ { \ - ATOMIC_MMU_DECLS; \ - XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ + XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ + PAGE_READ | PAGE_WRITE, retaddr); \ XDATA_TYPE ldo, ldn, old, new, val = xval; \ - uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ - false, ATOMIC_MMU_IDX); \ - atomic_trace_rmw_pre(env, addr, info); \ + uint16_t info = atomic_trace_rmw_pre(env, addr, oi); \ smp_mb(); \ ldn = qatomic_read__nocheck(haddr); \ do { \ |