diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/exec/cpu-all.h | 1 | ||||
| -rw-r--r-- | include/exec/exec-all.h | 1 | ||||
| -rw-r--r-- | include/qemu-common.h | 1 | ||||
| -rw-r--r-- | include/qemu/atomic.h | 67 | ||||
| -rw-r--r-- | include/qemu/int128.h | 171 |
5 files changed, 221 insertions, 20 deletions
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index 861260d3db..e9004e5798 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -31,6 +31,7 @@ #define EXCP_DEBUG 0x10002 /* cpu stopped after a breakpoint or singlestep */ #define EXCP_HALTED 0x10003 /* cpu is halted (waiting for external event) */ #define EXCP_YIELD 0x10004 /* cpu wants to yield timeslice to another */ +#define EXCP_ATOMIC 0x10005 /* stop-the-world and emulate atomic */ /* some important defines: * diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 9797d556e8..cb624e4acc 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -59,6 +59,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, void QEMU_NORETURN cpu_loop_exit(CPUState *cpu); void QEMU_NORETURN cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc); +void QEMU_NORETURN cpu_loop_exit_atomic(CPUState *cpu, uintptr_t pc); #if !defined(CONFIG_USER_ONLY) void cpu_reloading_memory_map(void); diff --git a/include/qemu-common.h b/include/qemu-common.h index 7e6e4feb4b..1430390eb6 100644 --- a/include/qemu-common.h +++ b/include/qemu-common.h @@ -80,6 +80,7 @@ void tcg_exec_init(unsigned long tb_size); bool tcg_enabled(void); void cpu_exec_init_all(void); +void cpu_exec_step_atomic(CPUState *cpu); /** * set_preferred_target_page_bits: diff --git a/include/qemu/atomic.h b/include/qemu/atomic.h index c09fce704f..878fa0700d 100644 --- a/include/qemu/atomic.h +++ b/include/qemu/atomic.h @@ -99,15 +99,21 @@ * no effect on the generated code but not using the atomic primitives * will get flagged by sanitizers as a violation. */ +#define atomic_read__nocheck(ptr) \ + __atomic_load_n(ptr, __ATOMIC_RELAXED) + #define atomic_read(ptr) \ ({ \ QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \ - __atomic_load_n(ptr, __ATOMIC_RELAXED); \ + atomic_read__nocheck(ptr); \ }) +#define atomic_set__nocheck(ptr, i) \ + __atomic_store_n(ptr, i, __ATOMIC_RELAXED) + #define atomic_set(ptr, i) do { \ QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \ - __atomic_store_n(ptr, i, __ATOMIC_RELAXED); \ + atomic_set__nocheck(ptr, i); \ } while(0) /* See above: most compilers currently treat consume and acquire the @@ -151,20 +157,27 @@ /* All the remaining operations are fully sequentially consistent */ +#define atomic_xchg__nocheck(ptr, i) ({ \ + __atomic_exchange_n(ptr, (i), __ATOMIC_SEQ_CST); \ +}) + #define atomic_xchg(ptr, i) ({ \ QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \ - __atomic_exchange_n(ptr, i, __ATOMIC_SEQ_CST); \ + atomic_xchg__nocheck(ptr, i); \ }) /* Returns the eventual value, failed or not */ -#define atomic_cmpxchg(ptr, old, new) \ - ({ \ - QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \ +#define atomic_cmpxchg__nocheck(ptr, old, new) ({ \ typeof_strip_qual(*ptr) _old = (old); \ __atomic_compare_exchange_n(ptr, &_old, new, false, \ __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \ _old; \ - }) +}) + +#define atomic_cmpxchg(ptr, old, new) ({ \ + QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \ + atomic_cmpxchg__nocheck(ptr, old, new); \ +}) /* Provide shorter names for GCC atomic builtins, return old value */ #define atomic_fetch_inc(ptr) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST) @@ -173,6 +186,15 @@ #define atomic_fetch_sub(ptr, n) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST) #define atomic_fetch_and(ptr, n) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST) #define atomic_fetch_or(ptr, n) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST) +#define atomic_fetch_xor(ptr, n) __atomic_fetch_xor(ptr, n, __ATOMIC_SEQ_CST) + +#define atomic_inc_fetch(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_SEQ_CST) +#define atomic_dec_fetch(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_SEQ_CST) +#define atomic_add_fetch(ptr, n) __atomic_add_fetch(ptr, n, __ATOMIC_SEQ_CST) +#define atomic_sub_fetch(ptr, n) __atomic_sub_fetch(ptr, n, __ATOMIC_SEQ_CST) +#define atomic_and_fetch(ptr, n) __atomic_and_fetch(ptr, n, __ATOMIC_SEQ_CST) +#define atomic_or_fetch(ptr, n) __atomic_or_fetch(ptr, n, __ATOMIC_SEQ_CST) +#define atomic_xor_fetch(ptr, n) __atomic_xor_fetch(ptr, n, __ATOMIC_SEQ_CST) /* And even shorter names that return void. */ #define atomic_inc(ptr) ((void) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST)) @@ -181,6 +203,7 @@ #define atomic_sub(ptr, n) ((void) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST)) #define atomic_and(ptr, n) ((void) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST)) #define atomic_or(ptr, n) ((void) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST)) +#define atomic_xor(ptr, n) ((void) __atomic_fetch_xor(ptr, n, __ATOMIC_SEQ_CST)) #else /* __ATOMIC_RELAXED */ @@ -269,8 +292,11 @@ /* These will only be atomic if the processor does the fetch or store * in a single issue memory operation */ -#define atomic_read(ptr) (*(__typeof__(*ptr) volatile*) (ptr)) -#define atomic_set(ptr, i) ((*(__typeof__(*ptr) volatile*) (ptr)) = (i)) +#define atomic_read__nocheck(p) (*(__typeof__(*(p)) volatile*) (p)) +#define atomic_set__nocheck(p, i) ((*(__typeof__(*(p)) volatile*) (p)) = (i)) + +#define atomic_read(ptr) atomic_read__nocheck(ptr) +#define atomic_set(ptr, i) atomic_set__nocheck(ptr,i) /** * atomic_rcu_read - reads a RCU-protected pointer to a local variable @@ -331,15 +357,27 @@ #define atomic_xchg(ptr, i) (smp_mb(), __sync_lock_test_and_set(ptr, i)) #endif #endif +#define atomic_xchg__nocheck atomic_xchg /* Provide shorter names for GCC atomic builtins. */ #define atomic_fetch_inc(ptr) __sync_fetch_and_add(ptr, 1) #define atomic_fetch_dec(ptr) __sync_fetch_and_add(ptr, -1) -#define atomic_fetch_add __sync_fetch_and_add -#define atomic_fetch_sub __sync_fetch_and_sub -#define atomic_fetch_and __sync_fetch_and_and -#define atomic_fetch_or __sync_fetch_and_or -#define atomic_cmpxchg __sync_val_compare_and_swap +#define atomic_fetch_add(ptr, n) __sync_fetch_and_add(ptr, n) +#define atomic_fetch_sub(ptr, n) __sync_fetch_and_sub(ptr, n) +#define atomic_fetch_and(ptr, n) __sync_fetch_and_and(ptr, n) +#define atomic_fetch_or(ptr, n) __sync_fetch_and_or(ptr, n) +#define atomic_fetch_xor(ptr, n) __sync_fetch_and_xor(ptr, n) + +#define atomic_inc_fetch(ptr) __sync_add_and_fetch(ptr, 1) +#define atomic_dec_fetch(ptr) __sync_add_and_fetch(ptr, -1) +#define atomic_add_fetch(ptr, n) __sync_add_and_fetch(ptr, n) +#define atomic_sub_fetch(ptr, n) __sync_sub_and_fetch(ptr, n) +#define atomic_and_fetch(ptr, n) __sync_and_and_fetch(ptr, n) +#define atomic_or_fetch(ptr, n) __sync_or_and_fetch(ptr, n) +#define atomic_xor_fetch(ptr, n) __sync_xor_and_fetch(ptr, n) + +#define atomic_cmpxchg(ptr, old, new) __sync_val_compare_and_swap(ptr, old, new) +#define atomic_cmpxchg__nocheck(ptr, old, new) atomic_cmpxchg(ptr, old, new) /* And even shorter names that return void. */ #define atomic_inc(ptr) ((void) __sync_fetch_and_add(ptr, 1)) @@ -348,6 +386,7 @@ #define atomic_sub(ptr, n) ((void) __sync_fetch_and_sub(ptr, n)) #define atomic_and(ptr, n) ((void) __sync_fetch_and_and(ptr, n)) #define atomic_or(ptr, n) ((void) __sync_fetch_and_or(ptr, n)) +#define atomic_xor(ptr, n) ((void) __sync_fetch_and_xor(ptr, n)) #endif /* __ATOMIC_RELAXED */ diff --git a/include/qemu/int128.h b/include/qemu/int128.h index c5988813df..5c9890db8b 100644 --- a/include/qemu/int128.h +++ b/include/qemu/int128.h @@ -1,6 +1,149 @@ #ifndef INT128_H #define INT128_H +#ifdef CONFIG_INT128 +#include "qemu/bswap.h" + +typedef __int128_t Int128; + +static inline Int128 int128_make64(uint64_t a) +{ + return a; +} + +static inline Int128 int128_make128(uint64_t lo, uint64_t hi) +{ + return (__uint128_t)hi << 64 | lo; +} + +static inline uint64_t int128_get64(Int128 a) +{ + uint64_t r = a; + assert(r == a); + return r; +} + +static inline uint64_t int128_getlo(Int128 a) +{ + return a; +} + +static inline int64_t int128_gethi(Int128 a) +{ + return a >> 64; +} + +static inline Int128 int128_zero(void) +{ + return 0; +} + +static inline Int128 int128_one(void) +{ + return 1; +} + +static inline Int128 int128_2_64(void) +{ + return (Int128)1 << 64; +} + +static inline Int128 int128_exts64(int64_t a) +{ + return a; +} + +static inline Int128 int128_and(Int128 a, Int128 b) +{ + return a & b; +} + +static inline Int128 int128_rshift(Int128 a, int n) +{ + return a >> n; +} + +static inline Int128 int128_add(Int128 a, Int128 b) +{ + return a + b; +} + +static inline Int128 int128_neg(Int128 a) +{ + return -a; +} + +static inline Int128 int128_sub(Int128 a, Int128 b) +{ + return a - b; +} + +static inline bool int128_nonneg(Int128 a) +{ + return a >= 0; +} + +static inline bool int128_eq(Int128 a, Int128 b) +{ + return a == b; +} + +static inline bool int128_ne(Int128 a, Int128 b) +{ + return a != b; +} + +static inline bool int128_ge(Int128 a, Int128 b) +{ + return a >= b; +} + +static inline bool int128_lt(Int128 a, Int128 b) +{ + return a < b; +} + +static inline bool int128_le(Int128 a, Int128 b) +{ + return a <= b; +} + +static inline bool int128_gt(Int128 a, Int128 b) +{ + return a > b; +} + +static inline bool int128_nz(Int128 a) +{ + return a != 0; +} + +static inline Int128 int128_min(Int128 a, Int128 b) +{ + return a < b ? a : b; +} + +static inline Int128 int128_max(Int128 a, Int128 b) +{ + return a > b ? a : b; +} + +static inline void int128_addto(Int128 *a, Int128 b) +{ + *a += b; +} + +static inline void int128_subfrom(Int128 *a, Int128 b) +{ + *a -= b; +} + +static inline Int128 bswap128(Int128 a) +{ + return int128_make128(bswap64(int128_gethi(a)), bswap64(int128_getlo(a))); +} + +#else /* !CONFIG_INT128 */ typedef struct Int128 Int128; @@ -14,12 +157,27 @@ static inline Int128 int128_make64(uint64_t a) return (Int128) { a, 0 }; } +static inline Int128 int128_make128(uint64_t lo, uint64_t hi) +{ + return (Int128) { lo, hi }; +} + static inline uint64_t int128_get64(Int128 a) { assert(!a.hi); return a.lo; } +static inline uint64_t int128_getlo(Int128 a) +{ + return a.lo; +} + +static inline int64_t int128_gethi(Int128 a) +{ + return a.hi; +} + static inline Int128 int128_zero(void) { return int128_make64(0); @@ -53,9 +211,9 @@ static inline Int128 int128_rshift(Int128 a, int n) } h = a.hi >> (n & 63); if (n >= 64) { - return (Int128) { h, h >> 63 }; + return int128_make128(h, h >> 63); } else { - return (Int128) { (a.lo >> n) | ((uint64_t)a.hi << (64 - n)), h }; + return int128_make128((a.lo >> n) | ((uint64_t)a.hi << (64 - n)), h); } } @@ -69,18 +227,18 @@ static inline Int128 int128_add(Int128 a, Int128 b) * * So the carry is lo < a.lo. */ - return (Int128) { lo, (uint64_t)a.hi + b.hi + (lo < a.lo) }; + return int128_make128(lo, (uint64_t)a.hi + b.hi + (lo < a.lo)); } static inline Int128 int128_neg(Int128 a) { uint64_t lo = -a.lo; - return (Int128) { lo, ~(uint64_t)a.hi + !lo }; + return int128_make128(lo, ~(uint64_t)a.hi + !lo); } static inline Int128 int128_sub(Int128 a, Int128 b) { - return (Int128){ a.lo - b.lo, (uint64_t)a.hi - b.hi - (a.lo < b.lo) }; + return int128_make128(a.lo - b.lo, (uint64_t)a.hi - b.hi - (a.lo < b.lo)); } static inline bool int128_nonneg(Int128 a) @@ -143,4 +301,5 @@ static inline void int128_subfrom(Int128 *a, Int128 b) *a = int128_sub(*a, b); } -#endif +#endif /* CONFIG_INT128 */ +#endif /* INT128_H */ |