diff options
Diffstat (limited to 'accel/tcg/tcg-runtime-gvec.c')
| -rw-r--r-- | accel/tcg/tcg-runtime-gvec.c | 298 |
1 files changed, 116 insertions, 182 deletions
diff --git a/accel/tcg/tcg-runtime-gvec.c b/accel/tcg/tcg-runtime-gvec.c index 5b1902d591..ca449702e6 100644 --- a/accel/tcg/tcg-runtime-gvec.c +++ b/accel/tcg/tcg-runtime-gvec.c @@ -24,48 +24,6 @@ #include "tcg/tcg-gvec-desc.h" -/* Virtually all hosts support 16-byte vectors. Those that don't can emulate - * them via GCC's generic vector extension. This turns out to be simpler and - * more reliable than getting the compiler to autovectorize. - * - * In tcg-op-gvec.c, we asserted that both the size and alignment of the data - * are multiples of 16. - * - * When the compiler does not support all of the operations we require, the - * loops are written so that we can always fall back on the base types. - */ -#ifdef CONFIG_VECTOR16 -typedef uint8_t vec8 __attribute__((vector_size(16))); -typedef uint16_t vec16 __attribute__((vector_size(16))); -typedef uint32_t vec32 __attribute__((vector_size(16))); -typedef uint64_t vec64 __attribute__((vector_size(16))); - -typedef int8_t svec8 __attribute__((vector_size(16))); -typedef int16_t svec16 __attribute__((vector_size(16))); -typedef int32_t svec32 __attribute__((vector_size(16))); -typedef int64_t svec64 __attribute__((vector_size(16))); - -#define DUP16(X) { X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X } -#define DUP8(X) { X, X, X, X, X, X, X, X } -#define DUP4(X) { X, X, X, X } -#define DUP2(X) { X, X } -#else -typedef uint8_t vec8; -typedef uint16_t vec16; -typedef uint32_t vec32; -typedef uint64_t vec64; - -typedef int8_t svec8; -typedef int16_t svec16; -typedef int32_t svec32; -typedef int64_t svec64; - -#define DUP16(X) X -#define DUP8(X) X -#define DUP4(X) X -#define DUP2(X) X -#endif /* CONFIG_VECTOR16 */ - static inline void clear_high(void *d, intptr_t oprsz, uint32_t desc) { intptr_t maxsz = simd_maxsz(desc); @@ -83,8 +41,8 @@ void HELPER(gvec_add8)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(vec8 *)(d + i) = *(vec8 *)(a + i) + *(vec8 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(uint8_t *)(d + i) = *(uint8_t *)(a + i) + *(uint8_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -94,8 +52,8 @@ void HELPER(gvec_add16)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(vec16 *)(d + i) = *(vec16 *)(a + i) + *(vec16 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(uint16_t *)(d + i) = *(uint16_t *)(a + i) + *(uint16_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -105,8 +63,8 @@ void HELPER(gvec_add32)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(vec32 *)(d + i) = *(vec32 *)(a + i) + *(vec32 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(uint32_t *)(d + i) = *(uint32_t *)(a + i) + *(uint32_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -116,8 +74,8 @@ void HELPER(gvec_add64)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) + *(vec64 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) + *(uint64_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -125,11 +83,10 @@ void HELPER(gvec_add64)(void *d, void *a, void *b, uint32_t desc) void HELPER(gvec_adds8)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec8 vecb = (vec8)DUP16(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(vec8 *)(d + i) = *(vec8 *)(a + i) + vecb; + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(uint8_t *)(d + i) = *(uint8_t *)(a + i) + (uint8_t)b; } clear_high(d, oprsz, desc); } @@ -137,11 +94,10 @@ void HELPER(gvec_adds8)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_adds16)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec16 vecb = (vec16)DUP8(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(vec16 *)(d + i) = *(vec16 *)(a + i) + vecb; + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(uint16_t *)(d + i) = *(uint16_t *)(a + i) + (uint16_t)b; } clear_high(d, oprsz, desc); } @@ -149,11 +105,10 @@ void HELPER(gvec_adds16)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_adds32)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec32 vecb = (vec32)DUP4(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(vec32 *)(d + i) = *(vec32 *)(a + i) + vecb; + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(uint32_t *)(d + i) = *(uint32_t *)(a + i) + (uint32_t)b; } clear_high(d, oprsz, desc); } @@ -161,11 +116,10 @@ void HELPER(gvec_adds32)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_adds64)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec64 vecb = (vec64)DUP2(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) + vecb; + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) + b; } clear_high(d, oprsz, desc); } @@ -175,8 +129,8 @@ void HELPER(gvec_sub8)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(vec8 *)(d + i) = *(vec8 *)(a + i) - *(vec8 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(uint8_t *)(d + i) = *(uint8_t *)(a + i) - *(uint8_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -186,8 +140,8 @@ void HELPER(gvec_sub16)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(vec16 *)(d + i) = *(vec16 *)(a + i) - *(vec16 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(uint16_t *)(d + i) = *(uint16_t *)(a + i) - *(uint16_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -197,8 +151,8 @@ void HELPER(gvec_sub32)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(vec32 *)(d + i) = *(vec32 *)(a + i) - *(vec32 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(uint32_t *)(d + i) = *(uint32_t *)(a + i) - *(uint32_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -208,8 +162,8 @@ void HELPER(gvec_sub64)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) - *(vec64 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) - *(uint64_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -217,11 +171,10 @@ void HELPER(gvec_sub64)(void *d, void *a, void *b, uint32_t desc) void HELPER(gvec_subs8)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec8 vecb = (vec8)DUP16(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(vec8 *)(d + i) = *(vec8 *)(a + i) - vecb; + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(uint8_t *)(d + i) = *(uint8_t *)(a + i) - (uint8_t)b; } clear_high(d, oprsz, desc); } @@ -229,11 +182,10 @@ void HELPER(gvec_subs8)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_subs16)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec16 vecb = (vec16)DUP8(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(vec16 *)(d + i) = *(vec16 *)(a + i) - vecb; + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(uint16_t *)(d + i) = *(uint16_t *)(a + i) - (uint16_t)b; } clear_high(d, oprsz, desc); } @@ -241,11 +193,10 @@ void HELPER(gvec_subs16)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_subs32)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec32 vecb = (vec32)DUP4(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(vec32 *)(d + i) = *(vec32 *)(a + i) - vecb; + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(uint32_t *)(d + i) = *(uint32_t *)(a + i) - (uint32_t)b; } clear_high(d, oprsz, desc); } @@ -253,11 +204,10 @@ void HELPER(gvec_subs32)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_subs64)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec64 vecb = (vec64)DUP2(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) - vecb; + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) - b; } clear_high(d, oprsz, desc); } @@ -267,8 +217,8 @@ void HELPER(gvec_mul8)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(vec8 *)(d + i) = *(vec8 *)(a + i) * *(vec8 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(uint8_t *)(d + i) = *(uint8_t *)(a + i) * *(uint8_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -278,8 +228,8 @@ void HELPER(gvec_mul16)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(vec16 *)(d + i) = *(vec16 *)(a + i) * *(vec16 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(uint16_t *)(d + i) = *(uint16_t *)(a + i) * *(uint16_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -289,8 +239,8 @@ void HELPER(gvec_mul32)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(vec32 *)(d + i) = *(vec32 *)(a + i) * *(vec32 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(uint32_t *)(d + i) = *(uint32_t *)(a + i) * *(uint32_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -300,8 +250,8 @@ void HELPER(gvec_mul64)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) * *(vec64 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) * *(uint64_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -309,11 +259,10 @@ void HELPER(gvec_mul64)(void *d, void *a, void *b, uint32_t desc) void HELPER(gvec_muls8)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec8 vecb = (vec8)DUP16(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(vec8 *)(d + i) = *(vec8 *)(a + i) * vecb; + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(uint8_t *)(d + i) = *(uint8_t *)(a + i) * (uint8_t)b; } clear_high(d, oprsz, desc); } @@ -321,11 +270,10 @@ void HELPER(gvec_muls8)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_muls16)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec16 vecb = (vec16)DUP8(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(vec16 *)(d + i) = *(vec16 *)(a + i) * vecb; + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(uint16_t *)(d + i) = *(uint16_t *)(a + i) * (uint16_t)b; } clear_high(d, oprsz, desc); } @@ -333,11 +281,10 @@ void HELPER(gvec_muls16)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_muls32)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec32 vecb = (vec32)DUP4(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(vec32 *)(d + i) = *(vec32 *)(a + i) * vecb; + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(uint32_t *)(d + i) = *(uint32_t *)(a + i) * (uint32_t)b; } clear_high(d, oprsz, desc); } @@ -345,11 +292,10 @@ void HELPER(gvec_muls32)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_muls64)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec64 vecb = (vec64)DUP2(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) * vecb; + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) * b; } clear_high(d, oprsz, desc); } @@ -359,8 +305,8 @@ void HELPER(gvec_neg8)(void *d, void *a, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(vec8 *)(d + i) = -*(vec8 *)(a + i); + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(uint8_t *)(d + i) = -*(uint8_t *)(a + i); } clear_high(d, oprsz, desc); } @@ -370,8 +316,8 @@ void HELPER(gvec_neg16)(void *d, void *a, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(vec16 *)(d + i) = -*(vec16 *)(a + i); + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(uint16_t *)(d + i) = -*(uint16_t *)(a + i); } clear_high(d, oprsz, desc); } @@ -381,8 +327,8 @@ void HELPER(gvec_neg32)(void *d, void *a, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(vec32 *)(d + i) = -*(vec32 *)(a + i); + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(uint32_t *)(d + i) = -*(uint32_t *)(a + i); } clear_high(d, oprsz, desc); } @@ -392,8 +338,8 @@ void HELPER(gvec_neg64)(void *d, void *a, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = -*(vec64 *)(a + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = -*(uint64_t *)(a + i); } clear_high(d, oprsz, desc); } @@ -499,8 +445,8 @@ void HELPER(gvec_not)(void *d, void *a, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = ~*(vec64 *)(a + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = ~*(uint64_t *)(a + i); } clear_high(d, oprsz, desc); } @@ -510,8 +456,8 @@ void HELPER(gvec_and)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) & *(vec64 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) & *(uint64_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -521,8 +467,8 @@ void HELPER(gvec_or)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) | *(vec64 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) | *(uint64_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -532,8 +478,8 @@ void HELPER(gvec_xor)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) ^ *(vec64 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) ^ *(uint64_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -543,8 +489,8 @@ void HELPER(gvec_andc)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) &~ *(vec64 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) &~ *(uint64_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -554,8 +500,8 @@ void HELPER(gvec_orc)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) |~ *(vec64 *)(b + i); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) |~ *(uint64_t *)(b + i); } clear_high(d, oprsz, desc); } @@ -565,8 +511,8 @@ void HELPER(gvec_nand)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = ~(*(vec64 *)(a + i) & *(vec64 *)(b + i)); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = ~(*(uint64_t *)(a + i) & *(uint64_t *)(b + i)); } clear_high(d, oprsz, desc); } @@ -576,8 +522,8 @@ void HELPER(gvec_nor)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = ~(*(vec64 *)(a + i) | *(vec64 *)(b + i)); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = ~(*(uint64_t *)(a + i) | *(uint64_t *)(b + i)); } clear_high(d, oprsz, desc); } @@ -587,8 +533,8 @@ void HELPER(gvec_eqv)(void *d, void *a, void *b, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = ~(*(vec64 *)(a + i) ^ *(vec64 *)(b + i)); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = ~(*(uint64_t *)(a + i) ^ *(uint64_t *)(b + i)); } clear_high(d, oprsz, desc); } @@ -596,11 +542,10 @@ void HELPER(gvec_eqv)(void *d, void *a, void *b, uint32_t desc) void HELPER(gvec_ands)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec64 vecb = (vec64)DUP2(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) & vecb; + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) & b; } clear_high(d, oprsz, desc); } @@ -608,11 +553,10 @@ void HELPER(gvec_ands)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_xors)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec64 vecb = (vec64)DUP2(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) ^ vecb; + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) ^ b; } clear_high(d, oprsz, desc); } @@ -620,11 +564,10 @@ void HELPER(gvec_xors)(void *d, void *a, uint64_t b, uint32_t desc) void HELPER(gvec_ors)(void *d, void *a, uint64_t b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); - vec64 vecb = (vec64)DUP2(b); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) | vecb; + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) | b; } clear_high(d, oprsz, desc); } @@ -635,8 +578,8 @@ void HELPER(gvec_shl8i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(vec8 *)(d + i) = *(vec8 *)(a + i) << shift; + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(uint8_t *)(d + i) = *(uint8_t *)(a + i) << shift; } clear_high(d, oprsz, desc); } @@ -647,8 +590,8 @@ void HELPER(gvec_shl16i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(vec16 *)(d + i) = *(vec16 *)(a + i) << shift; + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(uint16_t *)(d + i) = *(uint16_t *)(a + i) << shift; } clear_high(d, oprsz, desc); } @@ -659,8 +602,8 @@ void HELPER(gvec_shl32i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(vec32 *)(d + i) = *(vec32 *)(a + i) << shift; + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(uint32_t *)(d + i) = *(uint32_t *)(a + i) << shift; } clear_high(d, oprsz, desc); } @@ -671,8 +614,8 @@ void HELPER(gvec_shl64i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) << shift; + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) << shift; } clear_high(d, oprsz, desc); } @@ -683,8 +626,8 @@ void HELPER(gvec_shr8i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(vec8 *)(d + i) = *(vec8 *)(a + i) >> shift; + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(uint8_t *)(d + i) = *(uint8_t *)(a + i) >> shift; } clear_high(d, oprsz, desc); } @@ -695,8 +638,8 @@ void HELPER(gvec_shr16i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(vec16 *)(d + i) = *(vec16 *)(a + i) >> shift; + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(uint16_t *)(d + i) = *(uint16_t *)(a + i) >> shift; } clear_high(d, oprsz, desc); } @@ -707,8 +650,8 @@ void HELPER(gvec_shr32i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(vec32 *)(d + i) = *(vec32 *)(a + i) >> shift; + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(uint32_t *)(d + i) = *(uint32_t *)(a + i) >> shift; } clear_high(d, oprsz, desc); } @@ -719,8 +662,8 @@ void HELPER(gvec_shr64i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(vec64 *)(d + i) = *(vec64 *)(a + i) >> shift; + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(uint64_t *)(d + i) = *(uint64_t *)(a + i) >> shift; } clear_high(d, oprsz, desc); } @@ -731,8 +674,8 @@ void HELPER(gvec_sar8i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec8)) { - *(svec8 *)(d + i) = *(svec8 *)(a + i) >> shift; + for (i = 0; i < oprsz; i += sizeof(uint8_t)) { + *(int8_t *)(d + i) = *(int8_t *)(a + i) >> shift; } clear_high(d, oprsz, desc); } @@ -743,8 +686,8 @@ void HELPER(gvec_sar16i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec16)) { - *(svec16 *)(d + i) = *(svec16 *)(a + i) >> shift; + for (i = 0; i < oprsz; i += sizeof(uint16_t)) { + *(int16_t *)(d + i) = *(int16_t *)(a + i) >> shift; } clear_high(d, oprsz, desc); } @@ -755,8 +698,8 @@ void HELPER(gvec_sar32i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec32)) { - *(svec32 *)(d + i) = *(svec32 *)(a + i) >> shift; + for (i = 0; i < oprsz; i += sizeof(uint32_t)) { + *(int32_t *)(d + i) = *(int32_t *)(a + i) >> shift; } clear_high(d, oprsz, desc); } @@ -767,8 +710,8 @@ void HELPER(gvec_sar64i)(void *d, void *a, uint32_t desc) int shift = simd_data(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - *(svec64 *)(d + i) = *(svec64 *)(a + i) >> shift; + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + *(int64_t *)(d + i) = *(int64_t *)(a + i) >> shift; } clear_high(d, oprsz, desc); } @@ -917,39 +860,30 @@ void HELPER(gvec_sar64v)(void *d, void *a, void *b, uint32_t desc) clear_high(d, oprsz, desc); } -/* If vectors are enabled, the compiler fills in -1 for true. - Otherwise, we must take care of this by hand. */ -#ifdef CONFIG_VECTOR16 -# define DO_CMP0(X) X -#else -# define DO_CMP0(X) -(X) -#endif - #define DO_CMP1(NAME, TYPE, OP) \ void HELPER(NAME)(void *d, void *a, void *b, uint32_t desc) \ { \ intptr_t oprsz = simd_oprsz(desc); \ intptr_t i; \ for (i = 0; i < oprsz; i += sizeof(TYPE)) { \ - *(TYPE *)(d + i) = DO_CMP0(*(TYPE *)(a + i) OP *(TYPE *)(b + i)); \ + *(TYPE *)(d + i) = -(*(TYPE *)(a + i) OP *(TYPE *)(b + i)); \ } \ clear_high(d, oprsz, desc); \ } #define DO_CMP2(SZ) \ - DO_CMP1(gvec_eq##SZ, vec##SZ, ==) \ - DO_CMP1(gvec_ne##SZ, vec##SZ, !=) \ - DO_CMP1(gvec_lt##SZ, svec##SZ, <) \ - DO_CMP1(gvec_le##SZ, svec##SZ, <=) \ - DO_CMP1(gvec_ltu##SZ, vec##SZ, <) \ - DO_CMP1(gvec_leu##SZ, vec##SZ, <=) + DO_CMP1(gvec_eq##SZ, uint##SZ##_t, ==) \ + DO_CMP1(gvec_ne##SZ, uint##SZ##_t, !=) \ + DO_CMP1(gvec_lt##SZ, int##SZ##_t, <) \ + DO_CMP1(gvec_le##SZ, int##SZ##_t, <=) \ + DO_CMP1(gvec_ltu##SZ, uint##SZ##_t, <) \ + DO_CMP1(gvec_leu##SZ, uint##SZ##_t, <=) DO_CMP2(8) DO_CMP2(16) DO_CMP2(32) DO_CMP2(64) -#undef DO_CMP0 #undef DO_CMP1 #undef DO_CMP2 @@ -1450,11 +1384,11 @@ void HELPER(gvec_bitsel)(void *d, void *a, void *b, void *c, uint32_t desc) intptr_t oprsz = simd_oprsz(desc); intptr_t i; - for (i = 0; i < oprsz; i += sizeof(vec64)) { - vec64 aa = *(vec64 *)(a + i); - vec64 bb = *(vec64 *)(b + i); - vec64 cc = *(vec64 *)(c + i); - *(vec64 *)(d + i) = (bb & aa) | (cc & ~aa); + for (i = 0; i < oprsz; i += sizeof(uint64_t)) { + uint64_t aa = *(uint64_t *)(a + i); + uint64_t bb = *(uint64_t *)(b + i); + uint64_t cc = *(uint64_t *)(c + i); + *(uint64_t *)(d + i) = (bb & aa) | (cc & ~aa); } clear_high(d, oprsz, desc); } |