diff options
Diffstat (limited to 'target/arm/tcg/translate-a64.c')
| -rw-r--r-- | target/arm/tcg/translate-a64.c | 2079 |
1 files changed, 878 insertions, 1201 deletions
diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index 6d5f12e8f5..071b6349fc 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -4680,6 +4680,88 @@ static bool trans_EXTR(DisasContext *s, arg_extract *a) return true; } +static bool trans_TBL_TBX(DisasContext *s, arg_TBL_TBX *a) +{ + if (fp_access_check(s)) { + int len = (a->len + 1) * 16; + + tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd), + vec_full_reg_offset(s, a->rm), tcg_env, + a->q ? 16 : 8, vec_full_reg_size(s), + (len << 6) | (a->tbx << 5) | a->rn, + gen_helper_simd_tblx); + } + return true; +} + +typedef int simd_permute_idx_fn(int i, int part, int elements); + +static bool do_simd_permute(DisasContext *s, arg_qrrr_e *a, + simd_permute_idx_fn *fn, int part) +{ + MemOp esz = a->esz; + int datasize = a->q ? 16 : 8; + int elements = datasize >> esz; + TCGv_i64 tcg_res[2], tcg_ele; + + if (esz == MO_64 && !a->q) { + return false; + } + if (!fp_access_check(s)) { + return true; + } + + tcg_res[0] = tcg_temp_new_i64(); + tcg_res[1] = a->q ? tcg_temp_new_i64() : NULL; + tcg_ele = tcg_temp_new_i64(); + + for (int i = 0; i < elements; i++) { + int o, w, idx; + + idx = fn(i, part, elements); + read_vec_element(s, tcg_ele, (idx & elements ? a->rm : a->rn), + idx & (elements - 1), esz); + + w = (i << (esz + 3)) / 64; + o = (i << (esz + 3)) % 64; + if (o == 0) { + tcg_gen_mov_i64(tcg_res[w], tcg_ele); + } else { + tcg_gen_deposit_i64(tcg_res[w], tcg_res[w], tcg_ele, o, 8 << esz); + } + } + + for (int i = a->q; i >= 0; --i) { + write_vec_element(s, tcg_res[i], a->rd, i, MO_64); + } + clear_vec_high(s, a->q, a->rd); + return true; +} + +static int permute_load_uzp(int i, int part, int elements) +{ + return 2 * i + part; +} + +TRANS(UZP1, do_simd_permute, a, permute_load_uzp, 0) +TRANS(UZP2, do_simd_permute, a, permute_load_uzp, 1) + +static int permute_load_trn(int i, int part, int elements) +{ + return (i & 1) * elements + (i & ~1) + part; +} + +TRANS(TRN1, do_simd_permute, a, permute_load_trn, 0) +TRANS(TRN2, do_simd_permute, a, permute_load_trn, 1) + +static int permute_load_zip(int i, int part, int elements) +{ + return (i & 1) * elements + ((part * elements + i) >> 1); +} + +TRANS(ZIP1, do_simd_permute, a, permute_load_zip, 0) +TRANS(ZIP2, do_simd_permute, a, permute_load_zip, 1) + /* * Cryptographic AES, SHA, SHA512 */ @@ -6583,6 +6665,54 @@ static bool trans_FCSEL(DisasContext *s, arg_FCSEL *a) } /* + * Advanced SIMD Extract + */ + +static bool trans_EXT_d(DisasContext *s, arg_EXT_d *a) +{ + if (fp_access_check(s)) { + TCGv_i64 lo = read_fp_dreg(s, a->rn); + if (a->imm != 0) { + TCGv_i64 hi = read_fp_dreg(s, a->rm); + tcg_gen_extract2_i64(lo, lo, hi, a->imm * 8); + } + write_fp_dreg(s, a->rd, lo); + } + return true; +} + +static bool trans_EXT_q(DisasContext *s, arg_EXT_q *a) +{ + TCGv_i64 lo, hi; + int pos = (a->imm & 7) * 8; + int elt = a->imm >> 3; + + if (!fp_access_check(s)) { + return true; + } + + lo = tcg_temp_new_i64(); + hi = tcg_temp_new_i64(); + + read_vec_element(s, lo, a->rn, elt, MO_64); + elt++; + read_vec_element(s, hi, elt & 2 ? a->rm : a->rn, elt & 1, MO_64); + elt++; + + if (pos != 0) { + TCGv_i64 hh = tcg_temp_new_i64(); + tcg_gen_extract2_i64(lo, lo, hi, pos); + read_vec_element(s, hh, a->rm, elt & 1, MO_64); + tcg_gen_extract2_i64(hi, hi, hh, pos); + } + + write_vec_element(s, lo, a->rd, 0, MO_64); + write_vec_element(s, hi, a->rd, 1, MO_64); + clear_vec_high(s, true, a->rd); + return true; +} + +/* * Floating-point data-processing (3 source) */ @@ -6664,6 +6794,697 @@ TRANS(FNMADD, do_fmadd, a, true, true) TRANS(FMSUB, do_fmadd, a, false, true) TRANS(FNMSUB, do_fmadd, a, true, false) +/* + * Advanced SIMD Across Lanes + */ + +static bool do_int_reduction(DisasContext *s, arg_qrr_e *a, bool widen, + MemOp src_sign, NeonGenTwo64OpFn *fn) +{ + TCGv_i64 tcg_res, tcg_elt; + MemOp src_mop = a->esz | src_sign; + int elements = (a->q ? 16 : 8) >> a->esz; + + /* Reject MO_64, and MO_32 without Q: a minimum of 4 elements. */ + if (elements < 4) { + return false; + } + if (!fp_access_check(s)) { + return true; + } + + tcg_res = tcg_temp_new_i64(); + tcg_elt = tcg_temp_new_i64(); + + read_vec_element(s, tcg_res, a->rn, 0, src_mop); + for (int i = 1; i < elements; i++) { + read_vec_element(s, tcg_elt, a->rn, i, src_mop); + fn(tcg_res, tcg_res, tcg_elt); + } + + tcg_gen_ext_i64(tcg_res, tcg_res, a->esz + widen); + write_fp_dreg(s, a->rd, tcg_res); + return true; +} + +TRANS(ADDV, do_int_reduction, a, false, 0, tcg_gen_add_i64) +TRANS(SADDLV, do_int_reduction, a, true, MO_SIGN, tcg_gen_add_i64) +TRANS(UADDLV, do_int_reduction, a, true, 0, tcg_gen_add_i64) +TRANS(SMAXV, do_int_reduction, a, false, MO_SIGN, tcg_gen_smax_i64) +TRANS(UMAXV, do_int_reduction, a, false, 0, tcg_gen_umax_i64) +TRANS(SMINV, do_int_reduction, a, false, MO_SIGN, tcg_gen_smin_i64) +TRANS(UMINV, do_int_reduction, a, false, 0, tcg_gen_umin_i64) + +/* + * do_fp_reduction helper + * + * This mirrors the Reduce() pseudocode in the ARM ARM. It is + * important for correct NaN propagation that we do these + * operations in exactly the order specified by the pseudocode. + * + * This is a recursive function. + */ +static TCGv_i32 do_reduction_op(DisasContext *s, int rn, MemOp esz, + int ebase, int ecount, TCGv_ptr fpst, + NeonGenTwoSingleOpFn *fn) +{ + if (ecount == 1) { + TCGv_i32 tcg_elem = tcg_temp_new_i32(); + read_vec_element_i32(s, tcg_elem, rn, ebase, esz); + return tcg_elem; + } else { + int half = ecount >> 1; + TCGv_i32 tcg_hi, tcg_lo, tcg_res; + + tcg_hi = do_reduction_op(s, rn, esz, ebase + half, half, fpst, fn); + tcg_lo = do_reduction_op(s, rn, esz, ebase, half, fpst, fn); + tcg_res = tcg_temp_new_i32(); + + fn(tcg_res, tcg_lo, tcg_hi, fpst); + return tcg_res; + } +} + +static bool do_fp_reduction(DisasContext *s, arg_qrr_e *a, + NeonGenTwoSingleOpFn *fn) +{ + if (fp_access_check(s)) { + MemOp esz = a->esz; + int elts = (a->q ? 16 : 8) >> esz; + TCGv_ptr fpst = fpstatus_ptr(esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); + TCGv_i32 res = do_reduction_op(s, a->rn, esz, 0, elts, fpst, fn); + write_fp_sreg(s, a->rd, res); + } + return true; +} + +TRANS_FEAT(FMAXNMV_h, aa64_fp16, do_fp_reduction, a, gen_helper_advsimd_maxnumh) +TRANS_FEAT(FMINNMV_h, aa64_fp16, do_fp_reduction, a, gen_helper_advsimd_minnumh) +TRANS_FEAT(FMAXV_h, aa64_fp16, do_fp_reduction, a, gen_helper_advsimd_maxh) +TRANS_FEAT(FMINV_h, aa64_fp16, do_fp_reduction, a, gen_helper_advsimd_minh) + +TRANS(FMAXNMV_s, do_fp_reduction, a, gen_helper_vfp_maxnums) +TRANS(FMINNMV_s, do_fp_reduction, a, gen_helper_vfp_minnums) +TRANS(FMAXV_s, do_fp_reduction, a, gen_helper_vfp_maxs) +TRANS(FMINV_s, do_fp_reduction, a, gen_helper_vfp_mins) + +/* + * Floating-point Immediate + */ + +static bool trans_FMOVI_s(DisasContext *s, arg_FMOVI_s *a) +{ + switch (a->esz) { + case MO_32: + case MO_64: + break; + case MO_16: + if (!dc_isar_feature(aa64_fp16, s)) { + return false; + } + break; + default: + return false; + } + if (fp_access_check(s)) { + uint64_t imm = vfp_expand_imm(a->esz, a->imm); + write_fp_dreg(s, a->rd, tcg_constant_i64(imm)); + } + return true; +} + +/* + * Advanced SIMD Modified Immediate + */ + +static bool trans_FMOVI_v_h(DisasContext *s, arg_FMOVI_v_h *a) +{ + if (!dc_isar_feature(aa64_fp16, s)) { + return false; + } + if (fp_access_check(s)) { + tcg_gen_gvec_dup_imm(MO_16, vec_full_reg_offset(s, a->rd), + a->q ? 16 : 8, vec_full_reg_size(s), + vfp_expand_imm(MO_16, a->abcdefgh)); + } + return true; +} + +static void gen_movi(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t c, uint32_t oprsz, uint32_t maxsz) +{ + tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, c); +} + +static bool trans_Vimm(DisasContext *s, arg_Vimm *a) +{ + GVecGen2iFn *fn; + + /* Handle decode of cmode/op here between ORR/BIC/MOVI */ + if ((a->cmode & 1) && a->cmode < 12) { + /* For op=1, the imm will be inverted, so BIC becomes AND. */ + fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori; + } else { + /* There is one unallocated cmode/op combination in this space */ + if (a->cmode == 15 && a->op == 1 && a->q == 0) { + return false; + } + fn = gen_movi; + } + + if (fp_access_check(s)) { + uint64_t imm = asimd_imm_const(a->abcdefgh, a->cmode, a->op); + gen_gvec_fn2i(s, a->q, a->rd, a->rd, imm, fn, MO_64); + } + return true; +} + +/* + * Advanced SIMD Shift by Immediate + */ + +static bool do_vec_shift_imm(DisasContext *s, arg_qrri_e *a, GVecGen2iFn *fn) +{ + if (fp_access_check(s)) { + gen_gvec_fn2i(s, a->q, a->rd, a->rn, a->imm, fn, a->esz); + } + return true; +} + +TRANS(SSHR_v, do_vec_shift_imm, a, gen_gvec_sshr) +TRANS(USHR_v, do_vec_shift_imm, a, gen_gvec_ushr) +TRANS(SSRA_v, do_vec_shift_imm, a, gen_gvec_ssra) +TRANS(USRA_v, do_vec_shift_imm, a, gen_gvec_usra) +TRANS(SRSHR_v, do_vec_shift_imm, a, gen_gvec_srshr) +TRANS(URSHR_v, do_vec_shift_imm, a, gen_gvec_urshr) +TRANS(SRSRA_v, do_vec_shift_imm, a, gen_gvec_srsra) +TRANS(URSRA_v, do_vec_shift_imm, a, gen_gvec_ursra) +TRANS(SRI_v, do_vec_shift_imm, a, gen_gvec_sri) +TRANS(SHL_v, do_vec_shift_imm, a, tcg_gen_gvec_shli) +TRANS(SLI_v, do_vec_shift_imm, a, gen_gvec_sli); +TRANS(SQSHL_vi, do_vec_shift_imm, a, gen_neon_sqshli) +TRANS(UQSHL_vi, do_vec_shift_imm, a, gen_neon_uqshli) +TRANS(SQSHLU_vi, do_vec_shift_imm, a, gen_neon_sqshlui) + +static bool do_vec_shift_imm_wide(DisasContext *s, arg_qrri_e *a, bool is_u) +{ + TCGv_i64 tcg_rn, tcg_rd; + int esz = a->esz; + int esize; + + if (!fp_access_check(s)) { + return true; + } + + /* + * For the LL variants the store is larger than the load, + * so if rd == rn we would overwrite parts of our input. + * So load everything right now and use shifts in the main loop. + */ + tcg_rd = tcg_temp_new_i64(); + tcg_rn = tcg_temp_new_i64(); + read_vec_element(s, tcg_rn, a->rn, a->q, MO_64); + + esize = 8 << esz; + for (int i = 0, elements = 8 >> esz; i < elements; i++) { + if (is_u) { + tcg_gen_extract_i64(tcg_rd, tcg_rn, i * esize, esize); + } else { + tcg_gen_sextract_i64(tcg_rd, tcg_rn, i * esize, esize); + } + tcg_gen_shli_i64(tcg_rd, tcg_rd, a->imm); + write_vec_element(s, tcg_rd, a->rd, i, esz + 1); + } + clear_vec_high(s, true, a->rd); + return true; +} + +TRANS(SSHLL_v, do_vec_shift_imm_wide, a, false) +TRANS(USHLL_v, do_vec_shift_imm_wide, a, true) + +static void gen_sshr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + assert(shift >= 0 && shift <= 64); + tcg_gen_sari_i64(dst, src, MIN(shift, 63)); +} + +static void gen_ushr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + assert(shift >= 0 && shift <= 64); + if (shift == 64) { + tcg_gen_movi_i64(dst, 0); + } else { + tcg_gen_shri_i64(dst, src, shift); + } +} + +static void gen_ssra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + gen_sshr_d(src, src, shift); + tcg_gen_add_i64(dst, dst, src); +} + +static void gen_usra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + gen_ushr_d(src, src, shift); + tcg_gen_add_i64(dst, dst, src); +} + +static void gen_srshr_bhs(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + assert(shift >= 0 && shift <= 32); + if (shift) { + TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); + tcg_gen_add_i64(dst, src, rnd); + tcg_gen_sari_i64(dst, dst, shift); + } else { + tcg_gen_mov_i64(dst, src); + } +} + +static void gen_urshr_bhs(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + assert(shift >= 0 && shift <= 32); + if (shift) { + TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); + tcg_gen_add_i64(dst, src, rnd); + tcg_gen_shri_i64(dst, dst, shift); + } else { + tcg_gen_mov_i64(dst, src); + } +} + +static void gen_srshr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + assert(shift >= 0 && shift <= 64); + if (shift == 0) { + tcg_gen_mov_i64(dst, src); + } else if (shift == 64) { + /* Extension of sign bit (0,-1) plus sign bit (0,1) is zero. */ + tcg_gen_movi_i64(dst, 0); + } else { + TCGv_i64 rnd = tcg_temp_new_i64(); + tcg_gen_extract_i64(rnd, src, shift - 1, 1); + tcg_gen_sari_i64(dst, src, shift); + tcg_gen_add_i64(dst, dst, rnd); + } +} + +static void gen_urshr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + assert(shift >= 0 && shift <= 64); + if (shift == 0) { + tcg_gen_mov_i64(dst, src); + } else if (shift == 64) { + /* Rounding will propagate bit 63 into bit 64. */ + tcg_gen_shri_i64(dst, src, 63); + } else { + TCGv_i64 rnd = tcg_temp_new_i64(); + tcg_gen_extract_i64(rnd, src, shift - 1, 1); + tcg_gen_shri_i64(dst, src, shift); + tcg_gen_add_i64(dst, dst, rnd); + } +} + +static void gen_srsra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + gen_srshr_d(src, src, shift); + tcg_gen_add_i64(dst, dst, src); +} + +static void gen_ursra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + gen_urshr_d(src, src, shift); + tcg_gen_add_i64(dst, dst, src); +} + +static void gen_sri_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + /* If shift is 64, dst is unchanged. */ + if (shift != 64) { + tcg_gen_shri_i64(src, src, shift); + tcg_gen_deposit_i64(dst, dst, src, 0, 64 - shift); + } +} + +static void gen_sli_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift) +{ + tcg_gen_deposit_i64(dst, dst, src, shift, 64 - shift); +} + +static bool do_vec_shift_imm_narrow(DisasContext *s, arg_qrri_e *a, + WideShiftImmFn * const fns[3], MemOp sign) +{ + TCGv_i64 tcg_rn, tcg_rd; + int esz = a->esz; + int esize; + WideShiftImmFn *fn; + + tcg_debug_assert(esz >= MO_8 && esz <= MO_32); + + if (!fp_access_check(s)) { + return true; + } + + tcg_rn = tcg_temp_new_i64(); + tcg_rd = tcg_temp_new_i64(); + tcg_gen_movi_i64(tcg_rd, 0); + + fn = fns[esz]; + esize = 8 << esz; + for (int i = 0, elements = 8 >> esz; i < elements; i++) { + read_vec_element(s, tcg_rn, a->rn, i, (esz + 1) | sign); + fn(tcg_rn, tcg_rn, a->imm); + tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, esize * i, esize); + } + + write_vec_element(s, tcg_rd, a->rd, a->q, MO_64); + clear_vec_high(s, a->q, a->rd); + return true; +} + +static void gen_sqshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + tcg_gen_sari_i64(d, s, i); + tcg_gen_ext16u_i64(d, d); + gen_helper_neon_narrow_sat_s8(d, tcg_env, d); +} + +static void gen_sqshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + tcg_gen_sari_i64(d, s, i); + tcg_gen_ext32u_i64(d, d); + gen_helper_neon_narrow_sat_s16(d, tcg_env, d); +} + +static void gen_sqshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_sshr_d(d, s, i); + gen_helper_neon_narrow_sat_s32(d, tcg_env, d); +} + +static void gen_uqshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + tcg_gen_shri_i64(d, s, i); + gen_helper_neon_narrow_sat_u8(d, tcg_env, d); +} + +static void gen_uqshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + tcg_gen_shri_i64(d, s, i); + gen_helper_neon_narrow_sat_u16(d, tcg_env, d); +} + +static void gen_uqshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_ushr_d(d, s, i); + gen_helper_neon_narrow_sat_u32(d, tcg_env, d); +} + +static void gen_sqshrun_b(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + tcg_gen_sari_i64(d, s, i); + tcg_gen_ext16u_i64(d, d); + gen_helper_neon_unarrow_sat8(d, tcg_env, d); +} + +static void gen_sqshrun_h(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + tcg_gen_sari_i64(d, s, i); + tcg_gen_ext32u_i64(d, d); + gen_helper_neon_unarrow_sat16(d, tcg_env, d); +} + +static void gen_sqshrun_s(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_sshr_d(d, s, i); + gen_helper_neon_unarrow_sat32(d, tcg_env, d); +} + +static void gen_sqrshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_srshr_bhs(d, s, i); + tcg_gen_ext16u_i64(d, d); + gen_helper_neon_narrow_sat_s8(d, tcg_env, d); +} + +static void gen_sqrshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_srshr_bhs(d, s, i); + tcg_gen_ext32u_i64(d, d); + gen_helper_neon_narrow_sat_s16(d, tcg_env, d); +} + +static void gen_sqrshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_srshr_d(d, s, i); + gen_helper_neon_narrow_sat_s32(d, tcg_env, d); +} + +static void gen_uqrshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_urshr_bhs(d, s, i); + gen_helper_neon_narrow_sat_u8(d, tcg_env, d); +} + +static void gen_uqrshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_urshr_bhs(d, s, i); + gen_helper_neon_narrow_sat_u16(d, tcg_env, d); +} + +static void gen_uqrshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_urshr_d(d, s, i); + gen_helper_neon_narrow_sat_u32(d, tcg_env, d); +} + +static void gen_sqrshrun_b(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_srshr_bhs(d, s, i); + tcg_gen_ext16u_i64(d, d); + gen_helper_neon_unarrow_sat8(d, tcg_env, d); +} + +static void gen_sqrshrun_h(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_srshr_bhs(d, s, i); + tcg_gen_ext32u_i64(d, d); + gen_helper_neon_unarrow_sat16(d, tcg_env, d); +} + +static void gen_sqrshrun_s(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_srshr_d(d, s, i); + gen_helper_neon_unarrow_sat32(d, tcg_env, d); +} + +static WideShiftImmFn * const shrn_fns[] = { + tcg_gen_shri_i64, + tcg_gen_shri_i64, + gen_ushr_d, +}; +TRANS(SHRN_v, do_vec_shift_imm_narrow, a, shrn_fns, 0) + +static WideShiftImmFn * const rshrn_fns[] = { + gen_urshr_bhs, + gen_urshr_bhs, + gen_urshr_d, +}; +TRANS(RSHRN_v, do_vec_shift_imm_narrow, a, rshrn_fns, 0) + +static WideShiftImmFn * const sqshrn_fns[] = { + gen_sqshrn_b, + gen_sqshrn_h, + gen_sqshrn_s, +}; +TRANS(SQSHRN_v, do_vec_shift_imm_narrow, a, sqshrn_fns, MO_SIGN) + +static WideShiftImmFn * const uqshrn_fns[] = { + gen_uqshrn_b, + gen_uqshrn_h, + gen_uqshrn_s, +}; +TRANS(UQSHRN_v, do_vec_shift_imm_narrow, a, uqshrn_fns, 0) + +static WideShiftImmFn * const sqshrun_fns[] = { + gen_sqshrun_b, + gen_sqshrun_h, + gen_sqshrun_s, +}; +TRANS(SQSHRUN_v, do_vec_shift_imm_narrow, a, sqshrun_fns, MO_SIGN) + +static WideShiftImmFn * const sqrshrn_fns[] = { + gen_sqrshrn_b, + gen_sqrshrn_h, + gen_sqrshrn_s, +}; +TRANS(SQRSHRN_v, do_vec_shift_imm_narrow, a, sqrshrn_fns, MO_SIGN) + +static WideShiftImmFn * const uqrshrn_fns[] = { + gen_uqrshrn_b, + gen_uqrshrn_h, + gen_uqrshrn_s, +}; +TRANS(UQRSHRN_v, do_vec_shift_imm_narrow, a, uqrshrn_fns, 0) + +static WideShiftImmFn * const sqrshrun_fns[] = { + gen_sqrshrun_b, + gen_sqrshrun_h, + gen_sqrshrun_s, +}; +TRANS(SQRSHRUN_v, do_vec_shift_imm_narrow, a, sqrshrun_fns, MO_SIGN) + +/* + * Advanced SIMD Scalar Shift by Immediate + */ + +static bool do_scalar_shift_imm(DisasContext *s, arg_rri_e *a, + WideShiftImmFn *fn, bool accumulate, + MemOp sign) +{ + if (fp_access_check(s)) { + TCGv_i64 rd = tcg_temp_new_i64(); + TCGv_i64 rn = tcg_temp_new_i64(); + + read_vec_element(s, rn, a->rn, 0, a->esz | sign); + if (accumulate) { + read_vec_element(s, rd, a->rd, 0, a->esz | sign); + } + fn(rd, rn, a->imm); + write_fp_dreg(s, a->rd, rd); + } + return true; +} + +TRANS(SSHR_s, do_scalar_shift_imm, a, gen_sshr_d, false, 0) +TRANS(USHR_s, do_scalar_shift_imm, a, gen_ushr_d, false, 0) +TRANS(SSRA_s, do_scalar_shift_imm, a, gen_ssra_d, true, 0) +TRANS(USRA_s, do_scalar_shift_imm, a, gen_usra_d, true, 0) +TRANS(SRSHR_s, do_scalar_shift_imm, a, gen_srshr_d, false, 0) +TRANS(URSHR_s, do_scalar_shift_imm, a, gen_urshr_d, false, 0) +TRANS(SRSRA_s, do_scalar_shift_imm, a, gen_srsra_d, true, 0) +TRANS(URSRA_s, do_scalar_shift_imm, a, gen_ursra_d, true, 0) +TRANS(SRI_s, do_scalar_shift_imm, a, gen_sri_d, true, 0) + +TRANS(SHL_s, do_scalar_shift_imm, a, tcg_gen_shli_i64, false, 0) +TRANS(SLI_s, do_scalar_shift_imm, a, gen_sli_d, true, 0) + +static void trunc_i64_env_imm(TCGv_i64 d, TCGv_i64 s, int64_t i, + NeonGenTwoOpEnvFn *fn) +{ + TCGv_i32 t = tcg_temp_new_i32(); + tcg_gen_extrl_i64_i32(t, s); + fn(t, tcg_env, t, tcg_constant_i32(i)); + tcg_gen_extu_i32_i64(d, t); +} + +static void gen_sqshli_b(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_s8); +} + +static void gen_sqshli_h(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_s16); +} + +static void gen_sqshli_s(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_s32); +} + +static void gen_sqshli_d(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_helper_neon_qshl_s64(d, tcg_env, s, tcg_constant_i64(i)); +} + +static void gen_uqshli_b(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_u8); +} + +static void gen_uqshli_h(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_u16); +} + +static void gen_uqshli_s(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_u32); +} + +static void gen_uqshli_d(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_helper_neon_qshl_u64(d, tcg_env, s, tcg_constant_i64(i)); +} + +static void gen_sqshlui_b(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + trunc_i64_env_imm(d, s, i, gen_helper_neon_qshlu_s8); +} + +static void gen_sqshlui_h(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + trunc_i64_env_imm(d, s, i, gen_helper_neon_qshlu_s16); +} + +static void gen_sqshlui_s(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + trunc_i64_env_imm(d, s, i, gen_helper_neon_qshlu_s32); +} + +static void gen_sqshlui_d(TCGv_i64 d, TCGv_i64 s, int64_t i) +{ + gen_helper_neon_qshlu_s64(d, tcg_env, s, tcg_constant_i64(i)); +} + +static WideShiftImmFn * const f_scalar_sqshli[] = { + gen_sqshli_b, gen_sqshli_h, gen_sqshli_s, gen_sqshli_d +}; + +static WideShiftImmFn * const f_scalar_uqshli[] = { + gen_uqshli_b, gen_uqshli_h, gen_uqshli_s, gen_uqshli_d +}; + +static WideShiftImmFn * const f_scalar_sqshlui[] = { + gen_sqshlui_b, gen_sqshlui_h, gen_sqshlui_s, gen_sqshlui_d +}; + +/* Note that the helpers sign-extend their inputs, so don't do it here. */ +TRANS(SQSHL_si, do_scalar_shift_imm, a, f_scalar_sqshli[a->esz], false, 0) +TRANS(UQSHL_si, do_scalar_shift_imm, a, f_scalar_uqshli[a->esz], false, 0) +TRANS(SQSHLU_si, do_scalar_shift_imm, a, f_scalar_sqshlui[a->esz], false, 0) + +static bool do_scalar_shift_imm_narrow(DisasContext *s, arg_rri_e *a, + WideShiftImmFn * const fns[3], + MemOp sign, bool zext) +{ + MemOp esz = a->esz; + + tcg_debug_assert(esz >= MO_8 && esz <= MO_32); + + if (fp_access_check(s)) { + TCGv_i64 rd = tcg_temp_new_i64(); + TCGv_i64 rn = tcg_temp_new_i64(); + + read_vec_element(s, rn, a->rn, 0, (esz + 1) | sign); + fns[esz](rd, rn, a->imm); + if (zext) { + tcg_gen_ext_i64(rd, rd, esz); + } + write_fp_dreg(s, a->rd, rd); + } + return true; +} + +TRANS(SQSHRN_si, do_scalar_shift_imm_narrow, a, sqshrn_fns, MO_SIGN, true) +TRANS(SQRSHRN_si, do_scalar_shift_imm_narrow, a, sqrshrn_fns, MO_SIGN, true) +TRANS(UQSHRN_si, do_scalar_shift_imm_narrow, a, uqshrn_fns, 0, false) +TRANS(UQRSHRN_si, do_scalar_shift_imm_narrow, a, uqrshrn_fns, 0, false) +TRANS(SQSHRUN_si, do_scalar_shift_imm_narrow, a, sqshrun_fns, MO_SIGN, false) +TRANS(SQRSHRUN_si, do_scalar_shift_imm_narrow, a, sqrshrun_fns, MO_SIGN, false) + /* Shift a TCGv src by TCGv shift_amount, put result in dst. * Note that it is the caller's responsibility to ensure that the * shift amount is in range (ie 0..31 or 0..63) and provide the ARM @@ -8401,53 +9222,6 @@ static void disas_fp_1src(DisasContext *s, uint32_t insn) } } -/* Floating point immediate - * 31 30 29 28 24 23 22 21 20 13 12 10 9 5 4 0 - * +---+---+---+-----------+------+---+------------+-------+------+------+ - * | M | 0 | S | 1 1 1 1 0 | type | 1 | imm8 | 1 0 0 | imm5 | Rd | - * +---+---+---+-----------+------+---+------------+-------+------+------+ - */ -static void disas_fp_imm(DisasContext *s, uint32_t insn) -{ - int rd = extract32(insn, 0, 5); - int imm5 = extract32(insn, 5, 5); - int imm8 = extract32(insn, 13, 8); - int type = extract32(insn, 22, 2); - int mos = extract32(insn, 29, 3); - uint64_t imm; - MemOp sz; - - if (mos || imm5) { - unallocated_encoding(s); - return; - } - - switch (type) { - case 0: - sz = MO_32; - break; - case 1: - sz = MO_64; - break; - case 3: - sz = MO_16; - if (dc_isar_feature(aa64_fp16, s)) { - break; - } - /* fallthru */ - default: - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - imm = vfp_expand_imm(sz, imm8); - write_fp_dreg(s, rd, tcg_constant_i64(imm)); -} - /* Handle floating point <=> fixed point conversions. Note that we can * also deal with fp <=> integer conversions as a special case (scale == 64) * OPTME: consider handling that special case specially or at least skipping @@ -8867,7 +9641,7 @@ static void disas_data_proc_fp(DisasContext *s, uint32_t insn) switch (ctz32(extract32(insn, 12, 4))) { case 0: /* [15:12] == xxx1 */ /* Floating point immediate */ - disas_fp_imm(s, insn); + unallocated_encoding(s); /* in decodetree */ break; case 1: /* [15:12] == xx10 */ /* Floating point compare */ @@ -8890,874 +9664,6 @@ static void disas_data_proc_fp(DisasContext *s, uint32_t insn) } } -static void do_ext64(DisasContext *s, TCGv_i64 tcg_left, TCGv_i64 tcg_right, - int pos) -{ - /* Extract 64 bits from the middle of two concatenated 64 bit - * vector register slices left:right. The extracted bits start - * at 'pos' bits into the right (least significant) side. - * We return the result in tcg_right, and guarantee not to - * trash tcg_left. - */ - TCGv_i64 tcg_tmp = tcg_temp_new_i64(); - assert(pos > 0 && pos < 64); - - tcg_gen_shri_i64(tcg_right, tcg_right, pos); - tcg_gen_shli_i64(tcg_tmp, tcg_left, 64 - pos); - tcg_gen_or_i64(tcg_right, tcg_right, tcg_tmp); -} - -/* EXT - * 31 30 29 24 23 22 21 20 16 15 14 11 10 9 5 4 0 - * +---+---+-------------+-----+---+------+---+------+---+------+------+ - * | 0 | Q | 1 0 1 1 1 0 | op2 | 0 | Rm | 0 | imm4 | 0 | Rn | Rd | - * +---+---+-------------+-----+---+------+---+------+---+------+------+ - */ -static void disas_simd_ext(DisasContext *s, uint32_t insn) -{ - int is_q = extract32(insn, 30, 1); - int op2 = extract32(insn, 22, 2); - int imm4 = extract32(insn, 11, 4); - int rm = extract32(insn, 16, 5); - int rn = extract32(insn, 5, 5); - int rd = extract32(insn, 0, 5); - int pos = imm4 << 3; - TCGv_i64 tcg_resl, tcg_resh; - - if (op2 != 0 || (!is_q && extract32(imm4, 3, 1))) { - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - tcg_resh = tcg_temp_new_i64(); - tcg_resl = tcg_temp_new_i64(); - - /* Vd gets bits starting at pos bits into Vm:Vn. This is - * either extracting 128 bits from a 128:128 concatenation, or - * extracting 64 bits from a 64:64 concatenation. - */ - if (!is_q) { - read_vec_element(s, tcg_resl, rn, 0, MO_64); - if (pos != 0) { - read_vec_element(s, tcg_resh, rm, 0, MO_64); - do_ext64(s, tcg_resh, tcg_resl, pos); - } - } else { - TCGv_i64 tcg_hh; - typedef struct { - int reg; - int elt; - } EltPosns; - EltPosns eltposns[] = { {rn, 0}, {rn, 1}, {rm, 0}, {rm, 1} }; - EltPosns *elt = eltposns; - - if (pos >= 64) { - elt++; - pos -= 64; - } - - read_vec_element(s, tcg_resl, elt->reg, elt->elt, MO_64); - elt++; - read_vec_element(s, tcg_resh, elt->reg, elt->elt, MO_64); - elt++; - if (pos != 0) { - do_ext64(s, tcg_resh, tcg_resl, pos); - tcg_hh = tcg_temp_new_i64(); - read_vec_element(s, tcg_hh, elt->reg, elt->elt, MO_64); - do_ext64(s, tcg_hh, tcg_resh, pos); - } - } - - write_vec_element(s, tcg_resl, rd, 0, MO_64); - if (is_q) { - write_vec_element(s, tcg_resh, rd, 1, MO_64); - } - clear_vec_high(s, is_q, rd); -} - -/* TBL/TBX - * 31 30 29 24 23 22 21 20 16 15 14 13 12 11 10 9 5 4 0 - * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+ - * | 0 | Q | 0 0 1 1 1 0 | op2 | 0 | Rm | 0 | len | op | 0 0 | Rn | Rd | - * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+ - */ -static void disas_simd_tb(DisasContext *s, uint32_t insn) -{ - int op2 = extract32(insn, 22, 2); - int is_q = extract32(insn, 30, 1); - int rm = extract32(insn, 16, 5); - int rn = extract32(insn, 5, 5); - int rd = extract32(insn, 0, 5); - int is_tbx = extract32(insn, 12, 1); - int len = (extract32(insn, 13, 2) + 1) * 16; - - if (op2 != 0) { - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd), - vec_full_reg_offset(s, rm), tcg_env, - is_q ? 16 : 8, vec_full_reg_size(s), - (len << 6) | (is_tbx << 5) | rn, - gen_helper_simd_tblx); -} - -/* ZIP/UZP/TRN - * 31 30 29 24 23 22 21 20 16 15 14 12 11 10 9 5 4 0 - * +---+---+-------------+------+---+------+---+------------------+------+ - * | 0 | Q | 0 0 1 1 1 0 | size | 0 | Rm | 0 | opc | 1 0 | Rn | Rd | - * +---+---+-------------+------+---+------+---+------------------+------+ - */ -static void disas_simd_zip_trn(DisasContext *s, uint32_t insn) -{ - int rd = extract32(insn, 0, 5); - int rn = extract32(insn, 5, 5); - int rm = extract32(insn, 16, 5); - int size = extract32(insn, 22, 2); - /* opc field bits [1:0] indicate ZIP/UZP/TRN; - * bit 2 indicates 1 vs 2 variant of the insn. - */ - int opcode = extract32(insn, 12, 2); - bool part = extract32(insn, 14, 1); - bool is_q = extract32(insn, 30, 1); - int esize = 8 << size; - int i; - int datasize = is_q ? 128 : 64; - int elements = datasize / esize; - TCGv_i64 tcg_res[2], tcg_ele; - - if (opcode == 0 || (size == 3 && !is_q)) { - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - tcg_res[0] = tcg_temp_new_i64(); - tcg_res[1] = is_q ? tcg_temp_new_i64() : NULL; - tcg_ele = tcg_temp_new_i64(); - - for (i = 0; i < elements; i++) { - int o, w; - - switch (opcode) { - case 1: /* UZP1/2 */ - { - int midpoint = elements / 2; - if (i < midpoint) { - read_vec_element(s, tcg_ele, rn, 2 * i + part, size); - } else { - read_vec_element(s, tcg_ele, rm, - 2 * (i - midpoint) + part, size); - } - break; - } - case 2: /* TRN1/2 */ - if (i & 1) { - read_vec_element(s, tcg_ele, rm, (i & ~1) + part, size); - } else { - read_vec_element(s, tcg_ele, rn, (i & ~1) + part, size); - } - break; - case 3: /* ZIP1/2 */ - { - int base = part * elements / 2; - if (i & 1) { - read_vec_element(s, tcg_ele, rm, base + (i >> 1), size); - } else { - read_vec_element(s, tcg_ele, rn, base + (i >> 1), size); - } - break; - } - default: - g_assert_not_reached(); - } - - w = (i * esize) / 64; - o = (i * esize) % 64; - if (o == 0) { - tcg_gen_mov_i64(tcg_res[w], tcg_ele); - } else { - tcg_gen_shli_i64(tcg_ele, tcg_ele, o); - tcg_gen_or_i64(tcg_res[w], tcg_res[w], tcg_ele); - } - } - - for (i = 0; i <= is_q; ++i) { - write_vec_element(s, tcg_res[i], rd, i, MO_64); - } - clear_vec_high(s, is_q, rd); -} - -/* - * do_reduction_op helper - * - * This mirrors the Reduce() pseudocode in the ARM ARM. It is - * important for correct NaN propagation that we do these - * operations in exactly the order specified by the pseudocode. - * - * This is a recursive function, TCG temps should be freed by the - * calling function once it is done with the values. - */ -static TCGv_i32 do_reduction_op(DisasContext *s, int fpopcode, int rn, - int esize, int size, int vmap, TCGv_ptr fpst) -{ - if (esize == size) { - int element; - MemOp msize = esize == 16 ? MO_16 : MO_32; - TCGv_i32 tcg_elem; - - /* We should have one register left here */ - assert(ctpop8(vmap) == 1); - element = ctz32(vmap); - assert(element < 8); - - tcg_elem = tcg_temp_new_i32(); - read_vec_element_i32(s, tcg_elem, rn, element, msize); - return tcg_elem; - } else { - int bits = size / 2; - int shift = ctpop8(vmap) / 2; - int vmap_lo = (vmap >> shift) & vmap; - int vmap_hi = (vmap & ~vmap_lo); - TCGv_i32 tcg_hi, tcg_lo, tcg_res; - - tcg_hi = do_reduction_op(s, fpopcode, rn, esize, bits, vmap_hi, fpst); - tcg_lo = do_reduction_op(s, fpopcode, rn, esize, bits, vmap_lo, fpst); - tcg_res = tcg_temp_new_i32(); - - switch (fpopcode) { - case 0x0c: /* fmaxnmv half-precision */ - gen_helper_advsimd_maxnumh(tcg_res, tcg_lo, tcg_hi, fpst); - break; - case 0x0f: /* fmaxv half-precision */ - gen_helper_advsimd_maxh(tcg_res, tcg_lo, tcg_hi, fpst); - break; - case 0x1c: /* fminnmv half-precision */ - gen_helper_advsimd_minnumh(tcg_res, tcg_lo, tcg_hi, fpst); - break; - case 0x1f: /* fminv half-precision */ - gen_helper_advsimd_minh(tcg_res, tcg_lo, tcg_hi, fpst); - break; - case 0x2c: /* fmaxnmv */ - gen_helper_vfp_maxnums(tcg_res, tcg_lo, tcg_hi, fpst); - break; - case 0x2f: /* fmaxv */ - gen_helper_vfp_maxs(tcg_res, tcg_lo, tcg_hi, fpst); - break; - case 0x3c: /* fminnmv */ - gen_helper_vfp_minnums(tcg_res, tcg_lo, tcg_hi, fpst); - break; - case 0x3f: /* fminv */ - gen_helper_vfp_mins(tcg_res, tcg_lo, tcg_hi, fpst); - break; - default: - g_assert_not_reached(); - } - return tcg_res; - } -} - -/* AdvSIMD across lanes - * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0 - * +---+---+---+-----------+------+-----------+--------+-----+------+------+ - * | 0 | Q | U | 0 1 1 1 0 | size | 1 1 0 0 0 | opcode | 1 0 | Rn | Rd | - * +---+---+---+-----------+------+-----------+--------+-----+------+------+ - */ -static void disas_simd_across_lanes(DisasContext *s, uint32_t insn) -{ - int rd = extract32(insn, 0, 5); - int rn = extract32(insn, 5, 5); - int size = extract32(insn, 22, 2); - int opcode = extract32(insn, 12, 5); - bool is_q = extract32(insn, 30, 1); - bool is_u = extract32(insn, 29, 1); - bool is_fp = false; - bool is_min = false; - int esize; - int elements; - int i; - TCGv_i64 tcg_res, tcg_elt; - - switch (opcode) { - case 0x1b: /* ADDV */ - if (is_u) { - unallocated_encoding(s); - return; - } - /* fall through */ - case 0x3: /* SADDLV, UADDLV */ - case 0xa: /* SMAXV, UMAXV */ - case 0x1a: /* SMINV, UMINV */ - if (size == 3 || (size == 2 && !is_q)) { - unallocated_encoding(s); - return; - } - break; - case 0xc: /* FMAXNMV, FMINNMV */ - case 0xf: /* FMAXV, FMINV */ - /* Bit 1 of size field encodes min vs max and the actual size - * depends on the encoding of the U bit. If not set (and FP16 - * enabled) then we do half-precision float instead of single - * precision. - */ - is_min = extract32(size, 1, 1); - is_fp = true; - if (!is_u && dc_isar_feature(aa64_fp16, s)) { - size = 1; - } else if (!is_u || !is_q || extract32(size, 0, 1)) { - unallocated_encoding(s); - return; - } else { - size = 2; - } - break; - default: - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - esize = 8 << size; - elements = (is_q ? 128 : 64) / esize; - - tcg_res = tcg_temp_new_i64(); - tcg_elt = tcg_temp_new_i64(); - - /* These instructions operate across all lanes of a vector - * to produce a single result. We can guarantee that a 64 - * bit intermediate is sufficient: - * + for [US]ADDLV the maximum element size is 32 bits, and - * the result type is 64 bits - * + for FMAX*V, FMIN*V, ADDV the intermediate type is the - * same as the element size, which is 32 bits at most - * For the integer operations we can choose to work at 64 - * or 32 bits and truncate at the end; for simplicity - * we use 64 bits always. The floating point - * ops do require 32 bit intermediates, though. - */ - if (!is_fp) { - read_vec_element(s, tcg_res, rn, 0, size | (is_u ? 0 : MO_SIGN)); - - for (i = 1; i < elements; i++) { - read_vec_element(s, tcg_elt, rn, i, size | (is_u ? 0 : MO_SIGN)); - - switch (opcode) { - case 0x03: /* SADDLV / UADDLV */ - case 0x1b: /* ADDV */ - tcg_gen_add_i64(tcg_res, tcg_res, tcg_elt); - break; - case 0x0a: /* SMAXV / UMAXV */ - if (is_u) { - tcg_gen_umax_i64(tcg_res, tcg_res, tcg_elt); - } else { - tcg_gen_smax_i64(tcg_res, tcg_res, tcg_elt); - } - break; - case 0x1a: /* SMINV / UMINV */ - if (is_u) { - tcg_gen_umin_i64(tcg_res, tcg_res, tcg_elt); - } else { - tcg_gen_smin_i64(tcg_res, tcg_res, tcg_elt); - } - break; - default: - g_assert_not_reached(); - } - - } - } else { - /* Floating point vector reduction ops which work across 32 - * bit (single) or 16 bit (half-precision) intermediates. - * Note that correct NaN propagation requires that we do these - * operations in exactly the order specified by the pseudocode. - */ - TCGv_ptr fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); - int fpopcode = opcode | is_min << 4 | is_u << 5; - int vmap = (1 << elements) - 1; - TCGv_i32 tcg_res32 = do_reduction_op(s, fpopcode, rn, esize, - (is_q ? 128 : 64), vmap, fpst); - tcg_gen_extu_i32_i64(tcg_res, tcg_res32); - } - - /* Now truncate the result to the width required for the final output */ - if (opcode == 0x03) { - /* SADDLV, UADDLV: result is 2*esize */ - size++; - } - - switch (size) { - case 0: - tcg_gen_ext8u_i64(tcg_res, tcg_res); - break; - case 1: - tcg_gen_ext16u_i64(tcg_res, tcg_res); - break; - case 2: - tcg_gen_ext32u_i64(tcg_res, tcg_res); - break; - case 3: - break; - default: - g_assert_not_reached(); - } - - write_fp_dreg(s, rd, tcg_res); -} - -/* AdvSIMD modified immediate - * 31 30 29 28 19 18 16 15 12 11 10 9 5 4 0 - * +---+---+----+---------------------+-----+-------+----+---+-------+------+ - * | 0 | Q | op | 0 1 1 1 1 0 0 0 0 0 | abc | cmode | o2 | 1 | defgh | Rd | - * +---+---+----+---------------------+-----+-------+----+---+-------+------+ - * - * There are a number of operations that can be carried out here: - * MOVI - move (shifted) imm into register - * MVNI - move inverted (shifted) imm into register - * ORR - bitwise OR of (shifted) imm with register - * BIC - bitwise clear of (shifted) imm with register - * With ARMv8.2 we also have: - * FMOV half-precision - */ -static void disas_simd_mod_imm(DisasContext *s, uint32_t insn) -{ - int rd = extract32(insn, 0, 5); - int cmode = extract32(insn, 12, 4); - int o2 = extract32(insn, 11, 1); - uint64_t abcdefgh = extract32(insn, 5, 5) | (extract32(insn, 16, 3) << 5); - bool is_neg = extract32(insn, 29, 1); - bool is_q = extract32(insn, 30, 1); - uint64_t imm = 0; - - if (o2) { - if (cmode != 0xf || is_neg) { - unallocated_encoding(s); - return; - } - /* FMOV (vector, immediate) - half-precision */ - if (!dc_isar_feature(aa64_fp16, s)) { - unallocated_encoding(s); - return; - } - imm = vfp_expand_imm(MO_16, abcdefgh); - /* now duplicate across the lanes */ - imm = dup_const(MO_16, imm); - } else { - if (cmode == 0xf && is_neg && !is_q) { - unallocated_encoding(s); - return; - } - imm = asimd_imm_const(abcdefgh, cmode, is_neg); - } - - if (!fp_access_check(s)) { - return; - } - - if (!((cmode & 0x9) == 0x1 || (cmode & 0xd) == 0x9)) { - /* MOVI or MVNI, with MVNI negation handled above. */ - tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), is_q ? 16 : 8, - vec_full_reg_size(s), imm); - } else { - /* ORR or BIC, with BIC negation to AND handled above. */ - if (is_neg) { - gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_andi, MO_64); - } else { - gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_ori, MO_64); - } - } -} - -/* - * Common SSHR[RA]/USHR[RA] - Shift right (optional rounding/accumulate) - * - * This code is handles the common shifting code and is used by both - * the vector and scalar code. - */ -static void handle_shri_with_rndacc(TCGv_i64 tcg_res, TCGv_i64 tcg_src, - TCGv_i64 tcg_rnd, bool accumulate, - bool is_u, int size, int shift) -{ - bool extended_result = false; - bool round = tcg_rnd != NULL; - int ext_lshift = 0; - TCGv_i64 tcg_src_hi; - - if (round && size == 3) { - extended_result = true; - ext_lshift = 64 - shift; - tcg_src_hi = tcg_temp_new_i64(); - } else if (shift == 64) { - if (!accumulate && is_u) { - /* result is zero */ - tcg_gen_movi_i64(tcg_res, 0); - return; - } - } - - /* Deal with the rounding step */ - if (round) { - if (extended_result) { - TCGv_i64 tcg_zero = tcg_constant_i64(0); - if (!is_u) { - /* take care of sign extending tcg_res */ - tcg_gen_sari_i64(tcg_src_hi, tcg_src, 63); - tcg_gen_add2_i64(tcg_src, tcg_src_hi, - tcg_src, tcg_src_hi, - tcg_rnd, tcg_zero); - } else { - tcg_gen_add2_i64(tcg_src, tcg_src_hi, - tcg_src, tcg_zero, - tcg_rnd, tcg_zero); - } - } else { - tcg_gen_add_i64(tcg_src, tcg_src, tcg_rnd); - } - } - - /* Now do the shift right */ - if (round && extended_result) { - /* extended case, >64 bit precision required */ - if (ext_lshift == 0) { - /* special case, only high bits matter */ - tcg_gen_mov_i64(tcg_src, tcg_src_hi); - } else { - tcg_gen_shri_i64(tcg_src, tcg_src, shift); - tcg_gen_shli_i64(tcg_src_hi, tcg_src_hi, ext_lshift); - tcg_gen_or_i64(tcg_src, tcg_src, tcg_src_hi); - } - } else { - if (is_u) { - if (shift == 64) { - /* essentially shifting in 64 zeros */ - tcg_gen_movi_i64(tcg_src, 0); - } else { - tcg_gen_shri_i64(tcg_src, tcg_src, shift); - } - } else { - if (shift == 64) { - /* effectively extending the sign-bit */ - tcg_gen_sari_i64(tcg_src, tcg_src, 63); - } else { - tcg_gen_sari_i64(tcg_src, tcg_src, shift); - } - } - } - - if (accumulate) { - tcg_gen_add_i64(tcg_res, tcg_res, tcg_src); - } else { - tcg_gen_mov_i64(tcg_res, tcg_src); - } -} - -/* SSHR[RA]/USHR[RA] - Scalar shift right (optional rounding/accumulate) */ -static void handle_scalar_simd_shri(DisasContext *s, - bool is_u, int immh, int immb, - int opcode, int rn, int rd) -{ - const int size = 3; - int immhb = immh << 3 | immb; - int shift = 2 * (8 << size) - immhb; - bool accumulate = false; - bool round = false; - bool insert = false; - TCGv_i64 tcg_rn; - TCGv_i64 tcg_rd; - TCGv_i64 tcg_round; - - if (!extract32(immh, 3, 1)) { - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - switch (opcode) { - case 0x02: /* SSRA / USRA (accumulate) */ - accumulate = true; - break; - case 0x04: /* SRSHR / URSHR (rounding) */ - round = true; - break; - case 0x06: /* SRSRA / URSRA (accum + rounding) */ - accumulate = round = true; - break; - case 0x08: /* SRI */ - insert = true; - break; - } - - if (round) { - tcg_round = tcg_constant_i64(1ULL << (shift - 1)); - } else { - tcg_round = NULL; - } - - tcg_rn = read_fp_dreg(s, rn); - tcg_rd = (accumulate || insert) ? read_fp_dreg(s, rd) : tcg_temp_new_i64(); - - if (insert) { - /* shift count same as element size is valid but does nothing; - * special case to avoid potential shift by 64. - */ - int esize = 8 << size; - if (shift != esize) { - tcg_gen_shri_i64(tcg_rn, tcg_rn, shift); - tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, 0, esize - shift); - } - } else { - handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round, - accumulate, is_u, size, shift); - } - - write_fp_dreg(s, rd, tcg_rd); -} - -/* SHL/SLI - Scalar shift left */ -static void handle_scalar_simd_shli(DisasContext *s, bool insert, - int immh, int immb, int opcode, - int rn, int rd) -{ - int size = 32 - clz32(immh) - 1; - int immhb = immh << 3 | immb; - int shift = immhb - (8 << size); - TCGv_i64 tcg_rn; - TCGv_i64 tcg_rd; - - if (!extract32(immh, 3, 1)) { - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - tcg_rn = read_fp_dreg(s, rn); - tcg_rd = insert ? read_fp_dreg(s, rd) : tcg_temp_new_i64(); - - if (insert) { - tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, shift, 64 - shift); - } else { - tcg_gen_shli_i64(tcg_rd, tcg_rn, shift); - } - - write_fp_dreg(s, rd, tcg_rd); -} - -/* SQSHRN/SQSHRUN - Saturating (signed/unsigned) shift right with - * (signed/unsigned) narrowing */ -static void handle_vec_simd_sqshrn(DisasContext *s, bool is_scalar, bool is_q, - bool is_u_shift, bool is_u_narrow, - int immh, int immb, int opcode, - int rn, int rd) -{ - int immhb = immh << 3 | immb; - int size = 32 - clz32(immh) - 1; - int esize = 8 << size; - int shift = (2 * esize) - immhb; - int elements = is_scalar ? 1 : (64 / esize); - bool round = extract32(opcode, 0, 1); - MemOp ldop = (size + 1) | (is_u_shift ? 0 : MO_SIGN); - TCGv_i64 tcg_rn, tcg_rd, tcg_round; - TCGv_i32 tcg_rd_narrowed; - TCGv_i64 tcg_final; - - static NeonGenNarrowEnvFn * const signed_narrow_fns[4][2] = { - { gen_helper_neon_narrow_sat_s8, - gen_helper_neon_unarrow_sat8 }, - { gen_helper_neon_narrow_sat_s16, - gen_helper_neon_unarrow_sat16 }, - { gen_helper_neon_narrow_sat_s32, - gen_helper_neon_unarrow_sat32 }, - { NULL, NULL }, - }; - static NeonGenNarrowEnvFn * const unsigned_narrow_fns[4] = { - gen_helper_neon_narrow_sat_u8, - gen_helper_neon_narrow_sat_u16, - gen_helper_neon_narrow_sat_u32, - NULL - }; - NeonGenNarrowEnvFn *narrowfn; - - int i; - - assert(size < 4); - - if (extract32(immh, 3, 1)) { - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - if (is_u_shift) { - narrowfn = unsigned_narrow_fns[size]; - } else { - narrowfn = signed_narrow_fns[size][is_u_narrow ? 1 : 0]; - } - - tcg_rn = tcg_temp_new_i64(); - tcg_rd = tcg_temp_new_i64(); - tcg_rd_narrowed = tcg_temp_new_i32(); - tcg_final = tcg_temp_new_i64(); - - if (round) { - tcg_round = tcg_constant_i64(1ULL << (shift - 1)); - } else { - tcg_round = NULL; - } - - for (i = 0; i < elements; i++) { - read_vec_element(s, tcg_rn, rn, i, ldop); - handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round, - false, is_u_shift, size+1, shift); - narrowfn(tcg_rd_narrowed, tcg_env, tcg_rd); - tcg_gen_extu_i32_i64(tcg_rd, tcg_rd_narrowed); - if (i == 0) { - tcg_gen_extract_i64(tcg_final, tcg_rd, 0, esize); - } else { - tcg_gen_deposit_i64(tcg_final, tcg_final, tcg_rd, esize * i, esize); - } - } - - if (!is_q) { - write_vec_element(s, tcg_final, rd, 0, MO_64); - } else { - write_vec_element(s, tcg_final, rd, 1, MO_64); - } - clear_vec_high(s, is_q, rd); -} - -/* SQSHLU, UQSHL, SQSHL: saturating left shifts */ -static void handle_simd_qshl(DisasContext *s, bool scalar, bool is_q, - bool src_unsigned, bool dst_unsigned, - int immh, int immb, int rn, int rd) -{ - int immhb = immh << 3 | immb; - int size = 32 - clz32(immh) - 1; - int shift = immhb - (8 << size); - int pass; - - assert(immh != 0); - assert(!(scalar && is_q)); - - if (!scalar) { - if (!is_q && extract32(immh, 3, 1)) { - unallocated_encoding(s); - return; - } - - /* Since we use the variable-shift helpers we must - * replicate the shift count into each element of - * the tcg_shift value. - */ - switch (size) { - case 0: - shift |= shift << 8; - /* fall through */ - case 1: - shift |= shift << 16; - break; - case 2: - case 3: - break; - default: - g_assert_not_reached(); - } - } - - if (!fp_access_check(s)) { - return; - } - - if (size == 3) { - TCGv_i64 tcg_shift = tcg_constant_i64(shift); - static NeonGenTwo64OpEnvFn * const fns[2][2] = { - { gen_helper_neon_qshl_s64, gen_helper_neon_qshlu_s64 }, - { NULL, gen_helper_neon_qshl_u64 }, - }; - NeonGenTwo64OpEnvFn *genfn = fns[src_unsigned][dst_unsigned]; - int maxpass = is_q ? 2 : 1; - - for (pass = 0; pass < maxpass; pass++) { - TCGv_i64 tcg_op = tcg_temp_new_i64(); - - read_vec_element(s, tcg_op, rn, pass, MO_64); - genfn(tcg_op, tcg_env, tcg_op, tcg_shift); - write_vec_element(s, tcg_op, rd, pass, MO_64); - } - clear_vec_high(s, is_q, rd); - } else { - TCGv_i32 tcg_shift = tcg_constant_i32(shift); - static NeonGenTwoOpEnvFn * const fns[2][2][3] = { - { - { gen_helper_neon_qshl_s8, - gen_helper_neon_qshl_s16, - gen_helper_neon_qshl_s32 }, - { gen_helper_neon_qshlu_s8, - gen_helper_neon_qshlu_s16, - gen_helper_neon_qshlu_s32 } - }, { - { NULL, NULL, NULL }, - { gen_helper_neon_qshl_u8, - gen_helper_neon_qshl_u16, - gen_helper_neon_qshl_u32 } - } - }; - NeonGenTwoOpEnvFn *genfn = fns[src_unsigned][dst_unsigned][size]; - MemOp memop = scalar ? size : MO_32; - int maxpass = scalar ? 1 : is_q ? 4 : 2; - - for (pass = 0; pass < maxpass; pass++) { - TCGv_i32 tcg_op = tcg_temp_new_i32(); - - read_vec_element_i32(s, tcg_op, rn, pass, memop); - genfn(tcg_op, tcg_env, tcg_op, tcg_shift); - if (scalar) { - switch (size) { - case 0: - tcg_gen_ext8u_i32(tcg_op, tcg_op); - break; - case 1: - tcg_gen_ext16u_i32(tcg_op, tcg_op); - break; - case 2: - break; - default: - g_assert_not_reached(); - } - write_fp_sreg(s, rd, tcg_op); - } else { - write_vec_element_i32(s, tcg_op, rd, pass, MO_32); - } - } - - if (!scalar) { - clear_vec_high(s, is_q, rd); - } - } -} - /* Common vector code for handling integer to FP conversion */ static void handle_simd_intfp_conv(DisasContext *s, int rd, int rn, int elements, int is_signed, @@ -10019,53 +9925,26 @@ static void disas_simd_scalar_shift_imm(DisasContext *s, uint32_t insn) } switch (opcode) { - case 0x08: /* SRI */ - if (!is_u) { - unallocated_encoding(s); - return; - } - /* fall through */ - case 0x00: /* SSHR / USHR */ - case 0x02: /* SSRA / USRA */ - case 0x04: /* SRSHR / URSHR */ - case 0x06: /* SRSRA / URSRA */ - handle_scalar_simd_shri(s, is_u, immh, immb, opcode, rn, rd); - break; - case 0x0a: /* SHL / SLI */ - handle_scalar_simd_shli(s, is_u, immh, immb, opcode, rn, rd); - break; case 0x1c: /* SCVTF, UCVTF */ handle_simd_shift_intfp_conv(s, true, false, is_u, immh, immb, opcode, rn, rd); break; - case 0x10: /* SQSHRUN, SQSHRUN2 */ - case 0x11: /* SQRSHRUN, SQRSHRUN2 */ - if (!is_u) { - unallocated_encoding(s); - return; - } - handle_vec_simd_sqshrn(s, true, false, false, true, - immh, immb, opcode, rn, rd); - break; - case 0x12: /* SQSHRN, SQSHRN2, UQSHRN */ - case 0x13: /* SQRSHRN, SQRSHRN2, UQRSHRN, UQRSHRN2 */ - handle_vec_simd_sqshrn(s, true, false, is_u, is_u, - immh, immb, opcode, rn, rd); - break; - case 0xc: /* SQSHLU */ - if (!is_u) { - unallocated_encoding(s); - return; - } - handle_simd_qshl(s, true, false, false, true, immh, immb, rn, rd); - break; - case 0xe: /* SQSHL, UQSHL */ - handle_simd_qshl(s, true, false, is_u, is_u, immh, immb, rn, rd); - break; case 0x1f: /* FCVTZS, FCVTZU */ handle_simd_shift_fpint_conv(s, true, false, is_u, immh, immb, rn, rd); break; default: + case 0x00: /* SSHR / USHR */ + case 0x02: /* SSRA / USRA */ + case 0x04: /* SRSHR / URSHR */ + case 0x06: /* SRSRA / URSRA */ + case 0x08: /* SRI */ + case 0x0a: /* SHL / SLI */ + case 0x0c: /* SQSHLU */ + case 0x0e: /* SQSHL, UQSHL */ + case 0x10: /* SQSHRUN */ + case 0x11: /* SQRSHRUN */ + case 0x12: /* SQSHRN, UQSHRN */ + case 0x13: /* SQRSHRN, UQRSHRN */ unallocated_encoding(s); break; } @@ -10380,35 +10259,35 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar, * in the source becomes a size element in the destination). */ int pass; - TCGv_i32 tcg_res[2]; + TCGv_i64 tcg_res[2]; int destelt = is_q ? 2 : 0; int passes = scalar ? 1 : 2; if (scalar) { - tcg_res[1] = tcg_constant_i32(0); + tcg_res[1] = tcg_constant_i64(0); } for (pass = 0; pass < passes; pass++) { TCGv_i64 tcg_op = tcg_temp_new_i64(); - NeonGenNarrowFn *genfn = NULL; - NeonGenNarrowEnvFn *genenvfn = NULL; + NeonGenOne64OpFn *genfn = NULL; + NeonGenOne64OpEnvFn *genenvfn = NULL; if (scalar) { read_vec_element(s, tcg_op, rn, pass, size + 1); } else { read_vec_element(s, tcg_op, rn, pass, MO_64); } - tcg_res[pass] = tcg_temp_new_i32(); + tcg_res[pass] = tcg_temp_new_i64(); switch (opcode) { case 0x12: /* XTN, SQXTUN */ { - static NeonGenNarrowFn * const xtnfns[3] = { + static NeonGenOne64OpFn * const xtnfns[3] = { gen_helper_neon_narrow_u8, gen_helper_neon_narrow_u16, - tcg_gen_extrl_i64_i32, + tcg_gen_ext32u_i64, }; - static NeonGenNarrowEnvFn * const sqxtunfns[3] = { + static NeonGenOne64OpEnvFn * const sqxtunfns[3] = { gen_helper_neon_unarrow_sat8, gen_helper_neon_unarrow_sat16, gen_helper_neon_unarrow_sat32, @@ -10422,7 +10301,7 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar, } case 0x14: /* SQXTN, UQXTN */ { - static NeonGenNarrowEnvFn * const fns[3][2] = { + static NeonGenOne64OpEnvFn * const fns[3][2] = { { gen_helper_neon_narrow_sat_s8, gen_helper_neon_narrow_sat_u8 }, { gen_helper_neon_narrow_sat_s16, @@ -10436,7 +10315,9 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar, case 0x16: /* FCVTN, FCVTN2 */ /* 32 bit to 16 bit or 64 bit to 32 bit float conversion */ if (size == 2) { - gen_helper_vfp_fcvtsd(tcg_res[pass], tcg_op, tcg_env); + TCGv_i32 tmp = tcg_temp_new_i32(); + gen_helper_vfp_fcvtsd(tmp, tcg_op, tcg_env); + tcg_gen_extu_i32_i64(tcg_res[pass], tmp); } else { TCGv_i32 tcg_lo = tcg_temp_new_i32(); TCGv_i32 tcg_hi = tcg_temp_new_i32(); @@ -10446,21 +10327,29 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar, tcg_gen_extr_i64_i32(tcg_lo, tcg_hi, tcg_op); gen_helper_vfp_fcvt_f32_to_f16(tcg_lo, tcg_lo, fpst, ahp); gen_helper_vfp_fcvt_f32_to_f16(tcg_hi, tcg_hi, fpst, ahp); - tcg_gen_deposit_i32(tcg_res[pass], tcg_lo, tcg_hi, 16, 16); + tcg_gen_deposit_i32(tcg_lo, tcg_lo, tcg_hi, 16, 16); + tcg_gen_extu_i32_i64(tcg_res[pass], tcg_lo); } break; case 0x36: /* BFCVTN, BFCVTN2 */ { TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); - gen_helper_bfcvt_pair(tcg_res[pass], tcg_op, fpst); + TCGv_i32 tmp = tcg_temp_new_i32(); + gen_helper_bfcvt_pair(tmp, tcg_op, fpst); + tcg_gen_extu_i32_i64(tcg_res[pass], tmp); } break; case 0x56: /* FCVTXN, FCVTXN2 */ - /* 64 bit to 32 bit float conversion - * with von Neumann rounding (round to odd) - */ - assert(size == 2); - gen_helper_fcvtx_f64_to_f32(tcg_res[pass], tcg_op, tcg_env); + { + /* + * 64 bit to 32 bit float conversion + * with von Neumann rounding (round to odd) + */ + TCGv_i32 tmp = tcg_temp_new_i32(); + assert(size == 2); + gen_helper_fcvtx_f64_to_f32(tmp, tcg_op, tcg_env); + tcg_gen_extu_i32_i64(tcg_res[pass], tmp); + } break; default: g_assert_not_reached(); @@ -10474,7 +10363,7 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar, } for (pass = 0; pass < 2; pass++) { - write_vec_element_i32(s, tcg_res[pass], rd, destelt + pass, MO_32); + write_vec_element(s, tcg_res[pass], rd, destelt + pass, MO_32); } clear_vec_high(s, is_q, rd); } @@ -10667,185 +10556,6 @@ static void disas_simd_scalar_two_reg_misc(DisasContext *s, uint32_t insn) } } -/* SSHR[RA]/USHR[RA] - Vector shift right (optional rounding/accumulate) */ -static void handle_vec_simd_shri(DisasContext *s, bool is_q, bool is_u, - int immh, int immb, int opcode, int rn, int rd) -{ - int size = 32 - clz32(immh) - 1; - int immhb = immh << 3 | immb; - int shift = 2 * (8 << size) - immhb; - GVecGen2iFn *gvec_fn; - - if (extract32(immh, 3, 1) && !is_q) { - unallocated_encoding(s); - return; - } - tcg_debug_assert(size <= 3); - - if (!fp_access_check(s)) { - return; - } - - switch (opcode) { - case 0x02: /* SSRA / USRA (accumulate) */ - gvec_fn = is_u ? gen_gvec_usra : gen_gvec_ssra; - break; - - case 0x08: /* SRI */ - gvec_fn = gen_gvec_sri; - break; - - case 0x00: /* SSHR / USHR */ - if (is_u) { - if (shift == 8 << size) { - /* Shift count the same size as element size produces zero. */ - tcg_gen_gvec_dup_imm(size, vec_full_reg_offset(s, rd), - is_q ? 16 : 8, vec_full_reg_size(s), 0); - return; - } - gvec_fn = tcg_gen_gvec_shri; - } else { - /* Shift count the same size as element size produces all sign. */ - if (shift == 8 << size) { - shift -= 1; - } - gvec_fn = tcg_gen_gvec_sari; - } - break; - - case 0x04: /* SRSHR / URSHR (rounding) */ - gvec_fn = is_u ? gen_gvec_urshr : gen_gvec_srshr; - break; - - case 0x06: /* SRSRA / URSRA (accum + rounding) */ - gvec_fn = is_u ? gen_gvec_ursra : gen_gvec_srsra; - break; - - default: - g_assert_not_reached(); - } - - gen_gvec_fn2i(s, is_q, rd, rn, shift, gvec_fn, size); -} - -/* SHL/SLI - Vector shift left */ -static void handle_vec_simd_shli(DisasContext *s, bool is_q, bool insert, - int immh, int immb, int opcode, int rn, int rd) -{ - int size = 32 - clz32(immh) - 1; - int immhb = immh << 3 | immb; - int shift = immhb - (8 << size); - - /* Range of size is limited by decode: immh is a non-zero 4 bit field */ - assert(size >= 0 && size <= 3); - - if (extract32(immh, 3, 1) && !is_q) { - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - if (insert) { - gen_gvec_fn2i(s, is_q, rd, rn, shift, gen_gvec_sli, size); - } else { - gen_gvec_fn2i(s, is_q, rd, rn, shift, tcg_gen_gvec_shli, size); - } -} - -/* USHLL/SHLL - Vector shift left with widening */ -static void handle_vec_simd_wshli(DisasContext *s, bool is_q, bool is_u, - int immh, int immb, int opcode, int rn, int rd) -{ - int size = 32 - clz32(immh) - 1; - int immhb = immh << 3 | immb; - int shift = immhb - (8 << size); - int dsize = 64; - int esize = 8 << size; - int elements = dsize/esize; - TCGv_i64 tcg_rn = tcg_temp_new_i64(); - TCGv_i64 tcg_rd = tcg_temp_new_i64(); - int i; - - if (size >= 3) { - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - /* For the LL variants the store is larger than the load, - * so if rd == rn we would overwrite parts of our input. - * So load everything right now and use shifts in the main loop. - */ - read_vec_element(s, tcg_rn, rn, is_q ? 1 : 0, MO_64); - - for (i = 0; i < elements; i++) { - tcg_gen_shri_i64(tcg_rd, tcg_rn, i * esize); - ext_and_shift_reg(tcg_rd, tcg_rd, size | (!is_u << 2), 0); - tcg_gen_shli_i64(tcg_rd, tcg_rd, shift); - write_vec_element(s, tcg_rd, rd, i, size + 1); - } - clear_vec_high(s, true, rd); -} - -/* SHRN/RSHRN - Shift right with narrowing (and potential rounding) */ -static void handle_vec_simd_shrn(DisasContext *s, bool is_q, - int immh, int immb, int opcode, int rn, int rd) -{ - int immhb = immh << 3 | immb; - int size = 32 - clz32(immh) - 1; - int dsize = 64; - int esize = 8 << size; - int elements = dsize/esize; - int shift = (2 * esize) - immhb; - bool round = extract32(opcode, 0, 1); - TCGv_i64 tcg_rn, tcg_rd, tcg_final; - TCGv_i64 tcg_round; - int i; - - if (extract32(immh, 3, 1)) { - unallocated_encoding(s); - return; - } - - if (!fp_access_check(s)) { - return; - } - - tcg_rn = tcg_temp_new_i64(); - tcg_rd = tcg_temp_new_i64(); - tcg_final = tcg_temp_new_i64(); - read_vec_element(s, tcg_final, rd, is_q ? 1 : 0, MO_64); - - if (round) { - tcg_round = tcg_constant_i64(1ULL << (shift - 1)); - } else { - tcg_round = NULL; - } - - for (i = 0; i < elements; i++) { - read_vec_element(s, tcg_rn, rn, i, size+1); - handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round, - false, true, size+1, shift); - - tcg_gen_deposit_i64(tcg_final, tcg_final, tcg_rd, esize * i, esize); - } - - if (!is_q) { - write_vec_element(s, tcg_final, rd, 0, MO_64); - } else { - write_vec_element(s, tcg_final, rd, 1, MO_64); - } - - clear_vec_high(s, is_q, rd); -} - - /* AdvSIMD shift by immediate * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0 * +---+---+---+-------------+------+------+--------+---+------+------+ @@ -10862,60 +10572,33 @@ static void disas_simd_shift_imm(DisasContext *s, uint32_t insn) bool is_u = extract32(insn, 29, 1); bool is_q = extract32(insn, 30, 1); - /* data_proc_simd[] has sent immh == 0 to disas_simd_mod_imm. */ - assert(immh != 0); + if (immh == 0) { + unallocated_encoding(s); + return; + } switch (opcode) { - case 0x08: /* SRI */ - if (!is_u) { - unallocated_encoding(s); - return; - } - /* fall through */ + case 0x1c: /* SCVTF / UCVTF */ + handle_simd_shift_intfp_conv(s, false, is_q, is_u, immh, immb, + opcode, rn, rd); + break; + case 0x1f: /* FCVTZS/ FCVTZU */ + handle_simd_shift_fpint_conv(s, false, is_q, is_u, immh, immb, rn, rd); + return; + default: case 0x00: /* SSHR / USHR */ case 0x02: /* SSRA / USRA (accumulate) */ case 0x04: /* SRSHR / URSHR (rounding) */ case 0x06: /* SRSRA / URSRA (accum + rounding) */ - handle_vec_simd_shri(s, is_q, is_u, immh, immb, opcode, rn, rd); - break; + case 0x08: /* SRI */ case 0x0a: /* SHL / SLI */ - handle_vec_simd_shli(s, is_q, is_u, immh, immb, opcode, rn, rd); - break; - case 0x10: /* SHRN */ + case 0x0c: /* SQSHLU */ + case 0x0e: /* SQSHL, UQSHL */ + case 0x10: /* SHRN / SQSHRUN */ case 0x11: /* RSHRN / SQRSHRUN */ - if (is_u) { - handle_vec_simd_sqshrn(s, false, is_q, false, true, immh, immb, - opcode, rn, rd); - } else { - handle_vec_simd_shrn(s, is_q, immh, immb, opcode, rn, rd); - } - break; case 0x12: /* SQSHRN / UQSHRN */ case 0x13: /* SQRSHRN / UQRSHRN */ - handle_vec_simd_sqshrn(s, false, is_q, is_u, is_u, immh, immb, - opcode, rn, rd); - break; case 0x14: /* SSHLL / USHLL */ - handle_vec_simd_wshli(s, is_q, is_u, immh, immb, opcode, rn, rd); - break; - case 0x1c: /* SCVTF / UCVTF */ - handle_simd_shift_intfp_conv(s, false, is_q, is_u, immh, immb, - opcode, rn, rd); - break; - case 0xc: /* SQSHLU */ - if (!is_u) { - unallocated_encoding(s); - return; - } - handle_simd_qshl(s, false, is_q, false, true, immh, immb, rn, rd); - break; - case 0xe: /* SQSHL, UQSHL */ - handle_simd_qshl(s, false, is_q, is_u, is_u, immh, immb, rn, rd); - break; - case 0x1f: /* FCVTZS/ FCVTZU */ - handle_simd_shift_fpint_conv(s, false, is_q, is_u, immh, immb, rn, rd); - return; - default: unallocated_encoding(s); return; } @@ -11871,13 +11554,7 @@ static void disas_simd_two_reg_misc_fp16(DisasContext *s, uint32_t insn) static const AArch64DecodeTable data_proc_simd[] = { /* pattern , mask , fn */ { 0x0e200800, 0x9f3e0c00, disas_simd_two_reg_misc }, - { 0x0e300800, 0x9f3e0c00, disas_simd_across_lanes }, - /* simd_mod_imm decode is a subset of simd_shift_imm, so must precede it */ - { 0x0f000400, 0x9ff80400, disas_simd_mod_imm }, { 0x0f000400, 0x9f800400, disas_simd_shift_imm }, - { 0x0e000000, 0xbf208c00, disas_simd_tb }, - { 0x0e000800, 0xbf208c00, disas_simd_zip_trn }, - { 0x2e000000, 0xbf208400, disas_simd_ext }, { 0x5e200800, 0xdf3e0c00, disas_simd_scalar_two_reg_misc }, { 0x5f000400, 0xdf800400, disas_simd_scalar_shift_imm }, { 0x0e780800, 0x8f7e0c00, disas_simd_two_reg_misc_fp16 }, |