diff options
Diffstat (limited to 'target')
50 files changed, 1890 insertions, 568 deletions
diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 8d41f783dc..521444a5a1 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -153,6 +153,49 @@ typedef struct { uint32_t base_mask; } TCR; +/* Define a maximum sized vector register. + * For 32-bit, this is a 128-bit NEON/AdvSIMD register. + * For 64-bit, this is a 2048-bit SVE register. + * + * Note that the mapping between S, D, and Q views of the register bank + * differs between AArch64 and AArch32. + * In AArch32: + * Qn = regs[n].d[1]:regs[n].d[0] + * Dn = regs[n / 2].d[n & 1] + * Sn = regs[n / 4].d[n % 4 / 2], + * bits 31..0 for even n, and bits 63..32 for odd n + * (and regs[16] to regs[31] are inaccessible) + * In AArch64: + * Zn = regs[n].d[*] + * Qn = regs[n].d[1]:regs[n].d[0] + * Dn = regs[n].d[0] + * Sn = regs[n].d[0] bits 31..0 + * + * This corresponds to the architecturally defined mapping between + * the two execution states, and means we do not need to explicitly + * map these registers when changing states. + * + * Align the data for use with TCG host vector operations. + */ + +#ifdef TARGET_AARCH64 +# define ARM_MAX_VQ 16 +#else +# define ARM_MAX_VQ 1 +#endif + +typedef struct ARMVectorReg { + uint64_t d[2 * ARM_MAX_VQ] QEMU_ALIGNED(16); +} ARMVectorReg; + +/* In AArch32 mode, predicate registers do not exist at all. */ +#ifdef TARGET_AARCH64 +typedef struct ARMPredicateReg { + uint64_t p[2 * ARM_MAX_VQ / 8] QEMU_ALIGNED(16); +} ARMPredicateReg; +#endif + + typedef struct CPUARMState { /* Regs for current mode. */ uint32_t regs[16]; @@ -477,22 +520,12 @@ typedef struct CPUARMState { /* VFP coprocessor state. */ struct { - /* VFP/Neon register state. Note that the mapping between S, D and Q - * views of the register bank differs between AArch64 and AArch32: - * In AArch32: - * Qn = regs[2n+1]:regs[2n] - * Dn = regs[n] - * Sn = regs[n/2] bits 31..0 for even n, and bits 63..32 for odd n - * (and regs[32] to regs[63] are inaccessible) - * In AArch64: - * Qn = regs[2n+1]:regs[2n] - * Dn = regs[2n] - * Sn = regs[2n] bits 31..0 - * This corresponds to the architecturally defined mapping between - * the two execution states, and means we do not need to explicitly - * map these registers when changing states. - */ - uint64_t regs[64] QEMU_ALIGNED(16); + ARMVectorReg zregs[32]; + +#ifdef TARGET_AARCH64 + /* Store FFR as pregs[16] to make it easier to treat as any other. */ + ARMPredicateReg pregs[17]; +#endif uint32_t xregs[16]; /* We store these fpcsr fields separately for convenience. */ @@ -516,6 +549,9 @@ typedef struct CPUARMState { */ float_status fp_status; float_status standard_fp_status; + + /* ZCR_EL[1-3] */ + uint64_t zcr_el[4]; } vfp; uint64_t exclusive_addr; uint64_t exclusive_val; @@ -890,6 +926,8 @@ void pmccntr_sync(CPUARMState *env); #define CPTR_TCPAC (1U << 31) #define CPTR_TTA (1U << 20) #define CPTR_TFP (1U << 10) +#define CPTR_TZ (1U << 8) /* CPTR_EL2 */ +#define CPTR_EZ (1U << 8) /* CPTR_EL3 */ #define MDCR_EPMAD (1U << 21) #define MDCR_EDAD (1U << 20) @@ -1341,6 +1379,10 @@ enum arm_features { ARM_FEATURE_M_SECURITY, /* M profile Security Extension */ ARM_FEATURE_JAZELLE, /* has (trivial) Jazelle implementation */ ARM_FEATURE_SVE, /* has Scalable Vector Extension */ + ARM_FEATURE_V8_SHA512, /* implements SHA512 part of v8 Crypto Extensions */ + ARM_FEATURE_V8_SHA3, /* implements SHA3 part of v8 Crypto Extensions */ + ARM_FEATURE_V8_SM3, /* implements SM3 part of v8 Crypto Extensions */ + ARM_FEATURE_V8_SM4, /* implements SM4 part of v8 Crypto Extensions */ }; static inline int arm_feature(CPUARMState *env, int feature) @@ -1506,16 +1548,42 @@ static inline bool armv7m_nvic_can_take_pending_exception(void *opaque) */ void armv7m_nvic_set_pending(void *opaque, int irq, bool secure); /** + * armv7m_nvic_set_pending_derived: mark this derived exception as pending + * @opaque: the NVIC + * @irq: the exception number to mark pending + * @secure: false for non-banked exceptions or for the nonsecure + * version of a banked exception, true for the secure version of a banked + * exception. + * + * Similar to armv7m_nvic_set_pending(), but specifically for derived + * exceptions (exceptions generated in the course of trying to take + * a different exception). + */ +void armv7m_nvic_set_pending_derived(void *opaque, int irq, bool secure); +/** + * armv7m_nvic_get_pending_irq_info: return highest priority pending + * exception, and whether it targets Secure state + * @opaque: the NVIC + * @pirq: set to pending exception number + * @ptargets_secure: set to whether pending exception targets Secure + * + * This function writes the number of the highest priority pending + * exception (the one which would be made active by + * armv7m_nvic_acknowledge_irq()) to @pirq, and sets @ptargets_secure + * to true if the current highest priority pending exception should + * be taken to Secure state, false for NS. + */ +void armv7m_nvic_get_pending_irq_info(void *opaque, int *pirq, + bool *ptargets_secure); +/** * armv7m_nvic_acknowledge_irq: make highest priority pending exception active * @opaque: the NVIC * * Move the current highest priority pending exception from the pending * state to the active state, and update v7m.exception to indicate that * it is the exception currently being handled. - * - * Returns: true if exception should be taken to Secure state, false for NS */ -bool armv7m_nvic_acknowledge_irq(void *opaque); +void armv7m_nvic_acknowledge_irq(void *opaque); /** * armv7m_nvic_complete_irq: complete specified interrupt or exception * @opaque: the NVIC @@ -2610,6 +2678,10 @@ static inline bool arm_cpu_data_is_big_endian(CPUARMState *env) #define ARM_TBFLAG_TBI0_MASK (0x1ull << ARM_TBFLAG_TBI0_SHIFT) #define ARM_TBFLAG_TBI1_SHIFT 1 /* TBI1 for EL0/1 */ #define ARM_TBFLAG_TBI1_MASK (0x1ull << ARM_TBFLAG_TBI1_SHIFT) +#define ARM_TBFLAG_SVEEXC_EL_SHIFT 2 +#define ARM_TBFLAG_SVEEXC_EL_MASK (0x3 << ARM_TBFLAG_SVEEXC_EL_SHIFT) +#define ARM_TBFLAG_ZCR_LEN_SHIFT 4 +#define ARM_TBFLAG_ZCR_LEN_MASK (0xf << ARM_TBFLAG_ZCR_LEN_SHIFT) /* some convenience accessor macros */ #define ARM_TBFLAG_AARCH64_STATE(F) \ @@ -2646,6 +2718,10 @@ static inline bool arm_cpu_data_is_big_endian(CPUARMState *env) (((F) & ARM_TBFLAG_TBI0_MASK) >> ARM_TBFLAG_TBI0_SHIFT) #define ARM_TBFLAG_TBI1(F) \ (((F) & ARM_TBFLAG_TBI1_MASK) >> ARM_TBFLAG_TBI1_SHIFT) +#define ARM_TBFLAG_SVEEXC_EL(F) \ + (((F) & ARM_TBFLAG_SVEEXC_EL_MASK) >> ARM_TBFLAG_SVEEXC_EL_SHIFT) +#define ARM_TBFLAG_ZCR_LEN(F) \ + (((F) & ARM_TBFLAG_ZCR_LEN_MASK) >> ARM_TBFLAG_ZCR_LEN_SHIFT) static inline bool bswap_code(bool sctlr_b) { @@ -2769,7 +2845,7 @@ static inline void *arm_get_el_change_hook_opaque(ARMCPU *cpu) */ static inline uint64_t *aa32_vfp_dreg(CPUARMState *env, unsigned regno) { - return &env->vfp.regs[regno]; + return &env->vfp.zregs[regno >> 1].d[regno & 1]; } /** @@ -2778,7 +2854,7 @@ static inline uint64_t *aa32_vfp_dreg(CPUARMState *env, unsigned regno) */ static inline uint64_t *aa32_vfp_qreg(CPUARMState *env, unsigned regno) { - return &env->vfp.regs[2 * regno]; + return &env->vfp.zregs[regno].d[0]; } /** @@ -2787,7 +2863,7 @@ static inline uint64_t *aa32_vfp_qreg(CPUARMState *env, unsigned regno) */ static inline uint64_t *aa64_vfp_qreg(CPUARMState *env, unsigned regno) { - return &env->vfp.regs[2 * regno]; + return &env->vfp.zregs[regno].d[0]; } #endif diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index 670c07ab6e..1c330adc28 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -224,6 +224,10 @@ static void aarch64_any_initfn(Object *obj) set_feature(&cpu->env, ARM_FEATURE_V8_AES); set_feature(&cpu->env, ARM_FEATURE_V8_SHA1); set_feature(&cpu->env, ARM_FEATURE_V8_SHA256); + set_feature(&cpu->env, ARM_FEATURE_V8_SHA512); + set_feature(&cpu->env, ARM_FEATURE_V8_SHA3); + set_feature(&cpu->env, ARM_FEATURE_V8_SM3); + set_feature(&cpu->env, ARM_FEATURE_V8_SM4); set_feature(&cpu->env, ARM_FEATURE_V8_PMULL); set_feature(&cpu->env, ARM_FEATURE_CRC); cpu->ctr = 0x80038003; /* 32 byte I and D cacheline size, VIPT icache */ diff --git a/target/arm/crypto_helper.c b/target/arm/crypto_helper.c index 9ca0bdead7..cc339ea7e0 100644 --- a/target/arm/crypto_helper.c +++ b/target/arm/crypto_helper.c @@ -1,7 +1,7 @@ /* * crypto_helper.c - emulate v8 Crypto Extensions instructions * - * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org> + * Copyright (C) 2013 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -419,3 +419,278 @@ void HELPER(crypto_sha256su1)(void *vd, void *vn, void *vm) rd[0] = d.l[0]; rd[1] = d.l[1]; } + +/* + * The SHA-512 logical functions (same as above but using 64-bit operands) + */ + +static uint64_t cho512(uint64_t x, uint64_t y, uint64_t z) +{ + return (x & (y ^ z)) ^ z; +} + +static uint64_t maj512(uint64_t x, uint64_t y, uint64_t z) +{ + return (x & y) | ((x | y) & z); +} + +static uint64_t S0_512(uint64_t x) +{ + return ror64(x, 28) ^ ror64(x, 34) ^ ror64(x, 39); +} + +static uint64_t S1_512(uint64_t x) +{ + return ror64(x, 14) ^ ror64(x, 18) ^ ror64(x, 41); +} + +static uint64_t s0_512(uint64_t x) +{ + return ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7); +} + +static uint64_t s1_512(uint64_t x) +{ + return ror64(x, 19) ^ ror64(x, 61) ^ (x >> 6); +} + +void HELPER(crypto_sha512h)(void *vd, void *vn, void *vm) +{ + uint64_t *rd = vd; + uint64_t *rn = vn; + uint64_t *rm = vm; + uint64_t d0 = rd[0]; + uint64_t d1 = rd[1]; + + d1 += S1_512(rm[1]) + cho512(rm[1], rn[0], rn[1]); + d0 += S1_512(d1 + rm[0]) + cho512(d1 + rm[0], rm[1], rn[0]); + + rd[0] = d0; + rd[1] = d1; +} + +void HELPER(crypto_sha512h2)(void *vd, void *vn, void *vm) +{ + uint64_t *rd = vd; + uint64_t *rn = vn; + uint64_t *rm = vm; + uint64_t d0 = rd[0]; + uint64_t d1 = rd[1]; + + d1 += S0_512(rm[0]) + maj512(rn[0], rm[1], rm[0]); + d0 += S0_512(d1) + maj512(d1, rm[0], rm[1]); + + rd[0] = d0; + rd[1] = d1; +} + +void HELPER(crypto_sha512su0)(void *vd, void *vn) +{ + uint64_t *rd = vd; + uint64_t *rn = vn; + uint64_t d0 = rd[0]; + uint64_t d1 = rd[1]; + + d0 += s0_512(rd[1]); + d1 += s0_512(rn[0]); + + rd[0] = d0; + rd[1] = d1; +} + +void HELPER(crypto_sha512su1)(void *vd, void *vn, void *vm) +{ + uint64_t *rd = vd; + uint64_t *rn = vn; + uint64_t *rm = vm; + + rd[0] += s1_512(rn[0]) + rm[0]; + rd[1] += s1_512(rn[1]) + rm[1]; +} + +void HELPER(crypto_sm3partw1)(void *vd, void *vn, void *vm) +{ + uint64_t *rd = vd; + uint64_t *rn = vn; + uint64_t *rm = vm; + union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; + union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; + union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; + uint32_t t; + + t = CR_ST_WORD(d, 0) ^ CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 1), 17); + CR_ST_WORD(d, 0) = t ^ ror32(t, 17) ^ ror32(t, 9); + + t = CR_ST_WORD(d, 1) ^ CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 2), 17); + CR_ST_WORD(d, 1) = t ^ ror32(t, 17) ^ ror32(t, 9); + + t = CR_ST_WORD(d, 2) ^ CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 3), 17); + CR_ST_WORD(d, 2) = t ^ ror32(t, 17) ^ ror32(t, 9); + + t = CR_ST_WORD(d, 3) ^ CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 0), 17); + CR_ST_WORD(d, 3) = t ^ ror32(t, 17) ^ ror32(t, 9); + + rd[0] = d.l[0]; + rd[1] = d.l[1]; +} + +void HELPER(crypto_sm3partw2)(void *vd, void *vn, void *vm) +{ + uint64_t *rd = vd; + uint64_t *rn = vn; + uint64_t *rm = vm; + union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; + union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; + union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; + uint32_t t = CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 0), 25); + + CR_ST_WORD(d, 0) ^= t; + CR_ST_WORD(d, 1) ^= CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 1), 25); + CR_ST_WORD(d, 2) ^= CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 2), 25); + CR_ST_WORD(d, 3) ^= CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(m, 3), 25) ^ + ror32(t, 17) ^ ror32(t, 2) ^ ror32(t, 26); + + rd[0] = d.l[0]; + rd[1] = d.l[1]; +} + +void HELPER(crypto_sm3tt)(void *vd, void *vn, void *vm, uint32_t imm2, + uint32_t opcode) +{ + uint64_t *rd = vd; + uint64_t *rn = vn; + uint64_t *rm = vm; + union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; + union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; + union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; + uint32_t t; + + assert(imm2 < 4); + + if (opcode == 0 || opcode == 2) { + /* SM3TT1A, SM3TT2A */ + t = par(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1)); + } else if (opcode == 1) { + /* SM3TT1B */ + t = maj(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1)); + } else if (opcode == 3) { + /* SM3TT2B */ + t = cho(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1)); + } else { + g_assert_not_reached(); + } + + t += CR_ST_WORD(d, 0) + CR_ST_WORD(m, imm2); + + CR_ST_WORD(d, 0) = CR_ST_WORD(d, 1); + + if (opcode < 2) { + /* SM3TT1A, SM3TT1B */ + t += CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 3), 20); + + CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 23); + } else { + /* SM3TT2A, SM3TT2B */ + t += CR_ST_WORD(n, 3); + t ^= rol32(t, 9) ^ rol32(t, 17); + + CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 13); + } + + CR_ST_WORD(d, 2) = CR_ST_WORD(d, 3); + CR_ST_WORD(d, 3) = t; + + rd[0] = d.l[0]; + rd[1] = d.l[1]; +} + +static uint8_t const sm4_sbox[] = { + 0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, + 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05, + 0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, + 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99, + 0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a, + 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62, + 0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95, + 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6, + 0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba, + 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8, + 0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b, + 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35, + 0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2, + 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87, + 0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52, + 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e, + 0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5, + 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1, + 0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55, + 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3, + 0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60, + 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f, + 0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f, + 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51, + 0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f, + 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8, + 0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd, + 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0, + 0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e, + 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84, + 0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20, + 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48, +}; + +void HELPER(crypto_sm4e)(void *vd, void *vn) +{ + uint64_t *rd = vd; + uint64_t *rn = vn; + union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; + union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; + uint32_t t, i; + + for (i = 0; i < 4; i++) { + t = CR_ST_WORD(d, (i + 1) % 4) ^ + CR_ST_WORD(d, (i + 2) % 4) ^ + CR_ST_WORD(d, (i + 3) % 4) ^ + CR_ST_WORD(n, i); + + t = sm4_sbox[t & 0xff] | + sm4_sbox[(t >> 8) & 0xff] << 8 | + sm4_sbox[(t >> 16) & 0xff] << 16 | + sm4_sbox[(t >> 24) & 0xff] << 24; + + CR_ST_WORD(d, i) ^= t ^ rol32(t, 2) ^ rol32(t, 10) ^ rol32(t, 18) ^ + rol32(t, 24); + } + + rd[0] = d.l[0]; + rd[1] = d.l[1]; +} + +void HELPER(crypto_sm4ekey)(void *vd, void *vn, void* vm) +{ + uint64_t *rd = vd; + uint64_t *rn = vn; + uint64_t *rm = vm; + union CRYPTO_STATE d; + union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; + union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; + uint32_t t, i; + + d = n; + for (i = 0; i < 4; i++) { + t = CR_ST_WORD(d, (i + 1) % 4) ^ + CR_ST_WORD(d, (i + 2) % 4) ^ + CR_ST_WORD(d, (i + 3) % 4) ^ + CR_ST_WORD(m, i); + + t = sm4_sbox[t & 0xff] | + sm4_sbox[(t >> 8) & 0xff] << 8 | + sm4_sbox[(t >> 16) & 0xff] << 16 | + sm4_sbox[(t >> 24) & 0xff] << 24; + + CR_ST_WORD(d, i) ^= t ^ rol32(t, 13) ^ rol32(t, 23); + } + + rd[0] = d.l[0]; + rd[1] = d.l[1]; +} diff --git a/target/arm/helper.c b/target/arm/helper.c index bfce09643b..180ab75458 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -4266,6 +4266,125 @@ static const ARMCPRegInfo debug_lpae_cp_reginfo[] = { REGINFO_SENTINEL }; +/* Return the exception level to which SVE-disabled exceptions should + * be taken, or 0 if SVE is enabled. + */ +static int sve_exception_el(CPUARMState *env) +{ +#ifndef CONFIG_USER_ONLY + unsigned current_el = arm_current_el(env); + + /* The CPACR.ZEN controls traps to EL1: + * 0, 2 : trap EL0 and EL1 accesses + * 1 : trap only EL0 accesses + * 3 : trap no accesses + */ + switch (extract32(env->cp15.cpacr_el1, 16, 2)) { + default: + if (current_el <= 1) { + /* Trap to PL1, which might be EL1 or EL3 */ + if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) { + return 3; + } + return 1; + } + break; + case 1: + if (current_el == 0) { + return 1; + } + break; + case 3: + break; + } + + /* Similarly for CPACR.FPEN, after having checked ZEN. */ + switch (extract32(env->cp15.cpacr_el1, 20, 2)) { + default: + if (current_el <= 1) { + if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) { + return 3; + } + return 1; + } + break; + case 1: + if (current_el == 0) { + return 1; + } + break; + case 3: + break; + } + + /* CPTR_EL2. Check both TZ and TFP. */ + if (current_el <= 2 + && (env->cp15.cptr_el[2] & (CPTR_TFP | CPTR_TZ)) + && !arm_is_secure_below_el3(env)) { + return 2; + } + + /* CPTR_EL3. Check both EZ and TFP. */ + if (!(env->cp15.cptr_el[3] & CPTR_EZ) + || (env->cp15.cptr_el[3] & CPTR_TFP)) { + return 3; + } +#endif + return 0; +} + +static CPAccessResult zcr_access(CPUARMState *env, const ARMCPRegInfo *ri, + bool isread) +{ + switch (sve_exception_el(env)) { + case 3: + return CP_ACCESS_TRAP_EL3; + case 2: + return CP_ACCESS_TRAP_EL2; + case 1: + return CP_ACCESS_TRAP; + } + return CP_ACCESS_OK; +} + +static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + /* Bits other than [3:0] are RAZ/WI. */ + raw_write(env, ri, value & 0xf); +} + +static const ARMCPRegInfo zcr_el1_reginfo = { + .name = "ZCR_EL1", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 0, + .access = PL1_RW, .accessfn = zcr_access, .type = ARM_CP_64BIT, + .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[1]), + .writefn = zcr_write, .raw_writefn = raw_write +}; + +static const ARMCPRegInfo zcr_el2_reginfo = { + .name = "ZCR_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 0, + .access = PL2_RW, .accessfn = zcr_access, .type = ARM_CP_64BIT, + .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[2]), + .writefn = zcr_write, .raw_writefn = raw_write +}; + +static const ARMCPRegInfo zcr_no_el2_reginfo = { + .name = "ZCR_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 0, + .access = PL2_RW, .type = ARM_CP_64BIT, + .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore +}; + +static const ARMCPRegInfo zcr_el3_reginfo = { + .name = "ZCR_EL3", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 2, .opc2 = 0, + .access = PL3_RW, .accessfn = zcr_access, .type = ARM_CP_64BIT, + .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[3]), + .writefn = zcr_write, .raw_writefn = raw_write +}; + void hw_watchpoint_update(ARMCPU *cpu, int n) { CPUARMState *env = &cpu->env; @@ -5332,6 +5451,18 @@ void register_cp_regs_for_features(ARMCPU *cpu) } define_one_arm_cp_reg(cpu, &sctlr); } + + if (arm_feature(env, ARM_FEATURE_SVE)) { + define_one_arm_cp_reg(cpu, &zcr_el1_reginfo); + if (arm_feature(env, ARM_FEATURE_EL2)) { + define_one_arm_cp_reg(cpu, &zcr_el2_reginfo); + } else { + define_one_arm_cp_reg(cpu, &zcr_no_el2_reginfo); + } + if (arm_feature(env, ARM_FEATURE_EL3)) { + define_one_arm_cp_reg(cpu, &zcr_el3_reginfo); + } + } } void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu) @@ -6161,12 +6292,127 @@ uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx, return target_el; } -static void v7m_push(CPUARMState *env, uint32_t val) +static bool v7m_stack_write(ARMCPU *cpu, uint32_t addr, uint32_t value, + ARMMMUIdx mmu_idx, bool ignfault) { - CPUState *cs = CPU(arm_env_get_cpu(env)); + CPUState *cs = CPU(cpu); + CPUARMState *env = &cpu->env; + MemTxAttrs attrs = {}; + MemTxResult txres; + target_ulong page_size; + hwaddr physaddr; + int prot; + ARMMMUFaultInfo fi; + bool secure = mmu_idx & ARM_MMU_IDX_M_S; + int exc; + bool exc_secure; + + if (get_phys_addr(env, addr, MMU_DATA_STORE, mmu_idx, &physaddr, + &attrs, &prot, &page_size, &fi, NULL)) { + /* MPU/SAU lookup failed */ + if (fi.type == ARMFault_QEMU_SFault) { + qemu_log_mask(CPU_LOG_INT, + "...SecureFault with SFSR.AUVIOL during stacking\n"); + env->v7m.sfsr |= R_V7M_SFSR_AUVIOL_MASK | R_V7M_SFSR_SFARVALID_MASK; + env->v7m.sfar = addr; + exc = ARMV7M_EXCP_SECURE; + exc_secure = false; + } else { + qemu_log_mask(CPU_LOG_INT, "...MemManageFault with CFSR.MSTKERR\n"); + env->v7m.cfsr[secure] |= R_V7M_CFSR_MSTKERR_MASK; + exc = ARMV7M_EXCP_MEM; + exc_secure = secure; + } + goto pend_fault; + } + address_space_stl_le(arm_addressspace(cs, attrs), physaddr, value, + attrs, &txres); + if (txres != MEMTX_OK) { + /* BusFault trying to write the data */ + qemu_log_mask(CPU_LOG_INT, "...BusFault with BFSR.STKERR\n"); + env->v7m.cfsr[M_REG_NS] |= R_V7M_CFSR_STKERR_MASK; + exc = ARMV7M_EXCP_BUS; + exc_secure = false; + goto pend_fault; + } + return true; + +pend_fault: + /* By pending the exception at this point we are making + * the IMPDEF choice "overridden exceptions pended" (see the + * MergeExcInfo() pseudocode). The other choice would be to not + * pend them now and then make a choice about which to throw away + * later if we have two derived exceptions. + * The only case when we must not pend the exception but instead + * throw it away is if we are doing the push of the callee registers + * and we've already generated a derived exception. Even in this + * case we will still update the fault status registers. + */ + if (!ignfault) { + armv7m_nvic_set_pending_derived(env->nvic, exc, exc_secure); + } + return false; +} + +static bool v7m_stack_read(ARMCPU *cpu, uint32_t *dest, uint32_t addr, + ARMMMUIdx mmu_idx) +{ + CPUState *cs = CPU(cpu); + CPUARMState *env = &cpu->env; + MemTxAttrs attrs = {}; + MemTxResult txres; + target_ulong page_size; + hwaddr physaddr; + int prot; + ARMMMUFaultInfo fi; + bool secure = mmu_idx & ARM_MMU_IDX_M_S; + int exc; + bool exc_secure; + uint32_t value; + + if (get_phys_addr(env, addr, MMU_DATA_LOAD, mmu_idx, &physaddr, + &attrs, &prot, &page_size, &fi, NULL)) { + /* MPU/SAU lookup failed */ + if (fi.type == ARMFault_QEMU_SFault) { + qemu_log_mask(CPU_LOG_INT, + "...SecureFault with SFSR.AUVIOL during unstack\n"); + env->v7m.sfsr |= R_V7M_SFSR_AUVIOL_MASK | R_V7M_SFSR_SFARVALID_MASK; + env->v7m.sfar = addr; + exc = ARMV7M_EXCP_SECURE; + exc_secure = false; + } else { + qemu_log_mask(CPU_LOG_INT, + "...MemManageFault with CFSR.MUNSTKERR\n"); + env->v7m.cfsr[secure] |= R_V7M_CFSR_MUNSTKERR_MASK; + exc = ARMV7M_EXCP_MEM; + exc_secure = secure; + } + goto pend_fault; + } - env->regs[13] -= 4; - stl_phys(cs->as, env->regs[13], val); + value = address_space_ldl(arm_addressspace(cs, attrs), physaddr, + attrs, &txres); + if (txres != MEMTX_OK) { + /* BusFault trying to read the data */ + qemu_log_mask(CPU_LOG_INT, "...BusFault with BFSR.UNSTKERR\n"); + env->v7m.cfsr[M_REG_NS] |= R_V7M_CFSR_UNSTKERR_MASK; + exc = ARMV7M_EXCP_BUS; + exc_secure = false; + goto pend_fault; + } + + *dest = value; + return true; + +pend_fault: + /* By pending the exception at this point we are making + * the IMPDEF choice "overridden exceptions pended" (see the + * MergeExcInfo() pseudocode). The other choice would be to not + * pend them now and then make a choice about which to throw away + * later if we have two derived exceptions. + */ + armv7m_nvic_set_pending(env->nvic, exc, exc_secure); + return false; } /* Return true if we're using the process stack pointer (not the MSP) */ @@ -6395,65 +6641,126 @@ static uint32_t *get_v7m_sp_ptr(CPUARMState *env, bool secure, bool threadmode, } } -static uint32_t arm_v7m_load_vector(ARMCPU *cpu, bool targets_secure) +static bool arm_v7m_load_vector(ARMCPU *cpu, int exc, bool targets_secure, + uint32_t *pvec) { CPUState *cs = CPU(cpu); CPUARMState *env = &cpu->env; MemTxResult result; - hwaddr vec = env->v7m.vecbase[targets_secure] + env->v7m.exception * 4; - uint32_t addr; + uint32_t addr = env->v7m.vecbase[targets_secure] + exc * 4; + uint32_t vector_entry; + MemTxAttrs attrs = {}; + ARMMMUIdx mmu_idx; + bool exc_secure; + + mmu_idx = arm_v7m_mmu_idx_for_secstate_and_priv(env, targets_secure, true); + + /* We don't do a get_phys_addr() here because the rules for vector + * loads are special: they always use the default memory map, and + * the default memory map permits reads from all addresses. + * Since there's no easy way to pass through to pmsav8_mpu_lookup() + * that we want this special case which would always say "yes", + * we just do the SAU lookup here followed by a direct physical load. + */ + attrs.secure = targets_secure; + attrs.user = false; - addr = address_space_ldl(cs->as, vec, - MEMTXATTRS_UNSPECIFIED, &result); + if (arm_feature(env, ARM_FEATURE_M_SECURITY)) { + V8M_SAttributes sattrs = {}; + + v8m_security_lookup(env, addr, MMU_DATA_LOAD, mmu_idx, &sattrs); + if (sattrs.ns) { + attrs.secure = false; + } else if (!targets_secure) { + /* NS access to S memory */ + goto load_fail; + } + } + + vector_entry = address_space_ldl(arm_addressspace(cs, attrs), addr, + attrs, &result); if (result != MEMTX_OK) { - /* Architecturally this should cause a HardFault setting HSFR.VECTTBL, - * which would then be immediately followed by our failing to load - * the entry vector for that HardFault, which is a Lockup case. - * Since we don't model Lockup, we just report this guest error - * via cpu_abort(). - */ - cpu_abort(cs, "Failed to read from %s exception vector table " - "entry %08x\n", targets_secure ? "secure" : "nonsecure", - (unsigned)vec); + goto load_fail; } - return addr; + *pvec = vector_entry; + return true; + +load_fail: + /* All vector table fetch fails are reported as HardFault, with + * HFSR.VECTTBL and .FORCED set. (FORCED is set because + * technically the underlying exception is a MemManage or BusFault + * that is escalated to HardFault.) This is a terminal exception, + * so we will either take the HardFault immediately or else enter + * lockup (the latter case is handled in armv7m_nvic_set_pending_derived()). + */ + exc_secure = targets_secure || + !(cpu->env.v7m.aircr & R_V7M_AIRCR_BFHFNMINS_MASK); + env->v7m.hfsr |= R_V7M_HFSR_VECTTBL_MASK | R_V7M_HFSR_FORCED_MASK; + armv7m_nvic_set_pending_derived(env->nvic, ARMV7M_EXCP_HARD, exc_secure); + return false; } -static void v7m_push_callee_stack(ARMCPU *cpu, uint32_t lr, bool dotailchain) +static bool v7m_push_callee_stack(ARMCPU *cpu, uint32_t lr, bool dotailchain, + bool ignore_faults) { /* For v8M, push the callee-saves register part of the stack frame. * Compare the v8M pseudocode PushCalleeStack(). * In the tailchaining case this may not be the current stack. */ CPUARMState *env = &cpu->env; - CPUState *cs = CPU(cpu); uint32_t *frame_sp_p; uint32_t frameptr; + ARMMMUIdx mmu_idx; + bool stacked_ok; if (dotailchain) { - frame_sp_p = get_v7m_sp_ptr(env, true, - lr & R_V7M_EXCRET_MODE_MASK, + bool mode = lr & R_V7M_EXCRET_MODE_MASK; + bool priv = !(env->v7m.control[M_REG_S] & R_V7M_CONTROL_NPRIV_MASK) || + !mode; + + mmu_idx = arm_v7m_mmu_idx_for_secstate_and_priv(env, M_REG_S, priv); + frame_sp_p = get_v7m_sp_ptr(env, M_REG_S, mode, lr & R_V7M_EXCRET_SPSEL_MASK); } else { + mmu_idx = core_to_arm_mmu_idx(env, cpu_mmu_index(env, false)); frame_sp_p = &env->regs[13]; } frameptr = *frame_sp_p - 0x28; - stl_phys(cs->as, frameptr, 0xfefa125b); - stl_phys(cs->as, frameptr + 0x8, env->regs[4]); - stl_phys(cs->as, frameptr + 0xc, env->regs[5]); - stl_phys(cs->as, frameptr + 0x10, env->regs[6]); - stl_phys(cs->as, frameptr + 0x14, env->regs[7]); - stl_phys(cs->as, frameptr + 0x18, env->regs[8]); - stl_phys(cs->as, frameptr + 0x1c, env->regs[9]); - stl_phys(cs->as, frameptr + 0x20, env->regs[10]); - stl_phys(cs->as, frameptr + 0x24, env->regs[11]); - + /* Write as much of the stack frame as we can. A write failure may + * cause us to pend a derived exception. + */ + stacked_ok = + v7m_stack_write(cpu, frameptr, 0xfefa125b, mmu_idx, ignore_faults) && + v7m_stack_write(cpu, frameptr + 0x8, env->regs[4], mmu_idx, + ignore_faults) && + v7m_stack_write(cpu, frameptr + 0xc, env->regs[5], mmu_idx, + ignore_faults) && + v7m_stack_write(cpu, frameptr + 0x10, env->regs[6], mmu_idx, + ignore_faults) && + v7m_stack_write(cpu, frameptr + 0x14, env->regs[7], mmu_idx, + ignore_faults) && + v7m_stack_write(cpu, frameptr + 0x18, env->regs[8], mmu_idx, + ignore_faults) && + v7m_stack_write(cpu, frameptr + 0x1c, env->regs[9], mmu_idx, + ignore_faults) && + v7m_stack_write(cpu, frameptr + 0x20, env->regs[10], mmu_idx, + ignore_faults) && + v7m_stack_write(cpu, frameptr + 0x24, env->regs[11], mmu_idx, + ignore_faults); + + /* Update SP regardless of whether any of the stack accesses failed. + * When we implement v8M stack limit checking then this attempt to + * update SP might also fail and result in a derived exception. + */ *frame_sp_p = frameptr; + + return !stacked_ok; } -static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain) +static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain, + bool ignore_stackfaults) { /* Do the "take the exception" parts of exception entry, * but not the pushing of state to the stack. This is @@ -6462,8 +6769,10 @@ static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain) CPUARMState *env = &cpu->env; uint32_t addr; bool targets_secure; + int exc; + bool push_failed = false; - targets_secure = armv7m_nvic_acknowledge_irq(env->nvic); + armv7m_nvic_get_pending_irq_info(env->nvic, &exc, &targets_secure); if (arm_feature(env, ARM_FEATURE_V8)) { if (arm_feature(env, ARM_FEATURE_M_SECURITY) && @@ -6489,7 +6798,8 @@ static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain) */ if (lr & R_V7M_EXCRET_DCRS_MASK && !(dotailchain && (lr & R_V7M_EXCRET_ES_MASK))) { - v7m_push_callee_stack(cpu, lr, dotailchain); + push_failed = v7m_push_callee_stack(cpu, lr, dotailchain, + ignore_stackfaults); } lr |= R_V7M_EXCRET_DCRS_MASK; } @@ -6531,6 +6841,27 @@ static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain) } } + if (push_failed && !ignore_stackfaults) { + /* Derived exception on callee-saves register stacking: + * we might now want to take a different exception which + * targets a different security state, so try again from the top. + */ + v7m_exception_taken(cpu, lr, true, true); + return; + } + + if (!arm_v7m_load_vector(cpu, exc, targets_secure, &addr)) { + /* Vector load failed: derived exception */ + v7m_exception_taken(cpu, lr, true, true); + return; + } + + /* Now we've done everything that might cause a derived exception + * we can go ahead and activate whichever exception we're going to + * take (which might now be the derived exception). + */ + armv7m_nvic_acknowledge_irq(env->nvic); + /* Switch to target security state -- must do this before writing SPSEL */ switch_v7m_security_state(env, targets_secure); write_v7m_control_spsel(env, 0); @@ -6538,34 +6869,55 @@ static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain) /* Clear IT bits */ env->condexec_bits = 0; env->regs[14] = lr; - addr = arm_v7m_load_vector(cpu, targets_secure); env->regs[15] = addr & 0xfffffffe; env->thumb = addr & 1; } -static void v7m_push_stack(ARMCPU *cpu) +static bool v7m_push_stack(ARMCPU *cpu) { /* Do the "set up stack frame" part of exception entry, * similar to pseudocode PushStack(). + * Return true if we generate a derived exception (and so + * should ignore further stack faults trying to process + * that derived exception.) */ + bool stacked_ok; CPUARMState *env = &cpu->env; uint32_t xpsr = xpsr_read(env); + uint32_t frameptr = env->regs[13]; + ARMMMUIdx mmu_idx = core_to_arm_mmu_idx(env, cpu_mmu_index(env, false)); /* Align stack pointer if the guest wants that */ - if ((env->regs[13] & 4) && + if ((frameptr & 4) && (env->v7m.ccr[env->v7m.secure] & R_V7M_CCR_STKALIGN_MASK)) { - env->regs[13] -= 4; + frameptr -= 4; xpsr |= XPSR_SPREALIGN; } - /* Switch to the handler mode. */ - v7m_push(env, xpsr); - v7m_push(env, env->regs[15]); - v7m_push(env, env->regs[14]); - v7m_push(env, env->regs[12]); - v7m_push(env, env->regs[3]); - v7m_push(env, env->regs[2]); - v7m_push(env, env->regs[1]); - v7m_push(env, env->regs[0]); + + frameptr -= 0x20; + + /* Write as much of the stack frame as we can. If we fail a stack + * write this will result in a derived exception being pended + * (which may be taken in preference to the one we started with + * if it has higher priority). + */ + stacked_ok = + v7m_stack_write(cpu, frameptr, env->regs[0], mmu_idx, false) && + v7m_stack_write(cpu, frameptr + 4, env->regs[1], mmu_idx, false) && + v7m_stack_write(cpu, frameptr + 8, env->regs[2], mmu_idx, false) && + v7m_stack_write(cpu, frameptr + 12, env->regs[3], mmu_idx, false) && + v7m_stack_write(cpu, frameptr + 16, env->regs[12], mmu_idx, false) && + v7m_stack_write(cpu, frameptr + 20, env->regs[14], mmu_idx, false) && + v7m_stack_write(cpu, frameptr + 24, env->regs[15], mmu_idx, false) && + v7m_stack_write(cpu, frameptr + 28, xpsr, mmu_idx, false); + + /* Update SP regardless of whether any of the stack accesses failed. + * When we implement v8M stack limit checking then this attempt to + * update SP might also fail and result in a derived exception. + */ + env->regs[13] = frameptr; + + return !stacked_ok; } static void do_v7m_exception_exit(ARMCPU *cpu) @@ -6711,7 +7063,7 @@ static void do_v7m_exception_exit(ARMCPU *cpu) if (sfault) { env->v7m.sfsr |= R_V7M_SFSR_INVER_MASK; armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SECURE, false); - v7m_exception_taken(cpu, excret, true); + v7m_exception_taken(cpu, excret, true, false); qemu_log_mask(CPU_LOG_INT, "...taking SecureFault on existing " "stackframe: failed EXC_RETURN.ES validity check\n"); return; @@ -6723,7 +7075,7 @@ static void do_v7m_exception_exit(ARMCPU *cpu) */ env->v7m.cfsr[env->v7m.secure] |= R_V7M_CFSR_INVPC_MASK; armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE, env->v7m.secure); - v7m_exception_taken(cpu, excret, true); + v7m_exception_taken(cpu, excret, true, false); qemu_log_mask(CPU_LOG_INT, "...taking UsageFault on existing " "stackframe: failed exception return integrity check\n"); return; @@ -6752,6 +7104,11 @@ static void do_v7m_exception_exit(ARMCPU *cpu) !return_to_handler, return_to_sp_process); uint32_t frameptr = *frame_sp_p; + bool pop_ok = true; + ARMMMUIdx mmu_idx; + + mmu_idx = arm_v7m_mmu_idx_for_secstate_and_priv(env, return_to_secure, + !return_to_handler); if (!QEMU_IS_ALIGNED(frameptr, 8) && arm_feature(env, ARM_FEATURE_V8)) { @@ -6771,36 +7128,45 @@ static void do_v7m_exception_exit(ARMCPU *cpu) /* Take a SecureFault on the current stack */ env->v7m.sfsr |= R_V7M_SFSR_INVIS_MASK; armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SECURE, false); - v7m_exception_taken(cpu, excret, true); + v7m_exception_taken(cpu, excret, true, false); qemu_log_mask(CPU_LOG_INT, "...taking SecureFault on existing " "stackframe: failed exception return integrity " "signature check\n"); return; } - env->regs[4] = ldl_phys(cs->as, frameptr + 0x8); - env->regs[5] = ldl_phys(cs->as, frameptr + 0xc); - env->regs[6] = ldl_phys(cs->as, frameptr + 0x10); - env->regs[7] = ldl_phys(cs->as, frameptr + 0x14); - env->regs[8] = ldl_phys(cs->as, frameptr + 0x18); - env->regs[9] = ldl_phys(cs->as, frameptr + 0x1c); - env->regs[10] = ldl_phys(cs->as, frameptr + 0x20); - env->regs[11] = ldl_phys(cs->as, frameptr + 0x24); + pop_ok = + v7m_stack_read(cpu, &env->regs[4], frameptr + 0x8, mmu_idx) && + v7m_stack_read(cpu, &env->regs[4], frameptr + 0x8, mmu_idx) && + v7m_stack_read(cpu, &env->regs[5], frameptr + 0xc, mmu_idx) && + v7m_stack_read(cpu, &env->regs[6], frameptr + 0x10, mmu_idx) && + v7m_stack_read(cpu, &env->regs[7], frameptr + 0x14, mmu_idx) && + v7m_stack_read(cpu, &env->regs[8], frameptr + 0x18, mmu_idx) && + v7m_stack_read(cpu, &env->regs[9], frameptr + 0x1c, mmu_idx) && + v7m_stack_read(cpu, &env->regs[10], frameptr + 0x20, mmu_idx) && + v7m_stack_read(cpu, &env->regs[11], frameptr + 0x24, mmu_idx); frameptr += 0x28; } - /* Pop registers. TODO: make these accesses use the correct - * attributes and address space (S/NS, priv/unpriv) and handle - * memory transaction failures. - */ - env->regs[0] = ldl_phys(cs->as, frameptr); - env->regs[1] = ldl_phys(cs->as, frameptr + 0x4); - env->regs[2] = ldl_phys(cs->as, frameptr + 0x8); - env->regs[3] = ldl_phys(cs->as, frameptr + 0xc); - env->regs[12] = ldl_phys(cs->as, frameptr + 0x10); - env->regs[14] = ldl_phys(cs->as, frameptr + 0x14); - env->regs[15] = ldl_phys(cs->as, frameptr + 0x18); + /* Pop registers */ + pop_ok = pop_ok && + v7m_stack_read(cpu, &env->regs[0], frameptr, mmu_idx) && + v7m_stack_read(cpu, &env->regs[1], frameptr + 0x4, mmu_idx) && + v7m_stack_read(cpu, &env->regs[2], frameptr + 0x8, mmu_idx) && + v7m_stack_read(cpu, &env->regs[3], frameptr + 0xc, mmu_idx) && + v7m_stack_read(cpu, &env->regs[12], frameptr + 0x10, mmu_idx) && + v7m_stack_read(cpu, &env->regs[14], frameptr + 0x14, mmu_idx) && + v7m_stack_read(cpu, &env->regs[15], frameptr + 0x18, mmu_idx) && + v7m_stack_read(cpu, &xpsr, frameptr + 0x1c, mmu_idx); + + if (!pop_ok) { + /* v7m_stack_read() pended a fault, so take it (as a tail + * chained exception on the same stack frame) + */ + v7m_exception_taken(cpu, excret, true, false); + return; + } /* Returning from an exception with a PC with bit 0 set is defined * behaviour on v8M (bit 0 is ignored), but for v7M it was specified @@ -6819,8 +7185,6 @@ static void do_v7m_exception_exit(ARMCPU *cpu) } } - xpsr = ldl_phys(cs->as, frameptr + 0x1c); - if (arm_feature(env, ARM_FEATURE_V8)) { /* For v8M we have to check whether the xPSR exception field * matches the EXCRET value for return to handler/thread @@ -6836,7 +7200,7 @@ static void do_v7m_exception_exit(ARMCPU *cpu) armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE, env->v7m.secure); env->v7m.cfsr[env->v7m.secure] |= R_V7M_CFSR_INVPC_MASK; - v7m_exception_taken(cpu, excret, true); + v7m_exception_taken(cpu, excret, true, false); qemu_log_mask(CPU_LOG_INT, "...taking UsageFault on existing " "stackframe: failed exception return integrity " "check\n"); @@ -6869,11 +7233,13 @@ static void do_v7m_exception_exit(ARMCPU *cpu) /* Take an INVPC UsageFault by pushing the stack again; * we know we're v7M so this is never a Secure UsageFault. */ + bool ignore_stackfaults; + assert(!arm_feature(env, ARM_FEATURE_V8)); armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE, false); env->v7m.cfsr[env->v7m.secure] |= R_V7M_CFSR_INVPC_MASK; - v7m_push_stack(cpu); - v7m_exception_taken(cpu, excret, false); + ignore_stackfaults = v7m_push_stack(cpu); + v7m_exception_taken(cpu, excret, false, ignore_stackfaults); qemu_log_mask(CPU_LOG_INT, "...taking UsageFault on new stackframe: " "failed exception return integrity check\n"); return; @@ -7114,6 +7480,7 @@ void arm_v7m_cpu_do_interrupt(CPUState *cs) ARMCPU *cpu = ARM_CPU(cs); CPUARMState *env = &cpu->env; uint32_t lr; + bool ignore_stackfaults; arm_log_exception(cs->exception_index); @@ -7288,8 +7655,8 @@ void arm_v7m_cpu_do_interrupt(CPUState *cs) lr |= R_V7M_EXCRET_MODE_MASK; } - v7m_push_stack(cpu); - v7m_exception_taken(cpu, lr, false); + ignore_stackfaults = v7m_push_stack(cpu); + v7m_exception_taken(cpu, lr, false, ignore_stackfaults); qemu_log_mask(CPU_LOG_INT, "... as %d\n", env->v7m.exception); } @@ -11692,14 +12059,37 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, target_ulong *cs_base, uint32_t *pflags) { ARMMMUIdx mmu_idx = core_to_arm_mmu_idx(env, cpu_mmu_index(env, false)); + int fp_el = fp_exception_el(env); uint32_t flags; if (is_a64(env)) { + int sve_el = sve_exception_el(env); + uint32_t zcr_len; + *pc = env->pc; flags = ARM_TBFLAG_AARCH64_STATE_MASK; /* Get control bits for tagged addresses */ flags |= (arm_regime_tbi0(env, mmu_idx) << ARM_TBFLAG_TBI0_SHIFT); flags |= (arm_regime_tbi1(env, mmu_idx) << ARM_TBFLAG_TBI1_SHIFT); + flags |= sve_el << ARM_TBFLAG_SVEEXC_EL_SHIFT; + + /* If SVE is disabled, but FP is enabled, + then the effective len is 0. */ + if (sve_el != 0 && fp_el == 0) { + zcr_len = 0; + } else { + int current_el = arm_current_el(env); + + zcr_len = env->vfp.zcr_el[current_el <= 1 ? 1 : current_el]; + zcr_len &= 0xf; + if (current_el < 2 && arm_feature(env, ARM_FEATURE_EL2)) { + zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[2]); + } + if (current_el < 3 && arm_feature(env, ARM_FEATURE_EL3)) { + zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[3]); + } + } + flags |= zcr_len << ARM_TBFLAG_ZCR_LEN_SHIFT; } else { *pc = env->regs[15]; flags = (env->thumb << ARM_TBFLAG_THUMB_SHIFT) @@ -11742,7 +12132,7 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, if (arm_cpu_data_is_big_endian(env)) { flags |= ARM_TBFLAG_BE_DATA_MASK; } - flags |= fp_exception_el(env) << ARM_TBFLAG_FPEXC_EL_SHIFT; + flags |= fp_el << ARM_TBFLAG_FPEXC_EL_SHIFT; if (arm_v7m_is_handler_mode(env)) { flags |= ARM_TBFLAG_HANDLER_MASK; diff --git a/target/arm/helper.h b/target/arm/helper.h index 5dec2e6262..6383d7d09e 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -534,6 +534,18 @@ DEF_HELPER_FLAGS_3(crypto_sha256h2, TCG_CALL_NO_RWG, void, ptr, ptr, ptr) DEF_HELPER_FLAGS_2(crypto_sha256su0, TCG_CALL_NO_RWG, void, ptr, ptr) DEF_HELPER_FLAGS_3(crypto_sha256su1, TCG_CALL_NO_RWG, void, ptr, ptr, ptr) +DEF_HELPER_FLAGS_3(crypto_sha512h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr) +DEF_HELPER_FLAGS_3(crypto_sha512h2, TCG_CALL_NO_RWG, void, ptr, ptr, ptr) +DEF_HELPER_FLAGS_2(crypto_sha512su0, TCG_CALL_NO_RWG, void, ptr, ptr) +DEF_HELPER_FLAGS_3(crypto_sha512su1, TCG_CALL_NO_RWG, void, ptr, ptr, ptr) + +DEF_HELPER_FLAGS_5(crypto_sm3tt, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32, i32) +DEF_HELPER_FLAGS_3(crypto_sm3partw1, TCG_CALL_NO_RWG, void, ptr, ptr, ptr) +DEF_HELPER_FLAGS_3(crypto_sm3partw2, TCG_CALL_NO_RWG, void, ptr, ptr, ptr) + +DEF_HELPER_FLAGS_2(crypto_sm4e, TCG_CALL_NO_RWG, void, ptr, ptr) +DEF_HELPER_FLAGS_3(crypto_sm4ekey, TCG_CALL_NO_RWG, void, ptr, ptr, ptr) + DEF_HELPER_FLAGS_3(crc32, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32) DEF_HELPER_FLAGS_3(crc32c, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32) DEF_HELPER_2(dc_zva, void, env, i64) diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h index ff53e9fafb..cfb7e5af72 100644 --- a/target/arm/kvm_arm.h +++ b/target/arm/kvm_arm.h @@ -234,6 +234,10 @@ static inline const char *gicv3_class_name(void) exit(1); #endif } else { + if (kvm_enabled()) { + error_report("Userspace GICv3 is not supported with KVM"); + exit(1); + } return "arm-gicv3"; } } diff --git a/target/arm/machine.c b/target/arm/machine.c index a85c2430d3..2c8b43062f 100644 --- a/target/arm/machine.c +++ b/target/arm/machine.c @@ -50,7 +50,40 @@ static const VMStateDescription vmstate_vfp = { .minimum_version_id = 3, .needed = vfp_needed, .fields = (VMStateField[]) { - VMSTATE_UINT64_ARRAY(env.vfp.regs, ARMCPU, 64), + /* For compatibility, store Qn out of Zn here. */ + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[0].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[1].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[2].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[3].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[4].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[5].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[6].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[7].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[8].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[9].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[10].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[11].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[12].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[13].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[14].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[15].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[16].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[17].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[18].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[19].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[20].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[21].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[22].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[23].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[24].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[25].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[26].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[27].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[28].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[29].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[30].d, ARMCPU, 0, 2), + VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[31].d, ARMCPU, 0, 2), + /* The xregs array is a little awkward because element 1 (FPSCR) * requires a specific accessor, so we have to split it up in * the vmstate: @@ -89,6 +122,56 @@ static const VMStateDescription vmstate_iwmmxt = { } }; +#ifdef TARGET_AARCH64 +/* The expression ARM_MAX_VQ - 2 is 0 for pure AArch32 build, + * and ARMPredicateReg is actively empty. This triggers errors + * in the expansion of the VMSTATE macros. + */ + +static bool sve_needed(void *opaque) +{ + ARMCPU *cpu = opaque; + CPUARMState *env = &cpu->env; + + return arm_feature(env, ARM_FEATURE_SVE); +} + +/* The first two words of each Zreg is stored in VFP state. */ +static const VMStateDescription vmstate_zreg_hi_reg = { + .name = "cpu/sve/zreg_hi", + .version_id = 1, + .minimum_version_id = 1, + .fields = (VMStateField[]) { + VMSTATE_UINT64_SUB_ARRAY(d, ARMVectorReg, 2, ARM_MAX_VQ - 2), + VMSTATE_END_OF_LIST() + } +}; + +static const VMStateDescription vmstate_preg_reg = { + .name = "cpu/sve/preg", + .version_id = 1, + .minimum_version_id = 1, + .fields = (VMStateField[]) { + VMSTATE_UINT64_ARRAY(p, ARMPredicateReg, 2 * ARM_MAX_VQ / 8), + VMSTATE_END_OF_LIST() + } +}; + +static const VMStateDescription vmstate_sve = { + .name = "cpu/sve", + .version_id = 1, + .minimum_version_id = 1, + .needed = sve_needed, + .fields = (VMStateField[]) { + VMSTATE_STRUCT_ARRAY(env.vfp.zregs, ARMCPU, 32, 0, + vmstate_zreg_hi_reg, ARMVectorReg), + VMSTATE_STRUCT_ARRAY(env.vfp.pregs, ARMCPU, 17, 0, + vmstate_preg_reg, ARMPredicateReg), + VMSTATE_END_OF_LIST() + } +}; +#endif /* AARCH64 */ + static bool m_needed(void *opaque) { ARMCPU *cpu = opaque; @@ -553,6 +636,9 @@ const VMStateDescription vmstate_arm_cpu = { &vmstate_pmsav7, &vmstate_pmsav8, &vmstate_m_security, +#ifdef TARGET_AARCH64 + &vmstate_sve, +#endif NULL } }; diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index 0830c3f1c8..fb1a4cb532 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -525,8 +525,8 @@ static inline int vec_reg_offset(DisasContext *s, int regno, { int offs = 0; #ifdef HOST_WORDS_BIGENDIAN - /* This is complicated slightly because vfp.regs[2n] is - * still the low half and vfp.regs[2n+1] the high half + /* This is complicated slightly because vfp.zregs[n].d[0] is + * still the low half and vfp.zregs[n].d[1] the high half * of the 128 bit vector, even on big endian systems. * Calculate the offset assuming a fully bigendian 128 bits, * then XOR to account for the order of the two 64 bit halves. @@ -536,7 +536,7 @@ static inline int vec_reg_offset(DisasContext *s, int regno, #else offs += element * (1 << size); #endif - offs += offsetof(CPUARMState, vfp.regs[regno * 2]); + offs += offsetof(CPUARMState, vfp.zregs[regno]); assert_fp_access_checked(s); return offs; } @@ -545,7 +545,7 @@ static inline int vec_reg_offset(DisasContext *s, int regno, static inline int vec_full_reg_offset(DisasContext *s, int regno) { assert_fp_access_checked(s); - return offsetof(CPUARMState, vfp.regs[regno * 2]); + return offsetof(CPUARMState, vfp.zregs[regno]); } /* Return a newly allocated pointer to the vector register. */ @@ -11587,6 +11587,341 @@ static void disas_crypto_two_reg_sha(DisasContext *s, uint32_t insn) tcg_temp_free_ptr(tcg_rn_ptr); } +/* Crypto three-reg SHA512 + * 31 21 20 16 15 14 13 12 11 10 9 5 4 0 + * +-----------------------+------+---+---+-----+--------+------+------+ + * | 1 1 0 0 1 1 1 0 0 1 1 | Rm | 1 | O | 0 0 | opcode | Rn | Rd | + * +-----------------------+------+---+---+-----+--------+------+------+ + */ +static void disas_crypto_three_reg_sha512(DisasContext *s, uint32_t insn) +{ + int opcode = extract32(insn, 10, 2); + int o = extract32(insn, 14, 1); + int rm = extract32(insn, 16, 5); + int rn = extract32(insn, 5, 5); + int rd = extract32(insn, 0, 5); + int feature; + CryptoThreeOpFn *genfn; + + if (o == 0) { + switch (opcode) { + case 0: /* SHA512H */ + feature = ARM_FEATURE_V8_SHA512; + genfn = gen_helper_crypto_sha512h; + break; + case 1: /* SHA512H2 */ + feature = ARM_FEATURE_V8_SHA512; + genfn = gen_helper_crypto_sha512h2; + break; + case 2: /* SHA512SU1 */ + feature = ARM_FEATURE_V8_SHA512; + genfn = gen_helper_crypto_sha512su1; + break; + case 3: /* RAX1 */ + feature = ARM_FEATURE_V8_SHA3; + genfn = NULL; + break; + } + } else { + switch (opcode) { + case 0: /* SM3PARTW1 */ + feature = ARM_FEATURE_V8_SM3; + genfn = gen_helper_crypto_sm3partw1; + break; + case 1: /* SM3PARTW2 */ + feature = ARM_FEATURE_V8_SM3; + genfn = gen_helper_crypto_sm3partw2; + break; + case 2: /* SM4EKEY */ + feature = ARM_FEATURE_V8_SM4; + genfn = gen_helper_crypto_sm4ekey; + break; + default: + unallocated_encoding(s); + return; + } + } + + if (!arm_dc_feature(s, feature)) { + unallocated_encoding(s); + return; + } + + if (!fp_access_check(s)) { + return; + } + + if (genfn) { + TCGv_ptr tcg_rd_ptr, tcg_rn_ptr, tcg_rm_ptr; + + tcg_rd_ptr = vec_full_reg_ptr(s, rd); + tcg_rn_ptr = vec_full_reg_ptr(s, rn); + tcg_rm_ptr = vec_full_reg_ptr(s, rm); + + genfn(tcg_rd_ptr, tcg_rn_ptr, tcg_rm_ptr); + + tcg_temp_free_ptr(tcg_rd_ptr); + tcg_temp_free_ptr(tcg_rn_ptr); + tcg_temp_free_ptr(tcg_rm_ptr); + } else { + TCGv_i64 tcg_op1, tcg_op2, tcg_res[2]; + int pass; + + tcg_op1 = tcg_temp_new_i64(); + tcg_op2 = tcg_temp_new_i64(); + tcg_res[0] = tcg_temp_new_i64(); + tcg_res[1] = tcg_temp_new_i64(); + + for (pass = 0; pass < 2; pass++) { + read_vec_element(s, tcg_op1, rn, pass, MO_64); + read_vec_element(s, tcg_op2, rm, pass, MO_64); + + tcg_gen_rotli_i64(tcg_res[pass], tcg_op2, 1); + tcg_gen_xor_i64(tcg_res[pass], tcg_res[pass], tcg_op1); + } + write_vec_element(s, tcg_res[0], rd, 0, MO_64); + write_vec_element(s, tcg_res[1], rd, 1, MO_64); + + tcg_temp_free_i64(tcg_op1); + tcg_temp_free_i64(tcg_op2); + tcg_temp_free_i64(tcg_res[0]); + tcg_temp_free_i64(tcg_res[1]); + } +} + +/* Crypto two-reg SHA512 + * 31 12 11 10 9 5 4 0 + * +-----------------------------------------+--------+------+------+ + * | 1 1 0 0 1 1 1 0 1 1 0 0 0 0 0 0 1 0 0 0 | opcode | Rn | Rd | + * +-----------------------------------------+--------+------+------+ + */ +static void disas_crypto_two_reg_sha512(DisasContext *s, uint32_t insn) +{ + int opcode = extract32(insn, 10, 2); + int rn = extract32(insn, 5, 5); + int rd = extract32(insn, 0, 5); + TCGv_ptr tcg_rd_ptr, tcg_rn_ptr; + int feature; + CryptoTwoOpFn *genfn; + + switch (opcode) { + case 0: /* SHA512SU0 */ + feature = ARM_FEATURE_V8_SHA512; + genfn = gen_helper_crypto_sha512su0; + break; + case 1: /* SM4E */ + feature = ARM_FEATURE_V8_SM4; + genfn = gen_helper_crypto_sm4e; + break; + default: + unallocated_encoding(s); + return; + } + + if (!arm_dc_feature(s, feature)) { + unallocated_encoding(s); + return; + } + + if (!fp_access_check(s)) { + return; + } + + tcg_rd_ptr = vec_full_reg_ptr(s, rd); + tcg_rn_ptr = vec_full_reg_ptr(s, rn); + + genfn(tcg_rd_ptr, tcg_rn_ptr); + + tcg_temp_free_ptr(tcg_rd_ptr); + tcg_temp_free_ptr(tcg_rn_ptr); +} + +/* Crypto four-register + * 31 23 22 21 20 16 15 14 10 9 5 4 0 + * +-------------------+-----+------+---+------+------+------+ + * | 1 1 0 0 1 1 1 0 0 | Op0 | Rm | 0 | Ra | Rn | Rd | + * +-------------------+-----+------+---+------+------+------+ + */ +static void disas_crypto_four_reg(DisasContext *s, uint32_t insn) +{ + int op0 = extract32(insn, 21, 2); + int rm = extract32(insn, 16, 5); + int ra = extract32(insn, 10, 5); + int rn = extract32(insn, 5, 5); + int rd = extract32(insn, 0, 5); + int feature; + + switch (op0) { + case 0: /* EOR3 */ + case 1: /* BCAX */ + feature = ARM_FEATURE_V8_SHA3; + break; + case 2: /* SM3SS1 */ + feature = ARM_FEATURE_V8_SM3; + break; + default: + unallocated_encoding(s); + return; + } + + if (!arm_dc_feature(s, feature)) { + unallocated_encoding(s); + return; + } + + if (!fp_access_check(s)) { + return; + } + + if (op0 < 2) { + TCGv_i64 tcg_op1, tcg_op2, tcg_op3, tcg_res[2]; + int pass; + + tcg_op1 = tcg_temp_new_i64(); + tcg_op2 = tcg_temp_new_i64(); + tcg_op3 = tcg_temp_new_i64(); + tcg_res[0] = tcg_temp_new_i64(); + tcg_res[1] = tcg_temp_new_i64(); + + for (pass = 0; pass < 2; pass++) { + read_vec_element(s, tcg_op1, rn, pass, MO_64); + read_vec_element(s, tcg_op2, rm, pass, MO_64); + read_vec_element(s, tcg_op3, ra, pass, MO_64); + + if (op0 == 0) { + /* EOR3 */ + tcg_gen_xor_i64(tcg_res[pass], tcg_op2, tcg_op3); + } else { + /* BCAX */ + tcg_gen_andc_i64(tcg_res[pass], tcg_op2, tcg_op3); + } + tcg_gen_xor_i64(tcg_res[pass], tcg_res[pass], tcg_op1); + } + write_vec_element(s, tcg_res[0], rd, 0, MO_64); + write_vec_element(s, tcg_res[1], rd, 1, MO_64); + + tcg_temp_free_i64(tcg_op1); + tcg_temp_free_i64(tcg_op2); + tcg_temp_free_i64(tcg_op3); + tcg_temp_free_i64(tcg_res[0]); + tcg_temp_free_i64(tcg_res[1]); + } else { + TCGv_i32 tcg_op1, tcg_op2, tcg_op3, tcg_res, tcg_zero; + + tcg_op1 = tcg_temp_new_i32(); + tcg_op2 = tcg_temp_new_i32(); + tcg_op3 = tcg_temp_new_i32(); + tcg_res = tcg_temp_new_i32(); + tcg_zero = tcg_const_i32(0); + + read_vec_element_i32(s, tcg_op1, rn, 3, MO_32); + read_vec_element_i32(s, tcg_op2, rm, 3, MO_32); + read_vec_element_i32(s, tcg_op3, ra, 3, MO_32); + + tcg_gen_rotri_i32(tcg_res, tcg_op1, 20); + tcg_gen_add_i32(tcg_res, tcg_res, tcg_op2); + tcg_gen_add_i32(tcg_res, tcg_res, tcg_op3); + tcg_gen_rotri_i32(tcg_res, tcg_res, 25); + + write_vec_element_i32(s, tcg_zero, rd, 0, MO_32); + write_vec_element_i32(s, tcg_zero, rd, 1, MO_32); + write_vec_element_i32(s, tcg_zero, rd, 2, MO_32); + write_vec_element_i32(s, tcg_res, rd, 3, MO_32); + + tcg_temp_free_i32(tcg_op1); + tcg_temp_free_i32(tcg_op2); + tcg_temp_free_i32(tcg_op3); + tcg_temp_free_i32(tcg_res); + tcg_temp_free_i32(tcg_zero); + } +} + +/* Crypto XAR + * 31 21 20 16 15 10 9 5 4 0 + * +-----------------------+------+--------+------+------+ + * | 1 1 0 0 1 1 1 0 1 0 0 | Rm | imm6 | Rn | Rd | + * +-----------------------+------+--------+------+------+ + */ +static void disas_crypto_xar(DisasContext *s, uint32_t insn) +{ + int rm = extract32(insn, 16, 5); + int imm6 = extract32(insn, 10, 6); + int rn = extract32(insn, 5, 5); + int rd = extract32(insn, 0, 5); + TCGv_i64 tcg_op1, tcg_op2, tcg_res[2]; + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_V8_SHA3)) { + unallocated_encoding(s); + return; + } + + if (!fp_access_check(s)) { + return; + } + + tcg_op1 = tcg_temp_new_i64(); + tcg_op2 = tcg_temp_new_i64(); + tcg_res[0] = tcg_temp_new_i64(); + tcg_res[1] = tcg_temp_new_i64(); + + for (pass = 0; pass < 2; pass++) { + read_vec_element(s, tcg_op1, rn, pass, MO_64); + read_vec_element(s, tcg_op2, rm, pass, MO_64); + + tcg_gen_xor_i64(tcg_res[pass], tcg_op1, tcg_op2); + tcg_gen_rotri_i64(tcg_res[pass], tcg_res[pass], imm6); + } + write_vec_element(s, tcg_res[0], rd, 0, MO_64); + write_vec_element(s, tcg_res[1], rd, 1, MO_64); + + tcg_temp_free_i64(tcg_op1); + tcg_temp_free_i64(tcg_op2); + tcg_temp_free_i64(tcg_res[0]); + tcg_temp_free_i64(tcg_res[1]); +} + +/* Crypto three-reg imm2 + * 31 21 20 16 15 14 13 12 11 10 9 5 4 0 + * +-----------------------+------+-----+------+--------+------+------+ + * | 1 1 0 0 1 1 1 0 0 1 0 | Rm | 1 0 | imm2 | opcode | Rn | Rd | + * +-----------------------+------+-----+------+--------+------+------+ + */ +static void disas_crypto_three_reg_imm2(DisasContext *s, uint32_t insn) +{ + int opcode = extract32(insn, 10, 2); + int imm2 = extract32(insn, 12, 2); + int rm = extract32(insn, 16, 5); + int rn = extract32(insn, 5, 5); + int rd = extract32(insn, 0, 5); + TCGv_ptr tcg_rd_ptr, tcg_rn_ptr, tcg_rm_ptr; + TCGv_i32 tcg_imm2, tcg_opcode; + + if (!arm_dc_feature(s, ARM_FEATURE_V8_SM3)) { + unallocated_encoding(s); + return; + } + + if (!fp_access_check(s)) { + return; + } + + tcg_rd_ptr = vec_full_reg_ptr(s, rd); + tcg_rn_ptr = vec_full_reg_ptr(s, rn); + tcg_rm_ptr = vec_full_reg_ptr(s, rm); + tcg_imm2 = tcg_const_i32(imm2); + tcg_opcode = tcg_const_i32(opcode); + + gen_helper_crypto_sm3tt(tcg_rd_ptr, tcg_rn_ptr, tcg_rm_ptr, tcg_imm2, + tcg_opcode); + + tcg_temp_free_ptr(tcg_rd_ptr); + tcg_temp_free_ptr(tcg_rn_ptr); + tcg_temp_free_ptr(tcg_rm_ptr); + tcg_temp_free_i32(tcg_imm2); + tcg_temp_free_i32(tcg_opcode); +} + /* C3.6 Data processing - SIMD, inc Crypto * * As the decode gets a little complex we are using a table based @@ -11616,6 +11951,11 @@ static const AArch64DecodeTable data_proc_simd[] = { { 0x4e280800, 0xff3e0c00, disas_crypto_aes }, { 0x5e000000, 0xff208c00, disas_crypto_three_reg_sha }, { 0x5e280800, 0xff3e0c00, disas_crypto_two_reg_sha }, + { 0xce608000, 0xffe0b000, disas_crypto_three_reg_sha512 }, + { 0xcec08000, 0xfffff000, disas_crypto_two_reg_sha512 }, + { 0xce000000, 0xff808000, disas_crypto_four_reg }, + { 0xce800000, 0xffe00000, disas_crypto_xar }, + { 0xce408000, 0xffe0c000, disas_crypto_three_reg_imm2 }, { 0x00000000, 0x00000000, NULL } }; @@ -11718,6 +12058,8 @@ static int aarch64_tr_init_disas_context(DisasContextBase *dcbase, dc->user = (dc->current_el == 0); #endif dc->fp_excp_el = ARM_TBFLAG_FPEXC_EL(dc->base.tb->flags); + dc->sve_excp_el = ARM_TBFLAG_SVEEXC_EL(dc->base.tb->flags); + dc->sve_len = (ARM_TBFLAG_ZCR_LEN(dc->base.tb->flags) + 1) * 16; dc->vec_len = 0; dc->vec_stride = 0; dc->cp_regs = arm_cpu->cp_regs; diff --git a/target/arm/translate.c b/target/arm/translate.c index 55826b7e5a..1270022289 100644 --- a/target/arm/translate.c +++ b/target/arm/translate.c @@ -1512,13 +1512,12 @@ static inline void gen_vfp_st(DisasContext *s, int dp, TCGv_i32 addr) } } -static inline long -vfp_reg_offset (int dp, int reg) +static inline long vfp_reg_offset(bool dp, unsigned reg) { if (dp) { - return offsetof(CPUARMState, vfp.regs[reg]); + return offsetof(CPUARMState, vfp.zregs[reg >> 1].d[reg & 1]); } else { - long ofs = offsetof(CPUARMState, vfp.regs[reg >> 1]); + long ofs = offsetof(CPUARMState, vfp.zregs[reg >> 2].d[(reg >> 1) & 1]); if (reg & 1) { ofs += offsetof(CPU_DoubleU, l.upper); } else { @@ -9926,6 +9925,7 @@ static void disas_thumb2_insn(DisasContext *s, uint32_t insn) tcg_temp_free_i32(addr); tcg_temp_free_i32(op); store_reg(s, rd, ttresp); + break; } goto illegal_op; } diff --git a/target/arm/translate.h b/target/arm/translate.h index 3f4df91e5e..c47febf99d 100644 --- a/target/arm/translate.h +++ b/target/arm/translate.h @@ -29,6 +29,8 @@ typedef struct DisasContext { bool tbi1; /* TBI1 for EL0/1, not used for EL2/3 */ bool ns; /* Use non-secure CPREG bank on access */ int fp_excp_el; /* FP exception EL or 0 if enabled */ + int sve_excp_el; /* SVE exception EL or 0 if enabled */ + int sve_len; /* SVE vector length in bytes */ /* Flag indicating that exceptions from secure mode are routed to EL3. */ bool secure_routed_to_el3; bool vfp_enabled; /* FP enabled via FPSCR.EN */ diff --git a/target/i386/cpu.c b/target/i386/cpu.c index d70954b8b7..b5e431e769 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -16,6 +16,7 @@ * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, see <http://www.gnu.org/licenses/>. */ + #include "qemu/osdep.h" #include "qemu/cutils.h" @@ -29,10 +30,10 @@ #include "qemu/error-report.h" #include "qemu/option.h" #include "qemu/config-file.h" +#include "qapi/error.h" +#include "qapi/qmp/qdict.h" #include "qapi/qmp/qerror.h" -#include "qapi/qmp/types.h" -#include "qapi-types.h" #include "qapi-visit.h" #include "qapi/visitor.h" #include "qom/qom-qobject.h" diff --git a/target/i386/hax-all.c b/target/i386/hax-all.c index 934ec4afd1..bc9a12c1ee 100644 --- a/target/i386/hax-all.c +++ b/target/i386/hax-all.c @@ -30,7 +30,6 @@ #include "exec/ioport.h" #include "qemu-common.h" -#include "strings.h" #include "hax-i386.h" #include "sysemu/accel.h" #include "sysemu/sysemu.h" diff --git a/target/i386/hvf/hvf.c b/target/i386/hvf/hvf.c index 85e5964365..15870a4f36 100644 --- a/target/i386/hvf/hvf.c +++ b/target/i386/hvf/hvf.c @@ -70,7 +70,6 @@ #include "hw/i386/apic_internal.h" #include "hw/boards.h" #include "qemu/main-loop.h" -#include "strings.h" #include "sysemu/accel.h" #include "sysemu/sysemu.h" #include "target/i386/cpu.h" diff --git a/target/i386/hvf/vmx.h b/target/i386/hvf/vmx.h index 162a7d51ae..5dc52ecad6 100644 --- a/target/i386/hvf/vmx.h +++ b/target/i386/hvf/vmx.h @@ -25,7 +25,6 @@ #ifndef VMX_H #define VMX_H -#include <stdint.h> #include <Hypervisor/hv.h> #include <Hypervisor/hv_vmx.h> #include "vmcs.h" diff --git a/target/i386/hvf/x86_decode.c b/target/i386/hvf/x86_decode.c index bf93e8207d..2d7540fe7c 100644 --- a/target/i386/hvf/x86_decode.c +++ b/target/i386/hvf/x86_decode.c @@ -21,7 +21,6 @@ #include "qemu-common.h" #include "panic.h" #include "x86_decode.h" -#include "string.h" #include "vmx.h" #include "x86_mmu.h" #include "x86_descr.h" diff --git a/target/i386/hvf/x86_mmu.c b/target/i386/hvf/x86_mmu.c index 5c1f35acd0..c6be2cca35 100644 --- a/target/i386/hvf/x86_mmu.c +++ b/target/i386/hvf/x86_mmu.c @@ -15,18 +15,16 @@ * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see <http://www.gnu.org/licenses/>. */ + #include "qemu/osdep.h" +#include <memory.h> #include "panic.h" - #include "qemu-common.h" #include "cpu.h" #include "x86.h" #include "x86_mmu.h" -#include "string.h" #include "vmcs.h" #include "vmx.h" - -#include "memory.h" #include "exec/address-spaces.h" #define pte_present(pte) (pte & PT_PRESENT) diff --git a/target/i386/hvf/x86_task.c b/target/i386/hvf/x86_task.c index d7f665f8fa..4abf3db25e 100644 --- a/target/i386/hvf/x86_task.c +++ b/target/i386/hvf/x86_task.c @@ -32,7 +32,6 @@ #include "hw/i386/apic_internal.h" #include "hw/boards.h" #include "qemu/main-loop.h" -#include "strings.h" #include "sysemu/accel.h" #include "sysemu/sysemu.h" #include "target/i386/cpu.h" diff --git a/target/i386/hvf/x86hvf.c b/target/i386/hvf/x86hvf.c index 7803e09a28..6c88939b96 100644 --- a/target/i386/hvf/x86hvf.c +++ b/target/i386/hvf/x86hvf.c @@ -29,11 +29,8 @@ #include "hw/i386/apic_internal.h" -#include <stdio.h> -#include <stdlib.h> #include <Hypervisor/hv.h> #include <Hypervisor/hv_vmx.h> -#include <stdint.h> void hvf_set_segment(struct CPUState *cpu, struct vmx_segment *vmx_seg, SegmentCache *qseg, bool is_tr) diff --git a/target/i386/monitor.c b/target/i386/monitor.c index 75e155ffb1..75429129fd 100644 --- a/target/i386/monitor.c +++ b/target/i386/monitor.c @@ -21,10 +21,12 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ + #include "qemu/osdep.h" #include "cpu.h" #include "monitor/monitor.h" #include "monitor/hmp-target.h" +#include "qapi/qmp/qdict.h" #include "hw/i386/pc.h" #include "sysemu/kvm.h" #include "hmp.h" diff --git a/target/i386/xsave_helper.c b/target/i386/xsave_helper.c index ca735eee77..52ea7e654b 100644 --- a/target/i386/xsave_helper.c +++ b/target/i386/xsave_helper.c @@ -3,7 +3,6 @@ * See the COPYING file in the top-level directory. */ #include "qemu/osdep.h" -#include "qapi/error.h" #include "qemu-common.h" #include "cpu.h" diff --git a/target/nios2/helper.c b/target/nios2/helper.c index a169c91eaa..a8b8ec662a 100644 --- a/target/nios2/helper.c +++ b/target/nios2/helper.c @@ -22,7 +22,6 @@ #include "cpu.h" #include "qemu/host-utils.h" -#include "qapi/error.h" #include "exec/exec-all.h" #include "exec/log.h" #include "exec/helper-proto.h" diff --git a/target/ppc/mmu-book3s-v3.c b/target/ppc/mmu-book3s-v3.c index e7798b3582..b60df4408f 100644 --- a/target/ppc/mmu-book3s-v3.c +++ b/target/ppc/mmu-book3s-v3.c @@ -18,7 +18,6 @@ */ #include "qemu/osdep.h" -#include "qapi/error.h" #include "cpu.h" #include "mmu-hash64.h" #include "mmu-book3s-v3.h" diff --git a/target/ppc/mmu-hash64.c b/target/ppc/mmu-hash64.c index 14d34e512f..c9b72b7429 100644 --- a/target/ppc/mmu-hash64.c +++ b/target/ppc/mmu-hash64.c @@ -18,7 +18,6 @@ * License along with this library; if not, see <http://www.gnu.org/licenses/>. */ #include "qemu/osdep.h" -#include "qapi/error.h" #include "cpu.h" #include "exec/exec-all.h" #include "exec/helper-proto.h" diff --git a/target/ppc/mmu-radix64.c b/target/ppc/mmu-radix64.c index bbd37e3c7d..ab76cbc835 100644 --- a/target/ppc/mmu-radix64.c +++ b/target/ppc/mmu-radix64.c @@ -18,7 +18,6 @@ */ #include "qemu/osdep.h" -#include "qapi/error.h" #include "cpu.h" #include "exec/exec-all.h" #include "exec/helper-proto.h" diff --git a/target/ppc/mmu_helper.c b/target/ppc/mmu_helper.c index 16ef5acaa2..5568d1642b 100644 --- a/target/ppc/mmu_helper.c +++ b/target/ppc/mmu_helper.c @@ -17,7 +17,6 @@ * License along with this library; if not, see <http://www.gnu.org/licenses/>. */ #include "qemu/osdep.h" -#include "qapi/error.h" #include "cpu.h" #include "exec/helper-proto.h" #include "sysemu/kvm.h" diff --git a/target/ppc/translate_init.c b/target/ppc/translate_init.c index e7b1044944..48f2c10156 100644 --- a/target/ppc/translate_init.c +++ b/target/ppc/translate_init.c @@ -29,6 +29,8 @@ #include "mmu-hash32.h" #include "mmu-hash64.h" #include "qemu/error-report.h" +#include "qapi/error.h" +#include "qapi/qmp/qnull.h" #include "qapi/visitor.h" #include "hw/qdev-properties.h" #include "hw/ppc/ppc.h" diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c index 979469dc3c..da7cb9c278 100644 --- a/target/s390x/cpu.c +++ b/target/s390x/cpu.c @@ -100,7 +100,6 @@ static void s390_cpu_initial_reset(CPUState *s) { S390CPU *cpu = S390_CPU(s); CPUS390XState *env = &cpu->env; - int i; s390_cpu_reset(s); /* initial reset does not clear everything! */ @@ -116,10 +115,6 @@ static void s390_cpu_initial_reset(CPUState *s) env->gbea = 1; env->pfault_token = -1UL; - for (i = 0; i < ARRAY_SIZE(env->io_index); i++) { - env->io_index[i] = -1; - } - env->mchk_index = -1; /* tininess for underflow is detected before rounding */ set_float_detect_tininess(float_tininess_before_rounding, @@ -137,7 +132,6 @@ static void s390_cpu_full_reset(CPUState *s) S390CPU *cpu = S390_CPU(s); S390CPUClass *scc = S390_CPU_GET_CLASS(cpu); CPUS390XState *env = &cpu->env; - int i; scc->parent_reset(s); cpu->env.sigp_order = 0; @@ -153,10 +147,6 @@ static void s390_cpu_full_reset(CPUState *s) env->gbea = 1; env->pfault_token = -1UL; - for (i = 0; i < ARRAY_SIZE(env->io_index); i++) { - env->io_index[i] = -1; - } - env->mchk_index = -1; /* tininess for underflow is detected before rounding */ set_float_detect_tininess(float_tininess_before_rounding, diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h index a1123ad621..21ce40d5b6 100644 --- a/target/s390x/cpu.h +++ b/target/s390x/cpu.h @@ -53,12 +53,6 @@ #define MMU_USER_IDX 0 -#define MAX_IO_QUEUE 16 -#define MAX_MCHK_QUEUE 16 - -#define PSW_MCHK_MASK 0x0004000000000000 -#define PSW_IO_MASK 0x0200000000000000 - #define S390_MAX_CPUS 248 typedef struct PSW { @@ -66,17 +60,6 @@ typedef struct PSW { uint64_t addr; } PSW; -typedef struct IOIntQueue { - uint16_t id; - uint16_t nr; - uint32_t parm; - uint32_t word; -} IOIntQueue; - -typedef struct MchkQueue { - uint16_t type; -} MchkQueue; - struct CPUS390XState { uint64_t regs[16]; /* GP registers */ /* @@ -122,15 +105,9 @@ struct CPUS390XState { uint64_t cregs[16]; /* control registers */ - IOIntQueue io_queue[MAX_IO_QUEUE][8]; - MchkQueue mchk_queue[MAX_MCHK_QUEUE]; - int pending_int; - uint32_t service_param; uint16_t external_call_addr; DECLARE_BITMAP(emergency_signals, S390_MAX_CPUS); - int io_index[8]; - int mchk_index; uint64_t ckc; uint64_t cputm; @@ -409,9 +386,6 @@ static inline void cpu_get_tb_cpu_state(CPUS390XState* env, target_ulong *pc, #define EXCP_IO 7 /* I/O interrupt */ #define EXCP_MCHK 8 /* machine check */ -#define INTERRUPT_IO (1 << 0) -#define INTERRUPT_MCHK (1 << 1) -#define INTERRUPT_EXT_SERVICE (1 << 2) #define INTERRUPT_EXT_CPU_TIMER (1 << 3) #define INTERRUPT_EXT_CLOCK_COMPARATOR (1 << 4) #define INTERRUPT_EXTERNAL_CALL (1 << 5) @@ -452,62 +426,66 @@ static inline void setcc(S390CPU *cpu, uint64_t cc) } /* STSI */ -#define STSI_LEVEL_MASK 0x00000000f0000000ULL -#define STSI_LEVEL_CURRENT 0x0000000000000000ULL -#define STSI_LEVEL_1 0x0000000010000000ULL -#define STSI_LEVEL_2 0x0000000020000000ULL -#define STSI_LEVEL_3 0x0000000030000000ULL +#define STSI_R0_FC_MASK 0x00000000f0000000ULL +#define STSI_R0_FC_CURRENT 0x0000000000000000ULL +#define STSI_R0_FC_LEVEL_1 0x0000000010000000ULL +#define STSI_R0_FC_LEVEL_2 0x0000000020000000ULL +#define STSI_R0_FC_LEVEL_3 0x0000000030000000ULL #define STSI_R0_RESERVED_MASK 0x000000000fffff00ULL #define STSI_R0_SEL1_MASK 0x00000000000000ffULL #define STSI_R1_RESERVED_MASK 0x00000000ffff0000ULL #define STSI_R1_SEL2_MASK 0x000000000000ffffULL /* Basic Machine Configuration */ -struct sysib_111 { - uint32_t res1[8]; +typedef struct SysIB_111 { + uint8_t res1[32]; uint8_t manuf[16]; uint8_t type[4]; uint8_t res2[12]; uint8_t model[16]; uint8_t sequence[16]; uint8_t plant[4]; - uint8_t res3[156]; -}; + uint8_t res3[3996]; +} SysIB_111; +QEMU_BUILD_BUG_ON(sizeof(SysIB_111) != 4096); /* Basic Machine CPU */ -struct sysib_121 { - uint32_t res1[80]; +typedef struct SysIB_121 { + uint8_t res1[80]; uint8_t sequence[16]; uint8_t plant[4]; uint8_t res2[2]; uint16_t cpu_addr; - uint8_t res3[152]; -}; + uint8_t res3[3992]; +} SysIB_121; +QEMU_BUILD_BUG_ON(sizeof(SysIB_121) != 4096); /* Basic Machine CPUs */ -struct sysib_122 { +typedef struct SysIB_122 { uint8_t res1[32]; uint32_t capability; uint16_t total_cpus; - uint16_t active_cpus; + uint16_t conf_cpus; uint16_t standby_cpus; uint16_t reserved_cpus; uint16_t adjustments[2026]; -}; +} SysIB_122; +QEMU_BUILD_BUG_ON(sizeof(SysIB_122) != 4096); /* LPAR CPU */ -struct sysib_221 { - uint32_t res1[80]; +typedef struct SysIB_221 { + uint8_t res1[80]; uint8_t sequence[16]; uint8_t plant[4]; uint16_t cpu_id; uint16_t cpu_addr; - uint8_t res3[152]; -}; + uint8_t res3[3992]; +} SysIB_221; +QEMU_BUILD_BUG_ON(sizeof(SysIB_221) != 4096); /* LPAR CPUs */ -struct sysib_222 { - uint32_t res1[32]; +typedef struct SysIB_222 { + uint8_t res1[32]; uint16_t lpar_num; uint8_t res2; uint8_t lcpuc; @@ -520,11 +498,12 @@ struct sysib_222 { uint8_t res3[16]; uint16_t dedicated_cpus; uint16_t shared_cpus; - uint8_t res4[180]; -}; + uint8_t res4[4020]; +} SysIB_222; +QEMU_BUILD_BUG_ON(sizeof(SysIB_222) != 4096); /* VM CPUs */ -struct sysib_322 { +typedef struct SysIB_322 { uint8_t res1[31]; uint8_t count; struct { @@ -543,7 +522,18 @@ struct sysib_322 { } vm[8]; uint8_t res4[1504]; uint8_t ext_names[8][256]; -}; +} SysIB_322; +QEMU_BUILD_BUG_ON(sizeof(SysIB_322) != 4096); + +typedef union SysIB { + SysIB_111 sysib_111; + SysIB_121 sysib_121; + SysIB_122 sysib_122; + SysIB_221 sysib_221; + SysIB_222 sysib_222; + SysIB_322 sysib_322; +} SysIB; +QEMU_BUILD_BUG_ON(sizeof(SysIB) != 4096); /* MMU defines */ #define _ASCE_ORIGIN ~0xfffULL /* segment table origin */ @@ -718,6 +708,10 @@ static inline unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU *cpu) return 0; } #endif /* CONFIG_USER_ONLY */ +static inline uint8_t s390_cpu_get_state(S390CPU *cpu) +{ + return cpu->env.cpu_state; +} /* cpu_models.c */ @@ -752,7 +746,6 @@ void s390_program_interrupt(CPUS390XState *env, uint32_t code, int ilen, /* service interrupts are floating therefore we must not pass an cpustate */ void s390_sclp_extint(uint32_t parm); - /* mmu_helper.c */ int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf, int len, bool is_write); diff --git a/target/s390x/cpu_features.c b/target/s390x/cpu_features.c index 85d10b5710..a5619f2893 100644 --- a/target/s390x/cpu_features.c +++ b/target/s390x/cpu_features.c @@ -156,8 +156,12 @@ static const S390FeatDef s390_features[] = { FEAT_INIT("ptff-qpc", S390_FEAT_TYPE_PTFF, 3, "PTFF Query Physical Clock"), FEAT_INIT("ptff-qui", S390_FEAT_TYPE_PTFF, 4, "PTFF Query UTC Information"), FEAT_INIT("ptff-qtou", S390_FEAT_TYPE_PTFF, 5, "PTFF Query TOD Offset User"), + FEAT_INIT("ptff-qsie", S390_FEAT_TYPE_PTFF, 10, "PTFF Query Steering Information Extended"), + FEAT_INIT("ptff-qtoue", S390_FEAT_TYPE_PTFF, 13, "PTFF Query TOD Offset User Extended"), FEAT_INIT("ptff-sto", S390_FEAT_TYPE_PTFF, 65, "PTFF Set TOD Offset"), FEAT_INIT("ptff-stou", S390_FEAT_TYPE_PTFF, 69, "PTFF Set TOD Offset User"), + FEAT_INIT("ptff-stoe", S390_FEAT_TYPE_PTFF, 73, "PTFF Set TOD Offset Extended"), + FEAT_INIT("ptff-stoue", S390_FEAT_TYPE_PTFF, 77, "PTFF Set TOD Offset User Extended"), FEAT_INIT("kmac-dea", S390_FEAT_TYPE_KMAC, 1, "KMAC DEA"), FEAT_INIT("kmac-tdea-128", S390_FEAT_TYPE_KMAC, 2, "KMAC TDEA-128"), @@ -445,6 +449,7 @@ static S390FeatGroupDef s390_feature_groups[] = { FEAT_GROUP_INIT("plo", PLO, "Perform-locked-operation facility"), FEAT_GROUP_INIT("tods", TOD_CLOCK_STEERING, "Tod-clock-steering facility"), FEAT_GROUP_INIT("gen13ptff", GEN13_PTFF, "PTFF enhancements introduced with z13"), + FEAT_GROUP_INIT("mepochptff", MULTIPLE_EPOCH_PTFF, "PTFF enhancements introduced with Multiple-epoch facility"), FEAT_GROUP_INIT("msa", MSA, "Message-security-assist facility"), FEAT_GROUP_INIT("msa1", MSA_EXT_1, "Message-security-assist-extension 1 facility"), FEAT_GROUP_INIT("msa2", MSA_EXT_2, "Message-security-assist-extension 2 facility"), diff --git a/target/s390x/cpu_features_def.h b/target/s390x/cpu_features_def.h index 4d930871b4..7c5915c7b2 100644 --- a/target/s390x/cpu_features_def.h +++ b/target/s390x/cpu_features_def.h @@ -151,8 +151,12 @@ typedef enum { S390_FEAT_PTFF_QPT, S390_FEAT_PTFF_QUI, S390_FEAT_PTFF_QTOU, + S390_FEAT_PTFF_QSIE, + S390_FEAT_PTFF_QTOUE, S390_FEAT_PTFF_STO, S390_FEAT_PTFF_STOU, + S390_FEAT_PTFF_STOE, + S390_FEAT_PTFF_STOUE, /* KMAC */ S390_FEAT_KMAC_DEA, diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c index 212a5f0697..1d5f0da4fe 100644 --- a/target/s390x/cpu_models.c +++ b/target/s390x/cpu_models.c @@ -20,9 +20,10 @@ #include "qemu/error-report.h" #include "qapi/qmp/qerror.h" #include "qapi/qobject-input-visitor.h" -#include "qapi/qmp/qbool.h" +#include "qapi/qmp/qdict.h" #ifndef CONFIG_USER_ONLY #include "sysemu/arch_init.h" +#include "hw/pci/pci.h" #endif #define CPUDEF_INIT(_type, _gen, _ec_ga, _mha_pow, _hmfai, _name, _desc) \ @@ -1271,6 +1272,11 @@ static void register_types(void) /* init all bitmaps from gnerated data initially */ s390_init_feat_bitmap(qemu_max_cpu_feat_init, qemu_max_cpu_feat); +#ifndef CONFIG_USER_ONLY + if (!pci_available) { + clear_bit(S390_FEAT_ZPCI, qemu_max_cpu_feat); + } +#endif for (i = 0; i < ARRAY_SIZE(s390_cpu_defs); i++) { s390_init_feat_bitmap(s390_cpu_defs[i].base_init, s390_cpu_defs[i].base_feat); diff --git a/target/s390x/excp_helper.c b/target/s390x/excp_helper.c index e8f7a40c2b..411051edc3 100644 --- a/target/s390x/excp_helper.c +++ b/target/s390x/excp_helper.c @@ -19,7 +19,6 @@ */ #include "qemu/osdep.h" -#include "qapi/error.h" #include "cpu.h" #include "internal.h" #include "qemu/timer.h" @@ -29,6 +28,7 @@ #include "exec/address-spaces.h" #ifndef CONFIG_USER_ONLY #include "sysemu/sysemu.h" +#include "hw/s390x/s390_flic.h" #endif /* #define DEBUG_S390 */ @@ -237,6 +237,7 @@ static void do_svc_interrupt(CPUS390XState *env) static void do_ext_interrupt(CPUS390XState *env) { + QEMUS390FLICState *flic = QEMU_S390_FLIC(s390_get_flic()); S390CPU *cpu = s390_env_get_cpu(env); uint64_t mask, addr; uint16_t cpu_addr; @@ -273,17 +274,14 @@ static void do_ext_interrupt(CPUS390XState *env) lowcore->ext_int_code = cpu_to_be16(EXT_CPU_TIMER); lowcore->cpu_addr = 0; env->pending_int &= ~INTERRUPT_EXT_CPU_TIMER; - } else if ((env->pending_int & INTERRUPT_EXT_SERVICE) && + } else if (qemu_s390_flic_has_service(flic) && (env->cregs[0] & CR0_SERVICE_SC)) { - /* - * FIXME: floating IRQs should be considered by all CPUs and - * shuld not get cleared by CPU reset. - */ + uint32_t param; + + param = qemu_s390_flic_dequeue_service(flic); lowcore->ext_int_code = cpu_to_be16(EXT_SERVICE); - lowcore->ext_params = cpu_to_be32(env->service_param); + lowcore->ext_params = cpu_to_be32(param); lowcore->cpu_addr = 0; - env->service_param = 0; - env->pending_int &= ~INTERRUPT_EXT_SERVICE; } else { g_assert_not_reached(); } @@ -303,95 +301,46 @@ static void do_ext_interrupt(CPUS390XState *env) static void do_io_interrupt(CPUS390XState *env) { - S390CPU *cpu = s390_env_get_cpu(env); + QEMUS390FLICState *flic = QEMU_S390_FLIC(s390_get_flic()); + uint64_t mask, addr; + QEMUS390FlicIO *io; LowCore *lowcore; - IOIntQueue *q; - uint8_t isc; - int disable = 1; - int found = 0; - - if (!(env->psw.mask & PSW_MASK_IO)) { - cpu_abort(CPU(cpu), "I/O int w/o I/O mask\n"); - } - - for (isc = 0; isc < ARRAY_SIZE(env->io_index); isc++) { - uint64_t isc_bits; - - if (env->io_index[isc] < 0) { - continue; - } - if (env->io_index[isc] >= MAX_IO_QUEUE) { - cpu_abort(CPU(cpu), "I/O queue overrun for isc %d: %d\n", - isc, env->io_index[isc]); - } - - q = &env->io_queue[env->io_index[isc]][isc]; - isc_bits = ISC_TO_ISC_BITS(IO_INT_WORD_ISC(q->word)); - if (!(env->cregs[6] & isc_bits)) { - disable = 0; - continue; - } - if (!found) { - uint64_t mask, addr; - - found = 1; - lowcore = cpu_map_lowcore(env); - - lowcore->subchannel_id = cpu_to_be16(q->id); - lowcore->subchannel_nr = cpu_to_be16(q->nr); - lowcore->io_int_parm = cpu_to_be32(q->parm); - lowcore->io_int_word = cpu_to_be32(q->word); - lowcore->io_old_psw.mask = cpu_to_be64(get_psw_mask(env)); - lowcore->io_old_psw.addr = cpu_to_be64(env->psw.addr); - mask = be64_to_cpu(lowcore->io_new_psw.mask); - addr = be64_to_cpu(lowcore->io_new_psw.addr); - cpu_unmap_lowcore(lowcore); + g_assert(env->psw.mask & PSW_MASK_IO); + io = qemu_s390_flic_dequeue_io(flic, env->cregs[6]); + g_assert(io); - env->io_index[isc]--; + lowcore = cpu_map_lowcore(env); - DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__, - env->psw.mask, env->psw.addr); - load_psw(env, mask, addr); - } - if (env->io_index[isc] >= 0) { - disable = 0; - } - continue; - } + lowcore->subchannel_id = cpu_to_be16(io->id); + lowcore->subchannel_nr = cpu_to_be16(io->nr); + lowcore->io_int_parm = cpu_to_be32(io->parm); + lowcore->io_int_word = cpu_to_be32(io->word); + lowcore->io_old_psw.mask = cpu_to_be64(get_psw_mask(env)); + lowcore->io_old_psw.addr = cpu_to_be64(env->psw.addr); + mask = be64_to_cpu(lowcore->io_new_psw.mask); + addr = be64_to_cpu(lowcore->io_new_psw.addr); - if (disable) { - env->pending_int &= ~INTERRUPT_IO; - } + cpu_unmap_lowcore(lowcore); + g_free(io); + DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__, env->psw.mask, + env->psw.addr); + load_psw(env, mask, addr); } static void do_mchk_interrupt(CPUS390XState *env) { - S390CPU *cpu = s390_env_get_cpu(env); + QEMUS390FLICState *flic = QEMU_S390_FLIC(s390_get_flic()); uint64_t mask, addr; LowCore *lowcore; - MchkQueue *q; int i; - if (!(env->psw.mask & PSW_MASK_MCHECK)) { - cpu_abort(CPU(cpu), "Machine check w/o mchk mask\n"); - } - - if (env->mchk_index < 0 || env->mchk_index >= MAX_MCHK_QUEUE) { - cpu_abort(CPU(cpu), "Mchk queue overrun: %d\n", env->mchk_index); - } - - q = &env->mchk_queue[env->mchk_index]; + /* for now we only support channel report machine checks (floating) */ + g_assert(env->psw.mask & PSW_MASK_MCHECK); + g_assert(env->cregs[14] & CR14_CHANNEL_REPORT_SC); - if (q->type != 1) { - /* Don't know how to handle this... */ - cpu_abort(CPU(cpu), "Unknown machine check type %d\n", q->type); - } - if (!(env->cregs[14] & (1 << 28))) { - /* CRW machine checks disabled */ - return; - } + qemu_s390_flic_dequeue_crw_mchk(flic); lowcore = cpu_map_lowcore(env); @@ -418,11 +367,6 @@ static void do_mchk_interrupt(CPUS390XState *env) cpu_unmap_lowcore(lowcore); - env->mchk_index--; - if (env->mchk_index == -1) { - env->pending_int &= ~INTERRUPT_MCHK; - } - DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__, env->psw.mask, env->psw.addr); @@ -431,12 +375,15 @@ static void do_mchk_interrupt(CPUS390XState *env) void s390_cpu_do_interrupt(CPUState *cs) { + QEMUS390FLICState *flic = QEMU_S390_FLIC(s390_get_flic()); S390CPU *cpu = S390_CPU(cs); CPUS390XState *env = &cpu->env; + bool stopped = false; qemu_log_mask(CPU_LOG_INT, "%s: %d at pc=%" PRIx64 "\n", __func__, cs->exception_index, env->psw.addr); +try_deliver: /* handle machine checks */ if (cs->exception_index == -1 && s390_cpu_has_mcck_int(cpu)) { cs->exception_index = EXCP_MCHK; @@ -479,20 +426,30 @@ void s390_cpu_do_interrupt(CPUState *cs) break; case EXCP_STOP: do_stop_interrupt(env); + stopped = true; break; } - /* WAIT PSW during interrupt injection or STOP interrupt */ - if (cs->exception_index == EXCP_HLT) { - /* don't trigger a cpu_loop_exit(), use an interrupt instead */ - cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HALT); + if (cs->exception_index != -1 && !stopped) { + /* check if there are more pending interrupts to deliver */ + cs->exception_index = -1; + goto try_deliver; } cs->exception_index = -1; /* we might still have pending interrupts, but not deliverable */ - if (!env->pending_int) { + if (!env->pending_int && !qemu_s390_flic_has_any(flic)) { cs->interrupt_request &= ~CPU_INTERRUPT_HARD; } + + /* WAIT PSW during interrupt injection or STOP interrupt */ + if ((env->psw.mask & PSW_MASK_WAIT) || stopped) { + /* don't trigger a cpu_loop_exit(), use an interrupt instead */ + cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HALT); + } else if (cs->halted) { + /* unhalt if we had a WAIT PSW somehwere in our injection chain */ + s390_cpu_unhalt(cpu); + } } bool s390_cpu_exec_interrupt(CPUState *cs, int interrupt_request) @@ -510,6 +467,11 @@ bool s390_cpu_exec_interrupt(CPUState *cs, int interrupt_request) s390_cpu_do_interrupt(cs); return true; } + if (env->psw.mask & PSW_MASK_WAIT) { + /* Woken up because of a floating interrupt but it has already + * been delivered. Go back to sleep. */ + cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HALT); + } } return false; } diff --git a/target/s390x/gen-features.c b/target/s390x/gen-features.c index 0570f597ec..0cdbc15378 100644 --- a/target/s390x/gen-features.c +++ b/target/s390x/gen-features.c @@ -9,12 +9,10 @@ * This work is licensed under the terms of the GNU GPL, version 2 or (at * your option) any later version. See the COPYING file in the top-level * directory. - * */ - -#include "inttypes.h" -#include "stdio.h" +#include <inttypes.h> +#include <stdio.h> #include "cpu_features_def.h" #define ARRAY_SIZE(array) (sizeof(array) / sizeof(array[0])) @@ -59,6 +57,12 @@ S390_FEAT_PTFF_QTOU, \ S390_FEAT_PTFF_STOU +#define S390_FEAT_GROUP_MULTIPLE_EPOCH_PTFF \ + S390_FEAT_PTFF_QSIE, \ + S390_FEAT_PTFF_QTOUE, \ + S390_FEAT_PTFF_STOE, \ + S390_FEAT_PTFF_STOUE + #define S390_FEAT_GROUP_MSA \ S390_FEAT_MSA, \ S390_FEAT_KMAC_DEA, \ @@ -219,6 +223,9 @@ static uint16_t group_TOD_CLOCK_STEERING[] = { static uint16_t group_GEN13_PTFF[] = { S390_FEAT_GROUP_GEN13_PTFF, }; +static uint16_t group_MULTIPLE_EPOCH_PTFF[] = { + S390_FEAT_GROUP_MULTIPLE_EPOCH_PTFF, +}; static uint16_t group_MSA[] = { S390_FEAT_GROUP_MSA, }; @@ -466,6 +473,7 @@ static uint16_t full_GEN14_GA1[] = { S390_FEAT_CMM_NT, S390_FEAT_HPMA2, S390_FEAT_SIE_KSS, + S390_FEAT_GROUP_MULTIPLE_EPOCH_PTFF, }; /* Default features (in order of release) @@ -572,8 +580,10 @@ static uint16_t qemu_LATEST[] = { S390_FEAT_STFLE_49, S390_FEAT_LOCAL_TLB_CLEARING, S390_FEAT_INTERLOCKED_ACCESS_2, - S390_FEAT_MSA_EXT_4, + S390_FEAT_ADAPTER_EVENT_NOTIFICATION, + S390_FEAT_ADAPTER_INT_SUPPRESSION, S390_FEAT_MSA_EXT_3, + S390_FEAT_MSA_EXT_4, }; /* add all new definitions before this point */ @@ -582,6 +592,8 @@ static uint16_t qemu_MAX[] = { S390_FEAT_STFLE_53, /* generates a dependency warning, leave it out for now */ S390_FEAT_MSA_EXT_5, + /* only with CONFIG_PCI */ + S390_FEAT_ZPCI, }; /****** END FEATURE DEFS ******/ @@ -664,6 +676,7 @@ static FeatGroupDefSpec FeatGroupDef[] = { FEAT_GROUP_INITIALIZER(PLO), FEAT_GROUP_INITIALIZER(TOD_CLOCK_STEERING), FEAT_GROUP_INITIALIZER(GEN13_PTFF), + FEAT_GROUP_INITIALIZER(MULTIPLE_EPOCH_PTFF), FEAT_GROUP_INITIALIZER(MSA), FEAT_GROUP_INITIALIZER(MSA_EXT_1), FEAT_GROUP_INITIALIZER(MSA_EXT_2), diff --git a/target/s390x/helper.c b/target/s390x/helper.c index 35d9741918..84aaef3a53 100644 --- a/target/s390x/helper.c +++ b/target/s390x/helper.c @@ -19,7 +19,6 @@ */ #include "qemu/osdep.h" -#include "qapi/error.h" #include "cpu.h" #include "internal.h" #include "exec/gdbstub.h" diff --git a/target/s390x/helper.h b/target/s390x/helper.h index 59a1d9869b..59cba86a27 100644 --- a/target/s390x/helper.h +++ b/target/s390x/helper.h @@ -170,6 +170,16 @@ DEF_HELPER_4(schm, void, env, i64, i64, i64) DEF_HELPER_3(ssch, void, env, i64, i64) DEF_HELPER_2(stcrw, void, env, i64) DEF_HELPER_3(stsch, void, env, i64, i64) +DEF_HELPER_2(tpi, i32, env, i64) DEF_HELPER_3(tsch, void, env, i64, i64) DEF_HELPER_2(chsc, void, env, i64) + +DEF_HELPER_2(clp, void, env, i32) +DEF_HELPER_3(pcilg, void, env, i32, i32) +DEF_HELPER_3(pcistg, void, env, i32, i32) +DEF_HELPER_4(stpcifc, void, env, i32, i64, i32) +DEF_HELPER_3(sic, void, env, i64, i64) +DEF_HELPER_3(rpcit, void, env, i32, i32) +DEF_HELPER_5(pcistb, void, env, i32, i32, i64, i32) +DEF_HELPER_4(mpcifc, void, env, i32, i64, i32) #endif diff --git a/target/s390x/insn-data.def b/target/s390x/insn-data.def index 11ee43dcbc..621e10d615 100644 --- a/target/s390x/insn-data.def +++ b/target/s390x/insn-data.def @@ -1063,8 +1063,22 @@ C(0xb233, SSCH, S, Z, 0, insn, 0, 0, ssch, 0) C(0xb239, STCRW, S, Z, 0, insn, 0, 0, stcrw, 0) C(0xb234, STSCH, S, Z, 0, insn, 0, 0, stsch, 0) + C(0xb236, TPI , S, Z, la2, 0, 0, 0, tpi, 0) C(0xb235, TSCH, S, Z, 0, insn, 0, 0, tsch, 0) /* ??? Not listed in PoO ninth edition, but there's a linux driver that uses it: "A CHSC subchannel is usually present on LPAR only." */ C(0xb25f, CHSC, RRE, Z, 0, insn, 0, 0, chsc, 0) + +/* zPCI Instructions */ + /* None of these instructions are documented in the PoP, so this is all + based upon target/s390x/kvm.c and Linux code and likely incomplete */ + C(0xebd0, PCISTB, RSY_a, PCI, la2, 0, 0, 0, pcistb, 0) + C(0xebd1, SIC, RSY_a, AIS, r1, r3, 0, 0, sic, 0) + C(0xb9a0, CLP, RRF_c, PCI, 0, 0, 0, 0, clp, 0) + C(0xb9d0, PCISTG, RRE, PCI, 0, 0, 0, 0, pcistg, 0) + C(0xb9d2, PCILG, RRE, PCI, 0, 0, 0, 0, pcilg, 0) + C(0xb9d3, RPCIT, RRE, PCI, 0, 0, 0, 0, rpcit, 0) + C(0xe3d0, MPCIFC, RXY_a, PCI, la2, 0, 0, 0, mpcifc, 0) + C(0xe3d4, STPCIFC, RXY_a, PCI, la2, 0, 0, 0, stpcifc, 0) + #endif /* CONFIG_USER_ONLY */ diff --git a/target/s390x/internal.h b/target/s390x/internal.h index fea165ffe4..d911e84958 100644 --- a/target/s390x/internal.h +++ b/target/s390x/internal.h @@ -278,11 +278,6 @@ static inline void s390_do_cpu_full_reset(CPUState *cs, run_on_cpu_data arg) cpu_reset(cs); } -static inline uint8_t s390_cpu_get_state(S390CPU *cpu) -{ - return cpu->env.cpu_state; -} - /* arch_dump.c */ int s390_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs, diff --git a/target/s390x/interrupt.c b/target/s390x/interrupt.c index 39c026b8b5..25cfb3eef8 100644 --- a/target/s390x/interrupt.c +++ b/target/s390x/interrupt.c @@ -15,6 +15,9 @@ #include "exec/exec-all.h" #include "sysemu/kvm.h" #include "hw/s390x/ioinst.h" +#if !defined(CONFIG_USER_ONLY) +#include "hw/s390x/s390_flic.h" +#endif /* Ensure to exit the TB after this call! */ void trigger_pgm_exception(CPUS390XState *env, uint32_t code, uint32_t ilen) @@ -55,17 +58,6 @@ void s390_program_interrupt(CPUS390XState *env, uint32_t code, int ilen, } #if !defined(CONFIG_USER_ONLY) -static void cpu_inject_service(S390CPU *cpu, uint32_t param) -{ - CPUS390XState *env = &cpu->env; - - /* multiplexing is good enough for sclp - kvm does it internally as well*/ - env->service_param |= param; - - env->pending_int |= INTERRUPT_EXT_SERVICE; - cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD); -} - void cpu_inject_clock_comparator(S390CPU *cpu) { CPUS390XState *env = &cpu->env; @@ -134,48 +126,6 @@ void cpu_inject_stop(S390CPU *cpu) cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD); } -static void cpu_inject_io(S390CPU *cpu, uint16_t subchannel_id, - uint16_t subchannel_number, - uint32_t io_int_parm, uint32_t io_int_word) -{ - CPUS390XState *env = &cpu->env; - int isc = IO_INT_WORD_ISC(io_int_word); - - if (env->io_index[isc] == MAX_IO_QUEUE - 1) { - /* ugh - can't queue anymore. Let's drop. */ - return; - } - - env->io_index[isc]++; - assert(env->io_index[isc] < MAX_IO_QUEUE); - - env->io_queue[env->io_index[isc]][isc].id = subchannel_id; - env->io_queue[env->io_index[isc]][isc].nr = subchannel_number; - env->io_queue[env->io_index[isc]][isc].parm = io_int_parm; - env->io_queue[env->io_index[isc]][isc].word = io_int_word; - - env->pending_int |= INTERRUPT_IO; - cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD); -} - -static void cpu_inject_crw_mchk(S390CPU *cpu) -{ - CPUS390XState *env = &cpu->env; - - if (env->mchk_index == MAX_MCHK_QUEUE - 1) { - /* ugh - can't queue anymore. Let's drop. */ - return; - } - - env->mchk_index++; - assert(env->mchk_index < MAX_MCHK_QUEUE); - - env->mchk_queue[env->mchk_index].type = 1; - - env->pending_int |= INTERRUPT_MCHK; - cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD); -} - /* * All of the following interrupts are floating, i.e. not per-vcpu. * We just need a dummy cpustate in order to be able to inject in the @@ -183,53 +133,50 @@ static void cpu_inject_crw_mchk(S390CPU *cpu) */ void s390_sclp_extint(uint32_t parm) { - if (kvm_enabled()) { - kvm_s390_service_interrupt(parm); - } else { - S390CPU *dummy_cpu = s390_cpu_addr2state(0); + S390FLICState *fs = s390_get_flic(); + S390FLICStateClass *fsc = s390_get_flic_class(fs); - cpu_inject_service(dummy_cpu, parm); - } + fsc->inject_service(fs, parm); } void s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr, uint32_t io_int_parm, uint32_t io_int_word) { - if (kvm_enabled()) { - kvm_s390_io_interrupt(subchannel_id, subchannel_nr, io_int_parm, - io_int_word); - } else { - S390CPU *dummy_cpu = s390_cpu_addr2state(0); + S390FLICState *fs = s390_get_flic(); + S390FLICStateClass *fsc = s390_get_flic_class(fs); - cpu_inject_io(dummy_cpu, subchannel_id, subchannel_nr, io_int_parm, - io_int_word); - } + fsc->inject_io(fs, subchannel_id, subchannel_nr, io_int_parm, io_int_word); } void s390_crw_mchk(void) { - if (kvm_enabled()) { - kvm_s390_crw_mchk(); - } else { - S390CPU *dummy_cpu = s390_cpu_addr2state(0); + S390FLICState *fs = s390_get_flic(); + S390FLICStateClass *fsc = s390_get_flic_class(fs); - cpu_inject_crw_mchk(dummy_cpu); - } + fsc->inject_crw_mchk(fs); } bool s390_cpu_has_mcck_int(S390CPU *cpu) { + QEMUS390FLICState *flic = s390_get_qemu_flic(s390_get_flic()); CPUS390XState *env = &cpu->env; if (!(env->psw.mask & PSW_MASK_MCHECK)) { return false; } - return env->pending_int & INTERRUPT_MCHK; + /* for now we only support channel report machine checks (floating) */ + if (qemu_s390_flic_has_crw_mchk(flic) && + (env->cregs[14] & CR14_CHANNEL_REPORT_SC)) { + return true; + } + + return false; } bool s390_cpu_has_ext_int(S390CPU *cpu) { + QEMUS390FLICState *flic = s390_get_qemu_flic(s390_get_flic()); CPUS390XState *env = &cpu->env; if (!(env->psw.mask & PSW_MASK_EXT)) { @@ -261,7 +208,7 @@ bool s390_cpu_has_ext_int(S390CPU *cpu) return true; } - if ((env->pending_int & INTERRUPT_EXT_SERVICE) && + if (qemu_s390_flic_has_service(flic) && (env->cregs[0] & CR0_SERVICE_SC)) { return true; } @@ -271,13 +218,14 @@ bool s390_cpu_has_ext_int(S390CPU *cpu) bool s390_cpu_has_io_int(S390CPU *cpu) { + QEMUS390FLICState *flic = s390_get_qemu_flic(s390_get_flic()); CPUS390XState *env = &cpu->env; if (!(env->psw.mask & PSW_MASK_IO)) { return false; } - return env->pending_int & INTERRUPT_IO; + return qemu_s390_flic_has_io(flic, env->cregs[6]); } bool s390_cpu_has_restart_int(S390CPU *cpu) diff --git a/target/s390x/kvm-stub.c b/target/s390x/kvm-stub.c index 6bae3e99d3..8cdcf83845 100644 --- a/target/s390x/kvm-stub.c +++ b/target/s390x/kvm-stub.c @@ -12,10 +12,6 @@ #include "cpu.h" #include "kvm_s390x.h" -void kvm_s390_service_interrupt(uint32_t parm) -{ -} - void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code) { } @@ -30,15 +26,6 @@ void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code) { } -void kvm_s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr, - uint32_t io_int_parm, uint32_t io_int_word) -{ -} - -void kvm_s390_crw_mchk(void) -{ -} - int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state) { return -ENOSYS; diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c index 8736001156..0301e9d519 100644 --- a/target/s390x/kvm.c +++ b/target/s390x/kvm.c @@ -31,13 +31,13 @@ #include "cpu.h" #include "internal.h" #include "kvm_s390x.h" +#include "qapi/error.h" #include "qemu/error-report.h" #include "qemu/timer.h" #include "sysemu/sysemu.h" #include "sysemu/hw_accel.h" #include "hw/hw.h" #include "sysemu/device_tree.h" -#include "qapi/qmp/qjson.h" #include "exec/gdbstub.h" #include "exec/address-spaces.h" #include "trace.h" @@ -1034,7 +1034,7 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq) inject_vcpu_irq_legacy(cs, irq); } -static void __kvm_s390_floating_interrupt(struct kvm_s390_irq *irq) +void kvm_s390_floating_interrupt_legacy(struct kvm_s390_irq *irq) { struct kvm_s390_interrupt kvmint = {}; int r; @@ -1052,33 +1052,6 @@ static void __kvm_s390_floating_interrupt(struct kvm_s390_irq *irq) } } -void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq) -{ - static bool use_flic = true; - int r; - - if (use_flic) { - r = kvm_s390_inject_flic(irq); - if (r == -ENOSYS) { - use_flic = false; - } - if (!r) { - return; - } - } - __kvm_s390_floating_interrupt(irq); -} - -void kvm_s390_service_interrupt(uint32_t parm) -{ - struct kvm_s390_irq irq = { - .type = KVM_S390_INT_SERVICE, - .u.ext.ext_params = parm, - }; - - kvm_s390_floating_interrupt(&irq); -} - void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code) { struct kvm_s390_irq irq = { @@ -1690,10 +1663,10 @@ static int handle_tsch(S390CPU *cpu) * If an I/O interrupt had been dequeued, we have to reinject it. */ if (run->s390_tsch.dequeued) { - kvm_s390_io_interrupt(run->s390_tsch.subchannel_id, - run->s390_tsch.subchannel_nr, - run->s390_tsch.io_int_parm, - run->s390_tsch.io_int_word); + s390_io_interrupt(run->s390_tsch.subchannel_id, + run->s390_tsch.subchannel_nr, + run->s390_tsch.io_int_parm, + run->s390_tsch.io_int_word); } ret = 0; } @@ -1702,7 +1675,7 @@ static int handle_tsch(S390CPU *cpu) static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar) { - struct sysib_322 sysib; + SysIB_322 sysib; int del; if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) { @@ -1840,37 +1813,6 @@ bool kvm_arch_stop_on_emulation_error(CPUState *cpu) return true; } -void kvm_s390_io_interrupt(uint16_t subchannel_id, - uint16_t subchannel_nr, uint32_t io_int_parm, - uint32_t io_int_word) -{ - struct kvm_s390_irq irq = { - .u.io.subchannel_id = subchannel_id, - .u.io.subchannel_nr = subchannel_nr, - .u.io.io_int_parm = io_int_parm, - .u.io.io_int_word = io_int_word, - }; - - if (io_int_word & IO_INT_WORD_AI) { - irq.type = KVM_S390_INT_IO(1, 0, 0, 0); - } else { - irq.type = KVM_S390_INT_IO(0, (subchannel_id & 0xff00) >> 8, - (subchannel_id & 0x0006), - subchannel_nr); - } - kvm_s390_floating_interrupt(&irq); -} - -void kvm_s390_crw_mchk(void) -{ - struct kvm_s390_irq irq = { - .type = KVM_S390_MCHK, - .u.mchk.cr14 = CR14_CHANNEL_REPORT_SC, - .u.mchk.mcic = s390_build_validity_mcic() | MCIC_SC_CP, - }; - kvm_s390_floating_interrupt(&irq); -} - void kvm_s390_enable_css_support(S390CPU *cpu) { int r; @@ -2279,6 +2221,14 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp) return; } + /* PTFF subfunctions might be indicated although kernel support missing */ + if (!test_bit(S390_FEAT_MULTIPLE_EPOCH, model->features)) { + clear_bit(S390_FEAT_PTFF_QSIE, model->features); + clear_bit(S390_FEAT_PTFF_QTOUE, model->features); + clear_bit(S390_FEAT_PTFF_STOE, model->features); + clear_bit(S390_FEAT_PTFF_STOUE, model->features); + } + /* with cpu model support, CMM is only indicated if really available */ if (kvm_s390_cmma_available()) { set_bit(S390_FEAT_CMM, model->features); diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h index 79b35946f3..7a3b862eea 100644 --- a/target/s390x/kvm_s390x.h +++ b/target/s390x/kvm_s390x.h @@ -12,17 +12,12 @@ struct kvm_s390_irq; -void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq); -void kvm_s390_service_interrupt(uint32_t parm); +void kvm_s390_floating_interrupt_legacy(struct kvm_s390_irq *irq); void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq); void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code); int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf, int len, bool is_write); void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code); -void kvm_s390_io_interrupt(uint16_t subchannel_id, - uint16_t subchannel_nr, uint32_t io_int_parm, - uint32_t io_int_word); -void kvm_s390_crw_mchk(void); int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state); void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu); int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu); @@ -44,7 +39,4 @@ void kvm_s390_crypto_reset(void); void kvm_s390_restart_interrupt(S390CPU *cpu); void kvm_s390_stop_interrupt(S390CPU *cpu); -/* implemented outside of target/s390x/ */ -int kvm_s390_inject_flic(struct kvm_s390_irq *irq); - #endif /* KVM_S390X_H */ diff --git a/target/s390x/misc_helper.c b/target/s390x/misc_helper.c index 86da6aab7e..e0b23c1fd1 100644 --- a/target/s390x/misc_helper.c +++ b/target/s390x/misc_helper.c @@ -36,6 +36,10 @@ #include "hw/s390x/ebcdic.h" #include "hw/s390x/s390-virtio-hcall.h" #include "hw/s390x/sclp.h" +#include "hw/s390x/s390_flic.h" +#include "hw/s390x/ioinst.h" +#include "hw/s390x/s390-pci-inst.h" +#include "hw/boards.h" #endif /* #define DEBUG_HELPER */ @@ -194,132 +198,148 @@ void HELPER(spt)(CPUS390XState *env, uint64_t time) } /* Store System Information */ -uint32_t HELPER(stsi)(CPUS390XState *env, uint64_t a0, - uint64_t r0, uint64_t r1) +uint32_t HELPER(stsi)(CPUS390XState *env, uint64_t a0, uint64_t r0, uint64_t r1) { + const uintptr_t ra = GETPC(); + const uint32_t sel1 = r0 & STSI_R0_SEL1_MASK; + const uint32_t sel2 = r1 & STSI_R1_SEL2_MASK; + const MachineState *ms = MACHINE(qdev_get_machine()); + uint16_t total_cpus = 0, conf_cpus = 0, reserved_cpus = 0; S390CPU *cpu = s390_env_get_cpu(env); - int cc = 0; - int sel1, sel2; + SysIB sysib = { 0 }; + int i, cc = 0; + + if ((r0 & STSI_R0_FC_MASK) > STSI_R0_FC_LEVEL_3) { + /* invalid function code: no other checks are performed */ + return 3; + } - if ((r0 & STSI_LEVEL_MASK) <= STSI_LEVEL_3 && - ((r0 & STSI_R0_RESERVED_MASK) || (r1 & STSI_R1_RESERVED_MASK))) { - /* valid function code, invalid reserved bits */ - s390_program_interrupt(env, PGM_SPECIFICATION, 4, GETPC()); + if ((r0 & STSI_R0_RESERVED_MASK) || (r1 & STSI_R1_RESERVED_MASK)) { + s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra); } - sel1 = r0 & STSI_R0_SEL1_MASK; - sel2 = r1 & STSI_R1_SEL2_MASK; + if ((r0 & STSI_R0_FC_MASK) == STSI_R0_FC_CURRENT) { + /* query the current level: no further checks are performed */ + env->regs[0] = STSI_R0_FC_LEVEL_3; + return 0; + } + + if (a0 & ~TARGET_PAGE_MASK) { + s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra); + } - /* XXX: spec exception if sysib is not 4k-aligned */ + /* count the cpus and split them into configured and reserved ones */ + for (i = 0; i < ms->possible_cpus->len; i++) { + total_cpus++; + if (ms->possible_cpus->cpus[i].cpu) { + conf_cpus++; + } else { + reserved_cpus++; + } + } - switch (r0 & STSI_LEVEL_MASK) { - case STSI_LEVEL_1: + /* + * In theory, we could report Level 1 / Level 2 as current. However, + * the Linux kernel will detect this as running under LPAR and assume + * that we have a sclp linemode console (which is always present on + * LPAR, but not the default for QEMU), therefore not displaying boot + * messages and making booting a Linux kernel under TCG harder. + * + * For now we fake the same SMP configuration on all levels. + * + * TODO: We could later make the level configurable via the machine + * and change defaults (linemode console) based on machine type + * and accelerator. + */ + switch (r0 & STSI_R0_FC_MASK) { + case STSI_R0_FC_LEVEL_1: if ((sel1 == 1) && (sel2 == 1)) { /* Basic Machine Configuration */ - struct sysib_111 sysib; char type[5] = {}; - memset(&sysib, 0, sizeof(sysib)); - ebcdic_put(sysib.manuf, "QEMU ", 16); + ebcdic_put(sysib.sysib_111.manuf, "QEMU ", 16); /* same as machine type number in STORE CPU ID, but in EBCDIC */ snprintf(type, ARRAY_SIZE(type), "%X", cpu->model->def->type); - ebcdic_put(sysib.type, type, 4); + ebcdic_put(sysib.sysib_111.type, type, 4); /* model number (not stored in STORE CPU ID for z/Architecure) */ - ebcdic_put(sysib.model, "QEMU ", 16); - ebcdic_put(sysib.sequence, "QEMU ", 16); - ebcdic_put(sysib.plant, "QEMU", 4); - cpu_physical_memory_write(a0, &sysib, sizeof(sysib)); + ebcdic_put(sysib.sysib_111.model, "QEMU ", 16); + ebcdic_put(sysib.sysib_111.sequence, "QEMU ", 16); + ebcdic_put(sysib.sysib_111.plant, "QEMU", 4); } else if ((sel1 == 2) && (sel2 == 1)) { /* Basic Machine CPU */ - struct sysib_121 sysib; - - memset(&sysib, 0, sizeof(sysib)); - /* XXX make different for different CPUs? */ - ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16); - ebcdic_put(sysib.plant, "QEMU", 4); - stw_p(&sysib.cpu_addr, env->core_id); - cpu_physical_memory_write(a0, &sysib, sizeof(sysib)); + ebcdic_put(sysib.sysib_121.sequence, "QEMUQEMUQEMUQEMU", 16); + ebcdic_put(sysib.sysib_121.plant, "QEMU", 4); + sysib.sysib_121.cpu_addr = cpu_to_be16(env->core_id); } else if ((sel1 == 2) && (sel2 == 2)) { /* Basic Machine CPUs */ - struct sysib_122 sysib; - - memset(&sysib, 0, sizeof(sysib)); - stl_p(&sysib.capability, 0x443afc29); - /* XXX change when SMP comes */ - stw_p(&sysib.total_cpus, 1); - stw_p(&sysib.active_cpus, 1); - stw_p(&sysib.standby_cpus, 0); - stw_p(&sysib.reserved_cpus, 0); - cpu_physical_memory_write(a0, &sysib, sizeof(sysib)); + sysib.sysib_122.capability = cpu_to_be32(0x443afc29); + sysib.sysib_122.total_cpus = cpu_to_be16(total_cpus); + sysib.sysib_122.conf_cpus = cpu_to_be16(conf_cpus); + sysib.sysib_122.reserved_cpus = cpu_to_be16(reserved_cpus); } else { cc = 3; } break; - case STSI_LEVEL_2: - { - if ((sel1 == 2) && (sel2 == 1)) { - /* LPAR CPU */ - struct sysib_221 sysib; - - memset(&sysib, 0, sizeof(sysib)); - /* XXX make different for different CPUs? */ - ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16); - ebcdic_put(sysib.plant, "QEMU", 4); - stw_p(&sysib.cpu_addr, env->core_id); - stw_p(&sysib.cpu_id, 0); - cpu_physical_memory_write(a0, &sysib, sizeof(sysib)); - } else if ((sel1 == 2) && (sel2 == 2)) { - /* LPAR CPUs */ - struct sysib_222 sysib; - - memset(&sysib, 0, sizeof(sysib)); - stw_p(&sysib.lpar_num, 0); - sysib.lcpuc = 0; - /* XXX change when SMP comes */ - stw_p(&sysib.total_cpus, 1); - stw_p(&sysib.conf_cpus, 1); - stw_p(&sysib.standby_cpus, 0); - stw_p(&sysib.reserved_cpus, 0); - ebcdic_put(sysib.name, "QEMU ", 8); - stl_p(&sysib.caf, 1000); - stw_p(&sysib.dedicated_cpus, 0); - stw_p(&sysib.shared_cpus, 0); - cpu_physical_memory_write(a0, &sysib, sizeof(sysib)); - } else { - cc = 3; - } - break; + case STSI_R0_FC_LEVEL_2: + if ((sel1 == 2) && (sel2 == 1)) { + /* LPAR CPU */ + ebcdic_put(sysib.sysib_221.sequence, "QEMUQEMUQEMUQEMU", 16); + ebcdic_put(sysib.sysib_221.plant, "QEMU", 4); + sysib.sysib_221.cpu_addr = cpu_to_be16(env->core_id); + } else if ((sel1 == 2) && (sel2 == 2)) { + /* LPAR CPUs */ + sysib.sysib_222.lcpuc = 0x80; /* dedicated */ + sysib.sysib_222.total_cpus = cpu_to_be16(total_cpus); + sysib.sysib_222.conf_cpus = cpu_to_be16(conf_cpus); + sysib.sysib_222.reserved_cpus = cpu_to_be16(reserved_cpus); + ebcdic_put(sysib.sysib_222.name, "QEMU ", 8); + sysib.sysib_222.caf = cpu_to_be32(1000); + sysib.sysib_222.dedicated_cpus = cpu_to_be16(conf_cpus); + } else { + cc = 3; } - case STSI_LEVEL_3: - { - if ((sel1 == 2) && (sel2 == 2)) { - /* VM CPUs */ - struct sysib_322 sysib; - - memset(&sysib, 0, sizeof(sysib)); - sysib.count = 1; - /* XXX change when SMP comes */ - stw_p(&sysib.vm[0].total_cpus, 1); - stw_p(&sysib.vm[0].conf_cpus, 1); - stw_p(&sysib.vm[0].standby_cpus, 0); - stw_p(&sysib.vm[0].reserved_cpus, 0); - ebcdic_put(sysib.vm[0].name, "KVMguest", 8); - stl_p(&sysib.vm[0].caf, 1000); - ebcdic_put(sysib.vm[0].cpi, "KVM/Linux ", 16); - cpu_physical_memory_write(a0, &sysib, sizeof(sysib)); + break; + case STSI_R0_FC_LEVEL_3: + if ((sel1 == 2) && (sel2 == 2)) { + /* VM CPUs */ + sysib.sysib_322.count = 1; + sysib.sysib_322.vm[0].total_cpus = cpu_to_be16(total_cpus); + sysib.sysib_322.vm[0].conf_cpus = cpu_to_be16(conf_cpus); + sysib.sysib_322.vm[0].reserved_cpus = cpu_to_be16(reserved_cpus); + sysib.sysib_322.vm[0].caf = cpu_to_be32(1000); + /* Linux kernel uses this to distinguish us from z/VM */ + ebcdic_put(sysib.sysib_322.vm[0].cpi, "KVM/Linux ", 16); + sysib.sysib_322.vm[0].ext_name_encoding = 2; /* UTF-8 */ + + /* If our VM has a name, use the real name */ + if (qemu_name) { + memset(sysib.sysib_322.vm[0].name, 0x40, + sizeof(sysib.sysib_322.vm[0].name)); + ebcdic_put(sysib.sysib_322.vm[0].name, qemu_name, + MIN(sizeof(sysib.sysib_322.vm[0].name), + strlen(qemu_name))); + strncpy((char *)sysib.sysib_322.ext_names[0], qemu_name, + sizeof(sysib.sysib_322.ext_names[0])); } else { - cc = 3; + ebcdic_put(sysib.sysib_322.vm[0].name, "TCGguest", 8); + strcpy((char *)sysib.sysib_322.ext_names[0], "TCGguest"); } - break; + + /* add the uuid */ + memcpy(sysib.sysib_322.vm[0].uuid, &qemu_uuid, + sizeof(sysib.sysib_322.vm[0].uuid)); + } else { + cc = 3; } - case STSI_LEVEL_CURRENT: - env->regs[0] = STSI_LEVEL_3; - break; - default: - cc = 3; break; } + if (cc == 0) { + if (s390_cpu_virt_mem_write(cpu, a0, 0, &sysib, sizeof(sysib))) { + s390_cpu_virt_mem_handle_exc(cpu, ra); + } + } + return cc; } @@ -429,6 +449,59 @@ void HELPER(stsch)(CPUS390XState *env, uint64_t r1, uint64_t inst) qemu_mutex_unlock_iothread(); } +uint32_t HELPER(tpi)(CPUS390XState *env, uint64_t addr) +{ + const uintptr_t ra = GETPC(); + S390CPU *cpu = s390_env_get_cpu(env); + QEMUS390FLICState *flic = s390_get_qemu_flic(s390_get_flic()); + QEMUS390FlicIO *io = NULL; + LowCore *lowcore; + + if (addr & 0x3) { + s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra); + } + + qemu_mutex_lock_iothread(); + io = qemu_s390_flic_dequeue_io(flic, env->cregs[6]); + if (!io) { + qemu_mutex_unlock_iothread(); + return 0; + } + + if (addr) { + struct { + uint16_t id; + uint16_t nr; + uint32_t parm; + } intc = { + .id = cpu_to_be16(io->id), + .nr = cpu_to_be16(io->nr), + .parm = cpu_to_be32(io->parm), + }; + + if (s390_cpu_virt_mem_write(cpu, addr, 0, &intc, sizeof(intc))) { + /* writing failed, reinject and properly clean up */ + s390_io_interrupt(io->id, io->nr, io->parm, io->word); + qemu_mutex_unlock_iothread(); + g_free(io); + s390_cpu_virt_mem_handle_exc(cpu, ra); + return 0; + } + } else { + /* no protection applies */ + lowcore = cpu_map_lowcore(env); + lowcore->subchannel_id = cpu_to_be16(io->id); + lowcore->subchannel_nr = cpu_to_be16(io->nr); + lowcore->io_int_parm = cpu_to_be32(io->parm); + lowcore->io_int_word = cpu_to_be32(io->word); + cpu_unmap_lowcore(lowcore); + } + + g_free(io); + qemu_mutex_unlock_iothread(); + return 1; +} + void HELPER(tsch)(CPUS390XState *env, uint64_t r1, uint64_t inst) { S390CPU *cpu = s390_env_get_cpu(env); @@ -560,3 +633,91 @@ uint32_t HELPER(stfle)(CPUS390XState *env, uint64_t addr) env->regs[0] = deposit64(env->regs[0], 0, 8, (max_bytes / 8) - 1); return count_bytes >= max_bytes ? 0 : 3; } + +#ifndef CONFIG_USER_ONLY +/* + * Note: we ignore any return code of the functions called for the pci + * instructions, as the only time they return !0 is when the stub is + * called, and in that case we didn't even offer the zpci facility. + * The only exception is SIC, where program checks need to be handled + * by the caller. + */ +void HELPER(clp)(CPUS390XState *env, uint32_t r2) +{ + S390CPU *cpu = s390_env_get_cpu(env); + + qemu_mutex_lock_iothread(); + clp_service_call(cpu, r2, GETPC()); + qemu_mutex_unlock_iothread(); +} + +void HELPER(pcilg)(CPUS390XState *env, uint32_t r1, uint32_t r2) +{ + S390CPU *cpu = s390_env_get_cpu(env); + + qemu_mutex_lock_iothread(); + pcilg_service_call(cpu, r1, r2, GETPC()); + qemu_mutex_unlock_iothread(); +} + +void HELPER(pcistg)(CPUS390XState *env, uint32_t r1, uint32_t r2) +{ + S390CPU *cpu = s390_env_get_cpu(env); + + qemu_mutex_lock_iothread(); + pcistg_service_call(cpu, r1, r2, GETPC()); + qemu_mutex_unlock_iothread(); +} + +void HELPER(stpcifc)(CPUS390XState *env, uint32_t r1, uint64_t fiba, + uint32_t ar) +{ + S390CPU *cpu = s390_env_get_cpu(env); + + qemu_mutex_lock_iothread(); + stpcifc_service_call(cpu, r1, fiba, ar, GETPC()); + qemu_mutex_unlock_iothread(); +} + +void HELPER(sic)(CPUS390XState *env, uint64_t r1, uint64_t r3) +{ + int r; + + qemu_mutex_lock_iothread(); + r = css_do_sic(env, (r3 >> 27) & 0x7, r1 & 0xffff); + qemu_mutex_unlock_iothread(); + /* css_do_sic() may actually return a PGM_xxx value to inject */ + if (r) { + s390_program_interrupt(env, -r, 4, GETPC()); + } +} + +void HELPER(rpcit)(CPUS390XState *env, uint32_t r1, uint32_t r2) +{ + S390CPU *cpu = s390_env_get_cpu(env); + + qemu_mutex_lock_iothread(); + rpcit_service_call(cpu, r1, r2, GETPC()); + qemu_mutex_unlock_iothread(); +} + +void HELPER(pcistb)(CPUS390XState *env, uint32_t r1, uint32_t r3, + uint64_t gaddr, uint32_t ar) +{ + S390CPU *cpu = s390_env_get_cpu(env); + + qemu_mutex_lock_iothread(); + pcistb_service_call(cpu, r1, r3, gaddr, ar, GETPC()); + qemu_mutex_unlock_iothread(); +} + +void HELPER(mpcifc)(CPUS390XState *env, uint32_t r1, uint64_t fiba, + uint32_t ar) +{ + S390CPU *cpu = s390_env_get_cpu(env); + + qemu_mutex_lock_iothread(); + mpcifc_service_call(cpu, r1, fiba, ar, GETPC()); + qemu_mutex_unlock_iothread(); +} +#endif diff --git a/target/s390x/translate.c b/target/s390x/translate.c index df0b41606d..b470d691d3 100644 --- a/target/s390x/translate.c +++ b/target/s390x/translate.c @@ -4199,6 +4199,14 @@ static ExitStatus op_stcrw(DisasContext *s, DisasOps *o) return NO_EXIT; } +static ExitStatus op_tpi(DisasContext *s, DisasOps *o) +{ + check_privileged(s); + gen_helper_tpi(cc_op, cpu_env, o->addr1); + set_cc_static(s); + return NO_EXIT; +} + static ExitStatus op_tsch(DisasContext *s, DisasOps *o) { check_privileged(s); @@ -4777,6 +4785,106 @@ static ExitStatus op_zero2(DisasContext *s, DisasOps *o) return NO_EXIT; } +#ifndef CONFIG_USER_ONLY +static ExitStatus op_clp(DisasContext *s, DisasOps *o) +{ + TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2)); + + check_privileged(s); + gen_helper_clp(cpu_env, r2); + tcg_temp_free_i32(r2); + set_cc_static(s); + return NO_EXIT; +} + +static ExitStatus op_pcilg(DisasContext *s, DisasOps *o) +{ + TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1)); + TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2)); + + check_privileged(s); + gen_helper_pcilg(cpu_env, r1, r2); + tcg_temp_free_i32(r1); + tcg_temp_free_i32(r2); + set_cc_static(s); + return NO_EXIT; +} + +static ExitStatus op_pcistg(DisasContext *s, DisasOps *o) +{ + TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1)); + TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2)); + + check_privileged(s); + gen_helper_pcistg(cpu_env, r1, r2); + tcg_temp_free_i32(r1); + tcg_temp_free_i32(r2); + set_cc_static(s); + return NO_EXIT; +} + +static ExitStatus op_stpcifc(DisasContext *s, DisasOps *o) +{ + TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1)); + TCGv_i32 ar = tcg_const_i32(get_field(s->fields, b2)); + + check_privileged(s); + gen_helper_stpcifc(cpu_env, r1, o->addr1, ar); + tcg_temp_free_i32(ar); + tcg_temp_free_i32(r1); + set_cc_static(s); + return NO_EXIT; +} + +static ExitStatus op_sic(DisasContext *s, DisasOps *o) +{ + check_privileged(s); + gen_helper_sic(cpu_env, o->in1, o->in2); + return NO_EXIT; +} + +static ExitStatus op_rpcit(DisasContext *s, DisasOps *o) +{ + TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1)); + TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2)); + + check_privileged(s); + gen_helper_rpcit(cpu_env, r1, r2); + tcg_temp_free_i32(r1); + tcg_temp_free_i32(r2); + set_cc_static(s); + return NO_EXIT; +} + +static ExitStatus op_pcistb(DisasContext *s, DisasOps *o) +{ + TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1)); + TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3)); + TCGv_i32 ar = tcg_const_i32(get_field(s->fields, b2)); + + check_privileged(s); + gen_helper_pcistb(cpu_env, r1, r3, o->addr1, ar); + tcg_temp_free_i32(ar); + tcg_temp_free_i32(r1); + tcg_temp_free_i32(r3); + set_cc_static(s); + return NO_EXIT; +} + +static ExitStatus op_mpcifc(DisasContext *s, DisasOps *o) +{ + TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1)); + TCGv_i32 ar = tcg_const_i32(get_field(s->fields, b2)); + + check_privileged(s); + gen_helper_mpcifc(cpu_env, r1, o->addr1, ar); + tcg_temp_free_i32(ar); + tcg_temp_free_i32(r1); + set_cc_static(s); + return NO_EXIT; +} +#endif + /* ====================================================================== */ /* The "Cc OUTput" generators. Given the generated output (and in some cases the original inputs), update the various cc data structures in order to @@ -5708,6 +5816,8 @@ enum DisasInsnEnum { #define FAC_MSA4 S390_FEAT_MSA_EXT_4 /* msa-extension-4 facility */ #define FAC_MSA5 S390_FEAT_MSA_EXT_5 /* msa-extension-5 facility */ #define FAC_ECT S390_FEAT_EXTRACT_CPU_TIME +#define FAC_PCI S390_FEAT_ZPCI /* z/PCI facility */ +#define FAC_AIS S390_FEAT_ADAPTER_INT_SUPPRESSION static const DisasInsn insn_info[] = { #include "insn-data.def" diff --git a/target/xtensa/core-dc232b/xtensa-modules.c b/target/xtensa/core-dc232b/xtensa-modules.c index 2e103cd2f5..d322c3f52a 100644 --- a/target/xtensa/core-dc232b/xtensa-modules.c +++ b/target/xtensa/core-dc232b/xtensa-modules.c @@ -18,7 +18,8 @@ Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. */ -#include <xtensa-isa.h> +#include "qemu/osdep.h" +#include "xtensa-isa.h" #include "xtensa-isa-internal.h" diff --git a/target/xtensa/core-dc233c/xtensa-modules.c b/target/xtensa/core-dc233c/xtensa-modules.c index 2728311c9a..7c20f82349 100644 --- a/target/xtensa/core-dc233c/xtensa-modules.c +++ b/target/xtensa/core-dc233c/xtensa-modules.c @@ -21,7 +21,8 @@ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include <xtensa-isa.h> +#include "qemu/osdep.h" +#include "xtensa-isa.h" #include "xtensa-isa-internal.h" diff --git a/target/xtensa/core-de212/xtensa-modules.c b/target/xtensa/core-de212/xtensa-modules.c index 4a8735889e..ef7674de3a 100644 --- a/target/xtensa/core-de212/xtensa-modules.c +++ b/target/xtensa/core-de212/xtensa-modules.c @@ -21,7 +21,8 @@ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include <xtensa-isa.h> +#include "qemu/osdep.h" +#include "xtensa-isa.h" #include "xtensa-isa-internal.h" diff --git a/target/xtensa/core-fsf/xtensa-modules.c b/target/xtensa/core-fsf/xtensa-modules.c index 238800d823..f7de2dec15 100644 --- a/target/xtensa/core-fsf/xtensa-modules.c +++ b/target/xtensa/core-fsf/xtensa-modules.c @@ -18,7 +18,8 @@ Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. */ -#include <xtensa-isa.h> +#include "qemu/osdep.h" +#include "xtensa-isa.h" #include "xtensa-isa-internal.h" diff --git a/target/xtensa/core-sample_controller/xtensa-modules.c b/target/xtensa/core-sample_controller/xtensa-modules.c index 2f000199b8..fba41b99ae 100644 --- a/target/xtensa/core-sample_controller/xtensa-modules.c +++ b/target/xtensa/core-sample_controller/xtensa-modules.c @@ -21,7 +21,8 @@ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include <xtensa-isa.h> +#include "qemu/osdep.h" +#include "xtensa-isa.h" #include "xtensa-isa-internal.h" diff --git a/target/xtensa/xtensa-isa.c b/target/xtensa/xtensa-isa.c index e0076a694f..630b4f9da1 100644 --- a/target/xtensa/xtensa-isa.c +++ b/target/xtensa/xtensa-isa.c @@ -22,9 +22,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> +#include "qemu/osdep.h" #include "xtensa-isa.h" #include "xtensa-isa-internal.h" diff --git a/target/xtensa/xtensa-isa.h b/target/xtensa/xtensa-isa.h index d06614c187..0f0211f841 100644 --- a/target/xtensa/xtensa-isa.h +++ b/target/xtensa/xtensa-isa.h @@ -1 +1 @@ -#include <hw/xtensa/xtensa-isa.h> +#include "hw/xtensa/xtensa-isa.h" |