diff options
Diffstat (limited to 'target')
| -rw-r--r-- | target/arm/cpu-param.h | 2 | ||||
| -rw-r--r-- | target/arm/cpu-qom.h | 1 | ||||
| -rw-r--r-- | target/arm/cpu.c | 162 | ||||
| -rw-r--r-- | target/arm/cpu.h | 421 | ||||
| -rw-r--r-- | target/arm/cpu64.c | 1 | ||||
| -rw-r--r-- | target/arm/debug_helper.c | 50 | ||||
| -rw-r--r-- | target/arm/helper-a64.c | 2 | ||||
| -rw-r--r-- | target/arm/helper.c | 1207 | ||||
| -rw-r--r-- | target/arm/internals.h | 73 | ||||
| -rw-r--r-- | target/arm/monitor.c | 15 | ||||
| -rw-r--r-- | target/arm/pauth_helper.c | 14 | ||||
| -rw-r--r-- | target/arm/translate-a64.c | 47 | ||||
| -rw-r--r-- | target/arm/translate.c | 76 | ||||
| -rw-r--r-- | target/arm/translate.h | 4 |
14 files changed, 1390 insertions, 685 deletions
diff --git a/target/arm/cpu-param.h b/target/arm/cpu-param.h index 6e6948e960..18ac562346 100644 --- a/target/arm/cpu-param.h +++ b/target/arm/cpu-param.h @@ -29,6 +29,6 @@ # define TARGET_PAGE_BITS_MIN 10 #endif -#define NB_MMU_MODES 8 +#define NB_MMU_MODES 9 #endif diff --git a/target/arm/cpu-qom.h b/target/arm/cpu-qom.h index 7f5b244bde..3a9d31ea9d 100644 --- a/target/arm/cpu-qom.h +++ b/target/arm/cpu-qom.h @@ -76,6 +76,7 @@ void arm_gt_ptimer_cb(void *opaque); void arm_gt_vtimer_cb(void *opaque); void arm_gt_htimer_cb(void *opaque); void arm_gt_stimer_cb(void *opaque); +void arm_gt_hvtimer_cb(void *opaque); #define ARM_AFF0_SHIFT 0 #define ARM_AFF0_MASK (0xFFULL << ARM_AFF0_SHIFT) diff --git a/target/arm/cpu.c b/target/arm/cpu.c index f86e71a260..b0762a76c4 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -410,58 +410,173 @@ static void arm_cpu_reset(CPUState *s) arm_rebuild_hflags(env); } +static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, + unsigned int target_el, + unsigned int cur_el, bool secure, + uint64_t hcr_el2) +{ + CPUARMState *env = cs->env_ptr; + bool pstate_unmasked; + bool unmasked = false; + + /* + * Don't take exceptions if they target a lower EL. + * This check should catch any exceptions that would not be taken + * but left pending. + */ + if (cur_el > target_el) { + return false; + } + + switch (excp_idx) { + case EXCP_FIQ: + pstate_unmasked = !(env->daif & PSTATE_F); + break; + + case EXCP_IRQ: + pstate_unmasked = !(env->daif & PSTATE_I); + break; + + case EXCP_VFIQ: + if (secure || !(hcr_el2 & HCR_FMO) || (hcr_el2 & HCR_TGE)) { + /* VFIQs are only taken when hypervized and non-secure. */ + return false; + } + return !(env->daif & PSTATE_F); + case EXCP_VIRQ: + if (secure || !(hcr_el2 & HCR_IMO) || (hcr_el2 & HCR_TGE)) { + /* VIRQs are only taken when hypervized and non-secure. */ + return false; + } + return !(env->daif & PSTATE_I); + default: + g_assert_not_reached(); + } + + /* + * Use the target EL, current execution state and SCR/HCR settings to + * determine whether the corresponding CPSR bit is used to mask the + * interrupt. + */ + if ((target_el > cur_el) && (target_el != 1)) { + /* Exceptions targeting a higher EL may not be maskable */ + if (arm_feature(env, ARM_FEATURE_AARCH64)) { + /* + * 64-bit masking rules are simple: exceptions to EL3 + * can't be masked, and exceptions to EL2 can only be + * masked from Secure state. The HCR and SCR settings + * don't affect the masking logic, only the interrupt routing. + */ + if (target_el == 3 || !secure) { + unmasked = true; + } + } else { + /* + * The old 32-bit-only environment has a more complicated + * masking setup. HCR and SCR bits not only affect interrupt + * routing but also change the behaviour of masking. + */ + bool hcr, scr; + + switch (excp_idx) { + case EXCP_FIQ: + /* + * If FIQs are routed to EL3 or EL2 then there are cases where + * we override the CPSR.F in determining if the exception is + * masked or not. If neither of these are set then we fall back + * to the CPSR.F setting otherwise we further assess the state + * below. + */ + hcr = hcr_el2 & HCR_FMO; + scr = (env->cp15.scr_el3 & SCR_FIQ); + + /* + * When EL3 is 32-bit, the SCR.FW bit controls whether the + * CPSR.F bit masks FIQ interrupts when taken in non-secure + * state. If SCR.FW is set then FIQs can be masked by CPSR.F + * when non-secure but only when FIQs are only routed to EL3. + */ + scr = scr && !((env->cp15.scr_el3 & SCR_FW) && !hcr); + break; + case EXCP_IRQ: + /* + * When EL3 execution state is 32-bit, if HCR.IMO is set then + * we may override the CPSR.I masking when in non-secure state. + * The SCR.IRQ setting has already been taken into consideration + * when setting the target EL, so it does not have a further + * affect here. + */ + hcr = hcr_el2 & HCR_IMO; + scr = false; + break; + default: + g_assert_not_reached(); + } + + if ((scr || hcr) && !secure) { + unmasked = true; + } + } + } + + /* + * The PSTATE bits only mask the interrupt if we have not overriden the + * ability above. + */ + return unmasked || pstate_unmasked; +} + bool arm_cpu_exec_interrupt(CPUState *cs, int interrupt_request) { CPUClass *cc = CPU_GET_CLASS(cs); CPUARMState *env = cs->env_ptr; uint32_t cur_el = arm_current_el(env); bool secure = arm_is_secure(env); + uint64_t hcr_el2 = arm_hcr_el2_eff(env); uint32_t target_el; uint32_t excp_idx; - bool ret = false; + + /* The prioritization of interrupts is IMPLEMENTATION DEFINED. */ if (interrupt_request & CPU_INTERRUPT_FIQ) { excp_idx = EXCP_FIQ; target_el = arm_phys_excp_target_el(cs, excp_idx, cur_el, secure); - if (arm_excp_unmasked(cs, excp_idx, target_el)) { - cs->exception_index = excp_idx; - env->exception.target_el = target_el; - cc->do_interrupt(cs); - ret = true; + if (arm_excp_unmasked(cs, excp_idx, target_el, + cur_el, secure, hcr_el2)) { + goto found; } } if (interrupt_request & CPU_INTERRUPT_HARD) { excp_idx = EXCP_IRQ; target_el = arm_phys_excp_target_el(cs, excp_idx, cur_el, secure); - if (arm_excp_unmasked(cs, excp_idx, target_el)) { - cs->exception_index = excp_idx; - env->exception.target_el = target_el; - cc->do_interrupt(cs); - ret = true; + if (arm_excp_unmasked(cs, excp_idx, target_el, + cur_el, secure, hcr_el2)) { + goto found; } } if (interrupt_request & CPU_INTERRUPT_VIRQ) { excp_idx = EXCP_VIRQ; target_el = 1; - if (arm_excp_unmasked(cs, excp_idx, target_el)) { - cs->exception_index = excp_idx; - env->exception.target_el = target_el; - cc->do_interrupt(cs); - ret = true; + if (arm_excp_unmasked(cs, excp_idx, target_el, + cur_el, secure, hcr_el2)) { + goto found; } } if (interrupt_request & CPU_INTERRUPT_VFIQ) { excp_idx = EXCP_VFIQ; target_el = 1; - if (arm_excp_unmasked(cs, excp_idx, target_el)) { - cs->exception_index = excp_idx; - env->exception.target_el = target_el; - cc->do_interrupt(cs); - ret = true; + if (arm_excp_unmasked(cs, excp_idx, target_el, + cur_el, secure, hcr_el2)) { + goto found; } } + return false; - return ret; + found: + cs->exception_index = excp_idx; + env->exception.target_el = target_el; + cc->do_interrupt(cs); + return true; } #if !defined(CONFIG_USER_ONLY) || !defined(TARGET_AARCH64) @@ -1272,7 +1387,6 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) } } - { uint64_t scale; @@ -1295,6 +1409,8 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) arm_gt_htimer_cb, cpu); cpu->gt_timer[GTIMER_SEC] = timer_new(QEMU_CLOCK_VIRTUAL, scale, arm_gt_stimer_cb, cpu); + cpu->gt_timer[GTIMER_HYPVIRT] = timer_new(QEMU_CLOCK_VIRTUAL, scale, + arm_gt_hvtimer_cb, cpu); } #endif diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 608fcbd0b7..0b3036c484 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -144,11 +144,12 @@ typedef struct ARMGenericTimer { uint64_t ctl; /* Timer Control register */ } ARMGenericTimer; -#define GTIMER_PHYS 0 -#define GTIMER_VIRT 1 -#define GTIMER_HYP 2 -#define GTIMER_SEC 3 -#define NUM_GTIMERS 4 +#define GTIMER_PHYS 0 +#define GTIMER_VIRT 1 +#define GTIMER_HYP 2 +#define GTIMER_SEC 3 +#define GTIMER_HYPVIRT 4 +#define NUM_GTIMERS 5 typedef struct { uint64_t raw_tcr; @@ -1424,13 +1425,6 @@ static inline void xpsr_write(CPUARMState *env, uint32_t val, uint32_t mask) #define HCR_ATA (1ULL << 56) #define HCR_DCT (1ULL << 57) -/* - * When we actually implement ARMv8.1-VHE we should add HCR_E2H to - * HCR_MASK and then clear it again if the feature bit is not set in - * hcr_write(). - */ -#define HCR_MASK ((1ULL << 34) - 1) - #define SCR_NS (1U << 0) #define SCR_IRQ (1U << 1) #define SCR_FIQ (1U << 2) @@ -2582,6 +2576,19 @@ struct ARMCPRegInfo { * fieldoffset is 0 then no reset will be done. */ CPResetFn *resetfn; + + /* + * "Original" writefn and readfn. + * For ARMv8.1-VHE register aliases, we overwrite the read/write + * accessor functions of various EL1/EL0 to perform the runtime + * check for which sysreg should actually be modified, and then + * forwards the operation. Before overwriting the accessors, + * the original function is copied here, so that accesses that + * really do go to the EL1/EL0 version proceed normally. + * (The corresponding EL2 register is linked via opaque.) + */ + CPReadFn *orig_readfn; + CPWriteFn *orig_writefn; }; /* Macros which are lvalues for the field in CPUARMState for the @@ -2702,117 +2709,6 @@ bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync); #define ARM_CPUID_TI915T 0x54029152 #define ARM_CPUID_TI925T 0x54029252 -static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, - unsigned int target_el) -{ - CPUARMState *env = cs->env_ptr; - unsigned int cur_el = arm_current_el(env); - bool secure = arm_is_secure(env); - bool pstate_unmasked; - int8_t unmasked = 0; - uint64_t hcr_el2; - - /* Don't take exceptions if they target a lower EL. - * This check should catch any exceptions that would not be taken but left - * pending. - */ - if (cur_el > target_el) { - return false; - } - - hcr_el2 = arm_hcr_el2_eff(env); - - switch (excp_idx) { - case EXCP_FIQ: - pstate_unmasked = !(env->daif & PSTATE_F); - break; - - case EXCP_IRQ: - pstate_unmasked = !(env->daif & PSTATE_I); - break; - - case EXCP_VFIQ: - if (secure || !(hcr_el2 & HCR_FMO) || (hcr_el2 & HCR_TGE)) { - /* VFIQs are only taken when hypervized and non-secure. */ - return false; - } - return !(env->daif & PSTATE_F); - case EXCP_VIRQ: - if (secure || !(hcr_el2 & HCR_IMO) || (hcr_el2 & HCR_TGE)) { - /* VIRQs are only taken when hypervized and non-secure. */ - return false; - } - return !(env->daif & PSTATE_I); - default: - g_assert_not_reached(); - } - - /* Use the target EL, current execution state and SCR/HCR settings to - * determine whether the corresponding CPSR bit is used to mask the - * interrupt. - */ - if ((target_el > cur_el) && (target_el != 1)) { - /* Exceptions targeting a higher EL may not be maskable */ - if (arm_feature(env, ARM_FEATURE_AARCH64)) { - /* 64-bit masking rules are simple: exceptions to EL3 - * can't be masked, and exceptions to EL2 can only be - * masked from Secure state. The HCR and SCR settings - * don't affect the masking logic, only the interrupt routing. - */ - if (target_el == 3 || !secure) { - unmasked = 1; - } - } else { - /* The old 32-bit-only environment has a more complicated - * masking setup. HCR and SCR bits not only affect interrupt - * routing but also change the behaviour of masking. - */ - bool hcr, scr; - - switch (excp_idx) { - case EXCP_FIQ: - /* If FIQs are routed to EL3 or EL2 then there are cases where - * we override the CPSR.F in determining if the exception is - * masked or not. If neither of these are set then we fall back - * to the CPSR.F setting otherwise we further assess the state - * below. - */ - hcr = hcr_el2 & HCR_FMO; - scr = (env->cp15.scr_el3 & SCR_FIQ); - - /* When EL3 is 32-bit, the SCR.FW bit controls whether the - * CPSR.F bit masks FIQ interrupts when taken in non-secure - * state. If SCR.FW is set then FIQs can be masked by CPSR.F - * when non-secure but only when FIQs are only routed to EL3. - */ - scr = scr && !((env->cp15.scr_el3 & SCR_FW) && !hcr); - break; - case EXCP_IRQ: - /* When EL3 execution state is 32-bit, if HCR.IMO is set then - * we may override the CPSR.I masking when in non-secure state. - * The SCR.IRQ setting has already been taken into consideration - * when setting the target EL, so it does not have a further - * affect here. - */ - hcr = hcr_el2 & HCR_IMO; - scr = false; - break; - default: - g_assert_not_reached(); - } - - if ((scr || hcr) && !secure) { - unmasked = 1; - } - } - } - - /* The PSTATE bits only mask the interrupt if we have not overriden the - * ability above. - */ - return unmasked || pstate_unmasked; -} - #define ARM_CPU_TYPE_SUFFIX "-" TYPE_ARM_CPU #define ARM_CPU_TYPE_NAME(name) (name ARM_CPU_TYPE_SUFFIX) #define CPU_RESOLVING_TYPE TYPE_ARM_CPU @@ -2826,18 +2722,21 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, * + NonSecure EL1 & 0 stage 1 * + NonSecure EL1 & 0 stage 2 * + NonSecure EL2 - * + Secure EL1 & EL0 + * + NonSecure EL2 & 0 (ARMv8.1-VHE) + * + Secure EL1 & 0 * + Secure EL3 * If EL3 is 32-bit: * + NonSecure PL1 & 0 stage 1 * + NonSecure PL1 & 0 stage 2 * + NonSecure PL2 - * + Secure PL0 & PL1 + * + Secure PL0 + * + Secure PL1 * (reminder: for 32 bit EL3, Secure PL1 is *EL3*, not EL1.) * * For QEMU, an mmu_idx is not quite the same as a translation regime because: - * 1. we need to split the "EL1 & 0" regimes into two mmu_idxes, because they - * may differ in access permissions even if the VA->PA map is the same + * 1. we need to split the "EL1 & 0" and "EL2 & 0" regimes into two mmu_idxes, + * because they may differ in access permissions even if the VA->PA map is + * the same * 2. we want to cache in our TLB the full VA->IPA->PA lookup for a stage 1+2 * translation, which means that we have one mmu_idx that deals with two * concatenated translation regimes [this sort of combined s1+2 TLB is @@ -2849,19 +2748,23 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, * 4. we can also safely fold together the "32 bit EL3" and "64 bit EL3" * translation regimes, because they map reasonably well to each other * and they can't both be active at the same time. - * This gives us the following list of mmu_idx values: + * 5. we want to be able to use the TLB for accesses done as part of a + * stage1 page table walk, rather than having to walk the stage2 page + * table over and over. + * + * This gives us the following list of cases: * - * NS EL0 (aka NS PL0) stage 1+2 - * NS EL1 (aka NS PL1) stage 1+2 + * NS EL0 EL1&0 stage 1+2 (aka NS PL0) + * NS EL1 EL1&0 stage 1+2 (aka NS PL1) + * NS EL0 EL2&0 + * NS EL2 EL2&0 * NS EL2 (aka NS PL2) + * S EL0 EL1&0 (aka S PL0) + * S EL1 EL1&0 (not used if EL3 is 32 bit) * S EL3 (aka S PL1) - * S EL0 (aka S PL0) - * S EL1 (not used if EL3 is 32 bit) - * NS EL0+1 stage 2 + * NS EL1&0 stage 2 * - * (The last of these is an mmu_idx because we want to be able to use the TLB - * for the accesses done as part of a stage 1 page table walk, rather than - * having to walk the stage 2 page table over and over.) + * for a total of 9 different mmu_idx. * * R profile CPUs have an MPU, but can use the same set of MMU indexes * as A profile. They only need to distinguish NS EL0 and NS EL1 (and @@ -2899,106 +2802,88 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, * For M profile we arrange them to have a bit for priv, a bit for negpri * and a bit for secure. */ -#define ARM_MMU_IDX_A 0x10 /* A profile */ -#define ARM_MMU_IDX_NOTLB 0x20 /* does not have a TLB */ -#define ARM_MMU_IDX_M 0x40 /* M profile */ +#define ARM_MMU_IDX_A 0x10 /* A profile */ +#define ARM_MMU_IDX_NOTLB 0x20 /* does not have a TLB */ +#define ARM_MMU_IDX_M 0x40 /* M profile */ -/* meanings of the bits for M profile mmu idx values */ -#define ARM_MMU_IDX_M_PRIV 0x1 +/* Meanings of the bits for M profile mmu idx values */ +#define ARM_MMU_IDX_M_PRIV 0x1 #define ARM_MMU_IDX_M_NEGPRI 0x2 -#define ARM_MMU_IDX_M_S 0x4 +#define ARM_MMU_IDX_M_S 0x4 /* Secure */ -#define ARM_MMU_IDX_TYPE_MASK (~0x7) -#define ARM_MMU_IDX_COREIDX_MASK 0x7 +#define ARM_MMU_IDX_TYPE_MASK \ + (ARM_MMU_IDX_A | ARM_MMU_IDX_M | ARM_MMU_IDX_NOTLB) +#define ARM_MMU_IDX_COREIDX_MASK 0xf typedef enum ARMMMUIdx { - ARMMMUIdx_S12NSE0 = 0 | ARM_MMU_IDX_A, - ARMMMUIdx_S12NSE1 = 1 | ARM_MMU_IDX_A, - ARMMMUIdx_S1E2 = 2 | ARM_MMU_IDX_A, - ARMMMUIdx_S1E3 = 3 | ARM_MMU_IDX_A, - ARMMMUIdx_S1SE0 = 4 | ARM_MMU_IDX_A, - ARMMMUIdx_S1SE1 = 5 | ARM_MMU_IDX_A, - ARMMMUIdx_S2NS = 6 | ARM_MMU_IDX_A, - ARMMMUIdx_MUser = 0 | ARM_MMU_IDX_M, - ARMMMUIdx_MPriv = 1 | ARM_MMU_IDX_M, - ARMMMUIdx_MUserNegPri = 2 | ARM_MMU_IDX_M, - ARMMMUIdx_MPrivNegPri = 3 | ARM_MMU_IDX_M, - ARMMMUIdx_MSUser = 4 | ARM_MMU_IDX_M, - ARMMMUIdx_MSPriv = 5 | ARM_MMU_IDX_M, - ARMMMUIdx_MSUserNegPri = 6 | ARM_MMU_IDX_M, - ARMMMUIdx_MSPrivNegPri = 7 | ARM_MMU_IDX_M, - /* Indexes below here don't have TLBs and are used only for AT system - * instructions or for the first stage of an S12 page table walk. + /* + * A-profile. */ - ARMMMUIdx_S1NSE0 = 0 | ARM_MMU_IDX_NOTLB, - ARMMMUIdx_S1NSE1 = 1 | ARM_MMU_IDX_NOTLB, -} ARMMMUIdx; + ARMMMUIdx_E10_0 = 0 | ARM_MMU_IDX_A, + ARMMMUIdx_E20_0 = 1 | ARM_MMU_IDX_A, -/* Bit macros for the core-mmu-index values for each index, - * for use when calling tlb_flush_by_mmuidx() and friends. - */ -typedef enum ARMMMUIdxBit { - ARMMMUIdxBit_S12NSE0 = 1 << 0, - ARMMMUIdxBit_S12NSE1 = 1 << 1, - ARMMMUIdxBit_S1E2 = 1 << 2, - ARMMMUIdxBit_S1E3 = 1 << 3, - ARMMMUIdxBit_S1SE0 = 1 << 4, - ARMMMUIdxBit_S1SE1 = 1 << 5, - ARMMMUIdxBit_S2NS = 1 << 6, - ARMMMUIdxBit_MUser = 1 << 0, - ARMMMUIdxBit_MPriv = 1 << 1, - ARMMMUIdxBit_MUserNegPri = 1 << 2, - ARMMMUIdxBit_MPrivNegPri = 1 << 3, - ARMMMUIdxBit_MSUser = 1 << 4, - ARMMMUIdxBit_MSPriv = 1 << 5, - ARMMMUIdxBit_MSUserNegPri = 1 << 6, - ARMMMUIdxBit_MSPrivNegPri = 1 << 7, -} ARMMMUIdxBit; + ARMMMUIdx_E10_1 = 2 | ARM_MMU_IDX_A, -#define MMU_USER_IDX 0 + ARMMMUIdx_E2 = 3 | ARM_MMU_IDX_A, + ARMMMUIdx_E20_2 = 4 | ARM_MMU_IDX_A, -static inline int arm_to_core_mmu_idx(ARMMMUIdx mmu_idx) -{ - return mmu_idx & ARM_MMU_IDX_COREIDX_MASK; -} + ARMMMUIdx_SE10_0 = 5 | ARM_MMU_IDX_A, + ARMMMUIdx_SE10_1 = 6 | ARM_MMU_IDX_A, + ARMMMUIdx_SE3 = 7 | ARM_MMU_IDX_A, -static inline ARMMMUIdx core_to_arm_mmu_idx(CPUARMState *env, int mmu_idx) -{ - if (arm_feature(env, ARM_FEATURE_M)) { - return mmu_idx | ARM_MMU_IDX_M; - } else { - return mmu_idx | ARM_MMU_IDX_A; - } -} + ARMMMUIdx_Stage2 = 8 | ARM_MMU_IDX_A, -/* Return the exception level we're running at if this is our mmu_idx */ -static inline int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx) -{ - switch (mmu_idx & ARM_MMU_IDX_TYPE_MASK) { - case ARM_MMU_IDX_A: - return mmu_idx & 3; - case ARM_MMU_IDX_M: - return mmu_idx & ARM_MMU_IDX_M_PRIV; - default: - g_assert_not_reached(); - } -} + /* + * These are not allocated TLBs and are used only for AT system + * instructions or for the first stage of an S12 page table walk. + */ + ARMMMUIdx_Stage1_E0 = 0 | ARM_MMU_IDX_NOTLB, + ARMMMUIdx_Stage1_E1 = 1 | ARM_MMU_IDX_NOTLB, + + /* + * M-profile. + */ + ARMMMUIdx_MUser = ARM_MMU_IDX_M, + ARMMMUIdx_MPriv = ARM_MMU_IDX_M | ARM_MMU_IDX_M_PRIV, + ARMMMUIdx_MUserNegPri = ARMMMUIdx_MUser | ARM_MMU_IDX_M_NEGPRI, + ARMMMUIdx_MPrivNegPri = ARMMMUIdx_MPriv | ARM_MMU_IDX_M_NEGPRI, + ARMMMUIdx_MSUser = ARMMMUIdx_MUser | ARM_MMU_IDX_M_S, + ARMMMUIdx_MSPriv = ARMMMUIdx_MPriv | ARM_MMU_IDX_M_S, + ARMMMUIdx_MSUserNegPri = ARMMMUIdx_MUserNegPri | ARM_MMU_IDX_M_S, + ARMMMUIdx_MSPrivNegPri = ARMMMUIdx_MPrivNegPri | ARM_MMU_IDX_M_S, +} ARMMMUIdx; /* - * Return the MMU index for a v7M CPU with all relevant information - * manually specified. + * Bit macros for the core-mmu-index values for each index, + * for use when calling tlb_flush_by_mmuidx() and friends. */ -ARMMMUIdx arm_v7m_mmu_idx_all(CPUARMState *env, - bool secstate, bool priv, bool negpri); +#define TO_CORE_BIT(NAME) \ + ARMMMUIdxBit_##NAME = 1 << (ARMMMUIdx_##NAME & ARM_MMU_IDX_COREIDX_MASK) -/* Return the MMU index for a v7M CPU in the specified security and - * privilege state. - */ -ARMMMUIdx arm_v7m_mmu_idx_for_secstate_and_priv(CPUARMState *env, - bool secstate, bool priv); +typedef enum ARMMMUIdxBit { + TO_CORE_BIT(E10_0), + TO_CORE_BIT(E20_0), + TO_CORE_BIT(E10_1), + TO_CORE_BIT(E2), + TO_CORE_BIT(E20_2), + TO_CORE_BIT(SE10_0), + TO_CORE_BIT(SE10_1), + TO_CORE_BIT(SE3), + TO_CORE_BIT(Stage2), + + TO_CORE_BIT(MUser), + TO_CORE_BIT(MPriv), + TO_CORE_BIT(MUserNegPri), + TO_CORE_BIT(MPrivNegPri), + TO_CORE_BIT(MSUser), + TO_CORE_BIT(MSPriv), + TO_CORE_BIT(MSUserNegPri), + TO_CORE_BIT(MSPrivNegPri), +} ARMMMUIdxBit; -/* Return the MMU index for a v7M CPU in the specified security state */ -ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate); +#undef TO_CORE_BIT + +#define MMU_USER_IDX 0 /** * cpu_mmu_index: @@ -3159,15 +3044,7 @@ static inline bool arm_sctlr_b(CPUARMState *env) (env->cp15.sctlr_el[1] & SCTLR_B) != 0; } -static inline uint64_t arm_sctlr(CPUARMState *env, int el) -{ - if (el == 0) { - /* FIXME: ARMv8.1-VHE S2 translation regime. */ - return env->cp15.sctlr_el[1]; - } else { - return env->cp15.sctlr_el[el]; - } -} +uint64_t arm_sctlr(CPUARMState *env, int el); static inline bool arm_cpu_data_is_big_endian_a32(CPUARMState *env, bool sctlr_b) @@ -3221,55 +3098,73 @@ typedef ARMCPU ArchCPU; * We put flags which are shared between 32 and 64 bit mode at the top * of the word, and flags which apply to only one mode at the bottom. * + * 31 20 18 14 9 0 + * +--------------+-----+-----+----------+--------------+ + * | | | TBFLAG_A32 | | + * | | +-----+----------+ TBFLAG_AM32 | + * | TBFLAG_ANY | |TBFLAG_M32| | + * | | +-+----------+--------------| + * | | | TBFLAG_A64 | + * +--------------+---------+---------------------------+ + * 31 20 15 0 + * * Unless otherwise noted, these bits are cached in env->hflags. */ FIELD(TBFLAG_ANY, AARCH64_STATE, 31, 1) -FIELD(TBFLAG_ANY, MMUIDX, 28, 3) -FIELD(TBFLAG_ANY, SS_ACTIVE, 27, 1) -FIELD(TBFLAG_ANY, PSTATE_SS, 26, 1) /* Not cached. */ +FIELD(TBFLAG_ANY, SS_ACTIVE, 30, 1) +FIELD(TBFLAG_ANY, PSTATE_SS, 29, 1) /* Not cached. */ +FIELD(TBFLAG_ANY, BE_DATA, 28, 1) +FIELD(TBFLAG_ANY, MMUIDX, 24, 4) /* Target EL if we take a floating-point-disabled exception */ -FIELD(TBFLAG_ANY, FPEXC_EL, 24, 2) -FIELD(TBFLAG_ANY, BE_DATA, 23, 1) +FIELD(TBFLAG_ANY, FPEXC_EL, 22, 2) +/* For A-profile only, target EL for debug exceptions. */ +FIELD(TBFLAG_ANY, DEBUG_TARGET_EL, 20, 2) + /* - * For A-profile only, target EL for debug exceptions. - * Note that this overlaps with the M-profile-only HANDLER and STACKCHECK bits. + * Bit usage when in AArch32 state, both A- and M-profile. */ -FIELD(TBFLAG_ANY, DEBUG_TARGET_EL, 21, 2) +FIELD(TBFLAG_AM32, CONDEXEC, 0, 8) /* Not cached. */ +FIELD(TBFLAG_AM32, THUMB, 8, 1) /* Not cached. */ -/* Bit usage when in AArch32 state: */ -FIELD(TBFLAG_A32, THUMB, 0, 1) /* Not cached. */ -FIELD(TBFLAG_A32, VECLEN, 1, 3) /* Not cached. */ -FIELD(TBFLAG_A32, VECSTRIDE, 4, 2) /* Not cached. */ +/* + * Bit usage when in AArch32 state, for A-profile only. + */ +FIELD(TBFLAG_A32, VECLEN, 9, 3) /* Not cached. */ +FIELD(TBFLAG_A32, VECSTRIDE, 12, 2) /* Not cached. */ /* * We store the bottom two bits of the CPAR as TB flags and handle * checks on the other bits at runtime. This shares the same bits as * VECSTRIDE, which is OK as no XScale CPU has VFP. * Not cached, because VECLEN+VECSTRIDE are not cached. */ -FIELD(TBFLAG_A32, XSCALE_CPAR, 4, 2) +FIELD(TBFLAG_A32, XSCALE_CPAR, 12, 2) +FIELD(TBFLAG_A32, VFPEN, 14, 1) /* Partially cached, minus FPEXC. */ +FIELD(TBFLAG_A32, SCTLR_B, 15, 1) +FIELD(TBFLAG_A32, HSTR_ACTIVE, 16, 1) /* * Indicates whether cp register reads and writes by guest code should access * the secure or nonsecure bank of banked registers; note that this is not * the same thing as the current security state of the processor! */ -FIELD(TBFLAG_A32, NS, 6, 1) -FIELD(TBFLAG_A32, VFPEN, 7, 1) /* Partially cached, minus FPEXC. */ -FIELD(TBFLAG_A32, CONDEXEC, 8, 8) /* Not cached. */ -FIELD(TBFLAG_A32, SCTLR_B, 16, 1) -FIELD(TBFLAG_A32, HSTR_ACTIVE, 17, 1) - -/* For M profile only, set if FPCCR.LSPACT is set */ -FIELD(TBFLAG_A32, LSPACT, 18, 1) /* Not cached. */ -/* For M profile only, set if we must create a new FP context */ -FIELD(TBFLAG_A32, NEW_FP_CTXT_NEEDED, 19, 1) /* Not cached. */ -/* For M profile only, set if FPCCR.S does not match current security state */ -FIELD(TBFLAG_A32, FPCCR_S_WRONG, 20, 1) /* Not cached. */ -/* For M profile only, Handler (ie not Thread) mode */ -FIELD(TBFLAG_A32, HANDLER, 21, 1) -/* For M profile only, whether we should generate stack-limit checks */ -FIELD(TBFLAG_A32, STACKCHECK, 22, 1) - -/* Bit usage when in AArch64 state */ +FIELD(TBFLAG_A32, NS, 17, 1) + +/* + * Bit usage when in AArch32 state, for M-profile only. + */ +/* Handler (ie not Thread) mode */ +FIELD(TBFLAG_M32, HANDLER, 9, 1) +/* Whether we should generate stack-limit checks */ +FIELD(TBFLAG_M32, STACKCHECK, 10, 1) +/* Set if FPCCR.LSPACT is set */ +FIELD(TBFLAG_M32, LSPACT, 11, 1) /* Not cached. */ +/* Set if we must create a new FP context */ +FIELD(TBFLAG_M32, NEW_FP_CTXT_NEEDED, 12, 1) /* Not cached. */ +/* Set if FPCCR.S does not match current security state */ +FIELD(TBFLAG_M32, FPCCR_S_WRONG, 13, 1) /* Not cached. */ + +/* + * Bit usage when in AArch64 state + */ FIELD(TBFLAG_A64, TBII, 0, 2) FIELD(TBFLAG_A64, SVEEXC_EL, 2, 2) FIELD(TBFLAG_A64, ZCR_LEN, 4, 4) @@ -3277,6 +3172,7 @@ FIELD(TBFLAG_A64, PAUTH_ACTIVE, 8, 1) FIELD(TBFLAG_A64, BT, 9, 1) FIELD(TBFLAG_A64, BTYPE, 10, 2) /* Not cached. */ FIELD(TBFLAG_A64, TBID, 12, 2) +FIELD(TBFLAG_A64, UNPRIV, 14, 1) static inline bool bswap_code(bool sctlr_b) { @@ -3685,6 +3581,11 @@ static inline bool isar_feature_aa64_sve(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, SVE) != 0; } +static inline bool isar_feature_aa64_vh(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64mmfr1, ID_AA64MMFR1, VH) != 0; +} + static inline bool isar_feature_aa64_lor(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64mmfr1, ID_AA64MMFR1, LO) != 0; diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index 2d97bf45e1..c80fb5fd43 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -672,6 +672,7 @@ static void aarch64_max_initfn(Object *obj) t = cpu->isar.id_aa64mmfr1; t = FIELD_DP64(t, ID_AA64MMFR1, HPDS, 1); /* HPD */ t = FIELD_DP64(t, ID_AA64MMFR1, LO, 1); + t = FIELD_DP64(t, ID_AA64MMFR1, VH, 1); cpu->isar.id_aa64mmfr1 = t; /* Replicate the same data to the 32-bit id registers. */ diff --git a/target/arm/debug_helper.c b/target/arm/debug_helper.c index dde80273ff..2e3e90c6a5 100644 --- a/target/arm/debug_helper.c +++ b/target/arm/debug_helper.c @@ -20,6 +20,7 @@ static bool linked_bp_matches(ARMCPU *cpu, int lbn) int ctx_cmps = extract32(cpu->dbgdidr, 20, 4); int bt; uint32_t contextidr; + uint64_t hcr_el2; /* * Links to unimplemented or non-context aware breakpoints are @@ -40,24 +41,44 @@ static bool linked_bp_matches(ARMCPU *cpu, int lbn) } bt = extract64(bcr, 20, 4); - - /* - * We match the whole register even if this is AArch32 using the - * short descriptor format (in which case it holds both PROCID and ASID), - * since we don't implement the optional v7 context ID masking. - */ - contextidr = extract64(env->cp15.contextidr_el[1], 0, 32); + hcr_el2 = arm_hcr_el2_eff(env); switch (bt) { case 3: /* linked context ID match */ - if (arm_current_el(env) > 1) { - /* Context matches never fire in EL2 or (AArch64) EL3 */ + switch (arm_current_el(env)) { + default: + /* Context matches never fire in AArch64 EL3 */ return false; + case 2: + if (!(hcr_el2 & HCR_E2H)) { + /* Context matches never fire in EL2 without E2H enabled. */ + return false; + } + contextidr = env->cp15.contextidr_el[2]; + break; + case 1: + contextidr = env->cp15.contextidr_el[1]; + break; + case 0: + if ((hcr_el2 & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + contextidr = env->cp15.contextidr_el[2]; + } else { + contextidr = env->cp15.contextidr_el[1]; + } + break; } - return (contextidr == extract64(env->cp15.dbgbvr[lbn], 0, 32)); - case 5: /* linked address mismatch (reserved in AArch64) */ + break; + + case 7: /* linked contextidr_el1 match */ + contextidr = env->cp15.contextidr_el[1]; + break; + case 13: /* linked contextidr_el2 match */ + contextidr = env->cp15.contextidr_el[2]; + break; + case 9: /* linked VMID match (reserved if no EL2) */ case 11: /* linked context ID and VMID match (reserved if no EL2) */ + case 15: /* linked full context ID match */ default: /* * Links to Unlinked context breakpoints must generate no @@ -66,7 +87,12 @@ static bool linked_bp_matches(ARMCPU *cpu, int lbn) return false; } - return false; + /* + * We match the whole register even if this is AArch32 using the + * short descriptor format (in which case it holds both PROCID and ASID), + * since we don't implement the optional v7 context ID masking. + */ + return contextidr == (uint32_t)env->cp15.dbgbvr[lbn]; } static bool bp_wp_matches(ARMCPU *cpu, int n, bool is_wp) diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index 36aa6badfd..bf45f8a785 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -70,7 +70,7 @@ static void daif_check(CPUARMState *env, uint32_t op, uint32_t imm, uintptr_t ra) { /* DAIF update to PSTATE. This is OK from EL0 only if UMA is set. */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) { + if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) { raise_exception_ra(env, EXCP_UDEF, syn_aa64_sysregtrap(0, extract32(op, 0, 3), extract32(op, 3, 3), 4, diff --git a/target/arm/helper.c b/target/arm/helper.c index 19a57a17da..7d15d5c933 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -614,56 +614,54 @@ static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate all (TLBIALL) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); if (tlb_force_broadcast(env)) { - tlbiall_is_write(env, NULL, value); - return; + tlb_flush_all_cpus_synced(cs); + } else { + tlb_flush(cs); } - - tlb_flush(CPU(cpu)); } static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); + value &= TARGET_PAGE_MASK; if (tlb_force_broadcast(env)) { - tlbimva_is_write(env, NULL, value); - return; + tlb_flush_page_all_cpus_synced(cs, value); + } else { + tlb_flush_page(cs, value); } - - tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK); } static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate by ASID (TLBIASID) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); if (tlb_force_broadcast(env)) { - tlbiasid_is_write(env, NULL, value); - return; + tlb_flush_all_cpus_synced(cs); + } else { + tlb_flush(cs); } - - tlb_flush(CPU(cpu)); } static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); + value &= TARGET_PAGE_MASK; if (tlb_force_broadcast(env)) { - tlbimvaa_is_write(env, NULL, value); - return; + tlb_flush_page_all_cpus_synced(cs, value); + } else { + tlb_flush_page(cs, value); } - - tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK); } static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -672,9 +670,9 @@ static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = env_cpu(env); tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_E10_1 | + ARMMMUIdxBit_E10_0 | + ARMMMUIdxBit_Stage2); } static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -683,9 +681,9 @@ static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = env_cpu(env); tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_E10_1 | + ARMMMUIdxBit_E10_0 | + ARMMMUIdxBit_Stage2); } static void tlbiipas2_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -706,7 +704,7 @@ static void tlbiipas2_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 40); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S2NS); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_Stage2); } static void tlbiipas2_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -722,7 +720,7 @@ static void tlbiipas2_is_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 40); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_Stage2); } static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -730,7 +728,7 @@ static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, { CPUState *cs = env_cpu(env); - tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_E2); } static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -738,7 +736,7 @@ static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, { CPUState *cs = env_cpu(env); - tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_E2); } static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -747,7 +745,7 @@ static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = env_cpu(env); uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S1E2); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_E2); } static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -757,7 +755,7 @@ static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1E2); + ARMMMUIdxBit_E2); } static const ARMCPRegInfo cp_reginfo[] = { @@ -2326,10 +2324,18 @@ static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri, * Writable only at the highest implemented exception level. */ int el = arm_current_el(env); + uint64_t hcr; + uint32_t cntkctl; switch (el) { case 0: - if (!extract32(env->cp15.c14_cntkctl, 0, 2)) { + hcr = arm_hcr_el2_eff(env); + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + cntkctl = env->cp15.cnthctl_el2; + } else { + cntkctl = env->cp15.c14_cntkctl; + } + if (!extract32(cntkctl, 0, 2)) { return CP_ACCESS_TRAP; } break; @@ -2357,17 +2363,47 @@ static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx, { unsigned int cur_el = arm_current_el(env); bool secure = arm_is_secure(env); + uint64_t hcr = arm_hcr_el2_eff(env); - /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */ - if (cur_el == 0 && - !extract32(env->cp15.c14_cntkctl, timeridx, 1)) { - return CP_ACCESS_TRAP; - } + switch (cur_el) { + case 0: + /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]CTEN. */ + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + return (extract32(env->cp15.cnthctl_el2, timeridx, 1) + ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2); + } - if (arm_feature(env, ARM_FEATURE_EL2) && - timeridx == GTIMER_PHYS && !secure && cur_el < 2 && - !extract32(env->cp15.cnthctl_el2, 0, 1)) { - return CP_ACCESS_TRAP_EL2; + /* CNT[PV]CT: not visible from PL0 if EL0[PV]CTEN is zero */ + if (!extract32(env->cp15.c14_cntkctl, timeridx, 1)) { + return CP_ACCESS_TRAP; + } + + /* If HCR_EL2.<E2H,TGE> == '10': check CNTHCTL_EL2.EL1PCTEN. */ + if (hcr & HCR_E2H) { + if (timeridx == GTIMER_PHYS && + !extract32(env->cp15.cnthctl_el2, 10, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + /* If HCR_EL2.<E2H> == 0: check CNTHCTL_EL2.EL1PCEN. */ + if (arm_feature(env, ARM_FEATURE_EL2) && + timeridx == GTIMER_PHYS && !secure && + !extract32(env->cp15.cnthctl_el2, 1, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } + break; + + case 1: + /* Check CNTHCTL_EL2.EL1PCTEN, which changes location based on E2H. */ + if (arm_feature(env, ARM_FEATURE_EL2) && + timeridx == GTIMER_PHYS && !secure && + (hcr & HCR_E2H + ? !extract32(env->cp15.cnthctl_el2, 10, 1) + : !extract32(env->cp15.cnthctl_el2, 0, 1))) { + return CP_ACCESS_TRAP_EL2; + } + break; } return CP_ACCESS_OK; } @@ -2377,19 +2413,41 @@ static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx, { unsigned int cur_el = arm_current_el(env); bool secure = arm_is_secure(env); + uint64_t hcr = arm_hcr_el2_eff(env); - /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if - * EL0[PV]TEN is zero. - */ - if (cur_el == 0 && - !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) { - return CP_ACCESS_TRAP; - } + switch (cur_el) { + case 0: + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]TEN. */ + return (extract32(env->cp15.cnthctl_el2, 9 - timeridx, 1) + ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2); + } - if (arm_feature(env, ARM_FEATURE_EL2) && - timeridx == GTIMER_PHYS && !secure && cur_el < 2 && - !extract32(env->cp15.cnthctl_el2, 1, 1)) { - return CP_ACCESS_TRAP_EL2; + /* + * CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from + * EL0 if EL0[PV]TEN is zero. + */ + if (!extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) { + return CP_ACCESS_TRAP; + } + /* fall through */ + + case 1: + if (arm_feature(env, ARM_FEATURE_EL2) && + timeridx == GTIMER_PHYS && !secure) { + if (hcr & HCR_E2H) { + /* If HCR_EL2.<E2H,TGE> == '10': check CNTHCTL_EL2.EL1PTEN. */ + if (!extract32(env->cp15.cnthctl_el2, 11, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + /* If HCR_EL2.<E2H> == 0: check CNTHCTL_EL2.EL1PCEN. */ + if (!extract32(env->cp15.cnthctl_el2, 1, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } + } + break; } return CP_ACCESS_OK; } @@ -2515,9 +2573,31 @@ static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri) return gt_get_countervalue(env); } +static uint64_t gt_virt_cnt_offset(CPUARMState *env) +{ + uint64_t hcr; + + switch (arm_current_el(env)) { + case 2: + hcr = arm_hcr_el2_eff(env); + if (hcr & HCR_E2H) { + return 0; + } + break; + case 0: + hcr = arm_hcr_el2_eff(env); + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + return 0; + } + break; + } + + return env->cp15.cntvoff_el2; +} + static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri) { - return gt_get_countervalue(env) - env->cp15.cntvoff_el2; + return gt_get_countervalue(env) - gt_virt_cnt_offset(env); } static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -2532,7 +2612,14 @@ static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri, int timeridx) { - uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0; + uint64_t offset = 0; + + switch (timeridx) { + case GTIMER_VIRT: + case GTIMER_HYPVIRT: + offset = gt_virt_cnt_offset(env); + break; + } return (uint32_t)(env->cp15.c14_timer[timeridx].cval - (gt_get_countervalue(env) - offset)); @@ -2542,7 +2629,14 @@ static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, int timeridx, uint64_t value) { - uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0; + uint64_t offset = 0; + + switch (timeridx) { + case GTIMER_VIRT: + case GTIMER_HYPVIRT: + offset = gt_virt_cnt_offset(env); + break; + } trace_arm_gt_tval_write(timeridx, value); env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset + @@ -2601,6 +2695,70 @@ static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, gt_ctl_write(env, ri, GTIMER_PHYS, value); } +static int gt_phys_redir_timeridx(CPUARMState *env) +{ + switch (arm_mmu_idx(env)) { + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + return GTIMER_HYP; + default: + return GTIMER_PHYS; + } +} + +static int gt_virt_redir_timeridx(CPUARMState *env) +{ + switch (arm_mmu_idx(env)) { + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + return GTIMER_HYPVIRT; + default: + return GTIMER_VIRT; + } +} + +static uint64_t gt_phys_redir_cval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_phys_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].cval; +} + +static void gt_phys_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_phys_redir_timeridx(env); + gt_cval_write(env, ri, timeridx, value); +} + +static uint64_t gt_phys_redir_tval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_phys_redir_timeridx(env); + return gt_tval_read(env, ri, timeridx); +} + +static void gt_phys_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_phys_redir_timeridx(env); + gt_tval_write(env, ri, timeridx, value); +} + +static uint64_t gt_phys_redir_ctl_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_phys_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].ctl; +} + +static void gt_phys_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_phys_redir_timeridx(env); + gt_ctl_write(env, ri, timeridx, value); +} + static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri) { gt_timer_reset(env, ri, GTIMER_VIRT); @@ -2639,6 +2797,48 @@ static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri, gt_recalc_timer(cpu, GTIMER_VIRT); } +static uint64_t gt_virt_redir_cval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_virt_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].cval; +} + +static void gt_virt_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_virt_redir_timeridx(env); + gt_cval_write(env, ri, timeridx, value); +} + +static uint64_t gt_virt_redir_tval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_virt_redir_timeridx(env); + return gt_tval_read(env, ri, timeridx); +} + +static void gt_virt_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_virt_redir_timeridx(env); + gt_tval_write(env, ri, timeridx, value); +} + +static uint64_t gt_virt_redir_ctl_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_virt_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].ctl; +} + +static void gt_virt_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_virt_redir_timeridx(env); + gt_ctl_write(env, ri, timeridx, value); +} + static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri) { gt_timer_reset(env, ri, GTIMER_HYP); @@ -2695,6 +2895,34 @@ static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, gt_ctl_write(env, ri, GTIMER_SEC, value); } +static void gt_hv_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri) +{ + gt_timer_reset(env, ri, GTIMER_HYPVIRT); +} + +static void gt_hv_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + gt_cval_write(env, ri, GTIMER_HYPVIRT, value); +} + +static uint64_t gt_hv_tval_read(CPUARMState *env, const ARMCPRegInfo *ri) +{ + return gt_tval_read(env, ri, GTIMER_HYPVIRT); +} + +static void gt_hv_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + gt_tval_write(env, ri, GTIMER_HYPVIRT, value); +} + +static void gt_hv_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + gt_ctl_write(env, ri, GTIMER_HYPVIRT, value); +} + void arm_gt_ptimer_cb(void *opaque) { ARMCPU *cpu = opaque; @@ -2723,6 +2951,13 @@ void arm_gt_stimer_cb(void *opaque) gt_recalc_timer(cpu, GTIMER_SEC); } +void arm_gt_hvtimer_cb(void *opaque) +{ + ARMCPU *cpu = opaque; + + gt_recalc_timer(cpu, GTIMER_HYPVIRT); +} + static void arm_gt_cntfrq_reset(CPUARMState *env, const ARMCPRegInfo *opaque) { ARMCPU *cpu = env_archcpu(env); @@ -2760,7 +2995,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .accessfn = gt_ptimer_access, .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl), - .writefn = gt_phys_ctl_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write, }, { .name = "CNTP_CTL_S", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1, @@ -2777,14 +3013,16 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .accessfn = gt_ptimer_access, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl), .resetvalue = 0, - .writefn = gt_phys_ctl_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write, }, { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1, .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL0_RW, .accessfn = gt_vtimer_access, .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl), - .writefn = gt_virt_ctl_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write, }, { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1, @@ -2792,14 +3030,15 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .accessfn = gt_vtimer_access, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl), .resetvalue = 0, - .writefn = gt_virt_ctl_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write, }, /* TimerValue views: a 32 bit downcounting view of the underlying state */ { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0, .secure = ARM_CP_SECSTATE_NS, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_ptimer_access, - .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write, + .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write, }, { .name = "CNTP_TVAL_S", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0, @@ -2812,18 +3051,18 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset, - .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write, + .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write, }, { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_vtimer_access, - .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write, + .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write, }, { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset, - .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write, + .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write, }, /* The counter itself */ { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0, @@ -2853,7 +3092,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval), .accessfn = gt_ptimer_access, - .writefn = gt_phys_cval_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write, }, { .name = "CNTP_CVAL_S", .cp = 15, .crm = 14, .opc1 = 2, .secure = ARM_CP_SECSTATE_S, @@ -2869,14 +3109,16 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .type = ARM_CP_IO, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval), .resetvalue = 0, .accessfn = gt_ptimer_access, - .writefn = gt_phys_cval_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write, }, { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3, .access = PL0_RW, .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval), .accessfn = gt_vtimer_access, - .writefn = gt_virt_cval_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write, }, { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2, @@ -2884,7 +3126,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .type = ARM_CP_IO, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval), .resetvalue = 0, .accessfn = gt_vtimer_access, - .writefn = gt_virt_cval_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write, }, /* Secure timer -- this is actually restricted to only EL3 * and configurably Secure-EL1 via the accessfn. @@ -2915,6 +3158,15 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { REGINFO_SENTINEL }; +static CPAccessResult e2h_access(CPUARMState *env, const ARMCPRegInfo *ri, + bool isread) +{ + if (!(arm_hcr_el2_eff(env) & HCR_E2H)) { + return CP_ACCESS_TRAP; + } + return CP_ACCESS_OK; +} + #else /* In user-mode most of the generic timer registers are inaccessible @@ -3009,7 +3261,8 @@ static uint64_t do_ats_write(CPUARMState *env, uint64_t value, bool take_exc = false; if (fi.s1ptw && current_el == 1 && !arm_is_secure(env) - && (mmu_idx == ARMMMUIdx_S1NSE1 || mmu_idx == ARMMMUIdx_S1NSE0)) { + && (mmu_idx == ARMMMUIdx_Stage1_E1 || + mmu_idx == ARMMMUIdx_Stage1_E0)) { /* * Synchronous stage 2 fault on an access made as part of the * translation table walk for AT S1E0* or AT S1E1* insn @@ -3085,7 +3338,7 @@ static uint64_t do_ats_write(CPUARMState *env, uint64_t value, format64 = arm_s1_regime_using_lpae_format(env, mmu_idx); if (arm_feature(env, ARM_FEATURE_EL2)) { - if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) { + if (mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_E10_1) { format64 |= env->cp15.hcr_el2 & (HCR_VM | HCR_DC); } else { format64 |= arm_current_el(env) == 2; @@ -3154,13 +3407,13 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) /* stage 1 current state PL1: ATS1CPR, ATS1CPW */ switch (el) { case 3: - mmu_idx = ARMMMUIdx_S1E3; + mmu_idx = ARMMMUIdx_SE3; break; case 2: - mmu_idx = ARMMMUIdx_S1NSE1; + mmu_idx = ARMMMUIdx_Stage1_E1; break; case 1: - mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1; + mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_Stage1_E1; break; default: g_assert_not_reached(); @@ -3170,13 +3423,13 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) /* stage 1 current state PL0: ATS1CUR, ATS1CUW */ switch (el) { case 3: - mmu_idx = ARMMMUIdx_S1SE0; + mmu_idx = ARMMMUIdx_SE10_0; break; case 2: - mmu_idx = ARMMMUIdx_S1NSE0; + mmu_idx = ARMMMUIdx_Stage1_E0; break; case 1: - mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0; + mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_Stage1_E0; break; default: g_assert_not_reached(); @@ -3184,11 +3437,11 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) break; case 4: /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */ - mmu_idx = ARMMMUIdx_S12NSE1; + mmu_idx = ARMMMUIdx_E10_1; break; case 6: /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */ - mmu_idx = ARMMMUIdx_S12NSE0; + mmu_idx = ARMMMUIdx_E10_0; break; default: g_assert_not_reached(); @@ -3205,7 +3458,7 @@ static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri, MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD; uint64_t par64; - par64 = do_ats_write(env, value, access_type, ARMMMUIdx_S1E2); + par64 = do_ats_write(env, value, access_type, ARMMMUIdx_E2); A32_BANKED_CURRENT_REG_SET(env, par, par64); } @@ -3230,26 +3483,26 @@ static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri, case 0: switch (ri->opc1) { case 0: /* AT S1E1R, AT S1E1W */ - mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1; + mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_Stage1_E1; break; case 4: /* AT S1E2R, AT S1E2W */ - mmu_idx = ARMMMUIdx_S1E2; + mmu_idx = ARMMMUIdx_E2; break; case 6: /* AT S1E3R, AT S1E3W */ - mmu_idx = ARMMMUIdx_S1E3; + mmu_idx = ARMMMUIdx_SE3; break; default: g_assert_not_reached(); } break; case 2: /* AT S1E0R, AT S1E0W */ - mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0; + mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_Stage1_E0; break; case 4: /* AT S12E1R, AT S12E1W */ - mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S12NSE1; + mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_E10_1; break; case 6: /* AT S12E0R, AT S12E0W */ - mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S12NSE0; + mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_E10_0; break; default: g_assert_not_reached(); @@ -3510,7 +3763,7 @@ static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri) tcr->base_mask = 0xffffc000u; } -static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri, +static void vmsa_tcr_el12_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { ARMCPU *cpu = env_archcpu(env); @@ -3533,18 +3786,38 @@ static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri, raw_write(env, ri, value); } +static void vmsa_tcr_ttbr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + /* + * If we are running with E2&0 regime, then an ASID is active. + * Flush if that might be changing. Note we're not checking + * TCR_EL2.A1 to know if this is really the TTBRx_EL2 that + * holds the active ASID, only checking the field that might. + */ + if (extract64(raw_read(env, ri) ^ value, 48, 16) && + (arm_hcr_el2_eff(env) & HCR_E2H)) { + tlb_flush_by_mmuidx(env_cpu(env), + ARMMMUIdxBit_E20_2 | ARMMMUIdxBit_E20_0); + } + raw_write(env, ri, value); +} + static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); - /* Accesses to VTTBR may change the VMID so we must flush the TLB. */ + /* + * A change in VMID to the stage2 page table (Stage2) invalidates + * the combined stage 1&2 tlbs (EL10_1 and EL10_0). + */ if (raw_read(env, ri) != value) { tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_E10_1 | + ARMMMUIdxBit_E10_0 | + ARMMMUIdxBit_Stage2); raw_write(env, ri, value); } } @@ -3586,7 +3859,7 @@ static const ARMCPRegInfo vmsa_cp_reginfo[] = { offsetof(CPUARMState, cp15.ttbr1_ns) } }, { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64, .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2, - .access = PL1_RW, .writefn = vmsa_tcr_el1_write, + .access = PL1_RW, .writefn = vmsa_tcr_el12_write, .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write, .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) }, { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2, @@ -3870,7 +4143,7 @@ static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri, static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) { + if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) { return CP_ACCESS_TRAP; } return CP_ACCESS_OK; @@ -3889,7 +4162,7 @@ static CPAccessResult aa64_cacheop_access(CPUARMState *env, /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless * SCTLR_EL1.UCI is set. */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCI)) { + if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UCI)) { return CP_ACCESS_TRAP; } return CP_ACCESS_OK; @@ -3899,79 +4172,79 @@ static CPAccessResult aa64_cacheop_access(CPUARMState *env, * Page D4-1736 (DDI0487A.b) */ +static int vae1_tlbmask(CPUARMState *env) +{ + /* Since we exclude secure first, we may read HCR_EL2 directly. */ + if (arm_is_secure_below_el3(env)) { + return ARMMMUIdxBit_SE10_1 | ARMMMUIdxBit_SE10_0; + } else if ((env->cp15.hcr_el2 & (HCR_E2H | HCR_TGE)) + == (HCR_E2H | HCR_TGE)) { + return ARMMMUIdxBit_E20_2 | ARMMMUIdxBit_E20_0; + } else { + return ARMMMUIdxBit_E10_1 | ARMMMUIdxBit_E10_0; + } +} + static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { CPUState *cs = env_cpu(env); - bool sec = arm_is_secure_below_el3(env); + int mask = vae1_tlbmask(env); - if (sec) { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); - } else { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { CPUState *cs = env_cpu(env); + int mask = vae1_tlbmask(env); if (tlb_force_broadcast(env)) { - tlbi_aa64_vmalle1is_write(env, NULL, value); - return; - } - - if (arm_is_secure_below_el3(env)) { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } else { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); + tlb_flush_by_mmuidx(cs, mask); } } -static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri, - uint64_t value) +static int alle1_tlbmask(CPUARMState *env) { - /* Note that the 'ALL' scope must invalidate both stage 1 and + /* + * Note that the 'ALL' scope must invalidate both stage 1 and * stage 2 translations, whereas most other scopes only invalidate * stage 1 translations. */ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); - if (arm_is_secure_below_el3(env)) { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); + return ARMMMUIdxBit_SE10_1 | ARMMMUIdxBit_SE10_0; + } else if (arm_feature(env, ARM_FEATURE_EL2)) { + return ARMMMUIdxBit_E10_1 | ARMMMUIdxBit_E10_0 | ARMMMUIdxBit_Stage2; } else { - if (arm_feature(env, ARM_FEATURE_EL2)) { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); - } else { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + return ARMMMUIdxBit_E10_1 | ARMMMUIdxBit_E10_0; } } +static int e2_tlbmask(CPUARMState *env) +{ + /* TODO: ARMv8.4-SecEL2 */ + return ARMMMUIdxBit_E20_0 | ARMMMUIdxBit_E20_2 | ARMMMUIdxBit_E2; +} + +static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + CPUState *cs = env_cpu(env); + int mask = alle1_tlbmask(env); + + tlb_flush_by_mmuidx(cs, mask); +} + static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); + CPUState *cs = env_cpu(env); + int mask = e2_tlbmask(env); - tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx(cs, mask); } static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -3980,42 +4253,25 @@ static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri, ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); - tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_S1E3); + tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_SE3); } static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { - /* Note that the 'ALL' scope must invalidate both stage 1 and - * stage 2 translations, whereas most other scopes only invalidate - * stage 1 translations. - */ CPUState *cs = env_cpu(env); - bool sec = arm_is_secure_below_el3(env); - bool has_el2 = arm_feature(env, ARM_FEATURE_EL2); - - if (sec) { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); - } else if (has_el2) { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); - } else { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + int mask = alle1_tlbmask(env); + + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { CPUState *cs = env_cpu(env); + int mask = e2_tlbmask(env); - tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4023,7 +4279,7 @@ static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri, { CPUState *cs = env_cpu(env); - tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_S1E3); + tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_SE3); } static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4033,11 +4289,11 @@ static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri, * Currently handles both VAE2 and VALE2, since we don't support * flush-last-level-only. */ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); + CPUState *cs = env_cpu(env); + int mask = e2_tlbmask(env); uint64_t pageaddr = sextract64(value << 12, 0, 56); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S1E2); + tlb_flush_page_by_mmuidx(cs, pageaddr, mask); } static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4051,26 +4307,17 @@ static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = CPU(cpu); uint64_t pageaddr = sextract64(value << 12, 0, 56); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S1E3); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_SE3); } static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); - bool sec = arm_is_secure_below_el3(env); + CPUState *cs = env_cpu(env); + int mask = vae1_tlbmask(env); uint64_t pageaddr = sextract64(value << 12, 0, 56); - if (sec) { - tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); - } else { - tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask); } static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4081,23 +4328,14 @@ static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri, * since we don't support flush-for-specific-ASID-only or * flush-last-level-only. */ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); + CPUState *cs = env_cpu(env); + int mask = vae1_tlbmask(env); uint64_t pageaddr = sextract64(value << 12, 0, 56); if (tlb_force_broadcast(env)) { - tlbi_aa64_vae1is_write(env, NULL, value); - return; - } - - if (arm_is_secure_below_el3(env)) { - tlb_flush_page_by_mmuidx(cs, pageaddr, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); + tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask); } else { - tlb_flush_page_by_mmuidx(cs, pageaddr, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); + tlb_flush_page_by_mmuidx(cs, pageaddr, mask); } } @@ -4108,7 +4346,7 @@ static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t pageaddr = sextract64(value << 12, 0, 56); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1E2); + ARMMMUIdxBit_E2); } static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4118,7 +4356,7 @@ static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t pageaddr = sextract64(value << 12, 0, 56); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1E3); + ARMMMUIdxBit_SE3); } static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4140,7 +4378,7 @@ static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 48); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S2NS); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_Stage2); } static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4156,17 +4394,33 @@ static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 48); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_Stage2); } static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { - /* We don't implement EL2, so the only control on DC ZVA is the - * bit in the SCTLR which can prohibit access for EL0. - */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_DZE)) { - return CP_ACCESS_TRAP; + int cur_el = arm_current_el(env); + + if (cur_el < 2) { + uint64_t hcr = arm_hcr_el2_eff(env); + + if (cur_el == 0) { + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + if (!(env->cp15.sctlr_el[2] & SCTLR_DZE)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + if (!(env->cp15.sctlr_el[1] & SCTLR_DZE)) { + return CP_ACCESS_TRAP; + } + if (hcr & HCR_TDZ) { + return CP_ACCESS_TRAP_EL2; + } + } + } else if (hcr & HCR_TDZ) { + return CP_ACCESS_TRAP_EL2; + } } return CP_ACCESS_OK; } @@ -4721,7 +4975,8 @@ static const ARMCPRegInfo el3_no_el2_v8_cp_reginfo[] = { static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { ARMCPU *cpu = env_archcpu(env); - uint64_t valid_mask = HCR_MASK; + /* Begin with bits defined in base ARMv8.0. */ + uint64_t valid_mask = MAKE_64BIT_MASK(0, 34); if (arm_feature(env, ARM_FEATURE_EL3)) { valid_mask &= ~HCR_HCD; @@ -4735,6 +4990,9 @@ static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) */ valid_mask &= ~HCR_TSC; } + if (cpu_isar_feature(aa64_vh, cpu)) { + valid_mask |= HCR_E2H; + } if (cpu_isar_feature(aa64_lor, cpu)) { valid_mask |= HCR_TLOR; } @@ -4938,10 +5196,8 @@ static const ARMCPRegInfo el2_cp_reginfo[] = { .resetvalue = 0 }, { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH, .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2, - .access = PL2_RW, - /* no .writefn needed as this can't cause an ASID change; - * no .raw_writefn or .resetfn needed as we never use mask/base_mask - */ + .access = PL2_RW, .writefn = vmsa_tcr_el12_write, + /* no .raw_writefn or .resetfn needed as we never use mask/base_mask */ .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) }, { .name = "VTCR", .state = ARM_CP_STATE_AA32, .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2, @@ -4975,7 +5231,7 @@ static const ARMCPRegInfo el2_cp_reginfo[] = { .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) }, { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0, - .access = PL2_RW, .resetvalue = 0, + .access = PL2_RW, .resetvalue = 0, .writefn = vmsa_tcr_ttbr_el2_write, .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) }, { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2, .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS, @@ -5244,14 +5500,182 @@ static const ARMCPRegInfo el3_cp_reginfo[] = { REGINFO_SENTINEL }; +#ifndef CONFIG_USER_ONLY +/* Test if system register redirection is to occur in the current state. */ +static bool redirect_for_e2h(CPUARMState *env) +{ + return arm_current_el(env) == 2 && (arm_hcr_el2_eff(env) & HCR_E2H); +} + +static uint64_t el2_e2h_read(CPUARMState *env, const ARMCPRegInfo *ri) +{ + CPReadFn *readfn; + + if (redirect_for_e2h(env)) { + /* Switch to the saved EL2 version of the register. */ + ri = ri->opaque; + readfn = ri->readfn; + } else { + readfn = ri->orig_readfn; + } + if (readfn == NULL) { + readfn = raw_read; + } + return readfn(env, ri); +} + +static void el2_e2h_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + CPWriteFn *writefn; + + if (redirect_for_e2h(env)) { + /* Switch to the saved EL2 version of the register. */ + ri = ri->opaque; + writefn = ri->writefn; + } else { + writefn = ri->orig_writefn; + } + if (writefn == NULL) { + writefn = raw_write; + } + writefn(env, ri, value); +} + +static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu) +{ + struct E2HAlias { + uint32_t src_key, dst_key, new_key; + const char *src_name, *dst_name, *new_name; + bool (*feature)(const ARMISARegisters *id); + }; + +#define K(op0, op1, crn, crm, op2) \ + ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP, crn, crm, op0, op1, op2) + + static const struct E2HAlias aliases[] = { + { K(3, 0, 1, 0, 0), K(3, 4, 1, 0, 0), K(3, 5, 1, 0, 0), + "SCTLR", "SCTLR_EL2", "SCTLR_EL12" }, + { K(3, 0, 1, 0, 2), K(3, 4, 1, 1, 2), K(3, 5, 1, 0, 2), + "CPACR", "CPTR_EL2", "CPACR_EL12" }, + { K(3, 0, 2, 0, 0), K(3, 4, 2, 0, 0), K(3, 5, 2, 0, 0), + "TTBR0_EL1", "TTBR0_EL2", "TTBR0_EL12" }, + { K(3, 0, 2, 0, 1), K(3, 4, 2, 0, 1), K(3, 5, 2, 0, 1), + "TTBR1_EL1", "TTBR1_EL2", "TTBR1_EL12" }, + { K(3, 0, 2, 0, 2), K(3, 4, 2, 0, 2), K(3, 5, 2, 0, 2), + "TCR_EL1", "TCR_EL2", "TCR_EL12" }, + { K(3, 0, 4, 0, 0), K(3, 4, 4, 0, 0), K(3, 5, 4, 0, 0), + "SPSR_EL1", "SPSR_EL2", "SPSR_EL12" }, + { K(3, 0, 4, 0, 1), K(3, 4, 4, 0, 1), K(3, 5, 4, 0, 1), + "ELR_EL1", "ELR_EL2", "ELR_EL12" }, + { K(3, 0, 5, 1, 0), K(3, 4, 5, 1, 0), K(3, 5, 5, 1, 0), + "AFSR0_EL1", "AFSR0_EL2", "AFSR0_EL12" }, + { K(3, 0, 5, 1, 1), K(3, 4, 5, 1, 1), K(3, 5, 5, 1, 1), + "AFSR1_EL1", "AFSR1_EL2", "AFSR1_EL12" }, + { K(3, 0, 5, 2, 0), K(3, 4, 5, 2, 0), K(3, 5, 5, 2, 0), + "ESR_EL1", "ESR_EL2", "ESR_EL12" }, + { K(3, 0, 6, 0, 0), K(3, 4, 6, 0, 0), K(3, 5, 6, 0, 0), + "FAR_EL1", "FAR_EL2", "FAR_EL12" }, + { K(3, 0, 10, 2, 0), K(3, 4, 10, 2, 0), K(3, 5, 10, 2, 0), + "MAIR_EL1", "MAIR_EL2", "MAIR_EL12" }, + { K(3, 0, 10, 3, 0), K(3, 4, 10, 3, 0), K(3, 5, 10, 3, 0), + "AMAIR0", "AMAIR_EL2", "AMAIR_EL12" }, + { K(3, 0, 12, 0, 0), K(3, 4, 12, 0, 0), K(3, 5, 12, 0, 0), + "VBAR", "VBAR_EL2", "VBAR_EL12" }, + { K(3, 0, 13, 0, 1), K(3, 4, 13, 0, 1), K(3, 5, 13, 0, 1), + "CONTEXTIDR_EL1", "CONTEXTIDR_EL2", "CONTEXTIDR_EL12" }, + { K(3, 0, 14, 1, 0), K(3, 4, 14, 1, 0), K(3, 5, 14, 1, 0), + "CNTKCTL", "CNTHCTL_EL2", "CNTKCTL_EL12" }, + + /* + * Note that redirection of ZCR is mentioned in the description + * of ZCR_EL2, and aliasing in the description of ZCR_EL1, but + * not in the summary table. + */ + { K(3, 0, 1, 2, 0), K(3, 4, 1, 2, 0), K(3, 5, 1, 2, 0), + "ZCR_EL1", "ZCR_EL2", "ZCR_EL12", isar_feature_aa64_sve }, + + /* TODO: ARMv8.2-SPE -- PMSCR_EL2 */ + /* TODO: ARMv8.4-Trace -- TRFCR_EL2 */ + }; +#undef K + + size_t i; + + for (i = 0; i < ARRAY_SIZE(aliases); i++) { + const struct E2HAlias *a = &aliases[i]; + ARMCPRegInfo *src_reg, *dst_reg; + + if (a->feature && !a->feature(&cpu->isar)) { + continue; + } + + src_reg = g_hash_table_lookup(cpu->cp_regs, &a->src_key); + dst_reg = g_hash_table_lookup(cpu->cp_regs, &a->dst_key); + g_assert(src_reg != NULL); + g_assert(dst_reg != NULL); + + /* Cross-compare names to detect typos in the keys. */ + g_assert(strcmp(src_reg->name, a->src_name) == 0); + g_assert(strcmp(dst_reg->name, a->dst_name) == 0); + + /* None of the core system registers use opaque; we will. */ + g_assert(src_reg->opaque == NULL); + + /* Create alias before redirection so we dup the right data. */ + if (a->new_key) { + ARMCPRegInfo *new_reg = g_memdup(src_reg, sizeof(ARMCPRegInfo)); + uint32_t *new_key = g_memdup(&a->new_key, sizeof(uint32_t)); + bool ok; + + new_reg->name = a->new_name; + new_reg->type |= ARM_CP_ALIAS; + /* Remove PL1/PL0 access, leaving PL2/PL3 R/W in place. */ + new_reg->access &= PL2_RW | PL3_RW; + + ok = g_hash_table_insert(cpu->cp_regs, new_key, new_reg); + g_assert(ok); + } + + src_reg->opaque = dst_reg; + src_reg->orig_readfn = src_reg->readfn ?: raw_read; + src_reg->orig_writefn = src_reg->writefn ?: raw_write; + if (!src_reg->raw_readfn) { + src_reg->raw_readfn = raw_read; + } + if (!src_reg->raw_writefn) { + src_reg->raw_writefn = raw_write; + } + src_reg->readfn = el2_e2h_read; + src_reg->writefn = el2_e2h_write; + } +} +#endif + static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { - /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64, - * but the AArch32 CTR has its own reginfo struct) - */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCT)) { - return CP_ACCESS_TRAP; + int cur_el = arm_current_el(env); + + if (cur_el < 2) { + uint64_t hcr = arm_hcr_el2_eff(env); + + if (cur_el == 0) { + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + if (!(env->cp15.sctlr_el[2] & SCTLR_UCT)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + if (!(env->cp15.sctlr_el[1] & SCTLR_UCT)) { + return CP_ACCESS_TRAP; + } + if (hcr & HCR_TID2) { + return CP_ACCESS_TRAP_EL2; + } + } + } else if (hcr & HCR_TID2) { + return CP_ACCESS_TRAP_EL2; + } } if (arm_current_el(env) < 2 && arm_hcr_el2_eff(env) & HCR_TID2) { @@ -5367,7 +5791,9 @@ static const ARMCPRegInfo debug_lpae_cp_reginfo[] = { int sve_exception_el(CPUARMState *env, int el) { #ifndef CONFIG_USER_ONLY - if (el <= 1) { + uint64_t hcr_el2 = arm_hcr_el2_eff(env); + + if (el <= 1 && (hcr_el2 & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { bool disabled = false; /* The CPACR.ZEN controls traps to EL1: @@ -5382,8 +5808,7 @@ int sve_exception_el(CPUARMState *env, int el) } if (disabled) { /* route_to_el2 */ - return (arm_feature(env, ARM_FEATURE_EL2) - && (arm_hcr_el2_eff(env) & HCR_TGE) ? 2 : 1); + return hcr_el2 & HCR_TGE ? 2 : 1; } /* Check CPACR.FPEN. */ @@ -6122,6 +6547,71 @@ static const ARMCPRegInfo jazelle_regs[] = { REGINFO_SENTINEL }; +static const ARMCPRegInfo vhe_reginfo[] = { + { .name = "CONTEXTIDR_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 1, + .access = PL2_RW, + .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[2]) }, + { .name = "TTBR1_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 1, + .access = PL2_RW, .writefn = vmsa_tcr_ttbr_el2_write, + .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[2]) }, +#ifndef CONFIG_USER_ONLY + { .name = "CNTHV_CVAL_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 2, + .fieldoffset = + offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].cval), + .type = ARM_CP_IO, .access = PL2_RW, + .writefn = gt_hv_cval_write, .raw_writefn = raw_write }, + { .name = "CNTHV_TVAL_EL2", .state = ARM_CP_STATE_BOTH, + .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 0, + .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW, + .resetfn = gt_hv_timer_reset, + .readfn = gt_hv_tval_read, .writefn = gt_hv_tval_write }, + { .name = "CNTHV_CTL_EL2", .state = ARM_CP_STATE_BOTH, + .type = ARM_CP_IO, + .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 1, + .access = PL2_RW, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].ctl), + .writefn = gt_hv_ctl_write, .raw_writefn = raw_write }, + { .name = "CNTP_CTL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 1, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl), + .writefn = gt_phys_ctl_write, .raw_writefn = raw_write }, + { .name = "CNTV_CTL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 1, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl), + .writefn = gt_virt_ctl_write, .raw_writefn = raw_write }, + { .name = "CNTP_TVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 0, + .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write }, + { .name = "CNTV_TVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 0, + .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write }, + { .name = "CNTP_CVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 2, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval), + .access = PL2_RW, .accessfn = e2h_access, + .writefn = gt_phys_cval_write, .raw_writefn = raw_write }, + { .name = "CNTV_CVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 2, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval), + .access = PL2_RW, .accessfn = e2h_access, + .writefn = gt_virt_cval_write, .raw_writefn = raw_write }, +#endif + REGINFO_SENTINEL +}; + void register_cp_regs_for_features(ARMCPU *cpu) { /* Register all the coprocessor registers based on feature bits */ @@ -7085,6 +7575,10 @@ void register_cp_regs_for_features(ARMCPU *cpu) define_arm_cp_regs(cpu, lor_reginfo); } + if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) { + define_arm_cp_regs(cpu, vhe_reginfo); + } + if (cpu_isar_feature(aa64_sve, cpu)) { define_one_arm_cp_reg(cpu, &zcr_el1_reginfo); if (arm_feature(env, ARM_FEATURE_EL2)) { @@ -7126,6 +7620,16 @@ void register_cp_regs_for_features(ARMCPU *cpu) : cpu_isar_feature(aa32_predinv, cpu)) { define_arm_cp_regs(cpu, predinv_reginfo); } + +#ifndef CONFIG_USER_ONLY + /* + * Register redirections and aliases must be done last, + * after the registers from the other extensions have been defined. + */ + if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) { + define_arm_vh_e2h_redirects_aliases(cpu); + } +#endif } void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu) @@ -7421,13 +7925,10 @@ void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu, mask = PL0_RW; break; case 4: + case 5: /* min_EL EL2 */ mask = PL2_RW; break; - case 5: - /* unallocated encoding, so not possible */ - assert(false); - break; case 6: /* min_EL EL3 */ mask = PL3_RW; @@ -7946,6 +8447,12 @@ uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx, break; }; + /* + * For these purposes, TGE and AMO/IMO/FMO both force the + * interrupt to EL2. Fold TGE into the bit extracted above. + */ + hcr |= (hcr_el2 & HCR_TGE) != 0; + /* Perform a table-lookup for the target EL given the current state */ target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el]; @@ -8510,14 +9017,19 @@ static void arm_cpu_do_interrupt_aarch64(CPUState *cs) * immediately lower than the target level is using AArch32 or AArch64 */ bool is_aa64; + uint64_t hcr; switch (new_el) { case 3: is_aa64 = (env->cp15.scr_el3 & SCR_RW) != 0; break; case 2: - is_aa64 = (env->cp15.hcr_el2 & HCR_RW) != 0; - break; + hcr = arm_hcr_el2_eff(env); + if ((hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + is_aa64 = (hcr & HCR_RW) != 0; + break; + } + /* fall through */ case 1: is_aa64 = is_a64(env); break; @@ -8690,19 +9202,23 @@ void arm_cpu_do_interrupt(CPUState *cs) #endif /* !CONFIG_USER_ONLY */ /* Return the exception level which controls this address translation regime */ -static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) +static uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { - case ARMMMUIdx_S2NS: - case ARMMMUIdx_S1E2: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + case ARMMMUIdx_Stage2: + case ARMMMUIdx_E2: return 2; - case ARMMMUIdx_S1E3: + case ARMMMUIdx_SE3: return 3; - case ARMMMUIdx_S1SE0: + case ARMMMUIdx_SE10_0: return arm_el_is_aa64(env, 3) ? 1 : 3; - case ARMMMUIdx_S1SE1: - case ARMMMUIdx_S1NSE0: - case ARMMMUIdx_S1NSE1: + case ARMMMUIdx_SE10_1: + case ARMMMUIdx_Stage1_E0: + case ARMMMUIdx_Stage1_E1: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: case ARMMMUIdx_MPrivNegPri: case ARMMMUIdx_MUserNegPri: case ARMMMUIdx_MPriv: @@ -8717,14 +9233,24 @@ static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) } } -#ifndef CONFIG_USER_ONLY +uint64_t arm_sctlr(CPUARMState *env, int el) +{ + /* Only EL0 needs to be adjusted for EL1&0 or EL2&0. */ + if (el == 0) { + ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, 0); + el = (mmu_idx == ARMMMUIdx_E20_0 ? 2 : 1); + } + return env->cp15.sctlr_el[el]; +} /* Return the SCTLR value which controls this address translation regime */ -static inline uint32_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) +static inline uint64_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) { return env->cp15.sctlr_el[regime_el(env, mmu_idx)]; } +#ifndef CONFIG_USER_ONLY + /* Return true if the specified stage of address translation is disabled */ static inline bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) @@ -8747,7 +9273,7 @@ static inline bool regime_translation_disabled(CPUARMState *env, } } - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* HCR.DC means HCR.VM behaves as 1 */ return (env->cp15.hcr_el2 & (HCR_DC | HCR_VM)) == 0; } @@ -8760,7 +9286,7 @@ static inline bool regime_translation_disabled(CPUARMState *env, } if ((env->cp15.hcr_el2 & HCR_DC) && - (mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1)) { + (mmu_idx == ARMMMUIdx_Stage1_E0 || mmu_idx == ARMMMUIdx_Stage1_E1)) { /* HCR.DC means SCTLR_EL1.M behaves as 0 */ return true; } @@ -8778,7 +9304,7 @@ static inline bool regime_translation_big_endian(CPUARMState *env, static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) { - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { return env->cp15.vttbr_el2; } if (ttbrn == 0) { @@ -8793,7 +9319,7 @@ static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, /* Return the TCR controlling this translation regime */ static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx) { - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { return &env->cp15.vtcr_el2; } return &env->cp15.tcr_el[regime_el(env, mmu_idx)]; @@ -8804,10 +9330,14 @@ static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx) */ static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) { - if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) { - mmu_idx += (ARMMMUIdx_S1NSE0 - ARMMMUIdx_S12NSE0); + switch (mmu_idx) { + case ARMMMUIdx_E10_0: + return ARMMMUIdx_Stage1_E0; + case ARMMMUIdx_E10_1: + return ARMMMUIdx_Stage1_E1; + default: + return mmu_idx; } - return mmu_idx; } /* Return true if the translation regime is using LPAE format page tables */ @@ -8839,8 +9369,9 @@ bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { - case ARMMMUIdx_S1SE0: - case ARMMMUIdx_S1NSE0: + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_Stage1_E0: case ARMMMUIdx_MUser: case ARMMMUIdx_MSUser: case ARMMMUIdx_MUserNegPri: @@ -8848,8 +9379,8 @@ static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) return true; default: return false; - case ARMMMUIdx_S12NSE0: - case ARMMMUIdx_S12NSE1: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: g_assert_not_reached(); } } @@ -8980,7 +9511,7 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, bool have_wxn; int wxn = 0; - assert(mmu_idx != ARMMMUIdx_S2NS); + assert(mmu_idx != ARMMMUIdx_Stage2); user_rw = simple_ap_to_rw_prot_is_user(ap, true); if (is_user) { @@ -9005,15 +9536,8 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, } if (is_aa64) { - switch (regime_el(env, mmu_idx)) { - case 1: - if (!is_user) { - xn = pxn || (user_rw & PAGE_WRITE); - } - break; - case 2: - case 3: - break; + if (regime_has_2_ranges(mmu_idx) && !is_user) { + xn = pxn || (user_rw & PAGE_WRITE); } } else if (arm_feature(env, ARM_FEATURE_V7)) { switch (regime_el(env, mmu_idx)) { @@ -9071,8 +9595,8 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, hwaddr addr, MemTxAttrs txattrs, ARMMMUFaultInfo *fi) { - if ((mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1) && - !regime_translation_disabled(env, ARMMMUIdx_S2NS)) { + if ((mmu_idx == ARMMMUIdx_Stage1_E0 || mmu_idx == ARMMMUIdx_Stage1_E1) && + !regime_translation_disabled(env, ARMMMUIdx_Stage2)) { target_ulong s2size; hwaddr s2pa; int s2prot; @@ -9089,7 +9613,7 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, pcacheattrs = &cacheattrs; } - ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_S2NS, &s2pa, + ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_Stage2, &s2pa, &txattrs, &s2prot, &s2size, fi, pcacheattrs); if (ret) { assert(fi->type != ARMFault_None); @@ -9547,7 +10071,6 @@ ARMVAParameters aa64_va_parameters_both(CPUARMState *env, uint64_t va, ARMMMUIdx mmu_idx) { uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; - uint32_t el = regime_el(env, mmu_idx); bool tbi, tbid, epd, hpd, using16k, using64k; int select, tsz; @@ -9557,11 +10080,11 @@ ARMVAParameters aa64_va_parameters_both(CPUARMState *env, uint64_t va, */ select = extract64(va, 55, 1); - if (el > 1) { + if (!regime_has_2_ranges(mmu_idx)) { tsz = extract32(tcr, 0, 6); using64k = extract32(tcr, 14, 1); using16k = extract32(tcr, 15, 1); - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* VTCR_EL2 */ tbi = tbid = hpd = false; } else { @@ -9622,7 +10145,7 @@ static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, int select, tsz; bool epd, hpd; - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* VTCR */ bool sext = extract32(tcr, 4, 1); bool sign = extract32(tcr, 3, 1); @@ -9713,10 +10236,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, param = aa64_va_parameters(env, address, mmu_idx, access_type != MMU_INST_FETCH); level = 0; - /* If we are in 64-bit EL2 or EL3 then there is no TTBR1, so mark it - * invalid. - */ - ttbr1_valid = (el < 2); + ttbr1_valid = regime_has_2_ranges(mmu_idx); addrsize = 64 - 8 * param.tbi; inputsize = 64 - param.tsz; } else { @@ -9724,7 +10244,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, level = 1; /* There is no TTBR1 for EL2 */ ttbr1_valid = (el != 2); - addrsize = (mmu_idx == ARMMMUIdx_S2NS ? 40 : 32); + addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32); inputsize = addrsize - param.tsz; } @@ -9775,7 +10295,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, goto do_fault; } - if (mmu_idx != ARMMMUIdx_S2NS) { + if (mmu_idx != ARMMMUIdx_Stage2) { /* The starting level depends on the virtual address size (which can * be up to 48 bits) and the translation granule size. It indicates * the number of strides (stride bits at a time) needed to @@ -9875,7 +10395,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, attrs = extract64(descriptor, 2, 10) | (extract64(descriptor, 52, 12) << 10); - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* Stage 2 table descriptors do not include any attribute fields */ break; } @@ -9906,7 +10426,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, ap = extract32(attrs, 4, 2); xn = extract32(attrs, 12, 1); - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { ns = true; *prot = get_S2prot(env, ap, xn); } else { @@ -9933,7 +10453,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, } if (cacheattrs != NULL) { - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { cacheattrs->attrs = convert_stage2_attrs(env, extract32(attrs, 0, 4)); } else { @@ -9954,7 +10474,7 @@ do_fault: fi->type = fault_type; fi->level = level; /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2. */ - fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_S2NS); + fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_Stage2); return true; } @@ -10753,7 +11273,7 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, target_ulong *page_size, ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) { - if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) { + if (mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_E10_1) { /* Call ourselves recursively to do the stage 1 and then stage 2 * translations. */ @@ -10768,13 +11288,13 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, prot, page_size, fi, cacheattrs); /* If S1 fails or S2 is disabled, return early. */ - if (ret || regime_translation_disabled(env, ARMMMUIdx_S2NS)) { + if (ret || regime_translation_disabled(env, ARMMMUIdx_Stage2)) { *phys_ptr = ipa; return ret; } /* S1 is done. Now do S2 translation. */ - ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_S2NS, + ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_Stage2, phys_ptr, attrs, &s2_prot, page_size, fi, cacheattrs != NULL ? &cacheattrs2 : NULL); @@ -10816,7 +11336,7 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, /* Fast Context Switch Extension. This doesn't exist at all in v8. * In v7 and earlier it affects all stage 1 translations. */ - if (address < 0x02000000 && mmu_idx != ARMMMUIdx_S2NS + if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2 && !arm_feature(env, ARM_FEATURE_V8)) { if (regime_el(env, mmu_idx) == 3) { address += env->cp15.fcseidr_s; @@ -11177,8 +11697,6 @@ uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes) int fp_exception_el(CPUARMState *env, int cur_el) { #ifndef CONFIG_USER_ONLY - int fpen; - /* CPACR and the CPTR registers don't exist before v6, so FP is * always accessible */ @@ -11206,30 +11724,34 @@ int fp_exception_el(CPUARMState *env, int cur_el) * 0, 2 : trap EL0 and EL1/PL1 accesses * 1 : trap only EL0 accesses * 3 : trap no accesses + * This register is ignored if E2H+TGE are both set. */ - fpen = extract32(env->cp15.cpacr_el1, 20, 2); - switch (fpen) { - case 0: - case 2: - if (cur_el == 0 || cur_el == 1) { - /* Trap to PL1, which might be EL1 or EL3 */ - if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) { + if ((arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + int fpen = extract32(env->cp15.cpacr_el1, 20, 2); + + switch (fpen) { + case 0: + case 2: + if (cur_el == 0 || cur_el == 1) { + /* Trap to PL1, which might be EL1 or EL3 */ + if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) { + return 3; + } + return 1; + } + if (cur_el == 3 && !is_a64(env)) { + /* Secure PL1 running at EL3 */ return 3; } - return 1; - } - if (cur_el == 3 && !is_a64(env)) { - /* Secure PL1 running at EL3 */ - return 3; - } - break; - case 1: - if (cur_el == 0) { - return 1; + break; + case 1: + if (cur_el == 0) { + return 1; + } + break; + case 3: + break; } - break; - case 3: - break; } /* @@ -11265,6 +11787,31 @@ int fp_exception_el(CPUARMState *env, int cur_el) return 0; } +/* Return the exception level we're running at if this is our mmu_idx */ +int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx) +{ + if (mmu_idx & ARM_MMU_IDX_M) { + return mmu_idx & ARM_MMU_IDX_M_PRIV; + } + + switch (mmu_idx) { + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_SE10_0: + return 0; + case ARMMMUIdx_E10_1: + case ARMMMUIdx_SE10_1: + return 1; + case ARMMMUIdx_E2: + case ARMMMUIdx_E20_2: + return 2; + case ARMMMUIdx_SE3: + return 3; + default: + g_assert_not_reached(); + } +} + #ifndef CONFIG_TCG ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate) { @@ -11278,10 +11825,33 @@ ARMMMUIdx arm_mmu_idx_el(CPUARMState *env, int el) return arm_v7m_mmu_idx_for_secstate(env, env->v7m.secure); } - if (el < 2 && arm_is_secure_below_el3(env)) { - return ARMMMUIdx_S1SE0 + el; - } else { - return ARMMMUIdx_S12NSE0 + el; + /* See ARM pseudo-function ELIsInHost. */ + switch (el) { + case 0: + if (arm_is_secure_below_el3(env)) { + return ARMMMUIdx_SE10_0; + } + if ((env->cp15.hcr_el2 & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE) + && arm_el_is_aa64(env, 2)) { + return ARMMMUIdx_E20_0; + } + return ARMMMUIdx_E10_0; + case 1: + if (arm_is_secure_below_el3(env)) { + return ARMMMUIdx_SE10_1; + } + return ARMMMUIdx_E10_1; + case 2: + /* TODO: ARMv8.4-SecEL2 */ + /* Note that TGE does not apply at EL2. */ + if ((env->cp15.hcr_el2 & HCR_E2H) && arm_el_is_aa64(env, 2)) { + return ARMMMUIdx_E20_2; + } + return ARMMMUIdx_E2; + case 3: + return ARMMMUIdx_SE3; + default: + g_assert_not_reached(); } } @@ -11336,11 +11906,8 @@ static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, { uint32_t flags = 0; - /* v8M always enables the fpu. */ - flags = FIELD_DP32(flags, TBFLAG_A32, VFPEN, 1); - if (arm_v7m_is_handler_mode(env)) { - flags = FIELD_DP32(flags, TBFLAG_A32, HANDLER, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, HANDLER, 1); } /* @@ -11351,7 +11918,7 @@ static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, if (arm_feature(env, ARM_FEATURE_V8) && !((mmu_idx & ARM_MMU_IDX_M_NEGPRI) && (env->v7m.ccr[env->v7m.secure] & R_V7M_CCR_STKOFHFNMIGN_MASK))) { - flags = FIELD_DP32(flags, TBFLAG_A32, STACKCHECK, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, STACKCHECK, 1); } return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags); @@ -11394,8 +11961,8 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, flags = FIELD_DP32(flags, TBFLAG_ANY, AARCH64_STATE, 1); - /* FIXME: ARMv8.1-VHE S2 translation regime. */ - if (regime_el(env, stage1) < 2) { + /* Get control bits for tagged addresses. */ + if (regime_has_2_ranges(mmu_idx)) { ARMVAParameters p1 = aa64_va_parameters_both(env, -1, stage1); tbid = (p1.tbi << 1) | p0.tbi; tbii = tbid & ~((p1.tbid << 1) | p0.tbid); @@ -11424,7 +11991,7 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, flags = FIELD_DP32(flags, TBFLAG_A64, ZCR_LEN, zcr_len); } - sctlr = arm_sctlr(env, el); + sctlr = regime_sctlr(env, stage1); if (arm_cpu_data_is_big_endian_a64(el, sctlr)) { flags = FIELD_DP32(flags, TBFLAG_ANY, BE_DATA, 1); @@ -11449,6 +12016,28 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, } } + /* Compute the condition for using AccType_UNPRIV for LDTR et al. */ + /* TODO: ARMv8.2-UAO */ + switch (mmu_idx) { + case ARMMMUIdx_E10_1: + case ARMMMUIdx_SE10_1: + /* TODO: ARMv8.3-NV */ + flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + break; + case ARMMMUIdx_E20_2: + /* TODO: ARMv8.4-SecEL2 */ + /* + * Note that E20_2 is gated by HCR_EL2.E2H == 1, but E20_0 is + * gated by HCR_EL2.<E2H,TGE> == '11', and so is LDTR. + */ + if (env->cp15.hcr_el2 & HCR_TGE) { + flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + } + break; + default: + break; + } + return rebuild_hflags_common(env, fp_el, mmu_idx, flags); } @@ -11544,7 +12133,7 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, if (arm_feature(env, ARM_FEATURE_M_SECURITY) && FIELD_EX32(env->v7m.fpccr[M_REG_S], V7M_FPCCR, S) != env->v7m.secure) { - flags = FIELD_DP32(flags, TBFLAG_A32, FPCCR_S_WRONG, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, FPCCR_S_WRONG, 1); } if ((env->v7m.fpccr[env->v7m.secure] & R_V7M_FPCCR_ASPEN_MASK) && @@ -11556,12 +12145,12 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, * active FP context; we must create a new FP context before * executing any FP insn. */ - flags = FIELD_DP32(flags, TBFLAG_A32, NEW_FP_CTXT_NEEDED, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED, 1); } bool is_secure = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK; if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) { - flags = FIELD_DP32(flags, TBFLAG_A32, LSPACT, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, LSPACT, 1); } } else { /* @@ -11582,8 +12171,8 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, } } - flags = FIELD_DP32(flags, TBFLAG_A32, THUMB, env->thumb); - flags = FIELD_DP32(flags, TBFLAG_A32, CONDEXEC, env->condexec_bits); + flags = FIELD_DP32(flags, TBFLAG_AM32, THUMB, env->thumb); + flags = FIELD_DP32(flags, TBFLAG_AM32, CONDEXEC, env->condexec_bits); pstate_for_ss = env->uncached_cpsr; } diff --git a/target/arm/internals.h b/target/arm/internals.h index f5313dd3d4..6d4a942bde 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -769,6 +769,39 @@ bool arm_cpu_tlb_fill(CPUState *cs, vaddr address, int size, MMUAccessType access_type, int mmu_idx, bool probe, uintptr_t retaddr); +static inline int arm_to_core_mmu_idx(ARMMMUIdx mmu_idx) +{ + return mmu_idx & ARM_MMU_IDX_COREIDX_MASK; +} + +static inline ARMMMUIdx core_to_arm_mmu_idx(CPUARMState *env, int mmu_idx) +{ + if (arm_feature(env, ARM_FEATURE_M)) { + return mmu_idx | ARM_MMU_IDX_M; + } else { + return mmu_idx | ARM_MMU_IDX_A; + } +} + +int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx); + +/* + * Return the MMU index for a v7M CPU with all relevant information + * manually specified. + */ +ARMMMUIdx arm_v7m_mmu_idx_all(CPUARMState *env, + bool secstate, bool priv, bool negpri); + +/* + * Return the MMU index for a v7M CPU in the specified security and + * privilege state. + */ +ARMMMUIdx arm_v7m_mmu_idx_for_secstate_and_priv(CPUARMState *env, + bool secstate, bool priv); + +/* Return the MMU index for a v7M CPU in the specified security state */ +ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate); + /* Return true if the stage 1 translation regime is using LPAE format page * tables */ bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx); @@ -804,24 +837,44 @@ static inline void arm_call_el_change_hook(ARMCPU *cpu) } } +/* Return true if this address translation regime has two ranges. */ +static inline bool regime_has_2_ranges(ARMMMUIdx mmu_idx) +{ + switch (mmu_idx) { + case ARMMMUIdx_Stage1_E0: + case ARMMMUIdx_Stage1_E1: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_SE10_1: + return true; + default: + return false; + } +} + /* Return true if this address translation regime is secure */ static inline bool regime_is_secure(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { - case ARMMMUIdx_S12NSE0: - case ARMMMUIdx_S12NSE1: - case ARMMMUIdx_S1NSE0: - case ARMMMUIdx_S1NSE1: - case ARMMMUIdx_S1E2: - case ARMMMUIdx_S2NS: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + case ARMMMUIdx_Stage1_E0: + case ARMMMUIdx_Stage1_E1: + case ARMMMUIdx_E2: + case ARMMMUIdx_Stage2: case ARMMMUIdx_MPrivNegPri: case ARMMMUIdx_MUserNegPri: case ARMMMUIdx_MPriv: case ARMMMUIdx_MUser: return false; - case ARMMMUIdx_S1E3: - case ARMMMUIdx_S1SE0: - case ARMMMUIdx_S1SE1: + case ARMMMUIdx_SE3: + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_SE10_1: case ARMMMUIdx_MSPrivNegPri: case ARMMMUIdx_MSUserNegPri: case ARMMMUIdx_MSPriv: @@ -975,7 +1028,7 @@ ARMMMUIdx arm_mmu_idx(CPUARMState *env); #ifdef CONFIG_USER_ONLY static inline ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) { - return ARMMMUIdx_S1NSE0; + return ARMMMUIdx_Stage1_E0; } #else ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env); diff --git a/target/arm/monitor.c b/target/arm/monitor.c index 9725dfff16..c2dc7908de 100644 --- a/target/arm/monitor.c +++ b/target/arm/monitor.c @@ -137,17 +137,20 @@ CpuModelExpansionInfo *qmp_query_cpu_model_expansion(CpuModelExpansionType type, } if (kvm_enabled()) { - const char *cpu_type = current_machine->cpu_type; - int len = strlen(cpu_type) - strlen(ARM_CPU_TYPE_SUFFIX); bool supported = false; if (!strcmp(model->name, "host") || !strcmp(model->name, "max")) { /* These are kvmarm's recommended cpu types */ supported = true; - } else if (strlen(model->name) == len && - !strncmp(model->name, cpu_type, len)) { - /* KVM is enabled and we're using this type, so it works. */ - supported = true; + } else if (current_machine->cpu_type) { + const char *cpu_type = current_machine->cpu_type; + int len = strlen(cpu_type) - strlen(ARM_CPU_TYPE_SUFFIX); + + if (strlen(model->name) == len && + !strncmp(model->name, cpu_type, len)) { + /* KVM is enabled and we're using this type, so it works. */ + supported = true; + } } if (!supported) { error_setg(errp, "We cannot guarantee the CPU type '%s' works " diff --git a/target/arm/pauth_helper.c b/target/arm/pauth_helper.c index 0a5f41e10c..9746e32bf8 100644 --- a/target/arm/pauth_helper.c +++ b/target/arm/pauth_helper.c @@ -371,7 +371,10 @@ static void pauth_check_trap(CPUARMState *env, int el, uintptr_t ra) if (el < 2 && arm_feature(env, ARM_FEATURE_EL2)) { uint64_t hcr = arm_hcr_el2_eff(env); bool trap = !(hcr & HCR_API); - /* FIXME: ARMv8.1-VHE: trap only applies to EL1&0 regime. */ + if (el == 0) { + /* Trap only applies to EL1&0 regime. */ + trap &= (hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE); + } /* FIXME: ARMv8.3-NV: HCR_NV trap takes precedence for ERETA[AB]. */ if (trap) { pauth_trap(env, 2, ra); @@ -386,14 +389,7 @@ static void pauth_check_trap(CPUARMState *env, int el, uintptr_t ra) static bool pauth_key_enabled(CPUARMState *env, int el, uint32_t bit) { - uint32_t sctlr; - if (el == 0) { - /* FIXME: ARMv8.1-VHE S2 translation regime. */ - sctlr = env->cp15.sctlr_el[1]; - } else { - sctlr = env->cp15.sctlr_el[el]; - } - return (sctlr & bit) != 0; + return (arm_sctlr(env, el) & bit) != 0; } uint64_t HELPER(pacia)(CPUARMState *env, uint64_t x, uint64_t y) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index 766a03335b..6e82486884 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -105,25 +105,36 @@ void a64_translate_init(void) offsetof(CPUARMState, exclusive_high), "exclusive_high"); } -static inline int get_a64_user_mem_index(DisasContext *s) +/* + * Return the core mmu_idx to use for A64 "unprivileged load/store" insns + */ +static int get_a64_user_mem_index(DisasContext *s) { - /* Return the core mmu_idx to use for A64 "unprivileged load/store" insns: - * if EL1, access as if EL0; otherwise access at current EL + /* + * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL, + * which is the usual mmu_idx for this cpu state. */ - ARMMMUIdx useridx; + ARMMMUIdx useridx = s->mmu_idx; - switch (s->mmu_idx) { - case ARMMMUIdx_S12NSE1: - useridx = ARMMMUIdx_S12NSE0; - break; - case ARMMMUIdx_S1SE1: - useridx = ARMMMUIdx_S1SE0; - break; - case ARMMMUIdx_S2NS: - g_assert_not_reached(); - default: - useridx = s->mmu_idx; - break; + if (s->unpriv) { + /* + * We have pre-computed the condition for AccType_UNPRIV. + * Therefore we should never get here with a mmu_idx for + * which we do not know the corresponding user mmu_idx. + */ + switch (useridx) { + case ARMMMUIdx_E10_1: + useridx = ARMMMUIdx_E10_0; + break; + case ARMMMUIdx_E20_2: + useridx = ARMMMUIdx_E20_0; + break; + case ARMMMUIdx_SE10_1: + useridx = ARMMMUIdx_SE10_0; + break; + default: + g_assert_not_reached(); + } } return arm_to_core_mmu_idx(useridx); } @@ -175,8 +186,7 @@ static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst, if (tbi == 0) { /* Load unmodified address */ tcg_gen_mov_i64(dst, src); - } else if (s->current_el >= 2) { - /* FIXME: ARMv8.1-VHE S2 translation regime. */ + } else if (!regime_has_2_ranges(s->mmu_idx)) { /* Force tag byte to all zero */ tcg_gen_extract_i64(dst, src, 0, 56); } else { @@ -14172,6 +14182,7 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, dc->pauth_active = FIELD_EX32(tb_flags, TBFLAG_A64, PAUTH_ACTIVE); dc->bt = FIELD_EX32(tb_flags, TBFLAG_A64, BT); dc->btype = FIELD_EX32(tb_flags, TBFLAG_A64, BTYPE); + dc->unpriv = FIELD_EX32(tb_flags, TBFLAG_A64, UNPRIV); dc->vec_len = 0; dc->vec_stride = 0; dc->cp_regs = arm_cpu->cp_regs; diff --git a/target/arm/translate.c b/target/arm/translate.c index 2f4aea927f..e11a5871d0 100644 --- a/target/arm/translate.c +++ b/target/arm/translate.c @@ -152,14 +152,14 @@ static inline int get_a32_user_mem_index(DisasContext *s) * otherwise, access as if at PL0. */ switch (s->mmu_idx) { - case ARMMMUIdx_S1E2: /* this one is UNPREDICTABLE */ - case ARMMMUIdx_S12NSE0: - case ARMMMUIdx_S12NSE1: - return arm_to_core_mmu_idx(ARMMMUIdx_S12NSE0); - case ARMMMUIdx_S1E3: - case ARMMMUIdx_S1SE0: - case ARMMMUIdx_S1SE1: - return arm_to_core_mmu_idx(ARMMMUIdx_S1SE0); + case ARMMMUIdx_E2: /* this one is UNPREDICTABLE */ + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: + return arm_to_core_mmu_idx(ARMMMUIdx_E10_0); + case ARMMMUIdx_SE3: + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_SE10_1: + return arm_to_core_mmu_idx(ARMMMUIdx_SE10_0); case ARMMMUIdx_MUser: case ARMMMUIdx_MPriv: return arm_to_core_mmu_idx(ARMMMUIdx_MUser); @@ -172,7 +172,6 @@ static inline int get_a32_user_mem_index(DisasContext *s) case ARMMMUIdx_MSUserNegPri: case ARMMMUIdx_MSPrivNegPri: return arm_to_core_mmu_idx(ARMMMUIdx_MSUserNegPri); - case ARMMMUIdx_S2NS: default: g_assert_not_reached(); } @@ -10848,38 +10847,48 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) */ dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3); - dc->thumb = FIELD_EX32(tb_flags, TBFLAG_A32, THUMB); - dc->sctlr_b = FIELD_EX32(tb_flags, TBFLAG_A32, SCTLR_B); - dc->hstr_active = FIELD_EX32(tb_flags, TBFLAG_A32, HSTR_ACTIVE); + dc->thumb = FIELD_EX32(tb_flags, TBFLAG_AM32, THUMB); dc->be_data = FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; - condexec = FIELD_EX32(tb_flags, TBFLAG_A32, CONDEXEC); + condexec = FIELD_EX32(tb_flags, TBFLAG_AM32, CONDEXEC); dc->condexec_mask = (condexec & 0xf) << 1; dc->condexec_cond = condexec >> 4; + core_mmu_idx = FIELD_EX32(tb_flags, TBFLAG_ANY, MMUIDX); dc->mmu_idx = core_to_arm_mmu_idx(env, core_mmu_idx); dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); #if !defined(CONFIG_USER_ONLY) dc->user = (dc->current_el == 0); #endif - dc->ns = FIELD_EX32(tb_flags, TBFLAG_A32, NS); dc->fp_excp_el = FIELD_EX32(tb_flags, TBFLAG_ANY, FPEXC_EL); - dc->vfp_enabled = FIELD_EX32(tb_flags, TBFLAG_A32, VFPEN); - dc->vec_len = FIELD_EX32(tb_flags, TBFLAG_A32, VECLEN); - if (arm_feature(env, ARM_FEATURE_XSCALE)) { - dc->c15_cpar = FIELD_EX32(tb_flags, TBFLAG_A32, XSCALE_CPAR); - dc->vec_stride = 0; + + if (arm_feature(env, ARM_FEATURE_M)) { + dc->vfp_enabled = 1; + dc->be_data = MO_TE; + dc->v7m_handler_mode = FIELD_EX32(tb_flags, TBFLAG_M32, HANDLER); + dc->v8m_secure = arm_feature(env, ARM_FEATURE_M_SECURITY) && + regime_is_secure(env, dc->mmu_idx); + dc->v8m_stackcheck = FIELD_EX32(tb_flags, TBFLAG_M32, STACKCHECK); + dc->v8m_fpccr_s_wrong = + FIELD_EX32(tb_flags, TBFLAG_M32, FPCCR_S_WRONG); + dc->v7m_new_fp_ctxt_needed = + FIELD_EX32(tb_flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED); + dc->v7m_lspact = FIELD_EX32(tb_flags, TBFLAG_M32, LSPACT); } else { - dc->vec_stride = FIELD_EX32(tb_flags, TBFLAG_A32, VECSTRIDE); - dc->c15_cpar = 0; - } - dc->v7m_handler_mode = FIELD_EX32(tb_flags, TBFLAG_A32, HANDLER); - dc->v8m_secure = arm_feature(env, ARM_FEATURE_M_SECURITY) && - regime_is_secure(env, dc->mmu_idx); - dc->v8m_stackcheck = FIELD_EX32(tb_flags, TBFLAG_A32, STACKCHECK); - dc->v8m_fpccr_s_wrong = FIELD_EX32(tb_flags, TBFLAG_A32, FPCCR_S_WRONG); - dc->v7m_new_fp_ctxt_needed = - FIELD_EX32(tb_flags, TBFLAG_A32, NEW_FP_CTXT_NEEDED); - dc->v7m_lspact = FIELD_EX32(tb_flags, TBFLAG_A32, LSPACT); + dc->be_data = + FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; + dc->debug_target_el = + FIELD_EX32(tb_flags, TBFLAG_ANY, DEBUG_TARGET_EL); + dc->sctlr_b = FIELD_EX32(tb_flags, TBFLAG_A32, SCTLR_B); + dc->hstr_active = FIELD_EX32(tb_flags, TBFLAG_A32, HSTR_ACTIVE); + dc->ns = FIELD_EX32(tb_flags, TBFLAG_A32, NS); + dc->vfp_enabled = FIELD_EX32(tb_flags, TBFLAG_A32, VFPEN); + if (arm_feature(env, ARM_FEATURE_XSCALE)) { + dc->c15_cpar = FIELD_EX32(tb_flags, TBFLAG_A32, XSCALE_CPAR); + } else { + dc->vec_len = FIELD_EX32(tb_flags, TBFLAG_A32, VECLEN); + dc->vec_stride = FIELD_EX32(tb_flags, TBFLAG_A32, VECSTRIDE); + } + } dc->cp_regs = cpu->cp_regs; dc->features = env->features; @@ -10901,9 +10910,6 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) dc->ss_active = FIELD_EX32(tb_flags, TBFLAG_ANY, SS_ACTIVE); dc->pstate_ss = FIELD_EX32(tb_flags, TBFLAG_ANY, PSTATE_SS); dc->is_ldex = false; - if (!arm_feature(env, ARM_FEATURE_M)) { - dc->debug_target_el = FIELD_EX32(tb_flags, TBFLAG_ANY, DEBUG_TARGET_EL); - } dc->page_start = dc->base.pc_first & TARGET_PAGE_MASK; @@ -11340,10 +11346,10 @@ static const TranslatorOps thumb_translator_ops = { /* generate intermediate code for basic block 'tb'. */ void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) { - DisasContext dc; + DisasContext dc = { }; const TranslatorOps *ops = &arm_translator_ops; - if (FIELD_EX32(tb->flags, TBFLAG_A32, THUMB)) { + if (FIELD_EX32(tb->flags, TBFLAG_AM32, THUMB)) { ops = &thumb_translator_ops; } #ifdef TARGET_AARCH64 diff --git a/target/arm/translate.h b/target/arm/translate.h index b837b7fcbf..5b167c416a 100644 --- a/target/arm/translate.h +++ b/target/arm/translate.h @@ -73,6 +73,8 @@ typedef struct DisasContext { * ie A64 LDX*, LDAX*, A32/T32 LDREX*, LDAEX*. */ bool is_ldex; + /* True if AccType_UNPRIV should be used for LDTR et al */ + bool unpriv; /* True if v8.3-PAuth is active. */ bool pauth_active; /* True with v8.5-BTI and SCTLR_ELx.BT* set. */ @@ -126,7 +128,7 @@ static inline int default_exception_el(DisasContext *s) * exceptions can only be routed to ELs above 1, so we target the higher of * 1 or the current EL. */ - return (s->mmu_idx == ARMMMUIdx_S1SE0 && s->secure_routed_to_el3) + return (s->mmu_idx == ARMMMUIdx_SE10_0 && s->secure_routed_to_el3) ? 3 : MAX(1, s->current_el); } |