diff options
Diffstat (limited to 'target/arm/helper.c')
| -rw-r--r-- | target/arm/helper.c | 1207 |
1 files changed, 898 insertions, 309 deletions
diff --git a/target/arm/helper.c b/target/arm/helper.c index 19a57a17da..7d15d5c933 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -614,56 +614,54 @@ static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate all (TLBIALL) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); if (tlb_force_broadcast(env)) { - tlbiall_is_write(env, NULL, value); - return; + tlb_flush_all_cpus_synced(cs); + } else { + tlb_flush(cs); } - - tlb_flush(CPU(cpu)); } static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); + value &= TARGET_PAGE_MASK; if (tlb_force_broadcast(env)) { - tlbimva_is_write(env, NULL, value); - return; + tlb_flush_page_all_cpus_synced(cs, value); + } else { + tlb_flush_page(cs, value); } - - tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK); } static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate by ASID (TLBIASID) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); if (tlb_force_broadcast(env)) { - tlbiasid_is_write(env, NULL, value); - return; + tlb_flush_all_cpus_synced(cs); + } else { + tlb_flush(cs); } - - tlb_flush(CPU(cpu)); } static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); + value &= TARGET_PAGE_MASK; if (tlb_force_broadcast(env)) { - tlbimvaa_is_write(env, NULL, value); - return; + tlb_flush_page_all_cpus_synced(cs, value); + } else { + tlb_flush_page(cs, value); } - - tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK); } static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -672,9 +670,9 @@ static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = env_cpu(env); tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_E10_1 | + ARMMMUIdxBit_E10_0 | + ARMMMUIdxBit_Stage2); } static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -683,9 +681,9 @@ static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = env_cpu(env); tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_E10_1 | + ARMMMUIdxBit_E10_0 | + ARMMMUIdxBit_Stage2); } static void tlbiipas2_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -706,7 +704,7 @@ static void tlbiipas2_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 40); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S2NS); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_Stage2); } static void tlbiipas2_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -722,7 +720,7 @@ static void tlbiipas2_is_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 40); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_Stage2); } static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -730,7 +728,7 @@ static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, { CPUState *cs = env_cpu(env); - tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_E2); } static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -738,7 +736,7 @@ static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, { CPUState *cs = env_cpu(env); - tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_E2); } static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -747,7 +745,7 @@ static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = env_cpu(env); uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S1E2); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_E2); } static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -757,7 +755,7 @@ static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1E2); + ARMMMUIdxBit_E2); } static const ARMCPRegInfo cp_reginfo[] = { @@ -2326,10 +2324,18 @@ static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri, * Writable only at the highest implemented exception level. */ int el = arm_current_el(env); + uint64_t hcr; + uint32_t cntkctl; switch (el) { case 0: - if (!extract32(env->cp15.c14_cntkctl, 0, 2)) { + hcr = arm_hcr_el2_eff(env); + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + cntkctl = env->cp15.cnthctl_el2; + } else { + cntkctl = env->cp15.c14_cntkctl; + } + if (!extract32(cntkctl, 0, 2)) { return CP_ACCESS_TRAP; } break; @@ -2357,17 +2363,47 @@ static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx, { unsigned int cur_el = arm_current_el(env); bool secure = arm_is_secure(env); + uint64_t hcr = arm_hcr_el2_eff(env); - /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */ - if (cur_el == 0 && - !extract32(env->cp15.c14_cntkctl, timeridx, 1)) { - return CP_ACCESS_TRAP; - } + switch (cur_el) { + case 0: + /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]CTEN. */ + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + return (extract32(env->cp15.cnthctl_el2, timeridx, 1) + ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2); + } - if (arm_feature(env, ARM_FEATURE_EL2) && - timeridx == GTIMER_PHYS && !secure && cur_el < 2 && - !extract32(env->cp15.cnthctl_el2, 0, 1)) { - return CP_ACCESS_TRAP_EL2; + /* CNT[PV]CT: not visible from PL0 if EL0[PV]CTEN is zero */ + if (!extract32(env->cp15.c14_cntkctl, timeridx, 1)) { + return CP_ACCESS_TRAP; + } + + /* If HCR_EL2.<E2H,TGE> == '10': check CNTHCTL_EL2.EL1PCTEN. */ + if (hcr & HCR_E2H) { + if (timeridx == GTIMER_PHYS && + !extract32(env->cp15.cnthctl_el2, 10, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + /* If HCR_EL2.<E2H> == 0: check CNTHCTL_EL2.EL1PCEN. */ + if (arm_feature(env, ARM_FEATURE_EL2) && + timeridx == GTIMER_PHYS && !secure && + !extract32(env->cp15.cnthctl_el2, 1, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } + break; + + case 1: + /* Check CNTHCTL_EL2.EL1PCTEN, which changes location based on E2H. */ + if (arm_feature(env, ARM_FEATURE_EL2) && + timeridx == GTIMER_PHYS && !secure && + (hcr & HCR_E2H + ? !extract32(env->cp15.cnthctl_el2, 10, 1) + : !extract32(env->cp15.cnthctl_el2, 0, 1))) { + return CP_ACCESS_TRAP_EL2; + } + break; } return CP_ACCESS_OK; } @@ -2377,19 +2413,41 @@ static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx, { unsigned int cur_el = arm_current_el(env); bool secure = arm_is_secure(env); + uint64_t hcr = arm_hcr_el2_eff(env); - /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if - * EL0[PV]TEN is zero. - */ - if (cur_el == 0 && - !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) { - return CP_ACCESS_TRAP; - } + switch (cur_el) { + case 0: + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]TEN. */ + return (extract32(env->cp15.cnthctl_el2, 9 - timeridx, 1) + ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2); + } - if (arm_feature(env, ARM_FEATURE_EL2) && - timeridx == GTIMER_PHYS && !secure && cur_el < 2 && - !extract32(env->cp15.cnthctl_el2, 1, 1)) { - return CP_ACCESS_TRAP_EL2; + /* + * CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from + * EL0 if EL0[PV]TEN is zero. + */ + if (!extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) { + return CP_ACCESS_TRAP; + } + /* fall through */ + + case 1: + if (arm_feature(env, ARM_FEATURE_EL2) && + timeridx == GTIMER_PHYS && !secure) { + if (hcr & HCR_E2H) { + /* If HCR_EL2.<E2H,TGE> == '10': check CNTHCTL_EL2.EL1PTEN. */ + if (!extract32(env->cp15.cnthctl_el2, 11, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + /* If HCR_EL2.<E2H> == 0: check CNTHCTL_EL2.EL1PCEN. */ + if (!extract32(env->cp15.cnthctl_el2, 1, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } + } + break; } return CP_ACCESS_OK; } @@ -2515,9 +2573,31 @@ static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri) return gt_get_countervalue(env); } +static uint64_t gt_virt_cnt_offset(CPUARMState *env) +{ + uint64_t hcr; + + switch (arm_current_el(env)) { + case 2: + hcr = arm_hcr_el2_eff(env); + if (hcr & HCR_E2H) { + return 0; + } + break; + case 0: + hcr = arm_hcr_el2_eff(env); + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + return 0; + } + break; + } + + return env->cp15.cntvoff_el2; +} + static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri) { - return gt_get_countervalue(env) - env->cp15.cntvoff_el2; + return gt_get_countervalue(env) - gt_virt_cnt_offset(env); } static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -2532,7 +2612,14 @@ static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri, int timeridx) { - uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0; + uint64_t offset = 0; + + switch (timeridx) { + case GTIMER_VIRT: + case GTIMER_HYPVIRT: + offset = gt_virt_cnt_offset(env); + break; + } return (uint32_t)(env->cp15.c14_timer[timeridx].cval - (gt_get_countervalue(env) - offset)); @@ -2542,7 +2629,14 @@ static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, int timeridx, uint64_t value) { - uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0; + uint64_t offset = 0; + + switch (timeridx) { + case GTIMER_VIRT: + case GTIMER_HYPVIRT: + offset = gt_virt_cnt_offset(env); + break; + } trace_arm_gt_tval_write(timeridx, value); env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset + @@ -2601,6 +2695,70 @@ static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, gt_ctl_write(env, ri, GTIMER_PHYS, value); } +static int gt_phys_redir_timeridx(CPUARMState *env) +{ + switch (arm_mmu_idx(env)) { + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + return GTIMER_HYP; + default: + return GTIMER_PHYS; + } +} + +static int gt_virt_redir_timeridx(CPUARMState *env) +{ + switch (arm_mmu_idx(env)) { + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + return GTIMER_HYPVIRT; + default: + return GTIMER_VIRT; + } +} + +static uint64_t gt_phys_redir_cval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_phys_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].cval; +} + +static void gt_phys_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_phys_redir_timeridx(env); + gt_cval_write(env, ri, timeridx, value); +} + +static uint64_t gt_phys_redir_tval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_phys_redir_timeridx(env); + return gt_tval_read(env, ri, timeridx); +} + +static void gt_phys_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_phys_redir_timeridx(env); + gt_tval_write(env, ri, timeridx, value); +} + +static uint64_t gt_phys_redir_ctl_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_phys_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].ctl; +} + +static void gt_phys_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_phys_redir_timeridx(env); + gt_ctl_write(env, ri, timeridx, value); +} + static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri) { gt_timer_reset(env, ri, GTIMER_VIRT); @@ -2639,6 +2797,48 @@ static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri, gt_recalc_timer(cpu, GTIMER_VIRT); } +static uint64_t gt_virt_redir_cval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_virt_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].cval; +} + +static void gt_virt_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_virt_redir_timeridx(env); + gt_cval_write(env, ri, timeridx, value); +} + +static uint64_t gt_virt_redir_tval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_virt_redir_timeridx(env); + return gt_tval_read(env, ri, timeridx); +} + +static void gt_virt_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_virt_redir_timeridx(env); + gt_tval_write(env, ri, timeridx, value); +} + +static uint64_t gt_virt_redir_ctl_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_virt_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].ctl; +} + +static void gt_virt_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_virt_redir_timeridx(env); + gt_ctl_write(env, ri, timeridx, value); +} + static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri) { gt_timer_reset(env, ri, GTIMER_HYP); @@ -2695,6 +2895,34 @@ static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, gt_ctl_write(env, ri, GTIMER_SEC, value); } +static void gt_hv_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri) +{ + gt_timer_reset(env, ri, GTIMER_HYPVIRT); +} + +static void gt_hv_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + gt_cval_write(env, ri, GTIMER_HYPVIRT, value); +} + +static uint64_t gt_hv_tval_read(CPUARMState *env, const ARMCPRegInfo *ri) +{ + return gt_tval_read(env, ri, GTIMER_HYPVIRT); +} + +static void gt_hv_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + gt_tval_write(env, ri, GTIMER_HYPVIRT, value); +} + +static void gt_hv_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + gt_ctl_write(env, ri, GTIMER_HYPVIRT, value); +} + void arm_gt_ptimer_cb(void *opaque) { ARMCPU *cpu = opaque; @@ -2723,6 +2951,13 @@ void arm_gt_stimer_cb(void *opaque) gt_recalc_timer(cpu, GTIMER_SEC); } +void arm_gt_hvtimer_cb(void *opaque) +{ + ARMCPU *cpu = opaque; + + gt_recalc_timer(cpu, GTIMER_HYPVIRT); +} + static void arm_gt_cntfrq_reset(CPUARMState *env, const ARMCPRegInfo *opaque) { ARMCPU *cpu = env_archcpu(env); @@ -2760,7 +2995,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .accessfn = gt_ptimer_access, .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl), - .writefn = gt_phys_ctl_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write, }, { .name = "CNTP_CTL_S", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1, @@ -2777,14 +3013,16 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .accessfn = gt_ptimer_access, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl), .resetvalue = 0, - .writefn = gt_phys_ctl_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write, }, { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1, .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL0_RW, .accessfn = gt_vtimer_access, .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl), - .writefn = gt_virt_ctl_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write, }, { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1, @@ -2792,14 +3030,15 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .accessfn = gt_vtimer_access, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl), .resetvalue = 0, - .writefn = gt_virt_ctl_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write, }, /* TimerValue views: a 32 bit downcounting view of the underlying state */ { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0, .secure = ARM_CP_SECSTATE_NS, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_ptimer_access, - .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write, + .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write, }, { .name = "CNTP_TVAL_S", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0, @@ -2812,18 +3051,18 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset, - .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write, + .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write, }, { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_vtimer_access, - .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write, + .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write, }, { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset, - .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write, + .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write, }, /* The counter itself */ { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0, @@ -2853,7 +3092,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval), .accessfn = gt_ptimer_access, - .writefn = gt_phys_cval_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write, }, { .name = "CNTP_CVAL_S", .cp = 15, .crm = 14, .opc1 = 2, .secure = ARM_CP_SECSTATE_S, @@ -2869,14 +3109,16 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .type = ARM_CP_IO, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval), .resetvalue = 0, .accessfn = gt_ptimer_access, - .writefn = gt_phys_cval_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write, }, { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3, .access = PL0_RW, .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval), .accessfn = gt_vtimer_access, - .writefn = gt_virt_cval_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write, }, { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2, @@ -2884,7 +3126,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .type = ARM_CP_IO, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval), .resetvalue = 0, .accessfn = gt_vtimer_access, - .writefn = gt_virt_cval_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write, }, /* Secure timer -- this is actually restricted to only EL3 * and configurably Secure-EL1 via the accessfn. @@ -2915,6 +3158,15 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { REGINFO_SENTINEL }; +static CPAccessResult e2h_access(CPUARMState *env, const ARMCPRegInfo *ri, + bool isread) +{ + if (!(arm_hcr_el2_eff(env) & HCR_E2H)) { + return CP_ACCESS_TRAP; + } + return CP_ACCESS_OK; +} + #else /* In user-mode most of the generic timer registers are inaccessible @@ -3009,7 +3261,8 @@ static uint64_t do_ats_write(CPUARMState *env, uint64_t value, bool take_exc = false; if (fi.s1ptw && current_el == 1 && !arm_is_secure(env) - && (mmu_idx == ARMMMUIdx_S1NSE1 || mmu_idx == ARMMMUIdx_S1NSE0)) { + && (mmu_idx == ARMMMUIdx_Stage1_E1 || + mmu_idx == ARMMMUIdx_Stage1_E0)) { /* * Synchronous stage 2 fault on an access made as part of the * translation table walk for AT S1E0* or AT S1E1* insn @@ -3085,7 +3338,7 @@ static uint64_t do_ats_write(CPUARMState *env, uint64_t value, format64 = arm_s1_regime_using_lpae_format(env, mmu_idx); if (arm_feature(env, ARM_FEATURE_EL2)) { - if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) { + if (mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_E10_1) { format64 |= env->cp15.hcr_el2 & (HCR_VM | HCR_DC); } else { format64 |= arm_current_el(env) == 2; @@ -3154,13 +3407,13 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) /* stage 1 current state PL1: ATS1CPR, ATS1CPW */ switch (el) { case 3: - mmu_idx = ARMMMUIdx_S1E3; + mmu_idx = ARMMMUIdx_SE3; break; case 2: - mmu_idx = ARMMMUIdx_S1NSE1; + mmu_idx = ARMMMUIdx_Stage1_E1; break; case 1: - mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1; + mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_Stage1_E1; break; default: g_assert_not_reached(); @@ -3170,13 +3423,13 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) /* stage 1 current state PL0: ATS1CUR, ATS1CUW */ switch (el) { case 3: - mmu_idx = ARMMMUIdx_S1SE0; + mmu_idx = ARMMMUIdx_SE10_0; break; case 2: - mmu_idx = ARMMMUIdx_S1NSE0; + mmu_idx = ARMMMUIdx_Stage1_E0; break; case 1: - mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0; + mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_Stage1_E0; break; default: g_assert_not_reached(); @@ -3184,11 +3437,11 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) break; case 4: /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */ - mmu_idx = ARMMMUIdx_S12NSE1; + mmu_idx = ARMMMUIdx_E10_1; break; case 6: /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */ - mmu_idx = ARMMMUIdx_S12NSE0; + mmu_idx = ARMMMUIdx_E10_0; break; default: g_assert_not_reached(); @@ -3205,7 +3458,7 @@ static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri, MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD; uint64_t par64; - par64 = do_ats_write(env, value, access_type, ARMMMUIdx_S1E2); + par64 = do_ats_write(env, value, access_type, ARMMMUIdx_E2); A32_BANKED_CURRENT_REG_SET(env, par, par64); } @@ -3230,26 +3483,26 @@ static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri, case 0: switch (ri->opc1) { case 0: /* AT S1E1R, AT S1E1W */ - mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1; + mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_Stage1_E1; break; case 4: /* AT S1E2R, AT S1E2W */ - mmu_idx = ARMMMUIdx_S1E2; + mmu_idx = ARMMMUIdx_E2; break; case 6: /* AT S1E3R, AT S1E3W */ - mmu_idx = ARMMMUIdx_S1E3; + mmu_idx = ARMMMUIdx_SE3; break; default: g_assert_not_reached(); } break; case 2: /* AT S1E0R, AT S1E0W */ - mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0; + mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_Stage1_E0; break; case 4: /* AT S12E1R, AT S12E1W */ - mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S12NSE1; + mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_E10_1; break; case 6: /* AT S12E0R, AT S12E0W */ - mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S12NSE0; + mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_E10_0; break; default: g_assert_not_reached(); @@ -3510,7 +3763,7 @@ static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri) tcr->base_mask = 0xffffc000u; } -static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri, +static void vmsa_tcr_el12_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { ARMCPU *cpu = env_archcpu(env); @@ -3533,18 +3786,38 @@ static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri, raw_write(env, ri, value); } +static void vmsa_tcr_ttbr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + /* + * If we are running with E2&0 regime, then an ASID is active. + * Flush if that might be changing. Note we're not checking + * TCR_EL2.A1 to know if this is really the TTBRx_EL2 that + * holds the active ASID, only checking the field that might. + */ + if (extract64(raw_read(env, ri) ^ value, 48, 16) && + (arm_hcr_el2_eff(env) & HCR_E2H)) { + tlb_flush_by_mmuidx(env_cpu(env), + ARMMMUIdxBit_E20_2 | ARMMMUIdxBit_E20_0); + } + raw_write(env, ri, value); +} + static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); - /* Accesses to VTTBR may change the VMID so we must flush the TLB. */ + /* + * A change in VMID to the stage2 page table (Stage2) invalidates + * the combined stage 1&2 tlbs (EL10_1 and EL10_0). + */ if (raw_read(env, ri) != value) { tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_E10_1 | + ARMMMUIdxBit_E10_0 | + ARMMMUIdxBit_Stage2); raw_write(env, ri, value); } } @@ -3586,7 +3859,7 @@ static const ARMCPRegInfo vmsa_cp_reginfo[] = { offsetof(CPUARMState, cp15.ttbr1_ns) } }, { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64, .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2, - .access = PL1_RW, .writefn = vmsa_tcr_el1_write, + .access = PL1_RW, .writefn = vmsa_tcr_el12_write, .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write, .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) }, { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2, @@ -3870,7 +4143,7 @@ static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri, static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) { + if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) { return CP_ACCESS_TRAP; } return CP_ACCESS_OK; @@ -3889,7 +4162,7 @@ static CPAccessResult aa64_cacheop_access(CPUARMState *env, /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless * SCTLR_EL1.UCI is set. */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCI)) { + if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UCI)) { return CP_ACCESS_TRAP; } return CP_ACCESS_OK; @@ -3899,79 +4172,79 @@ static CPAccessResult aa64_cacheop_access(CPUARMState *env, * Page D4-1736 (DDI0487A.b) */ +static int vae1_tlbmask(CPUARMState *env) +{ + /* Since we exclude secure first, we may read HCR_EL2 directly. */ + if (arm_is_secure_below_el3(env)) { + return ARMMMUIdxBit_SE10_1 | ARMMMUIdxBit_SE10_0; + } else if ((env->cp15.hcr_el2 & (HCR_E2H | HCR_TGE)) + == (HCR_E2H | HCR_TGE)) { + return ARMMMUIdxBit_E20_2 | ARMMMUIdxBit_E20_0; + } else { + return ARMMMUIdxBit_E10_1 | ARMMMUIdxBit_E10_0; + } +} + static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { CPUState *cs = env_cpu(env); - bool sec = arm_is_secure_below_el3(env); + int mask = vae1_tlbmask(env); - if (sec) { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); - } else { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { CPUState *cs = env_cpu(env); + int mask = vae1_tlbmask(env); if (tlb_force_broadcast(env)) { - tlbi_aa64_vmalle1is_write(env, NULL, value); - return; - } - - if (arm_is_secure_below_el3(env)) { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } else { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); + tlb_flush_by_mmuidx(cs, mask); } } -static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri, - uint64_t value) +static int alle1_tlbmask(CPUARMState *env) { - /* Note that the 'ALL' scope must invalidate both stage 1 and + /* + * Note that the 'ALL' scope must invalidate both stage 1 and * stage 2 translations, whereas most other scopes only invalidate * stage 1 translations. */ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); - if (arm_is_secure_below_el3(env)) { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); + return ARMMMUIdxBit_SE10_1 | ARMMMUIdxBit_SE10_0; + } else if (arm_feature(env, ARM_FEATURE_EL2)) { + return ARMMMUIdxBit_E10_1 | ARMMMUIdxBit_E10_0 | ARMMMUIdxBit_Stage2; } else { - if (arm_feature(env, ARM_FEATURE_EL2)) { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); - } else { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + return ARMMMUIdxBit_E10_1 | ARMMMUIdxBit_E10_0; } } +static int e2_tlbmask(CPUARMState *env) +{ + /* TODO: ARMv8.4-SecEL2 */ + return ARMMMUIdxBit_E20_0 | ARMMMUIdxBit_E20_2 | ARMMMUIdxBit_E2; +} + +static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + CPUState *cs = env_cpu(env); + int mask = alle1_tlbmask(env); + + tlb_flush_by_mmuidx(cs, mask); +} + static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); + CPUState *cs = env_cpu(env); + int mask = e2_tlbmask(env); - tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx(cs, mask); } static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -3980,42 +4253,25 @@ static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri, ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); - tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_S1E3); + tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_SE3); } static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { - /* Note that the 'ALL' scope must invalidate both stage 1 and - * stage 2 translations, whereas most other scopes only invalidate - * stage 1 translations. - */ CPUState *cs = env_cpu(env); - bool sec = arm_is_secure_below_el3(env); - bool has_el2 = arm_feature(env, ARM_FEATURE_EL2); - - if (sec) { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); - } else if (has_el2) { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); - } else { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + int mask = alle1_tlbmask(env); + + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { CPUState *cs = env_cpu(env); + int mask = e2_tlbmask(env); - tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4023,7 +4279,7 @@ static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri, { CPUState *cs = env_cpu(env); - tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_S1E3); + tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_SE3); } static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4033,11 +4289,11 @@ static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri, * Currently handles both VAE2 and VALE2, since we don't support * flush-last-level-only. */ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); + CPUState *cs = env_cpu(env); + int mask = e2_tlbmask(env); uint64_t pageaddr = sextract64(value << 12, 0, 56); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S1E2); + tlb_flush_page_by_mmuidx(cs, pageaddr, mask); } static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4051,26 +4307,17 @@ static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = CPU(cpu); uint64_t pageaddr = sextract64(value << 12, 0, 56); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S1E3); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_SE3); } static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); - bool sec = arm_is_secure_below_el3(env); + CPUState *cs = env_cpu(env); + int mask = vae1_tlbmask(env); uint64_t pageaddr = sextract64(value << 12, 0, 56); - if (sec) { - tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); - } else { - tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask); } static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4081,23 +4328,14 @@ static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri, * since we don't support flush-for-specific-ASID-only or * flush-last-level-only. */ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); + CPUState *cs = env_cpu(env); + int mask = vae1_tlbmask(env); uint64_t pageaddr = sextract64(value << 12, 0, 56); if (tlb_force_broadcast(env)) { - tlbi_aa64_vae1is_write(env, NULL, value); - return; - } - - if (arm_is_secure_below_el3(env)) { - tlb_flush_page_by_mmuidx(cs, pageaddr, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); + tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask); } else { - tlb_flush_page_by_mmuidx(cs, pageaddr, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); + tlb_flush_page_by_mmuidx(cs, pageaddr, mask); } } @@ -4108,7 +4346,7 @@ static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t pageaddr = sextract64(value << 12, 0, 56); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1E2); + ARMMMUIdxBit_E2); } static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4118,7 +4356,7 @@ static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t pageaddr = sextract64(value << 12, 0, 56); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1E3); + ARMMMUIdxBit_SE3); } static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4140,7 +4378,7 @@ static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 48); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S2NS); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_Stage2); } static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4156,17 +4394,33 @@ static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 48); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_Stage2); } static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { - /* We don't implement EL2, so the only control on DC ZVA is the - * bit in the SCTLR which can prohibit access for EL0. - */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_DZE)) { - return CP_ACCESS_TRAP; + int cur_el = arm_current_el(env); + + if (cur_el < 2) { + uint64_t hcr = arm_hcr_el2_eff(env); + + if (cur_el == 0) { + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + if (!(env->cp15.sctlr_el[2] & SCTLR_DZE)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + if (!(env->cp15.sctlr_el[1] & SCTLR_DZE)) { + return CP_ACCESS_TRAP; + } + if (hcr & HCR_TDZ) { + return CP_ACCESS_TRAP_EL2; + } + } + } else if (hcr & HCR_TDZ) { + return CP_ACCESS_TRAP_EL2; + } } return CP_ACCESS_OK; } @@ -4721,7 +4975,8 @@ static const ARMCPRegInfo el3_no_el2_v8_cp_reginfo[] = { static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { ARMCPU *cpu = env_archcpu(env); - uint64_t valid_mask = HCR_MASK; + /* Begin with bits defined in base ARMv8.0. */ + uint64_t valid_mask = MAKE_64BIT_MASK(0, 34); if (arm_feature(env, ARM_FEATURE_EL3)) { valid_mask &= ~HCR_HCD; @@ -4735,6 +4990,9 @@ static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) */ valid_mask &= ~HCR_TSC; } + if (cpu_isar_feature(aa64_vh, cpu)) { + valid_mask |= HCR_E2H; + } if (cpu_isar_feature(aa64_lor, cpu)) { valid_mask |= HCR_TLOR; } @@ -4938,10 +5196,8 @@ static const ARMCPRegInfo el2_cp_reginfo[] = { .resetvalue = 0 }, { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH, .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2, - .access = PL2_RW, - /* no .writefn needed as this can't cause an ASID change; - * no .raw_writefn or .resetfn needed as we never use mask/base_mask - */ + .access = PL2_RW, .writefn = vmsa_tcr_el12_write, + /* no .raw_writefn or .resetfn needed as we never use mask/base_mask */ .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) }, { .name = "VTCR", .state = ARM_CP_STATE_AA32, .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2, @@ -4975,7 +5231,7 @@ static const ARMCPRegInfo el2_cp_reginfo[] = { .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) }, { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0, - .access = PL2_RW, .resetvalue = 0, + .access = PL2_RW, .resetvalue = 0, .writefn = vmsa_tcr_ttbr_el2_write, .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) }, { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2, .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS, @@ -5244,14 +5500,182 @@ static const ARMCPRegInfo el3_cp_reginfo[] = { REGINFO_SENTINEL }; +#ifndef CONFIG_USER_ONLY +/* Test if system register redirection is to occur in the current state. */ +static bool redirect_for_e2h(CPUARMState *env) +{ + return arm_current_el(env) == 2 && (arm_hcr_el2_eff(env) & HCR_E2H); +} + +static uint64_t el2_e2h_read(CPUARMState *env, const ARMCPRegInfo *ri) +{ + CPReadFn *readfn; + + if (redirect_for_e2h(env)) { + /* Switch to the saved EL2 version of the register. */ + ri = ri->opaque; + readfn = ri->readfn; + } else { + readfn = ri->orig_readfn; + } + if (readfn == NULL) { + readfn = raw_read; + } + return readfn(env, ri); +} + +static void el2_e2h_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + CPWriteFn *writefn; + + if (redirect_for_e2h(env)) { + /* Switch to the saved EL2 version of the register. */ + ri = ri->opaque; + writefn = ri->writefn; + } else { + writefn = ri->orig_writefn; + } + if (writefn == NULL) { + writefn = raw_write; + } + writefn(env, ri, value); +} + +static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu) +{ + struct E2HAlias { + uint32_t src_key, dst_key, new_key; + const char *src_name, *dst_name, *new_name; + bool (*feature)(const ARMISARegisters *id); + }; + +#define K(op0, op1, crn, crm, op2) \ + ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP, crn, crm, op0, op1, op2) + + static const struct E2HAlias aliases[] = { + { K(3, 0, 1, 0, 0), K(3, 4, 1, 0, 0), K(3, 5, 1, 0, 0), + "SCTLR", "SCTLR_EL2", "SCTLR_EL12" }, + { K(3, 0, 1, 0, 2), K(3, 4, 1, 1, 2), K(3, 5, 1, 0, 2), + "CPACR", "CPTR_EL2", "CPACR_EL12" }, + { K(3, 0, 2, 0, 0), K(3, 4, 2, 0, 0), K(3, 5, 2, 0, 0), + "TTBR0_EL1", "TTBR0_EL2", "TTBR0_EL12" }, + { K(3, 0, 2, 0, 1), K(3, 4, 2, 0, 1), K(3, 5, 2, 0, 1), + "TTBR1_EL1", "TTBR1_EL2", "TTBR1_EL12" }, + { K(3, 0, 2, 0, 2), K(3, 4, 2, 0, 2), K(3, 5, 2, 0, 2), + "TCR_EL1", "TCR_EL2", "TCR_EL12" }, + { K(3, 0, 4, 0, 0), K(3, 4, 4, 0, 0), K(3, 5, 4, 0, 0), + "SPSR_EL1", "SPSR_EL2", "SPSR_EL12" }, + { K(3, 0, 4, 0, 1), K(3, 4, 4, 0, 1), K(3, 5, 4, 0, 1), + "ELR_EL1", "ELR_EL2", "ELR_EL12" }, + { K(3, 0, 5, 1, 0), K(3, 4, 5, 1, 0), K(3, 5, 5, 1, 0), + "AFSR0_EL1", "AFSR0_EL2", "AFSR0_EL12" }, + { K(3, 0, 5, 1, 1), K(3, 4, 5, 1, 1), K(3, 5, 5, 1, 1), + "AFSR1_EL1", "AFSR1_EL2", "AFSR1_EL12" }, + { K(3, 0, 5, 2, 0), K(3, 4, 5, 2, 0), K(3, 5, 5, 2, 0), + "ESR_EL1", "ESR_EL2", "ESR_EL12" }, + { K(3, 0, 6, 0, 0), K(3, 4, 6, 0, 0), K(3, 5, 6, 0, 0), + "FAR_EL1", "FAR_EL2", "FAR_EL12" }, + { K(3, 0, 10, 2, 0), K(3, 4, 10, 2, 0), K(3, 5, 10, 2, 0), + "MAIR_EL1", "MAIR_EL2", "MAIR_EL12" }, + { K(3, 0, 10, 3, 0), K(3, 4, 10, 3, 0), K(3, 5, 10, 3, 0), + "AMAIR0", "AMAIR_EL2", "AMAIR_EL12" }, + { K(3, 0, 12, 0, 0), K(3, 4, 12, 0, 0), K(3, 5, 12, 0, 0), + "VBAR", "VBAR_EL2", "VBAR_EL12" }, + { K(3, 0, 13, 0, 1), K(3, 4, 13, 0, 1), K(3, 5, 13, 0, 1), + "CONTEXTIDR_EL1", "CONTEXTIDR_EL2", "CONTEXTIDR_EL12" }, + { K(3, 0, 14, 1, 0), K(3, 4, 14, 1, 0), K(3, 5, 14, 1, 0), + "CNTKCTL", "CNTHCTL_EL2", "CNTKCTL_EL12" }, + + /* + * Note that redirection of ZCR is mentioned in the description + * of ZCR_EL2, and aliasing in the description of ZCR_EL1, but + * not in the summary table. + */ + { K(3, 0, 1, 2, 0), K(3, 4, 1, 2, 0), K(3, 5, 1, 2, 0), + "ZCR_EL1", "ZCR_EL2", "ZCR_EL12", isar_feature_aa64_sve }, + + /* TODO: ARMv8.2-SPE -- PMSCR_EL2 */ + /* TODO: ARMv8.4-Trace -- TRFCR_EL2 */ + }; +#undef K + + size_t i; + + for (i = 0; i < ARRAY_SIZE(aliases); i++) { + const struct E2HAlias *a = &aliases[i]; + ARMCPRegInfo *src_reg, *dst_reg; + + if (a->feature && !a->feature(&cpu->isar)) { + continue; + } + + src_reg = g_hash_table_lookup(cpu->cp_regs, &a->src_key); + dst_reg = g_hash_table_lookup(cpu->cp_regs, &a->dst_key); + g_assert(src_reg != NULL); + g_assert(dst_reg != NULL); + + /* Cross-compare names to detect typos in the keys. */ + g_assert(strcmp(src_reg->name, a->src_name) == 0); + g_assert(strcmp(dst_reg->name, a->dst_name) == 0); + + /* None of the core system registers use opaque; we will. */ + g_assert(src_reg->opaque == NULL); + + /* Create alias before redirection so we dup the right data. */ + if (a->new_key) { + ARMCPRegInfo *new_reg = g_memdup(src_reg, sizeof(ARMCPRegInfo)); + uint32_t *new_key = g_memdup(&a->new_key, sizeof(uint32_t)); + bool ok; + + new_reg->name = a->new_name; + new_reg->type |= ARM_CP_ALIAS; + /* Remove PL1/PL0 access, leaving PL2/PL3 R/W in place. */ + new_reg->access &= PL2_RW | PL3_RW; + + ok = g_hash_table_insert(cpu->cp_regs, new_key, new_reg); + g_assert(ok); + } + + src_reg->opaque = dst_reg; + src_reg->orig_readfn = src_reg->readfn ?: raw_read; + src_reg->orig_writefn = src_reg->writefn ?: raw_write; + if (!src_reg->raw_readfn) { + src_reg->raw_readfn = raw_read; + } + if (!src_reg->raw_writefn) { + src_reg->raw_writefn = raw_write; + } + src_reg->readfn = el2_e2h_read; + src_reg->writefn = el2_e2h_write; + } +} +#endif + static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { - /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64, - * but the AArch32 CTR has its own reginfo struct) - */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCT)) { - return CP_ACCESS_TRAP; + int cur_el = arm_current_el(env); + + if (cur_el < 2) { + uint64_t hcr = arm_hcr_el2_eff(env); + + if (cur_el == 0) { + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + if (!(env->cp15.sctlr_el[2] & SCTLR_UCT)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + if (!(env->cp15.sctlr_el[1] & SCTLR_UCT)) { + return CP_ACCESS_TRAP; + } + if (hcr & HCR_TID2) { + return CP_ACCESS_TRAP_EL2; + } + } + } else if (hcr & HCR_TID2) { + return CP_ACCESS_TRAP_EL2; + } } if (arm_current_el(env) < 2 && arm_hcr_el2_eff(env) & HCR_TID2) { @@ -5367,7 +5791,9 @@ static const ARMCPRegInfo debug_lpae_cp_reginfo[] = { int sve_exception_el(CPUARMState *env, int el) { #ifndef CONFIG_USER_ONLY - if (el <= 1) { + uint64_t hcr_el2 = arm_hcr_el2_eff(env); + + if (el <= 1 && (hcr_el2 & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { bool disabled = false; /* The CPACR.ZEN controls traps to EL1: @@ -5382,8 +5808,7 @@ int sve_exception_el(CPUARMState *env, int el) } if (disabled) { /* route_to_el2 */ - return (arm_feature(env, ARM_FEATURE_EL2) - && (arm_hcr_el2_eff(env) & HCR_TGE) ? 2 : 1); + return hcr_el2 & HCR_TGE ? 2 : 1; } /* Check CPACR.FPEN. */ @@ -6122,6 +6547,71 @@ static const ARMCPRegInfo jazelle_regs[] = { REGINFO_SENTINEL }; +static const ARMCPRegInfo vhe_reginfo[] = { + { .name = "CONTEXTIDR_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 1, + .access = PL2_RW, + .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[2]) }, + { .name = "TTBR1_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 1, + .access = PL2_RW, .writefn = vmsa_tcr_ttbr_el2_write, + .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[2]) }, +#ifndef CONFIG_USER_ONLY + { .name = "CNTHV_CVAL_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 2, + .fieldoffset = + offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].cval), + .type = ARM_CP_IO, .access = PL2_RW, + .writefn = gt_hv_cval_write, .raw_writefn = raw_write }, + { .name = "CNTHV_TVAL_EL2", .state = ARM_CP_STATE_BOTH, + .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 0, + .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW, + .resetfn = gt_hv_timer_reset, + .readfn = gt_hv_tval_read, .writefn = gt_hv_tval_write }, + { .name = "CNTHV_CTL_EL2", .state = ARM_CP_STATE_BOTH, + .type = ARM_CP_IO, + .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 1, + .access = PL2_RW, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].ctl), + .writefn = gt_hv_ctl_write, .raw_writefn = raw_write }, + { .name = "CNTP_CTL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 1, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl), + .writefn = gt_phys_ctl_write, .raw_writefn = raw_write }, + { .name = "CNTV_CTL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 1, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl), + .writefn = gt_virt_ctl_write, .raw_writefn = raw_write }, + { .name = "CNTP_TVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 0, + .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write }, + { .name = "CNTV_TVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 0, + .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write }, + { .name = "CNTP_CVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 2, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval), + .access = PL2_RW, .accessfn = e2h_access, + .writefn = gt_phys_cval_write, .raw_writefn = raw_write }, + { .name = "CNTV_CVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 2, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval), + .access = PL2_RW, .accessfn = e2h_access, + .writefn = gt_virt_cval_write, .raw_writefn = raw_write }, +#endif + REGINFO_SENTINEL +}; + void register_cp_regs_for_features(ARMCPU *cpu) { /* Register all the coprocessor registers based on feature bits */ @@ -7085,6 +7575,10 @@ void register_cp_regs_for_features(ARMCPU *cpu) define_arm_cp_regs(cpu, lor_reginfo); } + if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) { + define_arm_cp_regs(cpu, vhe_reginfo); + } + if (cpu_isar_feature(aa64_sve, cpu)) { define_one_arm_cp_reg(cpu, &zcr_el1_reginfo); if (arm_feature(env, ARM_FEATURE_EL2)) { @@ -7126,6 +7620,16 @@ void register_cp_regs_for_features(ARMCPU *cpu) : cpu_isar_feature(aa32_predinv, cpu)) { define_arm_cp_regs(cpu, predinv_reginfo); } + +#ifndef CONFIG_USER_ONLY + /* + * Register redirections and aliases must be done last, + * after the registers from the other extensions have been defined. + */ + if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) { + define_arm_vh_e2h_redirects_aliases(cpu); + } +#endif } void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu) @@ -7421,13 +7925,10 @@ void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu, mask = PL0_RW; break; case 4: + case 5: /* min_EL EL2 */ mask = PL2_RW; break; - case 5: - /* unallocated encoding, so not possible */ - assert(false); - break; case 6: /* min_EL EL3 */ mask = PL3_RW; @@ -7946,6 +8447,12 @@ uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx, break; }; + /* + * For these purposes, TGE and AMO/IMO/FMO both force the + * interrupt to EL2. Fold TGE into the bit extracted above. + */ + hcr |= (hcr_el2 & HCR_TGE) != 0; + /* Perform a table-lookup for the target EL given the current state */ target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el]; @@ -8510,14 +9017,19 @@ static void arm_cpu_do_interrupt_aarch64(CPUState *cs) * immediately lower than the target level is using AArch32 or AArch64 */ bool is_aa64; + uint64_t hcr; switch (new_el) { case 3: is_aa64 = (env->cp15.scr_el3 & SCR_RW) != 0; break; case 2: - is_aa64 = (env->cp15.hcr_el2 & HCR_RW) != 0; - break; + hcr = arm_hcr_el2_eff(env); + if ((hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + is_aa64 = (hcr & HCR_RW) != 0; + break; + } + /* fall through */ case 1: is_aa64 = is_a64(env); break; @@ -8690,19 +9202,23 @@ void arm_cpu_do_interrupt(CPUState *cs) #endif /* !CONFIG_USER_ONLY */ /* Return the exception level which controls this address translation regime */ -static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) +static uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { - case ARMMMUIdx_S2NS: - case ARMMMUIdx_S1E2: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + case ARMMMUIdx_Stage2: + case ARMMMUIdx_E2: return 2; - case ARMMMUIdx_S1E3: + case ARMMMUIdx_SE3: return 3; - case ARMMMUIdx_S1SE0: + case ARMMMUIdx_SE10_0: return arm_el_is_aa64(env, 3) ? 1 : 3; - case ARMMMUIdx_S1SE1: - case ARMMMUIdx_S1NSE0: - case ARMMMUIdx_S1NSE1: + case ARMMMUIdx_SE10_1: + case ARMMMUIdx_Stage1_E0: + case ARMMMUIdx_Stage1_E1: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: case ARMMMUIdx_MPrivNegPri: case ARMMMUIdx_MUserNegPri: case ARMMMUIdx_MPriv: @@ -8717,14 +9233,24 @@ static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) } } -#ifndef CONFIG_USER_ONLY +uint64_t arm_sctlr(CPUARMState *env, int el) +{ + /* Only EL0 needs to be adjusted for EL1&0 or EL2&0. */ + if (el == 0) { + ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, 0); + el = (mmu_idx == ARMMMUIdx_E20_0 ? 2 : 1); + } + return env->cp15.sctlr_el[el]; +} /* Return the SCTLR value which controls this address translation regime */ -static inline uint32_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) +static inline uint64_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) { return env->cp15.sctlr_el[regime_el(env, mmu_idx)]; } +#ifndef CONFIG_USER_ONLY + /* Return true if the specified stage of address translation is disabled */ static inline bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) @@ -8747,7 +9273,7 @@ static inline bool regime_translation_disabled(CPUARMState *env, } } - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* HCR.DC means HCR.VM behaves as 1 */ return (env->cp15.hcr_el2 & (HCR_DC | HCR_VM)) == 0; } @@ -8760,7 +9286,7 @@ static inline bool regime_translation_disabled(CPUARMState *env, } if ((env->cp15.hcr_el2 & HCR_DC) && - (mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1)) { + (mmu_idx == ARMMMUIdx_Stage1_E0 || mmu_idx == ARMMMUIdx_Stage1_E1)) { /* HCR.DC means SCTLR_EL1.M behaves as 0 */ return true; } @@ -8778,7 +9304,7 @@ static inline bool regime_translation_big_endian(CPUARMState *env, static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) { - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { return env->cp15.vttbr_el2; } if (ttbrn == 0) { @@ -8793,7 +9319,7 @@ static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, /* Return the TCR controlling this translation regime */ static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx) { - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { return &env->cp15.vtcr_el2; } return &env->cp15.tcr_el[regime_el(env, mmu_idx)]; @@ -8804,10 +9330,14 @@ static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx) */ static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) { - if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) { - mmu_idx += (ARMMMUIdx_S1NSE0 - ARMMMUIdx_S12NSE0); + switch (mmu_idx) { + case ARMMMUIdx_E10_0: + return ARMMMUIdx_Stage1_E0; + case ARMMMUIdx_E10_1: + return ARMMMUIdx_Stage1_E1; + default: + return mmu_idx; } - return mmu_idx; } /* Return true if the translation regime is using LPAE format page tables */ @@ -8839,8 +9369,9 @@ bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { - case ARMMMUIdx_S1SE0: - case ARMMMUIdx_S1NSE0: + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_Stage1_E0: case ARMMMUIdx_MUser: case ARMMMUIdx_MSUser: case ARMMMUIdx_MUserNegPri: @@ -8848,8 +9379,8 @@ static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) return true; default: return false; - case ARMMMUIdx_S12NSE0: - case ARMMMUIdx_S12NSE1: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: g_assert_not_reached(); } } @@ -8980,7 +9511,7 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, bool have_wxn; int wxn = 0; - assert(mmu_idx != ARMMMUIdx_S2NS); + assert(mmu_idx != ARMMMUIdx_Stage2); user_rw = simple_ap_to_rw_prot_is_user(ap, true); if (is_user) { @@ -9005,15 +9536,8 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, } if (is_aa64) { - switch (regime_el(env, mmu_idx)) { - case 1: - if (!is_user) { - xn = pxn || (user_rw & PAGE_WRITE); - } - break; - case 2: - case 3: - break; + if (regime_has_2_ranges(mmu_idx) && !is_user) { + xn = pxn || (user_rw & PAGE_WRITE); } } else if (arm_feature(env, ARM_FEATURE_V7)) { switch (regime_el(env, mmu_idx)) { @@ -9071,8 +9595,8 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, hwaddr addr, MemTxAttrs txattrs, ARMMMUFaultInfo *fi) { - if ((mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1) && - !regime_translation_disabled(env, ARMMMUIdx_S2NS)) { + if ((mmu_idx == ARMMMUIdx_Stage1_E0 || mmu_idx == ARMMMUIdx_Stage1_E1) && + !regime_translation_disabled(env, ARMMMUIdx_Stage2)) { target_ulong s2size; hwaddr s2pa; int s2prot; @@ -9089,7 +9613,7 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, pcacheattrs = &cacheattrs; } - ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_S2NS, &s2pa, + ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_Stage2, &s2pa, &txattrs, &s2prot, &s2size, fi, pcacheattrs); if (ret) { assert(fi->type != ARMFault_None); @@ -9547,7 +10071,6 @@ ARMVAParameters aa64_va_parameters_both(CPUARMState *env, uint64_t va, ARMMMUIdx mmu_idx) { uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; - uint32_t el = regime_el(env, mmu_idx); bool tbi, tbid, epd, hpd, using16k, using64k; int select, tsz; @@ -9557,11 +10080,11 @@ ARMVAParameters aa64_va_parameters_both(CPUARMState *env, uint64_t va, */ select = extract64(va, 55, 1); - if (el > 1) { + if (!regime_has_2_ranges(mmu_idx)) { tsz = extract32(tcr, 0, 6); using64k = extract32(tcr, 14, 1); using16k = extract32(tcr, 15, 1); - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* VTCR_EL2 */ tbi = tbid = hpd = false; } else { @@ -9622,7 +10145,7 @@ static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, int select, tsz; bool epd, hpd; - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* VTCR */ bool sext = extract32(tcr, 4, 1); bool sign = extract32(tcr, 3, 1); @@ -9713,10 +10236,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, param = aa64_va_parameters(env, address, mmu_idx, access_type != MMU_INST_FETCH); level = 0; - /* If we are in 64-bit EL2 or EL3 then there is no TTBR1, so mark it - * invalid. - */ - ttbr1_valid = (el < 2); + ttbr1_valid = regime_has_2_ranges(mmu_idx); addrsize = 64 - 8 * param.tbi; inputsize = 64 - param.tsz; } else { @@ -9724,7 +10244,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, level = 1; /* There is no TTBR1 for EL2 */ ttbr1_valid = (el != 2); - addrsize = (mmu_idx == ARMMMUIdx_S2NS ? 40 : 32); + addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32); inputsize = addrsize - param.tsz; } @@ -9775,7 +10295,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, goto do_fault; } - if (mmu_idx != ARMMMUIdx_S2NS) { + if (mmu_idx != ARMMMUIdx_Stage2) { /* The starting level depends on the virtual address size (which can * be up to 48 bits) and the translation granule size. It indicates * the number of strides (stride bits at a time) needed to @@ -9875,7 +10395,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, attrs = extract64(descriptor, 2, 10) | (extract64(descriptor, 52, 12) << 10); - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* Stage 2 table descriptors do not include any attribute fields */ break; } @@ -9906,7 +10426,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, ap = extract32(attrs, 4, 2); xn = extract32(attrs, 12, 1); - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { ns = true; *prot = get_S2prot(env, ap, xn); } else { @@ -9933,7 +10453,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, } if (cacheattrs != NULL) { - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { cacheattrs->attrs = convert_stage2_attrs(env, extract32(attrs, 0, 4)); } else { @@ -9954,7 +10474,7 @@ do_fault: fi->type = fault_type; fi->level = level; /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2. */ - fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_S2NS); + fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_Stage2); return true; } @@ -10753,7 +11273,7 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, target_ulong *page_size, ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) { - if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) { + if (mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_E10_1) { /* Call ourselves recursively to do the stage 1 and then stage 2 * translations. */ @@ -10768,13 +11288,13 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, prot, page_size, fi, cacheattrs); /* If S1 fails or S2 is disabled, return early. */ - if (ret || regime_translation_disabled(env, ARMMMUIdx_S2NS)) { + if (ret || regime_translation_disabled(env, ARMMMUIdx_Stage2)) { *phys_ptr = ipa; return ret; } /* S1 is done. Now do S2 translation. */ - ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_S2NS, + ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_Stage2, phys_ptr, attrs, &s2_prot, page_size, fi, cacheattrs != NULL ? &cacheattrs2 : NULL); @@ -10816,7 +11336,7 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, /* Fast Context Switch Extension. This doesn't exist at all in v8. * In v7 and earlier it affects all stage 1 translations. */ - if (address < 0x02000000 && mmu_idx != ARMMMUIdx_S2NS + if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2 && !arm_feature(env, ARM_FEATURE_V8)) { if (regime_el(env, mmu_idx) == 3) { address += env->cp15.fcseidr_s; @@ -11177,8 +11697,6 @@ uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes) int fp_exception_el(CPUARMState *env, int cur_el) { #ifndef CONFIG_USER_ONLY - int fpen; - /* CPACR and the CPTR registers don't exist before v6, so FP is * always accessible */ @@ -11206,30 +11724,34 @@ int fp_exception_el(CPUARMState *env, int cur_el) * 0, 2 : trap EL0 and EL1/PL1 accesses * 1 : trap only EL0 accesses * 3 : trap no accesses + * This register is ignored if E2H+TGE are both set. */ - fpen = extract32(env->cp15.cpacr_el1, 20, 2); - switch (fpen) { - case 0: - case 2: - if (cur_el == 0 || cur_el == 1) { - /* Trap to PL1, which might be EL1 or EL3 */ - if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) { + if ((arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + int fpen = extract32(env->cp15.cpacr_el1, 20, 2); + + switch (fpen) { + case 0: + case 2: + if (cur_el == 0 || cur_el == 1) { + /* Trap to PL1, which might be EL1 or EL3 */ + if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) { + return 3; + } + return 1; + } + if (cur_el == 3 && !is_a64(env)) { + /* Secure PL1 running at EL3 */ return 3; } - return 1; - } - if (cur_el == 3 && !is_a64(env)) { - /* Secure PL1 running at EL3 */ - return 3; - } - break; - case 1: - if (cur_el == 0) { - return 1; + break; + case 1: + if (cur_el == 0) { + return 1; + } + break; + case 3: + break; } - break; - case 3: - break; } /* @@ -11265,6 +11787,31 @@ int fp_exception_el(CPUARMState *env, int cur_el) return 0; } +/* Return the exception level we're running at if this is our mmu_idx */ +int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx) +{ + if (mmu_idx & ARM_MMU_IDX_M) { + return mmu_idx & ARM_MMU_IDX_M_PRIV; + } + + switch (mmu_idx) { + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_SE10_0: + return 0; + case ARMMMUIdx_E10_1: + case ARMMMUIdx_SE10_1: + return 1; + case ARMMMUIdx_E2: + case ARMMMUIdx_E20_2: + return 2; + case ARMMMUIdx_SE3: + return 3; + default: + g_assert_not_reached(); + } +} + #ifndef CONFIG_TCG ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate) { @@ -11278,10 +11825,33 @@ ARMMMUIdx arm_mmu_idx_el(CPUARMState *env, int el) return arm_v7m_mmu_idx_for_secstate(env, env->v7m.secure); } - if (el < 2 && arm_is_secure_below_el3(env)) { - return ARMMMUIdx_S1SE0 + el; - } else { - return ARMMMUIdx_S12NSE0 + el; + /* See ARM pseudo-function ELIsInHost. */ + switch (el) { + case 0: + if (arm_is_secure_below_el3(env)) { + return ARMMMUIdx_SE10_0; + } + if ((env->cp15.hcr_el2 & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE) + && arm_el_is_aa64(env, 2)) { + return ARMMMUIdx_E20_0; + } + return ARMMMUIdx_E10_0; + case 1: + if (arm_is_secure_below_el3(env)) { + return ARMMMUIdx_SE10_1; + } + return ARMMMUIdx_E10_1; + case 2: + /* TODO: ARMv8.4-SecEL2 */ + /* Note that TGE does not apply at EL2. */ + if ((env->cp15.hcr_el2 & HCR_E2H) && arm_el_is_aa64(env, 2)) { + return ARMMMUIdx_E20_2; + } + return ARMMMUIdx_E2; + case 3: + return ARMMMUIdx_SE3; + default: + g_assert_not_reached(); } } @@ -11336,11 +11906,8 @@ static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, { uint32_t flags = 0; - /* v8M always enables the fpu. */ - flags = FIELD_DP32(flags, TBFLAG_A32, VFPEN, 1); - if (arm_v7m_is_handler_mode(env)) { - flags = FIELD_DP32(flags, TBFLAG_A32, HANDLER, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, HANDLER, 1); } /* @@ -11351,7 +11918,7 @@ static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, if (arm_feature(env, ARM_FEATURE_V8) && !((mmu_idx & ARM_MMU_IDX_M_NEGPRI) && (env->v7m.ccr[env->v7m.secure] & R_V7M_CCR_STKOFHFNMIGN_MASK))) { - flags = FIELD_DP32(flags, TBFLAG_A32, STACKCHECK, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, STACKCHECK, 1); } return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags); @@ -11394,8 +11961,8 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, flags = FIELD_DP32(flags, TBFLAG_ANY, AARCH64_STATE, 1); - /* FIXME: ARMv8.1-VHE S2 translation regime. */ - if (regime_el(env, stage1) < 2) { + /* Get control bits for tagged addresses. */ + if (regime_has_2_ranges(mmu_idx)) { ARMVAParameters p1 = aa64_va_parameters_both(env, -1, stage1); tbid = (p1.tbi << 1) | p0.tbi; tbii = tbid & ~((p1.tbid << 1) | p0.tbid); @@ -11424,7 +11991,7 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, flags = FIELD_DP32(flags, TBFLAG_A64, ZCR_LEN, zcr_len); } - sctlr = arm_sctlr(env, el); + sctlr = regime_sctlr(env, stage1); if (arm_cpu_data_is_big_endian_a64(el, sctlr)) { flags = FIELD_DP32(flags, TBFLAG_ANY, BE_DATA, 1); @@ -11449,6 +12016,28 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, } } + /* Compute the condition for using AccType_UNPRIV for LDTR et al. */ + /* TODO: ARMv8.2-UAO */ + switch (mmu_idx) { + case ARMMMUIdx_E10_1: + case ARMMMUIdx_SE10_1: + /* TODO: ARMv8.3-NV */ + flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + break; + case ARMMMUIdx_E20_2: + /* TODO: ARMv8.4-SecEL2 */ + /* + * Note that E20_2 is gated by HCR_EL2.E2H == 1, but E20_0 is + * gated by HCR_EL2.<E2H,TGE> == '11', and so is LDTR. + */ + if (env->cp15.hcr_el2 & HCR_TGE) { + flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + } + break; + default: + break; + } + return rebuild_hflags_common(env, fp_el, mmu_idx, flags); } @@ -11544,7 +12133,7 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, if (arm_feature(env, ARM_FEATURE_M_SECURITY) && FIELD_EX32(env->v7m.fpccr[M_REG_S], V7M_FPCCR, S) != env->v7m.secure) { - flags = FIELD_DP32(flags, TBFLAG_A32, FPCCR_S_WRONG, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, FPCCR_S_WRONG, 1); } if ((env->v7m.fpccr[env->v7m.secure] & R_V7M_FPCCR_ASPEN_MASK) && @@ -11556,12 +12145,12 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, * active FP context; we must create a new FP context before * executing any FP insn. */ - flags = FIELD_DP32(flags, TBFLAG_A32, NEW_FP_CTXT_NEEDED, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED, 1); } bool is_secure = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK; if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) { - flags = FIELD_DP32(flags, TBFLAG_A32, LSPACT, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, LSPACT, 1); } } else { /* @@ -11582,8 +12171,8 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, } } - flags = FIELD_DP32(flags, TBFLAG_A32, THUMB, env->thumb); - flags = FIELD_DP32(flags, TBFLAG_A32, CONDEXEC, env->condexec_bits); + flags = FIELD_DP32(flags, TBFLAG_AM32, THUMB, env->thumb); + flags = FIELD_DP32(flags, TBFLAG_AM32, CONDEXEC, env->condexec_bits); pstate_for_ss = env->uncached_cpsr; } |