diff options
Diffstat (limited to 'target')
54 files changed, 910 insertions, 489 deletions
diff --git a/target/alpha/cpu.c b/target/alpha/cpu.c index bf1787a69d..932cddac05 100644 --- a/target/alpha/cpu.c +++ b/target/alpha/cpu.c @@ -86,10 +86,10 @@ static bool alpha_cpu_has_work(CPUState *cs) assume that if a CPU really wants to stay asleep, it will mask interrupts at the chipset level, which will prevent these bits from being set in the first place. */ - return cs->interrupt_request & (CPU_INTERRUPT_HARD - | CPU_INTERRUPT_TIMER - | CPU_INTERRUPT_SMP - | CPU_INTERRUPT_MCHK); + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD + | CPU_INTERRUPT_TIMER + | CPU_INTERRUPT_SMP + | CPU_INTERRUPT_MCHK); } #endif /* !CONFIG_USER_ONLY */ diff --git a/target/arm/cpregs-pmu.c b/target/arm/cpregs-pmu.c index 9c4431c18b..31c01eddc8 100644 --- a/target/arm/cpregs-pmu.c +++ b/target/arm/cpregs-pmu.c @@ -228,22 +228,27 @@ static bool event_supported(uint16_t number) return supported_event_map[number] != UNSUPPORTED_EVENT; } -static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri, - bool isread) +static CPAccessResult do_pmreg_access(CPUARMState *env, bool is_pmcr) { /* * Performance monitor registers user accessibility is controlled - * by PMUSERENR. MDCR_EL2.TPM and MDCR_EL3.TPM allow configurable + * by PMUSERENR. MDCR_EL2.TPM/TPMCR and MDCR_EL3.TPM allow configurable * trapping to EL2 or EL3 for other accesses. */ int el = arm_current_el(env); - uint64_t mdcr_el2 = arm_mdcr_el2_eff(env); if (el == 0 && !(env->cp15.c9_pmuserenr & 1)) { return CP_ACCESS_TRAP_EL1; } - if (el < 2 && (mdcr_el2 & MDCR_TPM)) { - return CP_ACCESS_TRAP_EL2; + if (el < 2) { + uint64_t mdcr_el2 = arm_mdcr_el2_eff(env); + + if (mdcr_el2 & MDCR_TPM) { + return CP_ACCESS_TRAP_EL2; + } + if (is_pmcr && (mdcr_el2 & MDCR_TPMCR)) { + return CP_ACCESS_TRAP_EL2; + } } if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) { return CP_ACCESS_TRAP_EL3; @@ -252,6 +257,19 @@ static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri, return CP_ACCESS_OK; } +static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri, + bool isread) +{ + return do_pmreg_access(env, false); +} + +static CPAccessResult pmreg_access_pmcr(CPUARMState *env, + const ARMCPRegInfo *ri, + bool isread) +{ + return do_pmreg_access(env, true); +} + static CPAccessResult pmreg_access_xevcntr(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) @@ -1187,14 +1205,14 @@ void define_pm_cpregs(ARMCPU *cpu) .fgt = FGT_PMCR_EL0, .type = ARM_CP_IO | ARM_CP_ALIAS, .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr), - .accessfn = pmreg_access, + .accessfn = pmreg_access_pmcr, .readfn = pmcr_read, .raw_readfn = raw_read, .writefn = pmcr_write, .raw_writefn = raw_write, }; const ARMCPRegInfo pmcr64 = { .name = "PMCR_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 0, - .access = PL0_RW, .accessfn = pmreg_access, + .access = PL0_RW, .accessfn = pmreg_access_pmcr, .fgt = FGT_PMCR_EL0, .type = ARM_CP_IO, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr), diff --git a/target/arm/cpu-features.h b/target/arm/cpu-features.h index 5876162428..e49e0ae3af 100644 --- a/target/arm/cpu-features.h +++ b/target/arm/cpu-features.h @@ -406,9 +406,14 @@ static inline bool isar_feature_aa64_crc32(const ARMISARegisters *id) return FIELD_EX64_IDREG(id, ID_AA64ISAR0, CRC32) != 0; } -static inline bool isar_feature_aa64_atomics(const ARMISARegisters *id) +static inline bool isar_feature_aa64_lse(const ARMISARegisters *id) { - return FIELD_EX64_IDREG(id, ID_AA64ISAR0, ATOMIC) != 0; + return FIELD_EX64_IDREG(id, ID_AA64ISAR0, ATOMIC) >= 2; +} + +static inline bool isar_feature_aa64_lse128(const ARMISARegisters *id) +{ + return FIELD_EX64_IDREG(id, ID_AA64ISAR0, ATOMIC) >= 3; } static inline bool isar_feature_aa64_rdm(const ARMISARegisters *id) @@ -604,6 +609,11 @@ static inline bool isar_feature_aa64_rpres(const ARMISARegisters *id) return FIELD_EX64_IDREG(id, ID_AA64ISAR2, RPRES); } +static inline bool isar_feature_aa64_cssc(const ARMISARegisters *id) +{ + return FIELD_EX64_IDREG(id, ID_AA64ISAR2, CSSC) != 0; +} + static inline bool isar_feature_aa64_lut(const ARMISARegisters *id) { return FIELD_EX64_IDREG(id, ID_AA64ISAR2, LUT); @@ -904,6 +914,16 @@ static inline bool isar_feature_aa64_nv2(const ARMISARegisters *id) return FIELD_EX64_IDREG(id, ID_AA64MMFR2, NV) >= 2; } +static inline bool isar_feature_aa64_tcr2(const ARMISARegisters *id) +{ + return FIELD_EX64_IDREG(id, ID_AA64MMFR3, TCRX) != 0; +} + +static inline bool isar_feature_aa64_sctlr2(const ARMISARegisters *id) +{ + return FIELD_EX64_IDREG(id, ID_AA64MMFR3, SCTLRX) != 0; +} + static inline bool isar_feature_aa64_pmuv3p1(const ARMISARegisters *id) { return FIELD_EX64_IDREG(id, ID_AA64DFR0, PMUVER) >= 4 && diff --git a/target/arm/cpu.c b/target/arm/cpu.c index e2b2337399..d0f6fcdfce 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -142,11 +142,11 @@ static bool arm_cpu_has_work(CPUState *cs) ARMCPU *cpu = ARM_CPU(cs); return (cpu->power_state != PSCI_OFF) - && cs->interrupt_request & - (CPU_INTERRUPT_FIQ | CPU_INTERRUPT_HARD - | CPU_INTERRUPT_NMI | CPU_INTERRUPT_VINMI | CPU_INTERRUPT_VFNMI - | CPU_INTERRUPT_VFIQ | CPU_INTERRUPT_VIRQ | CPU_INTERRUPT_VSERR - | CPU_INTERRUPT_EXITTB); + && cpu_test_interrupt(cs, + CPU_INTERRUPT_FIQ | CPU_INTERRUPT_HARD + | CPU_INTERRUPT_NMI | CPU_INTERRUPT_VINMI | CPU_INTERRUPT_VFNMI + | CPU_INTERRUPT_VFIQ | CPU_INTERRUPT_VIRQ | CPU_INTERRUPT_VSERR + | CPU_INTERRUPT_EXITTB); } #endif /* !CONFIG_USER_ONLY */ @@ -644,6 +644,12 @@ void arm_emulate_firmware_reset(CPUState *cpustate, int target_el) if (cpu_isar_feature(aa64_fgt, cpu)) { env->cp15.scr_el3 |= SCR_FGTEN; } + if (cpu_isar_feature(aa64_tcr2, cpu)) { + env->cp15.scr_el3 |= SCR_TCR2EN; + } + if (cpu_isar_feature(aa64_sctlr2, cpu)) { + env->cp15.scr_el3 |= SCR_SCTLR2EN; + } } if (target_el == 2) { @@ -958,7 +964,7 @@ void arm_cpu_update_virq(ARMCPU *cpu) !(arm_hcrx_el2_eff(env) & HCRX_VINMI)) || (env->irq_line_state & CPU_INTERRUPT_VIRQ); - if (new_state != ((cs->interrupt_request & CPU_INTERRUPT_VIRQ) != 0)) { + if (new_state != cpu_test_interrupt(cs, CPU_INTERRUPT_VIRQ)) { if (new_state) { cpu_interrupt(cs, CPU_INTERRUPT_VIRQ); } else { @@ -980,7 +986,7 @@ void arm_cpu_update_vfiq(ARMCPU *cpu) !(arm_hcrx_el2_eff(env) & HCRX_VFNMI)) || (env->irq_line_state & CPU_INTERRUPT_VFIQ); - if (new_state != ((cs->interrupt_request & CPU_INTERRUPT_VFIQ) != 0)) { + if (new_state != cpu_test_interrupt(cs, CPU_INTERRUPT_VFIQ)) { if (new_state) { cpu_interrupt(cs, CPU_INTERRUPT_VFIQ); } else { @@ -1002,7 +1008,7 @@ void arm_cpu_update_vinmi(ARMCPU *cpu) (arm_hcrx_el2_eff(env) & HCRX_VINMI)) || (env->irq_line_state & CPU_INTERRUPT_VINMI); - if (new_state != ((cs->interrupt_request & CPU_INTERRUPT_VINMI) != 0)) { + if (new_state != cpu_test_interrupt(cs, CPU_INTERRUPT_VINMI)) { if (new_state) { cpu_interrupt(cs, CPU_INTERRUPT_VINMI); } else { @@ -1022,7 +1028,7 @@ void arm_cpu_update_vfnmi(ARMCPU *cpu) bool new_state = (arm_hcr_el2_eff(env) & HCR_VF) && (arm_hcrx_el2_eff(env) & HCRX_VFNMI); - if (new_state != ((cs->interrupt_request & CPU_INTERRUPT_VFNMI) != 0)) { + if (new_state != cpu_test_interrupt(cs, CPU_INTERRUPT_VFNMI)) { if (new_state) { cpu_interrupt(cs, CPU_INTERRUPT_VFNMI); } else { @@ -1041,7 +1047,7 @@ void arm_cpu_update_vserr(ARMCPU *cpu) bool new_state = env->cp15.hcr_el2 & HCR_VSE; - if (new_state != ((cs->interrupt_request & CPU_INTERRUPT_VSERR) != 0)) { + if (new_state != cpu_test_interrupt(cs, CPU_INTERRUPT_VSERR)) { if (new_state) { cpu_interrupt(cs, CPU_INTERRUPT_VSERR); } else { diff --git a/target/arm/cpu.h b/target/arm/cpu.h index dc9b6dce4c..c15d79a106 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -337,6 +337,7 @@ typedef struct CPUArchState { }; uint64_t sctlr_el[4]; }; + uint64_t sctlr2_el[4]; /* Extension to System control register. */ uint64_t vsctlr; /* Virtualization System control register. */ uint64_t cpacr_el1; /* Architectural feature access control register */ uint64_t cptr_el[4]; /* ARMv8 feature trap registers */ @@ -365,6 +366,7 @@ typedef struct CPUArchState { uint64_t vsttbr_el2; /* Secure Virtualization Translation Table. */ /* MMU translation table base control. */ uint64_t tcr_el[4]; + uint64_t tcr2_el[3]; uint64_t vtcr_el2; /* Virtualization Translation Control. */ uint64_t vstcr_el2; /* Secure Virtualization Translation Control. */ uint32_t c2_data; /* MPU data cacheable bits. */ @@ -1420,6 +1422,19 @@ void pmu_init(ARMCPU *cpu); #define SCTLR_SPINTMASK (1ULL << 62) /* FEAT_NMI */ #define SCTLR_TIDCP (1ULL << 63) /* FEAT_TIDCP1 */ +#define SCTLR2_EMEC (1ULL << 1) /* FEAT_MEC */ +#define SCTLR2_NMEA (1ULL << 2) /* FEAT_DoubleFault2 */ +#define SCTLR2_ENADERR (1ULL << 3) /* FEAT_ADERR */ +#define SCTLR2_ENANERR (1ULL << 4) /* FEAT_ANERR */ +#define SCTLR2_EASE (1ULL << 5) /* FEAT_DoubleFault2 */ +#define SCTLR2_ENIDCP128 (1ULL << 6) /* FEAT_SYSREG128 */ +#define SCTLR2_ENPACM (1ULL << 7) /* FEAT_PAuth_LR */ +#define SCTLR2_ENPACM0 (1ULL << 8) /* FEAT_PAuth_LR */ +#define SCTLR2_CPTA (1ULL << 9) /* FEAT_CPA2 */ +#define SCTLR2_CPTA0 (1ULL << 10) /* FEAT_CPA2 */ +#define SCTLR2_CPTM (1ULL << 11) /* FEAT_CPA2 */ +#define SCTLR2_CPTM0 (1ULL << 12) /* FEAT_CAP2 */ + #define CPSR_M (0x1fU) #define CPSR_T (1U << 5) #define CPSR_F (1U << 6) @@ -1712,6 +1727,8 @@ static inline void xpsr_write(CPUARMState *env, uint32_t val, uint32_t mask) #define SCR_HXEN (1ULL << 38) #define SCR_TRNDR (1ULL << 40) #define SCR_ENTP2 (1ULL << 41) +#define SCR_TCR2EN (1ULL << 43) +#define SCR_SCTLR2EN (1ULL << 44) #define SCR_GPF (1ULL << 48) #define SCR_NSE (1ULL << 62) diff --git a/target/arm/helper.c b/target/arm/helper.c index 0c1299ff84..19637e7301 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -741,6 +741,12 @@ static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) if (cpu_isar_feature(aa64_ecv, cpu)) { valid_mask |= SCR_ECVEN; } + if (cpu_isar_feature(aa64_tcr2, cpu)) { + valid_mask |= SCR_TCR2EN; + } + if (cpu_isar_feature(aa64_sctlr2, cpu)) { + valid_mask |= SCR_SCTLR2EN; + } } else { valid_mask &= ~(SCR_RW | SCR_ST); if (cpu_isar_feature(aa32_ras, cpu)) { @@ -833,40 +839,40 @@ static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri) uint64_t ret = 0; if (hcr_el2 & HCR_IMO) { - if (cs->interrupt_request & CPU_INTERRUPT_VIRQ) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_VIRQ)) { ret |= CPSR_I; } - if (cs->interrupt_request & CPU_INTERRUPT_VINMI) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_VINMI)) { ret |= ISR_IS; ret |= CPSR_I; } } else { - if (cs->interrupt_request & CPU_INTERRUPT_HARD) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_HARD)) { ret |= CPSR_I; } - if (cs->interrupt_request & CPU_INTERRUPT_NMI) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_NMI)) { ret |= ISR_IS; ret |= CPSR_I; } } if (hcr_el2 & HCR_FMO) { - if (cs->interrupt_request & CPU_INTERRUPT_VFIQ) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_VFIQ)) { ret |= CPSR_F; } - if (cs->interrupt_request & CPU_INTERRUPT_VFNMI) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_VFNMI)) { ret |= ISR_FS; ret |= CPSR_F; } } else { - if (cs->interrupt_request & CPU_INTERRUPT_FIQ) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_FIQ)) { ret |= CPSR_F; } } if (hcr_el2 & HCR_AMO) { - if (cs->interrupt_request & CPU_INTERRUPT_VSERR) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_VSERR)) { ret |= CPSR_A; } } @@ -3907,23 +3913,24 @@ static void hcrx_write(CPUARMState *env, const ARMCPRegInfo *ri, ARMCPU *cpu = env_archcpu(env); uint64_t valid_mask = 0; - /* FEAT_MOPS adds MSCEn and MCE2 */ if (cpu_isar_feature(aa64_mops, cpu)) { valid_mask |= HCRX_MSCEN | HCRX_MCE2; } - - /* FEAT_NMI adds TALLINT, VINMI and VFNMI */ if (cpu_isar_feature(aa64_nmi, cpu)) { valid_mask |= HCRX_TALLINT | HCRX_VINMI | HCRX_VFNMI; } - /* FEAT_CMOW adds CMOW */ if (cpu_isar_feature(aa64_cmow, cpu)) { valid_mask |= HCRX_CMOW; } - /* FEAT_XS adds FGTnXS, FnXS */ if (cpu_isar_feature(aa64_xs, cpu)) { valid_mask |= HCRX_FGTNXS | HCRX_FNXS; } + if (cpu_isar_feature(aa64_tcr2, cpu)) { + valid_mask |= HCRX_TCR2EN; + } + if (cpu_isar_feature(aa64_sctlr2, cpu)) { + valid_mask |= HCRX_SCTLR2EN; + } /* Clear RES0 bits. */ env->cp15.hcrx_el2 = value & valid_mask; @@ -3981,11 +3988,19 @@ uint64_t arm_hcrx_el2_eff(CPUARMState *env) * This may need to be revisited for future bits. */ if (!arm_is_el2_enabled(env)) { + ARMCPU *cpu = env_archcpu(env); uint64_t hcrx = 0; - if (cpu_isar_feature(aa64_mops, env_archcpu(env))) { - /* MSCEn behaves as 1 if EL2 is not enabled */ + + /* Bits which whose effective value is 1 if el2 not enabled. */ + if (cpu_isar_feature(aa64_mops, cpu)) { hcrx |= HCRX_MSCEN; } + if (cpu_isar_feature(aa64_tcr2, cpu)) { + hcrx |= HCRX_TCR2EN; + } + if (cpu_isar_feature(aa64_sctlr2, cpu)) { + hcrx |= HCRX_SCTLR2EN; + } return hcrx; } if (arm_feature(env, ARM_FEATURE_EL3) && !(env->cp15.scr_el3 & SCR_HXEN)) { @@ -4513,6 +4528,8 @@ static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu) static const struct E2HAlias aliases[] = { { K(3, 0, 1, 0, 0), K(3, 4, 1, 0, 0), K(3, 5, 1, 0, 0), "SCTLR", "SCTLR_EL2", "SCTLR_EL12" }, + { K(3, 0, 1, 0, 3), K(3, 4, 1, 0, 3), K(3, 5, 1, 0, 3), + "SCTLR2_EL1", "SCTLR2_EL2", "SCTLR2_EL12", isar_feature_aa64_sctlr2 }, { K(3, 0, 1, 0, 2), K(3, 4, 1, 1, 2), K(3, 5, 1, 0, 2), "CPACR", "CPTR_EL2", "CPACR_EL12" }, { K(3, 0, 2, 0, 0), K(3, 4, 2, 0, 0), K(3, 5, 2, 0, 0), @@ -4521,6 +4538,8 @@ static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu) "TTBR1_EL1", "TTBR1_EL2", "TTBR1_EL12" }, { K(3, 0, 2, 0, 2), K(3, 4, 2, 0, 2), K(3, 5, 2, 0, 2), "TCR_EL1", "TCR_EL2", "TCR_EL12" }, + { K(3, 0, 2, 0, 3), K(3, 4, 2, 0, 3), K(3, 5, 2, 0, 3), + "TCR2_EL1", "TCR2_EL2", "TCR2_EL12", isar_feature_aa64_tcr2 }, { K(3, 0, 4, 0, 0), K(3, 4, 4, 0, 0), K(3, 5, 4, 0, 0), "SPSR_EL1", "SPSR_EL2", "SPSR_EL12" }, { K(3, 0, 4, 0, 1), K(3, 4, 4, 0, 1), K(3, 5, 4, 0, 1), @@ -5994,6 +6013,133 @@ static const ARMCPRegInfo actlr2_hactlr2_reginfo[] = { .resetvalue = 0 }, }; +static CPAccessResult sctlr2_el2_access(CPUARMState *env, + const ARMCPRegInfo *ri, + bool isread) +{ + if (arm_current_el(env) < 3 + && arm_feature(env, ARM_FEATURE_EL3) + && !(env->cp15.scr_el3 & SCR_SCTLR2EN)) { + return CP_ACCESS_TRAP_EL3; + } + return CP_ACCESS_OK; +} + +static CPAccessResult sctlr2_el1_access(CPUARMState *env, + const ARMCPRegInfo *ri, + bool isread) +{ + CPAccessResult ret = access_tvm_trvm(env, ri, isread); + if (ret != CP_ACCESS_OK) { + return ret; + } + if (arm_current_el(env) < 2 && !(arm_hcrx_el2_eff(env) & HCRX_SCTLR2EN)) { + return CP_ACCESS_TRAP_EL2; + } + return sctlr2_el2_access(env, ri, isread); +} + +static void sctlr2_el1_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + uint64_t valid_mask = 0; + + value &= valid_mask; + raw_write(env, ri, value); +} + +static void sctlr2_el2_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + uint64_t valid_mask = 0; + + value &= valid_mask; + raw_write(env, ri, value); +} + +static void sctlr2_el3_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + uint64_t valid_mask = 0; + + value &= valid_mask; + raw_write(env, ri, value); +} + +static const ARMCPRegInfo sctlr2_reginfo[] = { + { .name = "SCTLR2_EL1", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 0, .opc2 = 3, .crn = 1, .crm = 0, + .access = PL1_RW, .accessfn = sctlr2_el1_access, + .writefn = sctlr2_el1_write, .fgt = FGT_SCTLR_EL1, + .nv2_redirect_offset = 0x278 | NV2_REDIR_NV1, + .fieldoffset = offsetof(CPUARMState, cp15.sctlr2_el[1]) }, + { .name = "SCTLR2_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .opc2 = 3, .crn = 1, .crm = 0, + .access = PL2_RW, .accessfn = sctlr2_el2_access, + .writefn = sctlr2_el2_write, + .fieldoffset = offsetof(CPUARMState, cp15.sctlr2_el[2]) }, + { .name = "SCTLR2_EL3", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 6, .opc2 = 3, .crn = 1, .crm = 0, + .access = PL3_RW, .writefn = sctlr2_el3_write, + .fieldoffset = offsetof(CPUARMState, cp15.sctlr2_el[3]) }, +}; + +static CPAccessResult tcr2_el2_access(CPUARMState *env, const ARMCPRegInfo *ri, + bool isread) +{ + if (arm_current_el(env) < 3 + && arm_feature(env, ARM_FEATURE_EL3) + && !(env->cp15.scr_el3 & SCR_TCR2EN)) { + return CP_ACCESS_TRAP_EL3; + } + return CP_ACCESS_OK; +} + +static CPAccessResult tcr2_el1_access(CPUARMState *env, const ARMCPRegInfo *ri, + bool isread) +{ + CPAccessResult ret = access_tvm_trvm(env, ri, isread); + if (ret != CP_ACCESS_OK) { + return ret; + } + if (arm_current_el(env) < 2 && !(arm_hcrx_el2_eff(env) & HCRX_TCR2EN)) { + return CP_ACCESS_TRAP_EL2; + } + return tcr2_el2_access(env, ri, isread); +} + +static void tcr2_el1_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + uint64_t valid_mask = 0; + + value &= valid_mask; + raw_write(env, ri, value); +} + +static void tcr2_el2_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + uint64_t valid_mask = 0; + + value &= valid_mask; + raw_write(env, ri, value); +} + +static const ARMCPRegInfo tcr2_reginfo[] = { + { .name = "TCR2_EL1", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 0, .opc2 = 3, .crn = 2, .crm = 0, + .access = PL1_RW, .accessfn = tcr2_el1_access, + .writefn = tcr2_el1_write, .fgt = FGT_TCR_EL1, + .nv2_redirect_offset = 0x270 | NV2_REDIR_NV1, + .fieldoffset = offsetof(CPUARMState, cp15.tcr2_el[1]) }, + { .name = "TCR2_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .opc2 = 3, .crn = 2, .crm = 0, + .access = PL2_RW, .accessfn = tcr2_el2_access, + .writefn = tcr2_el2_write, + .fieldoffset = offsetof(CPUARMState, cp15.tcr2_el[2]) }, +}; + void register_cp_regs_for_features(ARMCPU *cpu) { /* Register all the coprocessor registers based on feature bits */ @@ -7223,6 +7369,14 @@ void register_cp_regs_for_features(ARMCPU *cpu) define_arm_cp_regs(cpu, nmi_reginfo); } + if (cpu_isar_feature(aa64_sctlr2, cpu)) { + define_arm_cp_regs(cpu, sctlr2_reginfo); + } + + if (cpu_isar_feature(aa64_tcr2, cpu)) { + define_arm_cp_regs(cpu, tcr2_reginfo); + } + if (cpu_isar_feature(any_predinv, cpu)) { define_arm_cp_regs(cpu, predinv_reginfo); } @@ -9147,7 +9301,7 @@ void arm_cpu_do_interrupt(CPUState *cs) arm_call_el_change_hook(cpu); if (!kvm_enabled()) { - cs->interrupt_request |= CPU_INTERRUPT_EXITTB; + cpu_set_interrupt(cs, CPU_INTERRUPT_EXITTB); } } #endif /* !CONFIG_USER_ONLY */ diff --git a/target/arm/hvf/hvf.c b/target/arm/hvf/hvf.c index 47b0cd3a35..b77db99079 100644 --- a/target/arm/hvf/hvf.c +++ b/target/arm/hvf/hvf.c @@ -1782,13 +1782,13 @@ static int hvf_sysreg_write(CPUState *cpu, uint32_t reg, uint64_t val) static int hvf_inject_interrupts(CPUState *cpu) { - if (cpu->interrupt_request & CPU_INTERRUPT_FIQ) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_FIQ)) { trace_hvf_inject_fiq(); hv_vcpu_set_pending_interrupt(cpu->accel->fd, HV_INTERRUPT_TYPE_FIQ, true); } - if (cpu->interrupt_request & CPU_INTERRUPT_HARD) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) { trace_hvf_inject_irq(); hv_vcpu_set_pending_interrupt(cpu->accel->fd, HV_INTERRUPT_TYPE_IRQ, true); @@ -1840,7 +1840,7 @@ static void hvf_wfi(CPUState *cpu) uint64_t nanos; uint32_t cntfrq; - if (cpu->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_FIQ)) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD | CPU_INTERRUPT_FIQ)) { /* Interrupt pending, no need to wait */ return; } diff --git a/target/arm/internals.h b/target/arm/internals.h index 1b3d0244fd..f5a1e75db3 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -113,11 +113,6 @@ FIELD(DBGWCR, WT, 20, 1) FIELD(DBGWCR, MASK, 24, 5) FIELD(DBGWCR, SSCE, 29, 1) -#define VTCR_NSW (1u << 29) -#define VTCR_NSA (1u << 30) -#define VSTCR_SW VTCR_NSW -#define VSTCR_SA VTCR_NSA - /* Bit definitions for CPACR (AArch32 only) */ FIELD(CPACR, CP10, 20, 2) FIELD(CPACR, CP11, 22, 2) @@ -201,6 +196,24 @@ FIELD(CPTR_EL3, TCPAC, 31, 1) #define TTBCR_SH1 (1U << 28) #define TTBCR_EAE (1U << 31) +#define TCR2_PNCH (1ULL << 0) +#define TCR2_PIE (1ULL << 1) +#define TCR2_E0POE (1ULL << 2) +#define TCR2_POE (1ULL << 3) +#define TCR2_AIE (1ULL << 4) +#define TCR2_D128 (1ULL << 5) +#define TCR2_PTTWI (1ULL << 10) +#define TCR2_HAFT (1ULL << 11) +#define TCR2_AMEC0 (1ULL << 12) +#define TCR2_AMEC1 (1ULL << 13) +#define TCR2_DISCH0 (1ULL << 14) +#define TCR2_DISCH1 (1ULL << 15) +#define TCR2_A2 (1ULL << 16) +#define TCR2_FNG0 (1ULL << 17) +#define TCR2_FNG1 (1ULL << 18) +#define TCR2_FNGNA0 (1ULL << 20) +#define TCR2_FNGNA1 (1ULL << 21) + FIELD(VTCR, T0SZ, 0, 6) FIELD(VTCR, SL0, 6, 2) FIELD(VTCR, IRGN0, 8, 2) @@ -220,6 +233,9 @@ FIELD(VTCR, NSA, 30, 1) FIELD(VTCR, DS, 32, 1) FIELD(VTCR, SL2, 33, 1) +FIELD(VSTCR, SW, 29, 1) +FIELD(VSTCR, SA, 30, 1) + #define HCRX_ENAS0 (1ULL << 0) #define HCRX_ENALS (1ULL << 1) #define HCRX_ENASR (1ULL << 2) @@ -232,6 +248,8 @@ FIELD(VTCR, SL2, 33, 1) #define HCRX_CMOW (1ULL << 9) #define HCRX_MCE2 (1ULL << 10) #define HCRX_MSCEN (1ULL << 11) +#define HCRX_TCR2EN (1ULL << 14) +#define HCRX_SCTLR2EN (1ULL << 15) #define HPFAR_NS (1ULL << 63) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 561bf2678e..ed5c728eab 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -193,9 +193,9 @@ static ARMMMUIdx ptw_idx_for_stage_2(CPUARMState *env, ARMMMUIdx stage2idx) return ARMMMUIdx_Phys_Realm; case ARMSS_Secure: if (stage2idx == ARMMMUIdx_Stage2_S) { - s2walk_secure = !(env->cp15.vstcr_el2 & VSTCR_SW); + s2walk_secure = !(env->cp15.vstcr_el2 & R_VSTCR_SW_MASK); } else { - s2walk_secure = !(env->cp15.vtcr_el2 & VTCR_NSW); + s2walk_secure = !(env->cp15.vtcr_el2 & R_VTCR_NSW_MASK); } return s2walk_secure ? ARMMMUIdx_Phys_S : ARMMMUIdx_Phys_NS; default: @@ -3372,9 +3372,9 @@ static bool get_phys_addr_twostage(CPUARMState *env, S1Translate *ptw, */ if (in_space == ARMSS_Secure) { result->f.attrs.secure = - !(env->cp15.vstcr_el2 & (VSTCR_SA | VSTCR_SW)) + !(env->cp15.vstcr_el2 & (R_VSTCR_SA_MASK | R_VSTCR_SW_MASK)) && (ipa_secure - || !(env->cp15.vtcr_el2 & (VTCR_NSA | VTCR_NSW))); + || !(env->cp15.vtcr_el2 & (R_VTCR_NSA_MASK | R_VTCR_NSW_MASK))); result->f.attrs.space = arm_secure_to_space(result->f.attrs.secure); } diff --git a/target/arm/tcg/a64.decode b/target/arm/tcg/a64.decode index 8c798cde2b..55ff6c504f 100644 --- a/target/arm/tcg/a64.decode +++ b/target/arm/tcg/a64.decode @@ -156,6 +156,16 @@ MOVZ . 10 100101 .. ................ ..... @movw_32 MOVK . 11 100101 .. ................ ..... @movw_64 MOVK . 11 100101 .. ................ ..... @movw_32 +# Min/Max (immediate) + +@minmaxi_s sf:1 .. ........... imm:s8 rn:5 rd:5 &rri_sf +@minmaxi_u sf:1 .. ........... imm:8 rn:5 rd:5 &rri_sf + +SMAX_i . 00 1000111 0000 ........ ..... ..... @minmaxi_s +SMIN_i . 00 1000111 0010 ........ ..... ..... @minmaxi_s +UMAX_i . 00 1000111 0001 ........ ..... ..... @minmaxi_u +UMIN_i . 00 1000111 0011 ........ ..... ..... @minmaxi_u + # Bitfield &bitfield rd rn sf immr imms @@ -536,6 +546,13 @@ SWP .. 111 0 00 . . 1 ..... 1000 00 ..... ..... @atomic LDAPR sz:2 111 0 00 1 0 1 11111 1100 00 rn:5 rt:5 +# Atomic 128-bit memory operations +&atomic128 rn rt rt2 a r +@atomic128 ........ a:1 r:1 . rt2:5 ...... rn:5 rt:5 &atomic128 +LDCLRP 00011001 . . 1 ..... 000100 ..... ..... @atomic128 +LDSETP 00011001 . . 1 ..... 001100 ..... ..... @atomic128 +SWPP 00011001 . . 1 ..... 100000 ..... ..... @atomic128 + # Load/store register (pointer authentication) # LDRA immediate is 10 bits signed and scaled, but the bits aren't all contiguous @@ -698,6 +715,11 @@ GMI 1 00 11010110 ..... 000101 ..... ..... @rrr PACGA 1 00 11010110 ..... 001100 ..... ..... @rrr +SMAX . 00 11010110 ..... 011000 ..... ..... @rrr_sf +SMIN . 00 11010110 ..... 011010 ..... ..... @rrr_sf +UMAX . 00 11010110 ..... 011001 ..... ..... @rrr_sf +UMIN . 00 11010110 ..... 011011 ..... ..... @rrr_sf + # Data Processing (1-source) @rr . .......... ..... ...... rn:5 rd:5 &rr @@ -711,6 +733,10 @@ REV64 1 10 11010110 00000 000011 ..... ..... @rr CLZ . 10 11010110 00000 000100 ..... ..... @rr_sf CLS . 10 11010110 00000 000101 ..... ..... @rr_sf +CTZ . 10 11010110 00000 000110 ..... ..... @rr_sf +CNT . 10 11010110 00000 000111 ..... ..... @rr_sf +ABS . 10 11010110 00000 001000 ..... ..... @rr_sf + &pacaut rd rn z @pacaut . .. ........ ..... .. z:1 ... rn:5 rd:5 &pacaut diff --git a/target/arm/tcg/cpu64.c b/target/arm/tcg/cpu64.c index 35cddbafa4..b8b1981e70 100644 --- a/target/arm/tcg/cpu64.c +++ b/target/arm/tcg/cpu64.c @@ -1145,7 +1145,7 @@ void aarch64_max_tcg_initfn(Object *obj) t = FIELD_DP64(t, ID_AA64ISAR0, SHA1, 1); /* FEAT_SHA1 */ t = FIELD_DP64(t, ID_AA64ISAR0, SHA2, 2); /* FEAT_SHA512 */ t = FIELD_DP64(t, ID_AA64ISAR0, CRC32, 1); /* FEAT_CRC32 */ - t = FIELD_DP64(t, ID_AA64ISAR0, ATOMIC, 2); /* FEAT_LSE */ + t = FIELD_DP64(t, ID_AA64ISAR0, ATOMIC, 3); /* FEAT_LSE, FEAT_LSE128 */ t = FIELD_DP64(t, ID_AA64ISAR0, RDM, 1); /* FEAT_RDM */ t = FIELD_DP64(t, ID_AA64ISAR0, SHA3, 1); /* FEAT_SHA3 */ t = FIELD_DP64(t, ID_AA64ISAR0, SM3, 1); /* FEAT_SM3 */ @@ -1178,6 +1178,7 @@ void aarch64_max_tcg_initfn(Object *obj) t = FIELD_DP64(t, ID_AA64ISAR2, MOPS, 1); /* FEAT_MOPS */ t = FIELD_DP64(t, ID_AA64ISAR2, BC, 1); /* FEAT_HBC */ t = FIELD_DP64(t, ID_AA64ISAR2, WFXT, 2); /* FEAT_WFxT */ + t = FIELD_DP64(t, ID_AA64ISAR2, CSSC, 1); /* FEAT_CSSC */ SET_IDREG(isar, ID_AA64ISAR2, t); t = GET_IDREG(isar, ID_AA64PFR0); @@ -1247,7 +1248,11 @@ void aarch64_max_tcg_initfn(Object *obj) t = FIELD_DP64(t, ID_AA64MMFR2, E0PD, 1); /* FEAT_E0PD */ SET_IDREG(isar, ID_AA64MMFR2, t); - FIELD_DP64_IDREG(isar, ID_AA64MMFR3, SPEC_FPACC, 1); /* FEAT_FPACC_SPEC */ + t = GET_IDREG(isar, ID_AA64MMFR3); + t = FIELD_DP64(t, ID_AA64MMFR3, TCRX, 1); /* FEAT_TCR2 */ + t = FIELD_DP64(t, ID_AA64MMFR3, SCTLRX, 1); /* FEAT_SCTLR2 */ + t = FIELD_DP64(t, ID_AA64MMFR3, SPEC_FPACC, 1); /* FEAT_FPACC_SPEC */ + SET_IDREG(isar, ID_AA64MMFR3, t); t = GET_IDREG(isar, ID_AA64ZFR0); t = FIELD_DP64(t, ID_AA64ZFR0, SVEVER, 2); /* FEAT_SVE2p1 */ diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index dbf47595db..37bedc3780 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -3237,7 +3237,7 @@ static bool trans_LDXP(DisasContext *s, arg_stxr *a) static bool trans_CASP(DisasContext *s, arg_CASP *a) { - if (!dc_isar_feature(aa64_atomics, s)) { + if (!dc_isar_feature(aa64_lse, s)) { return false; } if (((a->rt | a->rs) & 1) != 0) { @@ -3250,7 +3250,7 @@ static bool trans_CASP(DisasContext *s, arg_CASP *a) static bool trans_CAS(DisasContext *s, arg_CAS *a) { - if (!dc_isar_feature(aa64_atomics, s)) { + if (!dc_isar_feature(aa64_lse, s)) { return false; } gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz); @@ -3743,15 +3743,64 @@ static bool do_atomic_ld(DisasContext *s, arg_atomic *a, AtomicThreeOpFn *fn, return true; } -TRANS_FEAT(LDADD, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_add_i64, 0, false) -TRANS_FEAT(LDCLR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_and_i64, 0, true) -TRANS_FEAT(LDEOR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_xor_i64, 0, false) -TRANS_FEAT(LDSET, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_or_i64, 0, false) -TRANS_FEAT(LDSMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smax_i64, MO_SIGN, false) -TRANS_FEAT(LDSMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smin_i64, MO_SIGN, false) -TRANS_FEAT(LDUMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umax_i64, 0, false) -TRANS_FEAT(LDUMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umin_i64, 0, false) -TRANS_FEAT(SWP, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_xchg_i64, 0, false) +TRANS_FEAT(LDADD, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_add_i64, 0, false) +TRANS_FEAT(LDCLR, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_and_i64, 0, true) +TRANS_FEAT(LDEOR, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_xor_i64, 0, false) +TRANS_FEAT(LDSET, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_or_i64, 0, false) +TRANS_FEAT(LDSMAX, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_smax_i64, MO_SIGN, false) +TRANS_FEAT(LDSMIN, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_smin_i64, MO_SIGN, false) +TRANS_FEAT(LDUMAX, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_umax_i64, 0, false) +TRANS_FEAT(LDUMIN, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_umin_i64, 0, false) +TRANS_FEAT(SWP, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_xchg_i64, 0, false) + +typedef void Atomic128ThreeOpFn(TCGv_i128, TCGv_i64, TCGv_i128, TCGArg, MemOp); + +static bool do_atomic128_ld(DisasContext *s, arg_atomic128 *a, + Atomic128ThreeOpFn *fn, bool invert) +{ + MemOp mop; + int rlo, rhi; + TCGv_i64 clean_addr, tlo, thi; + TCGv_i128 t16; + + if (a->rt == 31 || a->rt2 == 31 || a->rt == a->rt2) { + return false; + } + if (a->rn == 31) { + gen_check_sp_alignment(s); + } + mop = check_atomic_align(s, a->rn, MO_128); + clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, + a->rn != 31, mop); + + rlo = (s->be_data == MO_LE ? a->rt : a->rt2); + rhi = (s->be_data == MO_LE ? a->rt2 : a->rt); + + tlo = read_cpu_reg(s, rlo, true); + thi = read_cpu_reg(s, rhi, true); + if (invert) { + tcg_gen_not_i64(tlo, tlo); + tcg_gen_not_i64(thi, thi); + } + /* + * The tcg atomic primitives are all full barriers. Therefore we + * can ignore the Acquire and Release bits of this instruction. + */ + t16 = tcg_temp_new_i128(); + tcg_gen_concat_i64_i128(t16, tlo, thi); + + fn(t16, clean_addr, t16, get_mem_index(s), mop); + + tcg_gen_extr_i128_i64(cpu_reg(s, rlo), cpu_reg(s, rhi), t16); + return true; +} + +TRANS_FEAT(LDCLRP, aa64_lse128, do_atomic128_ld, + a, tcg_gen_atomic_fetch_and_i128, true) +TRANS_FEAT(LDSETP, aa64_lse128, do_atomic128_ld, + a, tcg_gen_atomic_fetch_or_i128, false) +TRANS_FEAT(SWPP, aa64_lse128, do_atomic128_ld, + a, tcg_gen_atomic_xchg_i128, false) static bool trans_LDAPR(DisasContext *s, arg_LDAPR *a) { @@ -3759,7 +3808,7 @@ static bool trans_LDAPR(DisasContext *s, arg_LDAPR *a) TCGv_i64 clean_addr; MemOp mop; - if (!dc_isar_feature(aa64_atomics, s) || + if (!dc_isar_feature(aa64_lse, s) || !dc_isar_feature(aa64_rcpc_8_3, s)) { return false; } @@ -4553,6 +4602,50 @@ TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC) TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC) /* + * Min/Max (immediate) + */ + +static void gen_wrap3_i32(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, NeonGenTwoOpFn fn) +{ + TCGv_i32 t1 = tcg_temp_new_i32(); + TCGv_i32 t2 = tcg_temp_new_i32(); + + tcg_gen_extrl_i64_i32(t1, n); + tcg_gen_extrl_i64_i32(t2, m); + fn(t1, t1, t2); + tcg_gen_extu_i32_i64(d, t1); +} + +static void gen_smax32_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) +{ + gen_wrap3_i32(d, n, m, tcg_gen_smax_i32); +} + +static void gen_smin32_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) +{ + gen_wrap3_i32(d, n, m, tcg_gen_smin_i32); +} + +static void gen_umax32_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) +{ + gen_wrap3_i32(d, n, m, tcg_gen_umax_i32); +} + +static void gen_umin32_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) +{ + gen_wrap3_i32(d, n, m, tcg_gen_umin_i32); +} + +TRANS_FEAT(SMAX_i, aa64_cssc, gen_rri, a, 0, 0, + a->sf ? tcg_gen_smax_i64 : gen_smax32_i64) +TRANS_FEAT(SMIN_i, aa64_cssc, gen_rri, a, 0, 0, + a->sf ? tcg_gen_smin_i64 : gen_smin32_i64) +TRANS_FEAT(UMAX_i, aa64_cssc, gen_rri, a, 0, 0, + a->sf ? tcg_gen_umax_i64 : gen_umax32_i64) +TRANS_FEAT(UMIN_i, aa64_cssc, gen_rri, a, 0, 0, + a->sf ? tcg_gen_umin_i64 : gen_umin32_i64) + +/* * Add/subtract (immediate, with tags) */ @@ -8157,6 +8250,28 @@ static bool trans_PACGA(DisasContext *s, arg_rrr *a) return false; } +static bool gen_rrr(DisasContext *s, arg_rrr_sf *a, ArithTwoOp fn) +{ + TCGv_i64 tcg_rm = cpu_reg(s, a->rm); + TCGv_i64 tcg_rn = cpu_reg(s, a->rn); + TCGv_i64 tcg_rd = cpu_reg(s, a->rd); + + fn(tcg_rd, tcg_rn, tcg_rm); + if (!a->sf) { + tcg_gen_ext32u_i64(tcg_rd, tcg_rd); + } + return true; +} + +TRANS_FEAT(SMAX, aa64_cssc, gen_rrr, a, + a->sf ? tcg_gen_smax_i64 : gen_smax32_i64) +TRANS_FEAT(SMIN, aa64_cssc, gen_rrr, a, + a->sf ? tcg_gen_smin_i64 : gen_smin32_i64) +TRANS_FEAT(UMAX, aa64_cssc, gen_rrr, a, + a->sf ? tcg_gen_umax_i64 : gen_umax32_i64) +TRANS_FEAT(UMIN, aa64_cssc, gen_rrr, a, + a->sf ? tcg_gen_umin_i64 : gen_umin32_i64) + typedef void ArithOneOp(TCGv_i64, TCGv_i64); static bool gen_rr(DisasContext *s, int rd, int rn, ArithOneOp fn) @@ -8165,13 +8280,22 @@ static bool gen_rr(DisasContext *s, int rd, int rn, ArithOneOp fn) return true; } -static void gen_rbit32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +/* + * Perform 32-bit operation fn on the low half of n; + * the high half of the output is zeroed. + */ +static void gen_wrap2_i32(TCGv_i64 d, TCGv_i64 n, NeonGenOneOpFn fn) { - TCGv_i32 t32 = tcg_temp_new_i32(); + TCGv_i32 t = tcg_temp_new_i32(); - tcg_gen_extrl_i64_i32(t32, tcg_rn); - gen_helper_rbit(t32, t32); - tcg_gen_extu_i32_i64(tcg_rd, t32); + tcg_gen_extrl_i64_i32(t, n); + fn(t, t); + tcg_gen_extu_i32_i64(d, t); +} + +static void gen_rbit32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ + gen_wrap2_i32(tcg_rd, tcg_rn, gen_helper_rbit); } static void gen_rev16_xx(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn, TCGv_i64 mask) @@ -8227,15 +8351,42 @@ static void gen_clz64(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) static void gen_cls32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) { + gen_wrap2_i32(tcg_rd, tcg_rn, tcg_gen_clrsb_i32); +} + +TRANS(CLZ, gen_rr, a->rd, a->rn, a->sf ? gen_clz64 : gen_clz32) +TRANS(CLS, gen_rr, a->rd, a->rn, a->sf ? tcg_gen_clrsb_i64 : gen_cls32) + +static void gen_ctz32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ TCGv_i32 t32 = tcg_temp_new_i32(); tcg_gen_extrl_i64_i32(t32, tcg_rn); - tcg_gen_clrsb_i32(t32, t32); + tcg_gen_ctzi_i32(t32, t32, 32); tcg_gen_extu_i32_i64(tcg_rd, t32); } -TRANS(CLZ, gen_rr, a->rd, a->rn, a->sf ? gen_clz64 : gen_clz32) -TRANS(CLS, gen_rr, a->rd, a->rn, a->sf ? tcg_gen_clrsb_i64 : gen_cls32) +static void gen_ctz64(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ + tcg_gen_ctzi_i64(tcg_rd, tcg_rn, 64); +} + +static void gen_cnt32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ + gen_wrap2_i32(tcg_rd, tcg_rn, tcg_gen_ctpop_i32); +} + +static void gen_abs32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ + gen_wrap2_i32(tcg_rd, tcg_rn, tcg_gen_abs_i32); +} + +TRANS_FEAT(CTZ, aa64_cssc, gen_rr, a->rd, a->rn, + a->sf ? gen_ctz64 : gen_ctz32) +TRANS_FEAT(CNT, aa64_cssc, gen_rr, a->rd, a->rn, + a->sf ? tcg_gen_ctpop_i64 : gen_cnt32) +TRANS_FEAT(ABS, aa64_cssc, gen_rr, a->rd, a->rn, + a->sf ? tcg_gen_abs_i64 : gen_abs32) static bool gen_pacaut(DisasContext *s, arg_pacaut *a, NeonGenTwo64OpEnvFn fn) { diff --git a/target/avr/cpu.c b/target/avr/cpu.c index 6995de6a12..a6df71d020 100644 --- a/target/avr/cpu.c +++ b/target/avr/cpu.c @@ -45,7 +45,7 @@ static vaddr avr_cpu_get_pc(CPUState *cs) static bool avr_cpu_has_work(CPUState *cs) { - return (cs->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_RESET)) + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD | CPU_INTERRUPT_RESET) && cpu_interrupts_enabled(cpu_env(cs)); } diff --git a/target/hppa/cpu.c b/target/hppa/cpu.c index 24777727e6..0ca79ee5e2 100644 --- a/target/hppa/cpu.c +++ b/target/hppa/cpu.c @@ -135,7 +135,7 @@ static void hppa_restore_state_to_opc(CPUState *cs, #ifndef CONFIG_USER_ONLY static bool hppa_cpu_has_work(CPUState *cs) { - return cs->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI); + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI); } #endif /* !CONFIG_USER_ONLY */ diff --git a/target/i386/hvf/hvf.c b/target/i386/hvf/hvf.c index 818b50419f..8445cadece 100644 --- a/target/i386/hvf/hvf.c +++ b/target/i386/hvf/hvf.c @@ -773,9 +773,9 @@ int hvf_vcpu_exec(CPUState *cpu) switch (exit_reason) { case EXIT_REASON_HLT: { macvm_set_rip(cpu, rip + ins_len); - if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) && + if (!(cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD) && (env->eflags & IF_MASK)) - && !(cpu->interrupt_request & CPU_INTERRUPT_NMI) && + && !cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI) && !(idtvec_info & VMCS_IDT_VEC_VALID)) { cpu->halted = 1; ret = EXCP_HLT; diff --git a/target/i386/hvf/x86hvf.c b/target/i386/hvf/x86hvf.c index 17fce1d3cd..9e05e0e576 100644 --- a/target/i386/hvf/x86hvf.c +++ b/target/i386/hvf/x86hvf.c @@ -395,7 +395,7 @@ bool hvf_inject_interrupts(CPUState *cs) }; } - if (cs->interrupt_request & CPU_INTERRUPT_NMI) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_NMI)) { if (!(env->hflags2 & HF2_NMI_MASK) && !(info & VMCS_INTR_VALID)) { cs->interrupt_request &= ~CPU_INTERRUPT_NMI; info = VMCS_INTR_VALID | VMCS_INTR_T_NMI | EXCP02_NMI; @@ -406,7 +406,7 @@ bool hvf_inject_interrupts(CPUState *cs) } if (!(env->hflags & HF_INHIBIT_IRQ_MASK) && - (cs->interrupt_request & CPU_INTERRUPT_HARD) && + cpu_test_interrupt(cs, CPU_INTERRUPT_HARD) && (env->eflags & IF_MASK) && !(info & VMCS_INTR_VALID)) { int line = cpu_get_pic_interrupt(env); cs->interrupt_request &= ~CPU_INTERRUPT_HARD; @@ -415,11 +415,10 @@ bool hvf_inject_interrupts(CPUState *cs) VMCS_INTR_VALID | VMCS_INTR_T_HWINTR); } } - if (cs->interrupt_request & CPU_INTERRUPT_HARD) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_HARD)) { vmx_set_int_window_exiting(cs); } - return (cs->interrupt_request - & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)); + return cpu_test_interrupt(cs, CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR); } int hvf_process_events(CPUState *cs) @@ -432,25 +431,25 @@ int hvf_process_events(CPUState *cs) env->eflags = rreg(cs->accel->fd, HV_X86_RFLAGS); } - if (cs->interrupt_request & CPU_INTERRUPT_INIT) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_INIT)) { cpu_synchronize_state(cs); do_cpu_init(cpu); } - if (cs->interrupt_request & CPU_INTERRUPT_POLL) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_POLL)) { cs->interrupt_request &= ~CPU_INTERRUPT_POLL; apic_poll_irq(cpu->apic_state); } - if (((cs->interrupt_request & CPU_INTERRUPT_HARD) && + if ((cpu_test_interrupt(cs, CPU_INTERRUPT_HARD) && (env->eflags & IF_MASK)) || - (cs->interrupt_request & CPU_INTERRUPT_NMI)) { + cpu_test_interrupt(cs, CPU_INTERRUPT_NMI)) { cs->halted = 0; } - if (cs->interrupt_request & CPU_INTERRUPT_SIPI) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_SIPI)) { cpu_synchronize_state(cs); do_cpu_sipi(cpu); } - if (cs->interrupt_request & CPU_INTERRUPT_TPR) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_TPR)) { cs->interrupt_request &= ~CPU_INTERRUPT_TPR; cpu_synchronize_state(cs); apic_handle_tpr_access_report(cpu->apic_state, env->eip, diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 369626f8c8..306430a052 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -5453,8 +5453,8 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run) int ret; /* Inject NMI */ - if (cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) { - if (cpu->interrupt_request & CPU_INTERRUPT_NMI) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) { bql_lock(); cpu->interrupt_request &= ~CPU_INTERRUPT_NMI; bql_unlock(); @@ -5465,7 +5465,7 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run) strerror(-ret)); } } - if (cpu->interrupt_request & CPU_INTERRUPT_SMI) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_SMI)) { bql_lock(); cpu->interrupt_request &= ~CPU_INTERRUPT_SMI; bql_unlock(); @@ -5478,31 +5478,30 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run) } } - if (!kvm_pic_in_kernel()) { - bql_lock(); - } /* Force the VCPU out of its inner loop to process any INIT requests * or (for userspace APIC, but it is cheap to combine the checks here) * pending TPR access reports. */ - if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { - if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) && + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT) && !(env->hflags & HF_SMM_MASK)) { - cpu->exit_request = 1; + qatomic_set(&cpu->exit_request, 1); } - if (cpu->interrupt_request & CPU_INTERRUPT_TPR) { - cpu->exit_request = 1; + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_TPR)) { + qatomic_set(&cpu->exit_request, 1); } } if (!kvm_pic_in_kernel()) { /* Try to inject an interrupt if the guest can accept it */ if (run->ready_for_interrupt_injection && - (cpu->interrupt_request & CPU_INTERRUPT_HARD) && + cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD) && (env->eflags & IF_MASK)) { int irq; + bql_lock(); + cpu->interrupt_request &= ~CPU_INTERRUPT_HARD; irq = cpu_get_pic_interrupt(env); if (irq >= 0) { @@ -5517,13 +5516,14 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run) strerror(-ret)); } } + bql_unlock(); } /* If we have an interrupt but the guest is not ready to receive an * interrupt, request an interrupt window exit. This will * cause a return to userspace as soon as the guest is ready to * receive interrupts. */ - if ((cpu->interrupt_request & CPU_INTERRUPT_HARD)) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) { run->request_interrupt_window = 1; } else { run->request_interrupt_window = 0; @@ -5531,8 +5531,6 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run) DPRINTF("setting tpr\n"); run->cr8 = cpu_get_apic_tpr(x86_cpu->apic_state); - - bql_unlock(); } } @@ -5595,7 +5593,7 @@ int kvm_arch_process_async_events(CPUState *cs) X86CPU *cpu = X86_CPU(cs); CPUX86State *env = &cpu->env; - if (cs->interrupt_request & CPU_INTERRUPT_MCE) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_MCE)) { /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */ assert(env->mcg_cap); @@ -5618,7 +5616,7 @@ int kvm_arch_process_async_events(CPUState *cs) } } - if ((cs->interrupt_request & CPU_INTERRUPT_INIT) && + if (cpu_test_interrupt(cs, CPU_INTERRUPT_INIT) && !(env->hflags & HF_SMM_MASK)) { kvm_cpu_synchronize_state(cs); do_cpu_init(cpu); @@ -5628,20 +5626,20 @@ int kvm_arch_process_async_events(CPUState *cs) return 0; } - if (cs->interrupt_request & CPU_INTERRUPT_POLL) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_POLL)) { cs->interrupt_request &= ~CPU_INTERRUPT_POLL; apic_poll_irq(cpu->apic_state); } - if (((cs->interrupt_request & CPU_INTERRUPT_HARD) && + if ((cpu_test_interrupt(cs, CPU_INTERRUPT_HARD) && (env->eflags & IF_MASK)) || - (cs->interrupt_request & CPU_INTERRUPT_NMI)) { + cpu_test_interrupt(cs, CPU_INTERRUPT_NMI)) { cs->halted = 0; } - if (cs->interrupt_request & CPU_INTERRUPT_SIPI) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_SIPI)) { kvm_cpu_synchronize_state(cs); do_cpu_sipi(cpu); } - if (cs->interrupt_request & CPU_INTERRUPT_TPR) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_TPR)) { cs->interrupt_request &= ~CPU_INTERRUPT_TPR; kvm_cpu_synchronize_state(cs); apic_handle_tpr_access_report(cpu->apic_state, env->eip, @@ -5656,9 +5654,9 @@ static int kvm_handle_halt(X86CPU *cpu) CPUState *cs = CPU(cpu); CPUX86State *env = &cpu->env; - if (!((cs->interrupt_request & CPU_INTERRUPT_HARD) && + if (!(cpu_test_interrupt(cs, CPU_INTERRUPT_HARD) && (env->eflags & IF_MASK)) && - !(cs->interrupt_request & CPU_INTERRUPT_NMI)) { + !cpu_test_interrupt(cs, CPU_INTERRUPT_NMI)) { cs->halted = 1; return EXCP_HLT; } diff --git a/target/i386/kvm/vmsr_energy.c b/target/i386/kvm/vmsr_energy.c index 58ce3df53a..890322ae37 100644 --- a/target/i386/kvm/vmsr_energy.c +++ b/target/i386/kvm/vmsr_energy.c @@ -57,13 +57,9 @@ QIOChannelSocket *vmsr_open_socket(const char *path) }; QIOChannelSocket *sioc = qio_channel_socket_new(); - Error *local_err = NULL; qio_channel_set_name(QIO_CHANNEL(sioc), "vmsr-helper"); - qio_channel_socket_connect_sync(sioc, - &saddr, - &local_err); - if (local_err) { + if (qio_channel_socket_connect_sync(sioc, &saddr, NULL) < 0) { /* Close socket. */ qio_channel_close(QIO_CHANNEL(sioc), NULL); object_unref(OBJECT(sioc)); diff --git a/target/i386/machine.c b/target/i386/machine.c index dd2dac1d44..45b7cea80a 100644 --- a/target/i386/machine.c +++ b/target/i386/machine.c @@ -462,6 +462,24 @@ static const VMStateDescription vmstate_exception_info = { } }; +static bool cpu_errcode_needed(void *opaque) +{ + X86CPU *cpu = opaque; + + return cpu->env.has_error_code != 0; +} + +static const VMStateDescription vmstate_error_code = { + .name = "cpu/error_code", + .version_id = 1, + .minimum_version_id = 1, + .needed = cpu_errcode_needed, + .fields = (const VMStateField[]) { + VMSTATE_INT32(env.error_code, X86CPU), + VMSTATE_END_OF_LIST() + } +}; + /* Poll control MSR enabled by default */ static bool poll_control_msr_needed(void *opaque) { @@ -1746,6 +1764,7 @@ const VMStateDescription vmstate_x86_cpu = { }, .subsections = (const VMStateDescription * const []) { &vmstate_exception_info, + &vmstate_error_code, &vmstate_async_pf_msr, &vmstate_async_pf_int_msr, &vmstate_pv_eoi_msr, diff --git a/target/i386/nvmm/nvmm-all.c b/target/i386/nvmm/nvmm-all.c index 92e3b8b2f4..c1ac74c4f0 100644 --- a/target/i386/nvmm/nvmm-all.c +++ b/target/i386/nvmm/nvmm-all.c @@ -413,11 +413,11 @@ nvmm_vcpu_pre_run(CPUState *cpu) * Force the VCPU out of its inner loop to process any INIT requests * or commit pending TPR access. */ - if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { cpu->exit_request = 1; } - if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_NMI)) { + if (!has_event && cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) { if (nvmm_can_take_nmi(cpu)) { cpu->interrupt_request &= ~CPU_INTERRUPT_NMI; event->type = NVMM_VCPU_EVENT_INTR; @@ -426,7 +426,7 @@ nvmm_vcpu_pre_run(CPUState *cpu) } } - if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_HARD)) { + if (!has_event && cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) { if (nvmm_can_take_int(cpu)) { cpu->interrupt_request &= ~CPU_INTERRUPT_HARD; event->type = NVMM_VCPU_EVENT_INTR; @@ -436,7 +436,7 @@ nvmm_vcpu_pre_run(CPUState *cpu) } /* Don't want SMIs. */ - if (cpu->interrupt_request & CPU_INTERRUPT_SMI) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_SMI)) { cpu->interrupt_request &= ~CPU_INTERRUPT_SMI; } @@ -651,9 +651,9 @@ nvmm_handle_halted(struct nvmm_machine *mach, CPUState *cpu, bql_lock(); - if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) && + if (!(cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD) && (cpu_env(cpu)->eflags & IF_MASK)) && - !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) { + !cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) { cpu->exception_index = EXCP_HLT; cpu->halted = true; ret = 1; @@ -691,25 +691,25 @@ nvmm_vcpu_loop(CPUState *cpu) * Some asynchronous events must be handled outside of the inner * VCPU loop. They are handled here. */ - if (cpu->interrupt_request & CPU_INTERRUPT_INIT) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT)) { nvmm_cpu_synchronize_state(cpu); do_cpu_init(x86_cpu); /* set int/nmi windows back to the reset state */ } - if (cpu->interrupt_request & CPU_INTERRUPT_POLL) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_POLL)) { cpu->interrupt_request &= ~CPU_INTERRUPT_POLL; apic_poll_irq(x86_cpu->apic_state); } - if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) && + if ((cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD) && (env->eflags & IF_MASK)) || - (cpu->interrupt_request & CPU_INTERRUPT_NMI)) { + cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) { cpu->halted = false; } - if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_SIPI)) { nvmm_cpu_synchronize_state(cpu); do_cpu_sipi(x86_cpu); } - if (cpu->interrupt_request & CPU_INTERRUPT_TPR) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_TPR)) { cpu->interrupt_request &= ~CPU_INTERRUPT_TPR; nvmm_cpu_synchronize_state(cpu); apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip, diff --git a/target/i386/tcg/system/seg_helper.c b/target/i386/tcg/system/seg_helper.c index d4ea890c12..794a23ddfc 100644 --- a/target/i386/tcg/system/seg_helper.c +++ b/target/i386/tcg/system/seg_helper.c @@ -133,7 +133,7 @@ bool x86_cpu_exec_halt(CPUState *cpu) X86CPU *x86_cpu = X86_CPU(cpu); CPUX86State *env = &x86_cpu->env; - if (cpu->interrupt_request & CPU_INTERRUPT_POLL) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_POLL)) { bql_lock(); apic_poll_irq(x86_cpu->apic_state); cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL); diff --git a/target/i386/tcg/system/svm_helper.c b/target/i386/tcg/system/svm_helper.c index b27049b9ed..3569196bdd 100644 --- a/target/i386/tcg/system/svm_helper.c +++ b/target/i386/tcg/system/svm_helper.c @@ -49,7 +49,7 @@ static void svm_save_seg(CPUX86State *env, int mmu_idx, hwaddr addr, static inline void svm_canonicalization(CPUX86State *env, target_ulong *seg_base) { uint16_t shift_amt = 64 - cpu_x86_virtual_addr_width(env); - *seg_base = ((((long) *seg_base) << shift_amt) >> shift_amt); + *seg_base = (((int64_t) *seg_base) << shift_amt) >> shift_amt; } static void svm_load_seg(CPUX86State *env, int mmu_idx, hwaddr addr, @@ -403,7 +403,7 @@ void helper_vmrun(CPUX86State *env, int aflag, int next_eip_addend) env->hflags2 |= HF2_GIF_MASK; if (ctl_has_irq(env)) { - cs->interrupt_request |= CPU_INTERRUPT_VIRQ; + cpu_set_interrupt(cs, CPU_INTERRUPT_VIRQ); } if (virtual_gif_set(env)) { diff --git a/target/i386/whpx/whpx-all.c b/target/i386/whpx/whpx-all.c index b72dcff3c8..878cdd1668 100644 --- a/target/i386/whpx/whpx-all.c +++ b/target/i386/whpx/whpx-all.c @@ -1436,9 +1436,9 @@ static int whpx_handle_halt(CPUState *cpu) int ret = 0; bql_lock(); - if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) && + if (!(cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD) && (cpu_env(cpu)->eflags & IF_MASK)) && - !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) { + !cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) { cpu->exception_index = EXCP_HLT; cpu->halted = true; ret = 1; @@ -1469,15 +1469,15 @@ static void whpx_vcpu_pre_run(CPUState *cpu) /* Inject NMI */ if (!vcpu->interruption_pending && - cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) { - if (cpu->interrupt_request & CPU_INTERRUPT_NMI) { + cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) { cpu->interrupt_request &= ~CPU_INTERRUPT_NMI; vcpu->interruptable = false; new_int.InterruptionType = WHvX64PendingNmi; new_int.InterruptionPending = 1; new_int.InterruptionVector = 2; } - if (cpu->interrupt_request & CPU_INTERRUPT_SMI) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_SMI)) { cpu->interrupt_request &= ~CPU_INTERRUPT_SMI; } } @@ -1486,12 +1486,12 @@ static void whpx_vcpu_pre_run(CPUState *cpu) * Force the VCPU out of its inner loop to process any INIT requests or * commit pending TPR access. */ - if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { - if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) && + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT) && !(env->hflags & HF_SMM_MASK)) { cpu->exit_request = 1; } - if (cpu->interrupt_request & CPU_INTERRUPT_TPR) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_TPR)) { cpu->exit_request = 1; } } @@ -1501,7 +1501,7 @@ static void whpx_vcpu_pre_run(CPUState *cpu) if (!vcpu->interruption_pending && vcpu->interruptable && (env->eflags & IF_MASK)) { assert(!new_int.InterruptionPending); - if (cpu->interrupt_request & CPU_INTERRUPT_HARD) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) { cpu->interrupt_request &= ~CPU_INTERRUPT_HARD; irq = cpu_get_pic_interrupt(env); if (irq >= 0) { @@ -1519,7 +1519,7 @@ static void whpx_vcpu_pre_run(CPUState *cpu) reg_count += 1; } } else if (vcpu->ready_for_pic_interrupt && - (cpu->interrupt_request & CPU_INTERRUPT_HARD)) { + cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) { cpu->interrupt_request &= ~CPU_INTERRUPT_HARD; irq = cpu_get_pic_interrupt(env); if (irq >= 0) { @@ -1546,7 +1546,7 @@ static void whpx_vcpu_pre_run(CPUState *cpu) /* Update the state of the interrupt delivery notification */ if (!vcpu->window_registered && - cpu->interrupt_request & CPU_INTERRUPT_HARD) { + cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) { reg_values[reg_count].DeliverabilityNotifications = (WHV_X64_DELIVERABILITY_NOTIFICATIONS_REGISTER) { .InterruptNotification = 1 @@ -1599,30 +1599,30 @@ static void whpx_vcpu_process_async_events(CPUState *cpu) CPUX86State *env = &x86_cpu->env; AccelCPUState *vcpu = cpu->accel; - if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) && + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT) && !(env->hflags & HF_SMM_MASK)) { whpx_cpu_synchronize_state(cpu); do_cpu_init(x86_cpu); vcpu->interruptable = true; } - if (cpu->interrupt_request & CPU_INTERRUPT_POLL) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_POLL)) { cpu->interrupt_request &= ~CPU_INTERRUPT_POLL; apic_poll_irq(x86_cpu->apic_state); } - if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) && + if ((cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD) && (env->eflags & IF_MASK)) || - (cpu->interrupt_request & CPU_INTERRUPT_NMI)) { + cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) { cpu->halted = false; } - if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_SIPI)) { whpx_cpu_synchronize_state(cpu); do_cpu_sipi(x86_cpu); } - if (cpu->interrupt_request & CPU_INTERRUPT_TPR) { + if (cpu_test_interrupt(cpu, CPU_INTERRUPT_TPR)) { cpu->interrupt_request &= ~CPU_INTERRUPT_TPR; whpx_cpu_synchronize_state(cpu); apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip, diff --git a/target/loongarch/cpu-mmu.h b/target/loongarch/cpu-mmu.h new file mode 100644 index 0000000000..0068d22efc --- /dev/null +++ b/target/loongarch/cpu-mmu.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * LoongArch CPU parameters for QEMU. + * + * Copyright (c) 2025 Loongson Technology Corporation Limited + */ + +#ifndef LOONGARCH_CPU_MMU_H +#define LOONGARCH_CPU_MMU_H + +typedef enum TLBRet { + TLBRET_MATCH, + TLBRET_BADADDR, + TLBRET_NOMATCH, + TLBRET_INVALID, + TLBRET_DIRTY, + TLBRET_RI, + TLBRET_XI, + TLBRET_PE, +} TLBRet; + +typedef struct MMUContext { + vaddr addr; + uint64_t pte; + hwaddr physical; + int ps; /* page size shift */ + int prot; +} MMUContext; + +bool check_ps(CPULoongArchState *ent, uint8_t ps); +TLBRet loongarch_check_pte(CPULoongArchState *env, MMUContext *context, + MMUAccessType access_type, int mmu_idx); +TLBRet get_physical_address(CPULoongArchState *env, MMUContext *context, + MMUAccessType access_type, int mmu_idx, + int is_debug); +void get_dir_base_width(CPULoongArchState *env, uint64_t *dir_base, + uint64_t *dir_width, target_ulong level); +hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr); + +#endif /* LOONGARCH_CPU_MMU_H */ diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c index abad84c054..55ee317bf2 100644 --- a/target/loongarch/cpu.c +++ b/target/loongarch/cpu.c @@ -17,6 +17,7 @@ #include "hw/qdev-properties.h" #include "exec/translation-block.h" #include "cpu.h" +#include "cpu-mmu.h" #include "internals.h" #include "fpu/softfloat-helpers.h" #include "csr.h" @@ -376,7 +377,7 @@ static bool loongarch_cpu_has_work(CPUState *cs) { bool has_work = false; - if ((cs->interrupt_request & CPU_INTERRUPT_HARD) && + if (cpu_test_interrupt(cs, CPU_INTERRUPT_HARD) && cpu_loongarch_hw_interrupts_pending(cpu_env(cs))) { has_work = true; } @@ -422,6 +423,96 @@ static void loongarch_la464_init_csr(Object *obj) #endif } +static bool loongarch_get_lsx(Object *obj, Error **errp) +{ + return LOONGARCH_CPU(obj)->lsx != ON_OFF_AUTO_OFF; +} + +static void loongarch_set_lsx(Object *obj, bool value, Error **errp) +{ + LoongArchCPU *cpu = LOONGARCH_CPU(obj); + uint32_t val; + + cpu->lsx = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; + if (cpu->lsx == ON_OFF_AUTO_OFF) { + cpu->lasx = ON_OFF_AUTO_OFF; + if (cpu->lasx == ON_OFF_AUTO_ON) { + error_setg(errp, "Failed to disable LSX since LASX is enabled"); + return; + } + } + + if (kvm_enabled()) { + /* kvm feature detection in function kvm_arch_init_vcpu */ + return; + } + + /* LSX feature detection in TCG mode */ + val = cpu->env.cpucfg[2]; + if (cpu->lsx == ON_OFF_AUTO_ON) { + if (FIELD_EX32(val, CPUCFG2, LSX) == 0) { + error_setg(errp, "Failed to enable LSX in TCG mode"); + return; + } + } else { + cpu->env.cpucfg[2] = FIELD_DP32(val, CPUCFG2, LASX, 0); + val = cpu->env.cpucfg[2]; + } + + cpu->env.cpucfg[2] = FIELD_DP32(val, CPUCFG2, LSX, value); +} + +static bool loongarch_get_lasx(Object *obj, Error **errp) +{ + return LOONGARCH_CPU(obj)->lasx != ON_OFF_AUTO_OFF; +} + +static void loongarch_set_lasx(Object *obj, bool value, Error **errp) +{ + LoongArchCPU *cpu = LOONGARCH_CPU(obj); + uint32_t val; + + cpu->lasx = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; + if ((cpu->lsx == ON_OFF_AUTO_OFF) && (cpu->lasx == ON_OFF_AUTO_ON)) { + error_setg(errp, "Failed to enable LASX since lSX is disabled"); + return; + } + + if (kvm_enabled()) { + /* kvm feature detection in function kvm_arch_init_vcpu */ + return; + } + + /* LASX feature detection in TCG mode */ + val = cpu->env.cpucfg[2]; + if (cpu->lasx == ON_OFF_AUTO_ON) { + if (FIELD_EX32(val, CPUCFG2, LASX) == 0) { + error_setg(errp, "Failed to enable LASX in TCG mode"); + return; + } + } + + cpu->env.cpucfg[2] = FIELD_DP32(val, CPUCFG2, LASX, value); +} + +static void loongarch_cpu_post_init(Object *obj) +{ + LoongArchCPU *cpu = LOONGARCH_CPU(obj); + + cpu->lbt = ON_OFF_AUTO_OFF; + cpu->pmu = ON_OFF_AUTO_OFF; + cpu->lsx = ON_OFF_AUTO_AUTO; + cpu->lasx = ON_OFF_AUTO_AUTO; + object_property_add_bool(obj, "lsx", loongarch_get_lsx, + loongarch_set_lsx); + object_property_add_bool(obj, "lasx", loongarch_get_lasx, + loongarch_set_lasx); + /* lbt is enabled only in kvm mode, not supported in tcg mode */ + if (kvm_enabled()) { + kvm_loongarch_cpu_post_init(cpu); + } +} + static void loongarch_la464_initfn(Object *obj) { LoongArchCPU *cpu = LOONGARCH_CPU(obj); @@ -683,96 +774,6 @@ static void loongarch_cpu_unrealizefn(DeviceState *dev) lacc->parent_unrealize(dev); } -static bool loongarch_get_lsx(Object *obj, Error **errp) -{ - return LOONGARCH_CPU(obj)->lsx != ON_OFF_AUTO_OFF; -} - -static void loongarch_set_lsx(Object *obj, bool value, Error **errp) -{ - LoongArchCPU *cpu = LOONGARCH_CPU(obj); - uint32_t val; - - cpu->lsx = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; - if (cpu->lsx == ON_OFF_AUTO_OFF) { - cpu->lasx = ON_OFF_AUTO_OFF; - if (cpu->lasx == ON_OFF_AUTO_ON) { - error_setg(errp, "Failed to disable LSX since LASX is enabled"); - return; - } - } - - if (kvm_enabled()) { - /* kvm feature detection in function kvm_arch_init_vcpu */ - return; - } - - /* LSX feature detection in TCG mode */ - val = cpu->env.cpucfg[2]; - if (cpu->lsx == ON_OFF_AUTO_ON) { - if (FIELD_EX32(val, CPUCFG2, LSX) == 0) { - error_setg(errp, "Failed to enable LSX in TCG mode"); - return; - } - } else { - cpu->env.cpucfg[2] = FIELD_DP32(val, CPUCFG2, LASX, 0); - val = cpu->env.cpucfg[2]; - } - - cpu->env.cpucfg[2] = FIELD_DP32(val, CPUCFG2, LSX, value); -} - -static bool loongarch_get_lasx(Object *obj, Error **errp) -{ - return LOONGARCH_CPU(obj)->lasx != ON_OFF_AUTO_OFF; -} - -static void loongarch_set_lasx(Object *obj, bool value, Error **errp) -{ - LoongArchCPU *cpu = LOONGARCH_CPU(obj); - uint32_t val; - - cpu->lasx = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; - if ((cpu->lsx == ON_OFF_AUTO_OFF) && (cpu->lasx == ON_OFF_AUTO_ON)) { - error_setg(errp, "Failed to enable LASX since lSX is disabled"); - return; - } - - if (kvm_enabled()) { - /* kvm feature detection in function kvm_arch_init_vcpu */ - return; - } - - /* LASX feature detection in TCG mode */ - val = cpu->env.cpucfg[2]; - if (cpu->lasx == ON_OFF_AUTO_ON) { - if (FIELD_EX32(val, CPUCFG2, LASX) == 0) { - error_setg(errp, "Failed to enable LASX in TCG mode"); - return; - } - } - - cpu->env.cpucfg[2] = FIELD_DP32(val, CPUCFG2, LASX, value); -} - -void loongarch_cpu_post_init(Object *obj) -{ - LoongArchCPU *cpu = LOONGARCH_CPU(obj); - - cpu->lbt = ON_OFF_AUTO_OFF; - cpu->pmu = ON_OFF_AUTO_OFF; - cpu->lsx = ON_OFF_AUTO_AUTO; - cpu->lasx = ON_OFF_AUTO_AUTO; - object_property_add_bool(obj, "lsx", loongarch_get_lsx, - loongarch_set_lsx); - object_property_add_bool(obj, "lasx", loongarch_get_lasx, - loongarch_set_lasx); - /* lbt is enabled only in kvm mode, not supported in tcg mode */ - if (kvm_enabled()) { - kvm_loongarch_cpu_post_init(cpu); - } -} - static void loongarch_cpu_init(Object *obj) { #ifndef CONFIG_USER_ONLY diff --git a/target/loongarch/cpu.h b/target/loongarch/cpu.h index 9538e8d61d..7731f6acdc 100644 --- a/target/loongarch/cpu.h +++ b/target/loongarch/cpu.h @@ -494,15 +494,4 @@ static inline void set_pc(CPULoongArchState *env, uint64_t value) #define CPU_RESOLVING_TYPE TYPE_LOONGARCH_CPU -void loongarch_cpu_post_init(Object *obj); - -#ifdef CONFIG_KVM -void kvm_loongarch_cpu_post_init(LoongArchCPU *cpu); -#else -static inline void kvm_loongarch_cpu_post_init(LoongArchCPU *cpu) -{ -} -#endif -void kvm_loongarch_init_irq_routing(void); - #endif /* LOONGARCH_CPU_H */ diff --git a/target/loongarch/cpu_helper.c b/target/loongarch/cpu_helper.c index b5f732f15b..4a9db3ea4c 100644 --- a/target/loongarch/cpu_helper.c +++ b/target/loongarch/cpu_helper.c @@ -13,6 +13,7 @@ #include "exec/target_page.h" #include "internals.h" #include "cpu-csr.h" +#include "cpu-mmu.h" #include "tcg/tcg_loongarch.h" void get_dir_base_width(CPULoongArchState *env, uint64_t *dir_base, @@ -43,15 +44,79 @@ void get_dir_base_width(CPULoongArchState *env, uint64_t *dir_base, } } -static int loongarch_page_table_walker(CPULoongArchState *env, hwaddr *physical, - int *prot, target_ulong address) +TLBRet loongarch_check_pte(CPULoongArchState *env, MMUContext *context, + MMUAccessType access_type, int mmu_idx) +{ + uint64_t plv = mmu_idx; + uint64_t tlb_entry, tlb_ppn; + uint8_t tlb_ps, tlb_v, tlb_d, tlb_plv, tlb_nx, tlb_nr, tlb_rplv; + + tlb_entry = context->pte; + tlb_ps = context->ps; + tlb_v = FIELD_EX64(tlb_entry, TLBENTRY, V); + tlb_d = FIELD_EX64(tlb_entry, TLBENTRY, D); + tlb_plv = FIELD_EX64(tlb_entry, TLBENTRY, PLV); + if (is_la64(env)) { + tlb_ppn = FIELD_EX64(tlb_entry, TLBENTRY_64, PPN); + tlb_nx = FIELD_EX64(tlb_entry, TLBENTRY_64, NX); + tlb_nr = FIELD_EX64(tlb_entry, TLBENTRY_64, NR); + tlb_rplv = FIELD_EX64(tlb_entry, TLBENTRY_64, RPLV); + } else { + tlb_ppn = FIELD_EX64(tlb_entry, TLBENTRY_32, PPN); + tlb_nx = 0; + tlb_nr = 0; + tlb_rplv = 0; + } + + /* Remove sw bit between bit12 -- bit PS*/ + tlb_ppn = tlb_ppn & ~(((0x1UL << (tlb_ps - 12)) - 1)); + + /* Check access rights */ + if (!tlb_v) { + return TLBRET_INVALID; + } + + if (access_type == MMU_INST_FETCH && tlb_nx) { + return TLBRET_XI; + } + + if (access_type == MMU_DATA_LOAD && tlb_nr) { + return TLBRET_RI; + } + + if (((tlb_rplv == 0) && (plv > tlb_plv)) || + ((tlb_rplv == 1) && (plv != tlb_plv))) { + return TLBRET_PE; + } + + if ((access_type == MMU_DATA_STORE) && !tlb_d) { + return TLBRET_DIRTY; + } + + context->physical = (tlb_ppn << R_TLBENTRY_64_PPN_SHIFT) | + (context->addr & MAKE_64BIT_MASK(0, tlb_ps)); + context->prot = PAGE_READ; + if (tlb_d) { + context->prot |= PAGE_WRITE; + } + if (!tlb_nx) { + context->prot |= PAGE_EXEC; + } + return TLBRET_MATCH; +} + +static TLBRet loongarch_page_table_walker(CPULoongArchState *env, + MMUContext *context, + int access_type, int mmu_idx) { CPUState *cs = env_cpu(env); target_ulong index, phys; uint64_t dir_base, dir_width; uint64_t base; int level; + vaddr address; + address = context->addr; if ((address >> 63) & 0x1) { base = env->CSR_PGDH; } else { @@ -93,41 +158,20 @@ static int loongarch_page_table_walker(CPULoongArchState *env, hwaddr *physical, base = ldq_phys(cs->as, phys); } - /* TODO: check plv and other bits? */ - - /* base is pte, in normal pte format */ - if (!FIELD_EX64(base, TLBENTRY, V)) { - return TLBRET_NOMATCH; - } - - if (!FIELD_EX64(base, TLBENTRY, D)) { - *prot = PAGE_READ; - } else { - *prot = PAGE_READ | PAGE_WRITE; - } - - /* get TARGET_PAGE_SIZE aligned physical address */ - base += (address & TARGET_PHYS_MASK) & ((1 << dir_base) - 1); - /* mask RPLV, NX, NR bits */ - base = FIELD_DP64(base, TLBENTRY_64, RPLV, 0); - base = FIELD_DP64(base, TLBENTRY_64, NX, 0); - base = FIELD_DP64(base, TLBENTRY_64, NR, 0); - /* mask other attribute bits */ - *physical = base & TARGET_PAGE_MASK; - - return 0; + context->ps = dir_base; + context->pte = base; + return loongarch_check_pte(env, context, access_type, mmu_idx); } -static int loongarch_map_address(CPULoongArchState *env, hwaddr *physical, - int *prot, target_ulong address, - MMUAccessType access_type, int mmu_idx, - int is_debug) +static TLBRet loongarch_map_address(CPULoongArchState *env, + MMUContext *context, + MMUAccessType access_type, int mmu_idx, + int is_debug) { - int ret; + TLBRet ret; if (tcg_enabled()) { - ret = loongarch_get_addr_from_tlb(env, physical, prot, address, - access_type, mmu_idx); + ret = loongarch_get_addr_from_tlb(env, context, access_type, mmu_idx); if (ret != TLBRET_NOMATCH) { return ret; } @@ -139,14 +183,13 @@ static int loongarch_map_address(CPULoongArchState *env, hwaddr *physical, * legal mapping, even if the mapping is not yet in TLB. return 0 if * there is a valid map, else none zero. */ - return loongarch_page_table_walker(env, physical, prot, address); + return loongarch_page_table_walker(env, context, access_type, mmu_idx); } return TLBRET_NOMATCH; } -static hwaddr dmw_va2pa(CPULoongArchState *env, target_ulong va, - target_ulong dmw) +static hwaddr dmw_va2pa(CPULoongArchState *env, vaddr va, target_ulong dmw) { if (is_la64(env)) { return va & TARGET_VIRT_MASK; @@ -157,9 +200,9 @@ static hwaddr dmw_va2pa(CPULoongArchState *env, target_ulong va, } } -int get_physical_address(CPULoongArchState *env, hwaddr *physical, - int *prot, target_ulong address, - MMUAccessType access_type, int mmu_idx, int is_debug) +TLBRet get_physical_address(CPULoongArchState *env, MMUContext *context, + MMUAccessType access_type, int mmu_idx, + int is_debug) { int user_mode = mmu_idx == MMU_USER_IDX; int kernel_mode = mmu_idx == MMU_KERNEL_IDX; @@ -167,11 +210,13 @@ int get_physical_address(CPULoongArchState *env, hwaddr *physical, int64_t addr_high; uint8_t da = FIELD_EX64(env->CSR_CRMD, CSR_CRMD, DA); uint8_t pg = FIELD_EX64(env->CSR_CRMD, CSR_CRMD, PG); + vaddr address; /* Check PG and DA */ + address = context->addr; if (da & !pg) { - *physical = address & TARGET_PHYS_MASK; - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + context->physical = address & TARGET_PHYS_MASK; + context->prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; return TLBRET_MATCH; } @@ -189,8 +234,8 @@ int get_physical_address(CPULoongArchState *env, hwaddr *physical, base_c = FIELD_EX64(env->CSR_DMW[i], CSR_DMW_32, VSEG); } if ((plv & env->CSR_DMW[i]) && (base_c == base_v)) { - *physical = dmw_va2pa(env, address, env->CSR_DMW[i]); - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + context->physical = dmw_va2pa(env, address, env->CSR_DMW[i]); + context->prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; return TLBRET_MATCH; } } @@ -202,19 +247,18 @@ int get_physical_address(CPULoongArchState *env, hwaddr *physical, } /* Mapped address */ - return loongarch_map_address(env, physical, prot, address, - access_type, mmu_idx, is_debug); + return loongarch_map_address(env, context, access_type, mmu_idx, is_debug); } hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cs, vaddr addr) { CPULoongArchState *env = cpu_env(cs); - hwaddr phys_addr; - int prot; + MMUContext context; - if (get_physical_address(env, &phys_addr, &prot, addr, MMU_DATA_LOAD, - cpu_mmu_index(cs, false), 1) != 0) { + context.addr = addr; + if (get_physical_address(env, &context, MMU_DATA_LOAD, + cpu_mmu_index(cs, false), 1) != TLBRET_MATCH) { return -1; } - return phys_addr; + return context.physical; } diff --git a/target/loongarch/internals.h b/target/loongarch/internals.h index a7384b0d31..e50d109767 100644 --- a/target/loongarch/internals.h +++ b/target/loongarch/internals.h @@ -32,19 +32,6 @@ void restore_fp_status(CPULoongArchState *env); #endif #ifndef CONFIG_USER_ONLY -enum { - TLBRET_MATCH = 0, - TLBRET_BADADDR = 1, - TLBRET_NOMATCH = 2, - TLBRET_INVALID = 3, - TLBRET_DIRTY = 4, - TLBRET_RI = 5, - TLBRET_XI = 6, - TLBRET_PE = 7, -}; - -bool check_ps(CPULoongArchState *ent, uint8_t ps); - extern const VMStateDescription vmstate_loongarch_cpu; void loongarch_cpu_set_irq(void *opaque, int irq, int level); @@ -54,13 +41,6 @@ uint64_t cpu_loongarch_get_constant_timer_counter(LoongArchCPU *cpu); uint64_t cpu_loongarch_get_constant_timer_ticks(LoongArchCPU *cpu); void cpu_loongarch_store_constant_timer_config(LoongArchCPU *cpu, uint64_t value); -int get_physical_address(CPULoongArchState *env, hwaddr *physical, - int *prot, target_ulong address, - MMUAccessType access_type, int mmu_idx, int is_debug); -void get_dir_base_width(CPULoongArchState *env, uint64_t *dir_base, - uint64_t *dir_width, target_ulong level); -hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr); - #endif /* !CONFIG_USER_ONLY */ uint64_t read_fcc(CPULoongArchState *env); diff --git a/target/loongarch/kvm/kvm_loongarch.h b/target/loongarch/kvm/kvm_loongarch.h index 1051a341ec..51475675d6 100644 --- a/target/loongarch/kvm/kvm_loongarch.h +++ b/target/loongarch/kvm/kvm_loongarch.h @@ -5,11 +5,11 @@ * Copyright (c) 2023 Loongson Technology Corporation Limited */ -#include "cpu.h" - #ifndef QEMU_KVM_LOONGARCH_H #define QEMU_KVM_LOONGARCH_H +void kvm_loongarch_cpu_post_init(LoongArchCPU *cpu); +void kvm_loongarch_init_irq_routing(void); int kvm_loongarch_set_interrupt(LoongArchCPU *cpu, int irq, int level); void kvm_arch_reset_vcpu(CPUState *cs); diff --git a/target/loongarch/tcg/csr_helper.c b/target/loongarch/tcg/csr_helper.c index 28b1bb86bd..0d99e2c92b 100644 --- a/target/loongarch/tcg/csr_helper.c +++ b/target/loongarch/tcg/csr_helper.c @@ -16,6 +16,7 @@ #include "accel/tcg/cpu-ldst.h" #include "hw/irq.h" #include "cpu-csr.h" +#include "cpu-mmu.h" target_ulong helper_csrwr_stlbps(CPULoongArchState *env, target_ulong val) { diff --git a/target/loongarch/tcg/insn_trans/trans_atomic.c.inc b/target/loongarch/tcg/insn_trans/trans_atomic.c.inc index 3d70d75941..77eeedbc42 100644 --- a/target/loongarch/tcg/insn_trans/trans_atomic.c.inc +++ b/target/loongarch/tcg/insn_trans/trans_atomic.c.inc @@ -74,38 +74,38 @@ TRANS(sc_w, ALL, gen_sc, MO_TESL) TRANS(ll_d, 64, gen_ll, MO_TEUQ) TRANS(sc_d, 64, gen_sc, MO_TEUQ) TRANS(amswap_w, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_TESL) -TRANS(amswap_d, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_TEUQ) +TRANS64(amswap_d, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_TEUQ) TRANS(amadd_w, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_TESL) -TRANS(amadd_d, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_TEUQ) +TRANS64(amadd_d, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_TEUQ) TRANS(amand_w, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_TESL) -TRANS(amand_d, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_TEUQ) +TRANS64(amand_d, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_TEUQ) TRANS(amor_w, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_TESL) -TRANS(amor_d, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_TEUQ) +TRANS64(amor_d, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_TEUQ) TRANS(amxor_w, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_TESL) -TRANS(amxor_d, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_TEUQ) +TRANS64(amxor_d, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_TEUQ) TRANS(ammax_w, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_TESL) -TRANS(ammax_d, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_TEUQ) +TRANS64(ammax_d, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_TEUQ) TRANS(ammin_w, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_TESL) -TRANS(ammin_d, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_TEUQ) +TRANS64(ammin_d, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_TEUQ) TRANS(ammax_wu, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_TESL) -TRANS(ammax_du, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_TEUQ) +TRANS64(ammax_du, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_TEUQ) TRANS(ammin_wu, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_TESL) -TRANS(ammin_du, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_TEUQ) +TRANS64(ammin_du, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_TEUQ) TRANS(amswap_db_w, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_TESL) -TRANS(amswap_db_d, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_TEUQ) +TRANS64(amswap_db_d, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_TEUQ) TRANS(amadd_db_w, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_TESL) -TRANS(amadd_db_d, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_TEUQ) +TRANS64(amadd_db_d, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_TEUQ) TRANS(amand_db_w, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_TESL) -TRANS(amand_db_d, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_TEUQ) +TRANS64(amand_db_d, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_TEUQ) TRANS(amor_db_w, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_TESL) -TRANS(amor_db_d, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_TEUQ) +TRANS64(amor_db_d, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_TEUQ) TRANS(amxor_db_w, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_TESL) -TRANS(amxor_db_d, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_TEUQ) +TRANS64(amxor_db_d, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_TEUQ) TRANS(ammax_db_w, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_TESL) -TRANS(ammax_db_d, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_TEUQ) +TRANS64(ammax_db_d, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_TEUQ) TRANS(ammin_db_w, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_TESL) -TRANS(ammin_db_d, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_TEUQ) +TRANS64(ammin_db_d, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_TEUQ) TRANS(ammax_db_wu, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_TESL) -TRANS(ammax_db_du, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_TEUQ) +TRANS64(ammax_db_du, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_TEUQ) TRANS(ammin_db_wu, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_TESL) -TRANS(ammin_db_du, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_TEUQ) +TRANS64(ammin_db_du, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_TEUQ) diff --git a/target/loongarch/tcg/insn_trans/trans_extra.c.inc b/target/loongarch/tcg/insn_trans/trans_extra.c.inc index eda3d6e561..298a80cff5 100644 --- a/target/loongarch/tcg/insn_trans/trans_extra.c.inc +++ b/target/loongarch/tcg/insn_trans/trans_extra.c.inc @@ -69,6 +69,10 @@ static bool trans_rdtimeh_w(DisasContext *ctx, arg_rdtimeh_w *a) static bool trans_rdtime_d(DisasContext *ctx, arg_rdtime_d *a) { + if (!avail_64(ctx)) { + return false; + } + return gen_rdtime(ctx, a, 0, 0); } @@ -100,8 +104,8 @@ static bool gen_crc(DisasContext *ctx, arg_rrr *a, TRANS(crc_w_b_w, CRC, gen_crc, gen_helper_crc32, tcg_constant_tl(1)) TRANS(crc_w_h_w, CRC, gen_crc, gen_helper_crc32, tcg_constant_tl(2)) TRANS(crc_w_w_w, CRC, gen_crc, gen_helper_crc32, tcg_constant_tl(4)) -TRANS(crc_w_d_w, CRC, gen_crc, gen_helper_crc32, tcg_constant_tl(8)) +TRANS64(crc_w_d_w, CRC, gen_crc, gen_helper_crc32, tcg_constant_tl(8)) TRANS(crcc_w_b_w, CRC, gen_crc, gen_helper_crc32c, tcg_constant_tl(1)) TRANS(crcc_w_h_w, CRC, gen_crc, gen_helper_crc32c, tcg_constant_tl(2)) TRANS(crcc_w_w_w, CRC, gen_crc, gen_helper_crc32c, tcg_constant_tl(4)) -TRANS(crcc_w_d_w, CRC, gen_crc, gen_helper_crc32c, tcg_constant_tl(8)) +TRANS64(crcc_w_d_w, CRC, gen_crc, gen_helper_crc32c, tcg_constant_tl(8)) diff --git a/target/loongarch/tcg/insn_trans/trans_farith.c.inc b/target/loongarch/tcg/insn_trans/trans_farith.c.inc index f4a0dea727..ff6cf3448e 100644 --- a/target/loongarch/tcg/insn_trans/trans_farith.c.inc +++ b/target/loongarch/tcg/insn_trans/trans_farith.c.inc @@ -183,16 +183,16 @@ TRANS(fmaxa_s, FP_SP, gen_fff, gen_helper_fmaxa_s) TRANS(fmaxa_d, FP_DP, gen_fff, gen_helper_fmaxa_d) TRANS(fmina_s, FP_SP, gen_fff, gen_helper_fmina_s) TRANS(fmina_d, FP_DP, gen_fff, gen_helper_fmina_d) -TRANS(fscaleb_s, FP_SP, gen_fff, gen_helper_fscaleb_s) -TRANS(fscaleb_d, FP_DP, gen_fff, gen_helper_fscaleb_d) +TRANS64(fscaleb_s, FP_SP, gen_fff, gen_helper_fscaleb_s) +TRANS64(fscaleb_d, FP_DP, gen_fff, gen_helper_fscaleb_d) TRANS(fsqrt_s, FP_SP, gen_ff, gen_helper_fsqrt_s) TRANS(fsqrt_d, FP_DP, gen_ff, gen_helper_fsqrt_d) TRANS(frecip_s, FP_SP, gen_ff, gen_helper_frecip_s) TRANS(frecip_d, FP_DP, gen_ff, gen_helper_frecip_d) TRANS(frsqrt_s, FP_SP, gen_ff, gen_helper_frsqrt_s) TRANS(frsqrt_d, FP_DP, gen_ff, gen_helper_frsqrt_d) -TRANS(flogb_s, FP_SP, gen_ff, gen_helper_flogb_s) -TRANS(flogb_d, FP_DP, gen_ff, gen_helper_flogb_d) +TRANS64(flogb_s, FP_SP, gen_ff, gen_helper_flogb_s) +TRANS64(flogb_d, FP_DP, gen_ff, gen_helper_flogb_d) TRANS(fclass_s, FP_SP, gen_ff, gen_helper_fclass_s) TRANS(fclass_d, FP_DP, gen_ff, gen_helper_fclass_d) TRANS(fmadd_s, FP_SP, gen_muladd, gen_helper_fmuladd_s, 0) diff --git a/target/loongarch/tcg/insn_trans/trans_fcnv.c.inc b/target/loongarch/tcg/insn_trans/trans_fcnv.c.inc index 833c059d6d..ca1d76a366 100644 --- a/target/loongarch/tcg/insn_trans/trans_fcnv.c.inc +++ b/target/loongarch/tcg/insn_trans/trans_fcnv.c.inc @@ -29,5 +29,5 @@ TRANS(ffint_s_w, FP_SP, gen_ff, gen_helper_ffint_s_w) TRANS(ffint_s_l, FP_SP, gen_ff, gen_helper_ffint_s_l) TRANS(ffint_d_w, FP_DP, gen_ff, gen_helper_ffint_d_w) TRANS(ffint_d_l, FP_DP, gen_ff, gen_helper_ffint_d_l) -TRANS(frint_s, FP_SP, gen_ff, gen_helper_frint_s) -TRANS(frint_d, FP_DP, gen_ff, gen_helper_frint_d) +TRANS64(frint_s, FP_SP, gen_ff, gen_helper_frint_s) +TRANS64(frint_d, FP_DP, gen_ff, gen_helper_frint_d) diff --git a/target/loongarch/tcg/insn_trans/trans_fmemory.c.inc b/target/loongarch/tcg/insn_trans/trans_fmemory.c.inc index 13452bc7e5..79da4718a5 100644 --- a/target/loongarch/tcg/insn_trans/trans_fmemory.c.inc +++ b/target/loongarch/tcg/insn_trans/trans_fmemory.c.inc @@ -148,11 +148,11 @@ TRANS(fldx_s, FP_SP, gen_floadx, MO_TEUL) TRANS(fldx_d, FP_DP, gen_floadx, MO_TEUQ) TRANS(fstx_s, FP_SP, gen_fstorex, MO_TEUL) TRANS(fstx_d, FP_DP, gen_fstorex, MO_TEUQ) -TRANS(fldgt_s, FP_SP, gen_fload_gt, MO_TEUL) -TRANS(fldgt_d, FP_DP, gen_fload_gt, MO_TEUQ) -TRANS(fldle_s, FP_SP, gen_fload_le, MO_TEUL) -TRANS(fldle_d, FP_DP, gen_fload_le, MO_TEUQ) -TRANS(fstgt_s, FP_SP, gen_fstore_gt, MO_TEUL) -TRANS(fstgt_d, FP_DP, gen_fstore_gt, MO_TEUQ) -TRANS(fstle_s, FP_SP, gen_fstore_le, MO_TEUL) -TRANS(fstle_d, FP_DP, gen_fstore_le, MO_TEUQ) +TRANS64(fldgt_s, FP_SP, gen_fload_gt, MO_TEUL) +TRANS64(fldgt_d, FP_DP, gen_fload_gt, MO_TEUQ) +TRANS64(fldle_s, FP_SP, gen_fload_le, MO_TEUL) +TRANS64(fldle_d, FP_DP, gen_fload_le, MO_TEUQ) +TRANS64(fstgt_s, FP_SP, gen_fstore_gt, MO_TEUL) +TRANS64(fstgt_d, FP_DP, gen_fstore_gt, MO_TEUQ) +TRANS64(fstle_s, FP_SP, gen_fstore_le, MO_TEUL) +TRANS64(fstle_d, FP_DP, gen_fstore_le, MO_TEUQ) diff --git a/target/loongarch/tcg/insn_trans/trans_privileged.c.inc b/target/loongarch/tcg/insn_trans/trans_privileged.c.inc index ecbfe23b63..34cfab8879 100644 --- a/target/loongarch/tcg/insn_trans/trans_privileged.c.inc +++ b/target/loongarch/tcg/insn_trans/trans_privileged.c.inc @@ -233,11 +233,11 @@ static bool gen_iocsrwr(DisasContext *ctx, arg_rr *a, TRANS(iocsrrd_b, IOCSR, gen_iocsrrd, gen_helper_iocsrrd_b) TRANS(iocsrrd_h, IOCSR, gen_iocsrrd, gen_helper_iocsrrd_h) TRANS(iocsrrd_w, IOCSR, gen_iocsrrd, gen_helper_iocsrrd_w) -TRANS(iocsrrd_d, IOCSR, gen_iocsrrd, gen_helper_iocsrrd_d) +TRANS64(iocsrrd_d, IOCSR, gen_iocsrrd, gen_helper_iocsrrd_d) TRANS(iocsrwr_b, IOCSR, gen_iocsrwr, gen_helper_iocsrwr_b) TRANS(iocsrwr_h, IOCSR, gen_iocsrwr, gen_helper_iocsrwr_h) TRANS(iocsrwr_w, IOCSR, gen_iocsrwr, gen_helper_iocsrwr_w) -TRANS(iocsrwr_d, IOCSR, gen_iocsrwr, gen_helper_iocsrwr_d) +TRANS64(iocsrwr_d, IOCSR, gen_iocsrwr, gen_helper_iocsrwr_d) static void check_mmu_idx(DisasContext *ctx) { diff --git a/target/loongarch/tcg/insn_trans/trans_shift.c.inc b/target/loongarch/tcg/insn_trans/trans_shift.c.inc index 377307785a..136c4c8455 100644 --- a/target/loongarch/tcg/insn_trans/trans_shift.c.inc +++ b/target/loongarch/tcg/insn_trans/trans_shift.c.inc @@ -78,7 +78,7 @@ TRANS(sra_w, ALL, gen_rrr, EXT_SIGN, EXT_NONE, EXT_SIGN, gen_sra_w) TRANS(sll_d, 64, gen_rrr, EXT_NONE, EXT_NONE, EXT_NONE, gen_sll_d) TRANS(srl_d, 64, gen_rrr, EXT_NONE, EXT_NONE, EXT_NONE, gen_srl_d) TRANS(sra_d, 64, gen_rrr, EXT_NONE, EXT_NONE, EXT_NONE, gen_sra_d) -TRANS(rotr_w, 64, gen_rrr, EXT_ZERO, EXT_NONE, EXT_SIGN, gen_rotr_w) +TRANS(rotr_w, ALL, gen_rrr, EXT_ZERO, EXT_NONE, EXT_SIGN, gen_rotr_w) TRANS(rotr_d, 64, gen_rrr, EXT_NONE, EXT_NONE, EXT_NONE, gen_rotr_d) TRANS(slli_w, ALL, gen_rri_c, EXT_NONE, EXT_SIGN, tcg_gen_shli_tl) TRANS(slli_d, 64, gen_rri_c, EXT_NONE, EXT_NONE, tcg_gen_shli_tl) @@ -86,5 +86,5 @@ TRANS(srli_w, ALL, gen_rri_c, EXT_ZERO, EXT_SIGN, tcg_gen_shri_tl) TRANS(srli_d, 64, gen_rri_c, EXT_NONE, EXT_NONE, tcg_gen_shri_tl) TRANS(srai_w, ALL, gen_rri_c, EXT_NONE, EXT_NONE, gen_sari_w) TRANS(srai_d, 64, gen_rri_c, EXT_NONE, EXT_NONE, tcg_gen_sari_tl) -TRANS(rotri_w, 64, gen_rri_v, EXT_NONE, EXT_NONE, gen_rotr_w) +TRANS(rotri_w, ALL, gen_rri_v, EXT_NONE, EXT_NONE, gen_rotr_w) TRANS(rotri_d, 64, gen_rri_c, EXT_NONE, EXT_NONE, tcg_gen_rotri_tl) diff --git a/target/loongarch/tcg/tcg_loongarch.h b/target/loongarch/tcg/tcg_loongarch.h index fd4e116022..47702893e3 100644 --- a/target/loongarch/tcg/tcg_loongarch.h +++ b/target/loongarch/tcg/tcg_loongarch.h @@ -7,6 +7,7 @@ #ifndef TARGET_LOONGARCH_TCG_LOONGARCH_H #define TARGET_LOONGARCH_TCG_LOONGARCH_H #include "cpu.h" +#include "cpu-mmu.h" void loongarch_csr_translate_init(void); @@ -14,8 +15,8 @@ bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size, MMUAccessType access_type, int mmu_idx, bool probe, uintptr_t retaddr); -int loongarch_get_addr_from_tlb(CPULoongArchState *env, hwaddr *physical, - int *prot, target_ulong address, - MMUAccessType access_type, int mmu_idx); +TLBRet loongarch_get_addr_from_tlb(CPULoongArchState *env, + MMUContext *context, + MMUAccessType access_type, int mmu_idx); #endif /* TARGET_LOONGARCH_TCG_LOONGARCH_H */ diff --git a/target/loongarch/tcg/tlb_helper.c b/target/loongarch/tcg/tlb_helper.c index 8872593ff0..9365860c8c 100644 --- a/target/loongarch/tcg/tlb_helper.c +++ b/target/loongarch/tcg/tlb_helper.c @@ -10,6 +10,7 @@ #include "qemu/guest-random.h" #include "cpu.h" +#include "cpu-mmu.h" #include "internals.h" #include "exec/helper-proto.h" #include "exec/cputlb.h" @@ -28,8 +29,8 @@ bool check_ps(CPULoongArchState *env, uint8_t tlb_ps) return BIT_ULL(tlb_ps) & (env->CSR_PRCFG2); } -static void raise_mmu_exception(CPULoongArchState *env, target_ulong address, - MMUAccessType access_type, int tlb_error) +static void raise_mmu_exception(CPULoongArchState *env, vaddr address, + MMUAccessType access_type, TLBRet tlb_error) { CPUState *cs = env_cpu(env); @@ -110,23 +111,20 @@ static void invalidate_tlb_entry(CPULoongArchState *env, int index) if (!tlb_e) { return; } - if (index >= LOONGARCH_STLB) { - tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); - } else { - tlb_ps = FIELD_EX64(env->CSR_STLBPS, CSR_STLBPS, PS); - } + + tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); pagesize = MAKE_64BIT_MASK(tlb_ps, 1); mask = MAKE_64BIT_MASK(0, tlb_ps + 1); + addr = (tlb_vppn << R_TLB_MISC_VPPN_SHIFT) & ~mask; + addr = sextract64(addr, 0, TARGET_VIRT_ADDR_SPACE_BITS); if (tlb_v0) { - addr = (tlb_vppn << R_TLB_MISC_VPPN_SHIFT) & ~mask; /* even */ tlb_flush_range_by_mmuidx(env_cpu(env), addr, pagesize, mmu_idx, TARGET_LONG_BITS); } if (tlb_v1) { - addr = (tlb_vppn << R_TLB_MISC_VPPN_SHIFT) & pagesize; /* odd */ - tlb_flush_range_by_mmuidx(env_cpu(env), addr, pagesize, + tlb_flush_range_by_mmuidx(env_cpu(env), addr + pagesize, pagesize, mmu_idx, TARGET_LONG_BITS); } } @@ -173,11 +171,8 @@ static void fill_tlb_entry(CPULoongArchState *env, int index) lo1 = env->CSR_TLBELO1; } - /* Only MTLB has the ps fields */ - if (index >= LOONGARCH_STLB) { - tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, PS, csr_ps); - } - + /* Store page size in field PS */ + tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, PS, csr_ps); tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, VPPN, csr_vppn); tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, E, 1); csr_asid = FIELD_EX64(env->CSR_ASID, CSR_ASID, ASID); @@ -203,7 +198,7 @@ static uint32_t get_random_tlb(uint32_t low, uint32_t high) * field in tlb entry contains bit[47:13], so need adjust. * virt_vpn = vaddr[47:13] */ -static bool loongarch_tlb_search(CPULoongArchState *env, target_ulong vaddr, +static bool loongarch_tlb_search(CPULoongArchState *env, vaddr vaddr, int *index) { LoongArchTLB *tlb; @@ -283,12 +278,7 @@ void helper_tlbrd(CPULoongArchState *env) index = FIELD_EX64(env->CSR_TLBIDX, CSR_TLBIDX, INDEX); tlb = &env->tlb[index]; - - if (index >= LOONGARCH_STLB) { - tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); - } else { - tlb_ps = FIELD_EX64(env->CSR_STLBPS, CSR_STLBPS, PS); - } + tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); tlb_e = FIELD_EX64(tlb->tlb_misc, TLB_MISC, E); if (!tlb_e) { @@ -476,11 +466,8 @@ void helper_invtlb_page_asid(CPULoongArchState *env, target_ulong info, if (!tlb_e) { continue; } - if (i >= LOONGARCH_STLB) { - tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); - } else { - tlb_ps = FIELD_EX64(env->CSR_STLBPS, CSR_STLBPS, PS); - } + + tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); tlb_vppn = FIELD_EX64(tlb->tlb_misc, TLB_MISC, VPPN); vpn = (addr & TARGET_VIRT_MASK) >> (tlb_ps + 1); compare_shift = tlb_ps + 1 - R_TLB_MISC_VPPN_SHIFT; @@ -509,11 +496,8 @@ void helper_invtlb_page_asid_or_g(CPULoongArchState *env, if (!tlb_e) { continue; } - if (i >= LOONGARCH_STLB) { - tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); - } else { - tlb_ps = FIELD_EX64(env->CSR_STLBPS, CSR_STLBPS, PS); - } + + tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); tlb_vppn = FIELD_EX64(tlb->tlb_misc, TLB_MISC, VPPN); vpn = (addr & TARGET_VIRT_MASK) >> (tlb_ps + 1); compare_shift = tlb_ps + 1 - R_TLB_MISC_VPPN_SHIFT; @@ -533,13 +517,15 @@ bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size, CPULoongArchState *env = cpu_env(cs); hwaddr physical; int prot; - int ret; + MMUContext context; + TLBRet ret; /* Data access */ - ret = get_physical_address(env, &physical, &prot, address, - access_type, mmu_idx, 0); - + context.addr = address; + ret = get_physical_address(env, &context, access_type, mmu_idx, 0); if (ret == TLBRET_MATCH) { + physical = context.physical; + prot = context.prot; tlb_set_page(cs, address & TARGET_PAGE_MASK, physical & TARGET_PAGE_MASK, prot, mmu_idx, TARGET_PAGE_SIZE); @@ -664,85 +650,31 @@ void helper_ldpte(CPULoongArchState *env, target_ulong base, target_ulong odd, env->CSR_TLBREHI = FIELD_DP64(env->CSR_TLBREHI, CSR_TLBREHI, PS, ps); } -static int loongarch_map_tlb_entry(CPULoongArchState *env, hwaddr *physical, - int *prot, target_ulong address, - int access_type, int index, int mmu_idx) +static TLBRet loongarch_map_tlb_entry(CPULoongArchState *env, + MMUContext *context, + MMUAccessType access_type, int index, + int mmu_idx) { LoongArchTLB *tlb = &env->tlb[index]; - uint64_t plv = mmu_idx; - uint64_t tlb_entry, tlb_ppn; - uint8_t tlb_ps, n, tlb_v, tlb_d, tlb_plv, tlb_nx, tlb_nr, tlb_rplv; - - if (index >= LOONGARCH_STLB) { - tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); - } else { - tlb_ps = FIELD_EX64(env->CSR_STLBPS, CSR_STLBPS, PS); - } - n = (address >> tlb_ps) & 0x1;/* Odd or even */ - - tlb_entry = n ? tlb->tlb_entry1 : tlb->tlb_entry0; - tlb_v = FIELD_EX64(tlb_entry, TLBENTRY, V); - tlb_d = FIELD_EX64(tlb_entry, TLBENTRY, D); - tlb_plv = FIELD_EX64(tlb_entry, TLBENTRY, PLV); - if (is_la64(env)) { - tlb_ppn = FIELD_EX64(tlb_entry, TLBENTRY_64, PPN); - tlb_nx = FIELD_EX64(tlb_entry, TLBENTRY_64, NX); - tlb_nr = FIELD_EX64(tlb_entry, TLBENTRY_64, NR); - tlb_rplv = FIELD_EX64(tlb_entry, TLBENTRY_64, RPLV); - } else { - tlb_ppn = FIELD_EX64(tlb_entry, TLBENTRY_32, PPN); - tlb_nx = 0; - tlb_nr = 0; - tlb_rplv = 0; - } - - /* Remove sw bit between bit12 -- bit PS*/ - tlb_ppn = tlb_ppn & ~(((0x1UL << (tlb_ps - 12)) - 1)); - - /* Check access rights */ - if (!tlb_v) { - return TLBRET_INVALID; - } + uint8_t tlb_ps, n; - if (access_type == MMU_INST_FETCH && tlb_nx) { - return TLBRET_XI; - } - - if (access_type == MMU_DATA_LOAD && tlb_nr) { - return TLBRET_RI; - } - - if (((tlb_rplv == 0) && (plv > tlb_plv)) || - ((tlb_rplv == 1) && (plv != tlb_plv))) { - return TLBRET_PE; - } - - if ((access_type == MMU_DATA_STORE) && !tlb_d) { - return TLBRET_DIRTY; - } - - *physical = (tlb_ppn << R_TLBENTRY_64_PPN_SHIFT) | - (address & MAKE_64BIT_MASK(0, tlb_ps)); - *prot = PAGE_READ; - if (tlb_d) { - *prot |= PAGE_WRITE; - } - if (!tlb_nx) { - *prot |= PAGE_EXEC; - } - return TLBRET_MATCH; + tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); + n = (context->addr >> tlb_ps) & 0x1;/* Odd or even */ + context->pte = n ? tlb->tlb_entry1 : tlb->tlb_entry0; + context->ps = tlb_ps; + return loongarch_check_pte(env, context, access_type, mmu_idx); } -int loongarch_get_addr_from_tlb(CPULoongArchState *env, hwaddr *physical, - int *prot, target_ulong address, - MMUAccessType access_type, int mmu_idx) +TLBRet loongarch_get_addr_from_tlb(CPULoongArchState *env, + MMUContext *context, + MMUAccessType access_type, int mmu_idx) { int index, match; - match = loongarch_tlb_search(env, address, &index); + match = loongarch_tlb_search(env, context->addr, &index); if (match) { - return loongarch_map_tlb_entry(env, physical, prot, - address, access_type, index, mmu_idx); + return loongarch_map_tlb_entry(env, context, access_type, index, + mmu_idx); } return TLBRET_NOMATCH; diff --git a/target/loongarch/translate.h b/target/loongarch/translate.h index 018dc5eb17..bbe015ba57 100644 --- a/target/loongarch/translate.h +++ b/target/loongarch/translate.h @@ -14,6 +14,10 @@ static bool trans_##NAME(DisasContext *ctx, arg_##NAME * a) \ { return avail_##AVAIL(ctx) && FUNC(ctx, a, __VA_ARGS__); } +#define TRANS64(NAME, AVAIL, FUNC, ...) \ + static bool trans_##NAME(DisasContext *ctx, arg_##NAME * a) \ + { return avail_64(ctx) && avail_##AVAIL(ctx) && FUNC(ctx, a, __VA_ARGS__); } + #define avail_ALL(C) true #define avail_64(C) (FIELD_EX32((C)->cpucfg1, CPUCFG1, ARCH) == \ CPUCFG1_ARCH_LA64) diff --git a/target/m68k/cpu.c b/target/m68k/cpu.c index 6a09db3a6f..f1b673119d 100644 --- a/target/m68k/cpu.c +++ b/target/m68k/cpu.c @@ -74,7 +74,7 @@ static void m68k_restore_state_to_opc(CPUState *cs, #ifndef CONFIG_USER_ONLY static bool m68k_cpu_has_work(CPUState *cs) { - return cs->interrupt_request & CPU_INTERRUPT_HARD; + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD); } #endif /* !CONFIG_USER_ONLY */ diff --git a/target/microblaze/cpu.c b/target/microblaze/cpu.c index ee0a869a94..22231f09e6 100644 --- a/target/microblaze/cpu.c +++ b/target/microblaze/cpu.c @@ -129,7 +129,7 @@ static void mb_restore_state_to_opc(CPUState *cs, #ifndef CONFIG_USER_ONLY static bool mb_cpu_has_work(CPUState *cs) { - return cs->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI); + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI); } #endif /* !CONFIG_USER_ONLY */ diff --git a/target/mips/cpu.c b/target/mips/cpu.c index 1f6c41fd34..5989c3ba17 100644 --- a/target/mips/cpu.c +++ b/target/mips/cpu.c @@ -145,7 +145,7 @@ static bool mips_cpu_has_work(CPUState *cs) * check for interrupts that can be taken. For pre-release 6 CPUs, * check for CP0 Config7 'Wait IE ignore' bit. */ - if ((cs->interrupt_request & CPU_INTERRUPT_HARD) && + if (cpu_test_interrupt(cs, CPU_INTERRUPT_HARD) && cpu_mips_hw_interrupts_pending(env)) { if (cpu_mips_hw_interrupts_enabled(env) || (env->CP0_Config7 & (1 << CP0C7_WII)) || @@ -160,7 +160,7 @@ static bool mips_cpu_has_work(CPUState *cs) * The QEMU model will issue an _WAKE request whenever the CPUs * should be woken up. */ - if (cs->interrupt_request & CPU_INTERRUPT_WAKE) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_WAKE)) { has_work = true; } @@ -170,7 +170,7 @@ static bool mips_cpu_has_work(CPUState *cs) } /* MIPS Release 6 has the ability to halt the CPU. */ if (env->CP0_Config5 & (1 << CP0C5_VP)) { - if (cs->interrupt_request & CPU_INTERRUPT_WAKE) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_WAKE)) { has_work = true; } if (!mips_vp_active(env)) { diff --git a/target/mips/kvm.c b/target/mips/kvm.c index ec53acb51a..450947c3fa 100644 --- a/target/mips/kvm.c +++ b/target/mips/kvm.c @@ -144,7 +144,7 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run) bql_lock(); - if ((cs->interrupt_request & CPU_INTERRUPT_HARD) && + if (cpu_test_interrupt(cs, CPU_INTERRUPT_HARD) && cpu_mips_io_interrupts_pending(cpu)) { intr.cpu = -1; intr.irq = 2; diff --git a/target/openrisc/cpu.c b/target/openrisc/cpu.c index dfbb2df643..9bbfe22ed3 100644 --- a/target/openrisc/cpu.c +++ b/target/openrisc/cpu.c @@ -78,8 +78,7 @@ static void openrisc_restore_state_to_opc(CPUState *cs, #ifndef CONFIG_USER_ONLY static bool openrisc_cpu_has_work(CPUState *cs) { - return cs->interrupt_request & (CPU_INTERRUPT_HARD | - CPU_INTERRUPT_TIMER); + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD | CPU_INTERRUPT_TIMER); } #endif /* !CONFIG_USER_ONLY */ diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c index a0e77f2673..db841f1260 100644 --- a/target/ppc/cpu_init.c +++ b/target/ppc/cpu_init.c @@ -7225,7 +7225,7 @@ static int ppc_cpu_mmu_index(CPUState *cs, bool ifetch) #ifndef CONFIG_USER_ONLY static bool ppc_cpu_has_work(CPUState *cs) { - return cs->interrupt_request & CPU_INTERRUPT_HARD; + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD); } #endif /* !CONFIG_USER_ONLY */ diff --git a/target/ppc/kvm.c b/target/ppc/kvm.c index 015658049e..d145774b09 100644 --- a/target/ppc/kvm.c +++ b/target/ppc/kvm.c @@ -1354,7 +1354,7 @@ static int kvmppc_handle_halt(PowerPCCPU *cpu) CPUState *cs = CPU(cpu); CPUPPCState *env = &cpu->env; - if (!(cs->interrupt_request & CPU_INTERRUPT_HARD) && + if (!cpu_test_interrupt(cs, CPU_INTERRUPT_HARD) && FIELD_EX64(env->msr, MSR, EE)) { cs->halted = 1; cs->exception_index = EXCP_HLT; diff --git a/target/rx/cpu.c b/target/rx/cpu.c index c6dd5d6f83..da02ae7bf8 100644 --- a/target/rx/cpu.c +++ b/target/rx/cpu.c @@ -75,8 +75,7 @@ static void rx_restore_state_to_opc(CPUState *cs, static bool rx_cpu_has_work(CPUState *cs) { - return cs->interrupt_request & - (CPU_INTERRUPT_HARD | CPU_INTERRUPT_FIR); + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD | CPU_INTERRUPT_FIR); } static int rx_cpu_mmu_index(CPUState *cs, bool ifunc) diff --git a/target/rx/helper.c b/target/rx/helper.c index 0640ab322b..ce003af421 100644 --- a/target/rx/helper.c +++ b/target/rx/helper.c @@ -44,7 +44,7 @@ void rx_cpu_unpack_psw(CPURXState *env, uint32_t psw, int rte) void rx_cpu_do_interrupt(CPUState *cs) { CPURXState *env = cpu_env(cs); - int do_irq = cs->interrupt_request & INT_FLAGS; + int do_irq = cpu_test_interrupt(cs, INT_FLAGS); uint32_t save_psw; env->in_sleep = 0; diff --git a/target/s390x/cpu-system.c b/target/s390x/cpu-system.c index 709ccd5299..f3a9ffb2a2 100644 --- a/target/s390x/cpu-system.c +++ b/target/s390x/cpu-system.c @@ -49,7 +49,7 @@ bool s390_cpu_has_work(CPUState *cs) return false; } - if (!(cs->interrupt_request & CPU_INTERRUPT_HARD)) { + if (!cpu_test_interrupt(cs, CPU_INTERRUPT_HARD)) { return false; } diff --git a/target/sh4/cpu.c b/target/sh4/cpu.c index 4f561e8c91..21ccb86df4 100644 --- a/target/sh4/cpu.c +++ b/target/sh4/cpu.c @@ -108,7 +108,7 @@ static bool superh_io_recompile_replay_branch(CPUState *cs, static bool superh_cpu_has_work(CPUState *cs) { - return cs->interrupt_request & CPU_INTERRUPT_HARD; + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD); } #endif /* !CONFIG_USER_ONLY */ diff --git a/target/sh4/helper.c b/target/sh4/helper.c index fb7642bda1..1744ef0e6d 100644 --- a/target/sh4/helper.c +++ b/target/sh4/helper.c @@ -58,7 +58,7 @@ int cpu_sh4_is_cached(CPUSH4State *env, target_ulong addr) void superh_cpu_do_interrupt(CPUState *cs) { CPUSH4State *env = cpu_env(cs); - int do_irq = cs->interrupt_request & CPU_INTERRUPT_HARD; + int do_irq = cpu_test_interrupt(cs, CPU_INTERRUPT_HARD); int do_exp, irq_vector = cs->exception_index; /* prioritize exceptions over interrupts */ diff --git a/target/sparc/cpu.c b/target/sparc/cpu.c index 245caf2de0..c9773f1540 100644 --- a/target/sparc/cpu.c +++ b/target/sparc/cpu.c @@ -783,7 +783,7 @@ static void sparc_restore_state_to_opc(CPUState *cs, #ifndef CONFIG_USER_ONLY static bool sparc_cpu_has_work(CPUState *cs) { - return (cs->interrupt_request & CPU_INTERRUPT_HARD) && + return cpu_test_interrupt(cs, CPU_INTERRUPT_HARD) && cpu_interrupts_enabled(cpu_env(cs)); } #endif /* !CONFIG_USER_ONLY */ diff --git a/target/sparc/int64_helper.c b/target/sparc/int64_helper.c index bd14c7a0db..49e4e51c6d 100644 --- a/target/sparc/int64_helper.c +++ b/target/sparc/int64_helper.c @@ -89,7 +89,7 @@ void cpu_check_irqs(CPUSPARCState *env) * the next bit is (2 << psrpil). */ if (pil < (2 << env->psrpil)) { - if (cs->interrupt_request & CPU_INTERRUPT_HARD) { + if (cpu_test_interrupt(cs, CPU_INTERRUPT_HARD)) { trace_sparc64_cpu_check_irqs_reset_irq(env->interrupt_index); env->interrupt_index = 0; cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD); @@ -120,7 +120,7 @@ void cpu_check_irqs(CPUSPARCState *env) break; } } - } else if (cs->interrupt_request & CPU_INTERRUPT_HARD) { + } else if (cpu_test_interrupt(cs, CPU_INTERRUPT_HARD)) { trace_sparc64_cpu_check_irqs_disabled(pil, env->pil_in, env->softint, env->interrupt_index); env->interrupt_index = 0; |