diff options
Diffstat (limited to 'target')
38 files changed, 2978 insertions, 686 deletions
diff --git a/target/arm/monitor.c b/target/arm/arm-qmp-cmds.c index ecdd5ee817..c8fa524002 100644 --- a/target/arm/monitor.c +++ b/target/arm/arm-qmp-cmds.c @@ -227,3 +227,31 @@ CpuModelExpansionInfo *qmp_query_cpu_model_expansion(CpuModelExpansionType type, return expansion_info; } + +static void arm_cpu_add_definition(gpointer data, gpointer user_data) +{ + ObjectClass *oc = data; + CpuDefinitionInfoList **cpu_list = user_data; + CpuDefinitionInfo *info; + const char *typename; + + typename = object_class_get_name(oc); + info = g_malloc0(sizeof(*info)); + info->name = g_strndup(typename, + strlen(typename) - strlen("-" TYPE_ARM_CPU)); + info->q_typename = g_strdup(typename); + + QAPI_LIST_PREPEND(*cpu_list, info); +} + +CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp) +{ + CpuDefinitionInfoList *cpu_list = NULL; + GSList *list; + + list = object_class_get_list(TYPE_ARM_CPU, false); + g_slist_foreach(list, arm_cpu_add_definition, &cpu_list); + g_slist_free(list); + + return cpu_list; +} diff --git a/target/arm/helper.c b/target/arm/helper.c index 14af7ba095..82c546f11a 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -23,7 +23,6 @@ #include "sysemu/cpu-timers.h" #include "sysemu/kvm.h" #include "sysemu/tcg.h" -#include "qapi/qapi-commands-machine-target.h" #include "qapi/error.h" #include "qemu/guest-random.h" #ifdef CONFIG_TCG @@ -9188,34 +9187,6 @@ void arm_cpu_list(void) g_slist_free(list); } -static void arm_cpu_add_definition(gpointer data, gpointer user_data) -{ - ObjectClass *oc = data; - CpuDefinitionInfoList **cpu_list = user_data; - CpuDefinitionInfo *info; - const char *typename; - - typename = object_class_get_name(oc); - info = g_malloc0(sizeof(*info)); - info->name = g_strndup(typename, - strlen(typename) - strlen("-" TYPE_ARM_CPU)); - info->q_typename = g_strdup(typename); - - QAPI_LIST_PREPEND(*cpu_list, info); -} - -CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp) -{ - CpuDefinitionInfoList *cpu_list = NULL; - GSList *list; - - list = object_class_get_list(TYPE_ARM_CPU, false); - g_slist_foreach(list, arm_cpu_add_definition, &cpu_list); - g_slist_free(list); - - return cpu_list; -} - /* * Private utility function for define_one_arm_cp_reg_with_opaque(): * add a single reginfo struct to the hash table. diff --git a/target/arm/meson.build b/target/arm/meson.build index a5191b57e1..6226098ad5 100644 --- a/target/arm/meson.build +++ b/target/arm/meson.build @@ -20,8 +20,8 @@ arm_softmmu_ss = ss.source_set() arm_softmmu_ss.add(files( 'arch_dump.c', 'arm-powerctl.c', + 'arm-qmp-cmds.c', 'machine.c', - 'monitor.c', 'ptw.c', )) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 62755bf511..cab1e2a957 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -31,11 +31,11 @@ #include "qapi/error.h" #include "qapi/qapi-visit-machine.h" #include "qapi/qmp/qerror.h" -#include "qapi/qapi-commands-machine-target.h" #include "standard-headers/asm-x86/kvm_para.h" #include "hw/qdev-properties.h" #include "hw/i386/topology.h" #ifndef CONFIG_USER_ONLY +#include "qapi/qapi-commands-machine-target.h" #include "exec/address-spaces.h" #include "hw/boards.h" #include "hw/i386/sgx-epc.h" @@ -4843,40 +4843,6 @@ static void x86_cpu_get_unavailable_features(Object *obj, Visitor *v, visit_type_strList(v, "unavailable-features", &result, errp); } -/* Check for missing features that may prevent the CPU class from - * running using the current machine and accelerator. - */ -static void x86_cpu_class_check_missing_features(X86CPUClass *xcc, - strList **list) -{ - strList **tail = list; - X86CPU *xc; - Error *err = NULL; - - if (xcc->host_cpuid_required && !accel_uses_host_cpuid()) { - QAPI_LIST_APPEND(tail, g_strdup("kvm")); - return; - } - - xc = X86_CPU(object_new_with_class(OBJECT_CLASS(xcc))); - - x86_cpu_expand_features(xc, &err); - if (err) { - /* Errors at x86_cpu_expand_features should never happen, - * but in case it does, just report the model as not - * runnable at all using the "type" property. - */ - QAPI_LIST_APPEND(tail, g_strdup("type")); - error_free(err); - } - - x86_cpu_filter_features(xc, false); - - x86_cpu_list_feature_names(xc->filtered_features, tail); - - object_unref(OBJECT(xc)); -} - /* Print all cpuid feature names in featureset */ static void listflags(GList *features) @@ -5005,6 +4971,42 @@ void x86_cpu_list(void) g_list_free(names); } +#ifndef CONFIG_USER_ONLY + +/* Check for missing features that may prevent the CPU class from + * running using the current machine and accelerator. + */ +static void x86_cpu_class_check_missing_features(X86CPUClass *xcc, + strList **list) +{ + strList **tail = list; + X86CPU *xc; + Error *err = NULL; + + if (xcc->host_cpuid_required && !accel_uses_host_cpuid()) { + QAPI_LIST_APPEND(tail, g_strdup("kvm")); + return; + } + + xc = X86_CPU(object_new_with_class(OBJECT_CLASS(xcc))); + + x86_cpu_expand_features(xc, &err); + if (err) { + /* Errors at x86_cpu_expand_features should never happen, + * but in case it does, just report the model as not + * runnable at all using the "type" property. + */ + QAPI_LIST_APPEND(tail, g_strdup("type")); + error_free(err); + } + + x86_cpu_filter_features(xc, false); + + x86_cpu_list_feature_names(xc->filtered_features, tail); + + object_unref(OBJECT(xc)); +} + static void x86_cpu_definition_entry(gpointer data, gpointer user_data) { ObjectClass *oc = data; @@ -5045,6 +5047,8 @@ CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp) return cpu_list; } +#endif /* !CONFIG_USER_ONLY */ + uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, bool migratable_only) { @@ -7205,6 +7209,7 @@ static Property x86_cpu_properties[] = { * own cache information (see x86_cpu_load_def()). */ DEFINE_PROP_BOOL("legacy-cache", X86CPU, legacy_cache, true), + DEFINE_PROP_BOOL("xen-vapic", X86CPU, xen_vapic, false), /* * From "Requirements for Implementing the Microsoft diff --git a/target/i386/cpu.h b/target/i386/cpu.h index ea650e68a3..d243e290d3 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -26,6 +26,9 @@ #include "exec/cpu-defs.h" #include "qapi/qapi-types-common.h" #include "qemu/cpu-float.h" +#include "qemu/timer.h" + +#define XEN_NR_VIRQS 24 /* The x86 has a strong memory model with some store-after-load re-ordering */ #define TCG_GUEST_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) @@ -1799,6 +1802,20 @@ typedef struct CPUArchState { #endif #if defined(CONFIG_KVM) struct kvm_nested_state *nested_state; + MemoryRegion *xen_vcpu_info_mr; + void *xen_vcpu_info_hva; + uint64_t xen_vcpu_info_gpa; + uint64_t xen_vcpu_info_default_gpa; + uint64_t xen_vcpu_time_info_gpa; + uint64_t xen_vcpu_runstate_gpa; + uint8_t xen_vcpu_callback_vector; + bool xen_callback_asserted; + uint16_t xen_virq[XEN_NR_VIRQS]; + uint64_t xen_singleshot_timer_ns; + QEMUTimer *xen_singleshot_timer; + uint64_t xen_periodic_timer_period; + QEMUTimer *xen_periodic_timer; + QemuMutex xen_timers_lock; #endif #if defined(CONFIG_HVF) HVFX86LazyFlags hvf_lflags; @@ -1975,6 +1992,8 @@ struct ArchCPU { int32_t thread_id; int32_t hv_max_vps; + + bool xen_vapic; }; diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index d18bd2f3e8..1aef54f87e 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -22,6 +22,7 @@ #include <linux/kvm.h> #include "standard-headers/asm-x86/kvm_para.h" +#include "hw/xen/interface/arch-x86/cpuid.h" #include "cpu.h" #include "host-cpu.h" @@ -31,6 +32,7 @@ #include "sysemu/runstate.h" #include "kvm_i386.h" #include "sev.h" +#include "xen-emu.h" #include "hyperv.h" #include "hyperv-proto.h" @@ -42,6 +44,8 @@ #include "qemu/error-report.h" #include "qemu/memalign.h" #include "hw/i386/x86.h" +#include "hw/i386/kvm/xen_evtchn.h" +#include "hw/i386/pc.h" #include "hw/i386/apic.h" #include "hw/i386/apic_internal.h" #include "hw/i386/apic-msidef.h" @@ -49,6 +53,8 @@ #include "hw/i386/x86-iommu.h" #include "hw/i386/e820_memory_layout.h" +#include "hw/xen/xen.h" + #include "hw/pci/pci.h" #include "hw/pci/msi.h" #include "hw/pci/msix.h" @@ -1815,7 +1821,82 @@ int kvm_arch_init_vcpu(CPUState *cs) has_msr_hv_hypercall = true; } - if (cpu->expose_kvm) { + if (cs->kvm_state->xen_version) { +#ifdef CONFIG_XEN_EMU + struct kvm_cpuid_entry2 *xen_max_leaf; + + memcpy(signature, "XenVMMXenVMM", 12); + + xen_max_leaf = c = &cpuid_data.entries[cpuid_i++]; + c->function = kvm_base + XEN_CPUID_SIGNATURE; + c->eax = kvm_base + XEN_CPUID_TIME; + c->ebx = signature[0]; + c->ecx = signature[1]; + c->edx = signature[2]; + + c = &cpuid_data.entries[cpuid_i++]; + c->function = kvm_base + XEN_CPUID_VENDOR; + c->eax = cs->kvm_state->xen_version; + c->ebx = 0; + c->ecx = 0; + c->edx = 0; + + c = &cpuid_data.entries[cpuid_i++]; + c->function = kvm_base + XEN_CPUID_HVM_MSR; + /* Number of hypercall-transfer pages */ + c->eax = 1; + /* Hypercall MSR base address */ + if (hyperv_enabled(cpu)) { + c->ebx = XEN_HYPERCALL_MSR_HYPERV; + kvm_xen_init(cs->kvm_state, c->ebx); + } else { + c->ebx = XEN_HYPERCALL_MSR; + } + c->ecx = 0; + c->edx = 0; + + c = &cpuid_data.entries[cpuid_i++]; + c->function = kvm_base + XEN_CPUID_TIME; + c->eax = ((!!tsc_is_stable_and_known(env) << 1) | + (!!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_RDTSCP) << 2)); + /* default=0 (emulate if necessary) */ + c->ebx = 0; + /* guest tsc frequency */ + c->ecx = env->user_tsc_khz; + /* guest tsc incarnation (migration count) */ + c->edx = 0; + + c = &cpuid_data.entries[cpuid_i++]; + c->function = kvm_base + XEN_CPUID_HVM; + xen_max_leaf->eax = kvm_base + XEN_CPUID_HVM; + if (cs->kvm_state->xen_version >= XEN_VERSION(4, 5)) { + c->function = kvm_base + XEN_CPUID_HVM; + + if (cpu->xen_vapic) { + c->eax |= XEN_HVM_CPUID_APIC_ACCESS_VIRT; + c->eax |= XEN_HVM_CPUID_X2APIC_VIRT; + } + + c->eax |= XEN_HVM_CPUID_IOMMU_MAPPINGS; + + if (cs->kvm_state->xen_version >= XEN_VERSION(4, 6)) { + c->eax |= XEN_HVM_CPUID_VCPU_ID_PRESENT; + c->ebx = cs->cpu_index; + } + } + + r = kvm_xen_init_vcpu(cs); + if (r) { + return r; + } + + kvm_base += 0x100; +#else /* CONFIG_XEN_EMU */ + /* This should never happen as kvm_arch_init() would have died first. */ + fprintf(stderr, "Cannot enable Xen CPUID without Xen support\n"); + abort(); +#endif + } else if (cpu->expose_kvm) { memcpy(signature, "KVMKVMKVM\0\0\0", 12); c = &cpuid_data.entries[cpuid_i++]; c->function = KVM_CPUID_SIGNATURE | kvm_base; @@ -2529,6 +2610,24 @@ int kvm_arch_init(MachineState *ms, KVMState *s) } } + if (s->xen_version) { +#ifdef CONFIG_XEN_EMU + if (!object_dynamic_cast(OBJECT(ms), TYPE_PC_MACHINE)) { + error_report("kvm: Xen support only available in PC machine"); + return -ENOTSUP; + } + /* hyperv_enabled() doesn't work yet. */ + uint32_t msr = XEN_HYPERCALL_MSR; + ret = kvm_xen_init(s, msr); + if (ret < 0) { + return ret; + } +#else + error_report("kvm: Xen support not enabled in qemu"); + return -ENOTSUP; +#endif + } + ret = kvm_get_supported_msrs(s); if (ret < 0) { return ret; @@ -4652,6 +4751,15 @@ int kvm_arch_put_registers(CPUState *cpu, int level) kvm_arch_set_tsc_khz(cpu); } +#ifdef CONFIG_XEN_EMU + if (xen_mode == XEN_EMULATE && level == KVM_PUT_FULL_STATE) { + ret = kvm_put_xen_state(cpu); + if (ret < 0) { + return ret; + } + } +#endif + ret = kvm_getput_regs(x86_cpu, 1); if (ret < 0) { return ret; @@ -4751,6 +4859,14 @@ int kvm_arch_get_registers(CPUState *cs) if (ret < 0) { goto out; } +#ifdef CONFIG_XEN_EMU + if (xen_mode == XEN_EMULATE) { + ret = kvm_get_xen_state(cs); + if (ret < 0) { + goto out; + } + } +#endif ret = 0; out: cpu_sync_bndcs_hflags(&cpu->env); @@ -4875,6 +4991,17 @@ MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run) kvm_rate_limit_on_bus_lock(); } + /* + * If the callback is asserted as a GSI (or PCI INTx) then check if + * vcpu_info->evtchn_upcall_pending has been cleared, and deassert + * the callback IRQ if so. Ideally we could hook into the PIC/IOAPIC + * EOI and only resample then, exactly how the VFIO eventfd pairs + * are designed to work for level triggered interrupts. + */ + if (x86_cpu->env.xen_callback_asserted) { + kvm_xen_maybe_deassert_callback(cpu); + } + /* We need to protect the apic state against concurrent accesses from * different threads in case the userspace irqchip is used. */ if (!kvm_irqchip_in_kernel()) { @@ -5395,6 +5522,11 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) assert(run->msr.reason == KVM_MSR_EXIT_REASON_FILTER); ret = kvm_handle_wrmsr(cpu, run); break; +#ifdef CONFIG_XEN_EMU + case KVM_EXIT_XEN: + ret = kvm_xen_handle_exit(cpu, &run->xen); + break; +#endif default: fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason); ret = -1; @@ -5523,6 +5655,20 @@ int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route, } } +#ifdef CONFIG_XEN_EMU + if (xen_mode == XEN_EMULATE) { + int handled = xen_evtchn_translate_pirq_msi(route, address, data); + + /* + * If it was a PIRQ and successfully routed (handled == 0) or it was + * an error (handled < 0), return. If it wasn't a PIRQ, keep going. + */ + if (handled <= 0) { + return handled; + } + } +#endif + address = kvm_swizzle_msi_ext_dest_id(address); route->u.msi.address_hi = address >> VTD_MSI_ADDR_HI_SHIFT; route->u.msi.address_lo = address & VTD_MSI_ADDR_LO_MASK; @@ -5542,8 +5688,8 @@ struct MSIRouteEntry { static QLIST_HEAD(, MSIRouteEntry) msi_route_list = \ QLIST_HEAD_INITIALIZER(msi_route_list); -static void kvm_update_msi_routes_all(void *private, bool global, - uint32_t index, uint32_t mask) +void kvm_update_msi_routes_all(void *private, bool global, + uint32_t index, uint32_t mask) { int cnt = 0, vector; MSIRouteEntry *entry; @@ -5719,6 +5865,90 @@ static void kvm_arch_set_notify_window(Object *obj, Visitor *v, s->notify_window = value; } +static void kvm_arch_get_xen_version(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + uint32_t value = s->xen_version; + + visit_type_uint32(v, name, &value, errp); +} + +static void kvm_arch_set_xen_version(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + Error *error = NULL; + uint32_t value; + + visit_type_uint32(v, name, &value, &error); + if (error) { + error_propagate(errp, error); + return; + } + + s->xen_version = value; + if (value && xen_mode == XEN_DISABLED) { + xen_mode = XEN_EMULATE; + } +} + +static void kvm_arch_get_xen_gnttab_max_frames(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + uint16_t value = s->xen_gnttab_max_frames; + + visit_type_uint16(v, name, &value, errp); +} + +static void kvm_arch_set_xen_gnttab_max_frames(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + Error *error = NULL; + uint16_t value; + + visit_type_uint16(v, name, &value, &error); + if (error) { + error_propagate(errp, error); + return; + } + + s->xen_gnttab_max_frames = value; +} + +static void kvm_arch_get_xen_evtchn_max_pirq(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + uint16_t value = s->xen_evtchn_max_pirq; + + visit_type_uint16(v, name, &value, errp); +} + +static void kvm_arch_set_xen_evtchn_max_pirq(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + Error *error = NULL; + uint16_t value; + + visit_type_uint16(v, name, &value, &error); + if (error) { + error_propagate(errp, error); + return; + } + + s->xen_evtchn_max_pirq = value; +} + void kvm_arch_accel_class_init(ObjectClass *oc) { object_class_property_add_enum(oc, "notify-vmexit", "NotifyVMexitOption", @@ -5735,6 +5965,29 @@ void kvm_arch_accel_class_init(ObjectClass *oc) object_class_property_set_description(oc, "notify-window", "Clock cycles without an event window " "after which a notification VM exit occurs"); + + object_class_property_add(oc, "xen-version", "uint32", + kvm_arch_get_xen_version, + kvm_arch_set_xen_version, + NULL, NULL); + object_class_property_set_description(oc, "xen-version", + "Xen version to be emulated " + "(in XENVER_version form " + "e.g. 0x4000a for 4.10)"); + + object_class_property_add(oc, "xen-gnttab-max-frames", "uint16", + kvm_arch_get_xen_gnttab_max_frames, + kvm_arch_set_xen_gnttab_max_frames, + NULL, NULL); + object_class_property_set_description(oc, "xen-gnttab-max-frames", + "Maximum number of grant table frames"); + + object_class_property_add(oc, "xen-evtchn-max-pirq", "uint16", + kvm_arch_get_xen_evtchn_max_pirq, + kvm_arch_set_xen_evtchn_max_pirq, + NULL, NULL); + object_class_property_set_description(oc, "xen-evtchn-max-pirq", + "Maximum number of Xen PIRQs"); } void kvm_set_max_apic_id(uint32_t max_apic_id) diff --git a/target/i386/kvm/kvm_i386.h b/target/i386/kvm/kvm_i386.h index 6a5c24e3dc..e24753abfe 100644 --- a/target/i386/kvm/kvm_i386.h +++ b/target/i386/kvm/kvm_i386.h @@ -51,6 +51,8 @@ bool kvm_hv_vpindex_settable(void); bool kvm_hyperv_expand_features(X86CPU *cpu, Error **errp); uint64_t kvm_swizzle_msi_ext_dest_id(uint64_t address); +void kvm_update_msi_routes_all(void *private, bool global, + uint32_t index, uint32_t mask); bool kvm_enable_sgx_provisioning(KVMState *s); void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask); diff --git a/target/i386/kvm/meson.build b/target/i386/kvm/meson.build index 736df8b72e..322272091b 100644 --- a/target/i386/kvm/meson.build +++ b/target/i386/kvm/meson.build @@ -7,6 +7,8 @@ i386_softmmu_kvm_ss.add(files( 'kvm-cpu.c', )) +i386_softmmu_kvm_ss.add(when: 'CONFIG_XEN_EMU', if_true: files('xen-emu.c')) + i386_softmmu_kvm_ss.add(when: 'CONFIG_SEV', if_false: files('sev-stub.c')) i386_softmmu_ss.add(when: 'CONFIG_HYPERV', if_true: files('hyperv.c'), if_false: files('hyperv-stub.c')) diff --git a/target/i386/kvm/trace-events b/target/i386/kvm/trace-events index 7c369db1e1..b365a8e8e2 100644 --- a/target/i386/kvm/trace-events +++ b/target/i386/kvm/trace-events @@ -5,3 +5,10 @@ kvm_x86_fixup_msi_error(uint32_t gsi) "VT-d failed to remap interrupt for GSI %" kvm_x86_add_msi_route(int virq) "Adding route entry for virq %d" kvm_x86_remove_msi_route(int virq) "Removing route entry for virq %d" kvm_x86_update_msi_routes(int num) "Updated %d MSI routes" + +# xen-emu.c +kvm_xen_hypercall(int cpu, uint8_t cpl, uint64_t input, uint64_t a0, uint64_t a1, uint64_t a2, uint64_t ret) "xen_hypercall: cpu %d cpl %d input %" PRIu64 " a0 0x%" PRIx64 " a1 0x%" PRIx64 " a2 0x%" PRIx64" ret 0x%" PRIx64 +kvm_xen_soft_reset(void) "" +kvm_xen_set_shared_info(uint64_t gfn) "shared info at gfn 0x%" PRIx64 +kvm_xen_set_vcpu_attr(int cpu, int type, uint64_t gpa) "vcpu attr cpu %d type %d gpa 0x%" PRIx64 +kvm_xen_set_vcpu_callback(int cpu, int vector) "callback vcpu %d vector %d" diff --git a/target/i386/kvm/xen-compat.h b/target/i386/kvm/xen-compat.h new file mode 100644 index 0000000000..7f30180cc2 --- /dev/null +++ b/target/i386/kvm/xen-compat.h @@ -0,0 +1,70 @@ +/* + * Xen HVM emulation support in KVM + * + * Copyright © 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ + +#ifndef QEMU_I386_KVM_XEN_COMPAT_H +#define QEMU_I386_KVM_XEN_COMPAT_H + +#include "hw/xen/interface/memory.h" + +typedef uint32_t compat_pfn_t; +typedef uint32_t compat_ulong_t; +typedef uint32_t compat_ptr_t; + +#define __DEFINE_COMPAT_HANDLE(name, type) \ + typedef struct { \ + compat_ptr_t c; \ + type *_[0] __attribute__((packed)); \ + } __compat_handle_ ## name; \ + +#define DEFINE_COMPAT_HANDLE(name) __DEFINE_COMPAT_HANDLE(name, name) +#define COMPAT_HANDLE(name) __compat_handle_ ## name + +DEFINE_COMPAT_HANDLE(compat_pfn_t); +DEFINE_COMPAT_HANDLE(compat_ulong_t); +DEFINE_COMPAT_HANDLE(int); + +struct compat_xen_add_to_physmap { + domid_t domid; + uint16_t size; + unsigned int space; + compat_ulong_t idx; + compat_pfn_t gpfn; +}; + +struct compat_xen_add_to_physmap_batch { + domid_t domid; + uint16_t space; + uint16_t size; + uint16_t extra; + COMPAT_HANDLE(compat_ulong_t) idxs; + COMPAT_HANDLE(compat_pfn_t) gpfns; + COMPAT_HANDLE(int) errs; +}; + +struct compat_physdev_map_pirq { + domid_t domid; + uint16_t pad; + /* IN */ + int type; + /* IN (ignored for ..._MULTI_MSI) */ + int index; + /* IN or OUT */ + int pirq; + /* IN - high 16 bits hold segment for ..._MSI_SEG and ..._MULTI_MSI */ + int bus; + /* IN */ + int devfn; + /* IN (also OUT for ..._MULTI_MSI) */ + int entry_nr; + /* IN */ + uint64_t table_base; +} __attribute__((packed)); + +#endif /* QEMU_I386_XEN_COMPAT_H */ diff --git a/target/i386/kvm/xen-emu.c b/target/i386/kvm/xen-emu.c new file mode 100644 index 0000000000..bad3131d08 --- /dev/null +++ b/target/i386/kvm/xen-emu.c @@ -0,0 +1,1897 @@ +/* + * Xen HVM emulation support in KVM + * + * Copyright © 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright © 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ + +#include "qemu/osdep.h" +#include "qemu/log.h" +#include "qemu/main-loop.h" +#include "hw/xen/xen.h" +#include "sysemu/kvm_int.h" +#include "sysemu/kvm_xen.h" +#include "kvm/kvm_i386.h" +#include "exec/address-spaces.h" +#include "xen-emu.h" +#include "trace.h" +#include "sysemu/runstate.h" + +#include "hw/pci/msi.h" +#include "hw/i386/apic-msidef.h" +#include "hw/i386/e820_memory_layout.h" +#include "hw/i386/kvm/xen_overlay.h" +#include "hw/i386/kvm/xen_evtchn.h" +#include "hw/i386/kvm/xen_gnttab.h" +#include "hw/i386/kvm/xen_xenstore.h" + +#include "hw/xen/interface/version.h" +#include "hw/xen/interface/sched.h" +#include "hw/xen/interface/memory.h" +#include "hw/xen/interface/hvm/hvm_op.h" +#include "hw/xen/interface/hvm/params.h" +#include "hw/xen/interface/vcpu.h" +#include "hw/xen/interface/event_channel.h" +#include "hw/xen/interface/grant_table.h" + +#include "xen-compat.h" + +static void xen_vcpu_singleshot_timer_event(void *opaque); +static void xen_vcpu_periodic_timer_event(void *opaque); + +#ifdef TARGET_X86_64 +#define hypercall_compat32(longmode) (!(longmode)) +#else +#define hypercall_compat32(longmode) (false) +#endif + +static bool kvm_gva_to_gpa(CPUState *cs, uint64_t gva, uint64_t *gpa, + size_t *len, bool is_write) +{ + struct kvm_translation tr = { + .linear_address = gva, + }; + + if (len) { + *len = TARGET_PAGE_SIZE - (gva & ~TARGET_PAGE_MASK); + } + + if (kvm_vcpu_ioctl(cs, KVM_TRANSLATE, &tr) || !tr.valid || + (is_write && !tr.writeable)) { + return false; + } + *gpa = tr.physical_address; + return true; +} + +static int kvm_gva_rw(CPUState *cs, uint64_t gva, void *_buf, size_t sz, + bool is_write) +{ + uint8_t *buf = (uint8_t *)_buf; + uint64_t gpa; + size_t len; + + while (sz) { + if (!kvm_gva_to_gpa(cs, gva, &gpa, &len, is_write)) { + return -EFAULT; + } + if (len > sz) { + len = sz; + } + + cpu_physical_memory_rw(gpa, buf, len, is_write); + + buf += len; + sz -= len; + gva += len; + } + + return 0; +} + +static inline int kvm_copy_from_gva(CPUState *cs, uint64_t gva, void *buf, + size_t sz) +{ + return kvm_gva_rw(cs, gva, buf, sz, false); +} + +static inline int kvm_copy_to_gva(CPUState *cs, uint64_t gva, void *buf, + size_t sz) +{ + return kvm_gva_rw(cs, gva, buf, sz, true); +} + +int kvm_xen_init(KVMState *s, uint32_t hypercall_msr) +{ + const int required_caps = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR | + KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL | KVM_XEN_HVM_CONFIG_SHARED_INFO; + struct kvm_xen_hvm_config cfg = { + .msr = hypercall_msr, + .flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL, + }; + int xen_caps, ret; + + xen_caps = kvm_check_extension(s, KVM_CAP_XEN_HVM); + if (required_caps & ~xen_caps) { + error_report("kvm: Xen HVM guest support not present or insufficient"); + return -ENOSYS; + } + + if (xen_caps & KVM_XEN_HVM_CONFIG_EVTCHN_SEND) { + struct kvm_xen_hvm_attr ha = { + .type = KVM_XEN_ATTR_TYPE_XEN_VERSION, + .u.xen_version = s->xen_version, + }; + (void)kvm_vm_ioctl(s, KVM_XEN_HVM_SET_ATTR, &ha); + + cfg.flags |= KVM_XEN_HVM_CONFIG_EVTCHN_SEND; + } + + ret = kvm_vm_ioctl(s, KVM_XEN_HVM_CONFIG, &cfg); + if (ret < 0) { + error_report("kvm: Failed to enable Xen HVM support: %s", + strerror(-ret)); + return ret; + } + + /* If called a second time, don't repeat the rest of the setup. */ + if (s->xen_caps) { + return 0; + } + + /* + * Event channel delivery via GSI/PCI_INTX needs to poll the vcpu_info + * of vCPU0 to deassert the IRQ when ->evtchn_upcall_pending is cleared. + * + * In the kernel, there's a notifier hook on the PIC/IOAPIC which allows + * such things to be polled at precisely the right time. We *could* do + * it nicely in the kernel: check vcpu_info[0]->evtchn_upcall_pending at + * the moment the IRQ is acked, and see if it should be reasserted. + * + * But the in-kernel irqchip is deprecated, so we're unlikely to add + * that support in the kernel. Insist on using the split irqchip mode + * instead. + * + * This leaves us polling for the level going low in QEMU, which lacks + * the appropriate hooks in its PIC/IOAPIC code. Even VFIO is sending a + * spurious 'ack' to an INTX IRQ every time there's any MMIO access to + * the device (for which it has to unmap the device and trap access, for + * some period after an IRQ!!). In the Xen case, we do it on exit from + * KVM_RUN, if the flag is set to say that the GSI is currently asserted. + * Which is kind of icky, but less so than the VFIO one. I may fix them + * both later... + */ + if (!kvm_kernel_irqchip_split()) { + error_report("kvm: Xen support requires kernel-irqchip=split"); + return -EINVAL; + } + + s->xen_caps = xen_caps; + + /* Tell fw_cfg to notify the BIOS to reserve the range. */ + ret = e820_add_entry(XEN_SPECIAL_AREA_ADDR, XEN_SPECIAL_AREA_SIZE, + E820_RESERVED); + if (ret < 0) { + fprintf(stderr, "e820_add_entry() table is full\n"); + return ret; + } + + /* The page couldn't be overlaid until KVM was initialized */ + xen_xenstore_reset(); + + return 0; +} + +int kvm_xen_init_vcpu(CPUState *cs) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + int err; + + /* + * The kernel needs to know the Xen/ACPI vCPU ID because that's + * what the guest uses in hypercalls such as timers. It doesn't + * match the APIC ID which is generally used for talking to the + * kernel about vCPUs. And if vCPU threads race with creating + * their KVM vCPUs out of order, it doesn't necessarily match + * with the kernel's internal vCPU indices either. + */ + if (kvm_xen_has_cap(EVTCHN_SEND)) { + struct kvm_xen_vcpu_attr va = { + .type = KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID, + .u.vcpu_id = cs->cpu_index, + }; + err = kvm_vcpu_ioctl(cs, KVM_XEN_VCPU_SET_ATTR, &va); + if (err) { + error_report("kvm: Failed to set Xen vCPU ID attribute: %s", + strerror(-err)); + return err; + } + } + + env->xen_vcpu_info_gpa = INVALID_GPA; + env->xen_vcpu_info_default_gpa = INVALID_GPA; + env->xen_vcpu_time_info_gpa = INVALID_GPA; + env->xen_vcpu_runstate_gpa = INVALID_GPA; + + qemu_mutex_init(&env->xen_timers_lock); + env->xen_singleshot_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, + xen_vcpu_singleshot_timer_event, + cpu); + if (!env->xen_singleshot_timer) { + return -ENOMEM; + } + env->xen_singleshot_timer->opaque = cs; + + env->xen_periodic_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, + xen_vcpu_periodic_timer_event, + cpu); + if (!env->xen_periodic_timer) { + return -ENOMEM; + } + env->xen_periodic_timer->opaque = cs; + + return 0; +} + +uint32_t kvm_xen_get_caps(void) +{ + return kvm_state->xen_caps; +} + +static bool kvm_xen_hcall_xen_version(struct kvm_xen_exit *exit, X86CPU *cpu, + int cmd, uint64_t arg) +{ + int err = 0; + + switch (cmd) { + case XENVER_get_features: { + struct xen_feature_info fi; + + /* No need for 32/64 compat handling */ + qemu_build_assert(sizeof(fi) == 8); + + err = kvm_copy_from_gva(CPU(cpu), arg, &fi, sizeof(fi)); + if (err) { + break; + } + + fi.submap = 0; + if (fi.submap_idx == 0) { + fi.submap |= 1 << XENFEAT_writable_page_tables | + 1 << XENFEAT_writable_descriptor_tables | + 1 << XENFEAT_auto_translated_physmap | + 1 << XENFEAT_supervisor_mode_kernel | + 1 << XENFEAT_hvm_callback_vector | + 1 << XENFEAT_hvm_safe_pvclock | + 1 << XENFEAT_hvm_pirqs; + } + + err = kvm_copy_to_gva(CPU(cpu), arg, &fi, sizeof(fi)); + break; + } + + default: + return false; + } + + exit->u.hcall.result = err; + return true; +} + +static int kvm_xen_set_vcpu_attr(CPUState *cs, uint16_t type, uint64_t gpa) +{ + struct kvm_xen_vcpu_attr xhsi; + + xhsi.type = type; + xhsi.u.gpa = gpa; + + trace_kvm_xen_set_vcpu_attr(cs->cpu_index, type, gpa); + + return kvm_vcpu_ioctl(cs, KVM_XEN_VCPU_SET_ATTR, &xhsi); +} + +static int kvm_xen_set_vcpu_callback_vector(CPUState *cs) +{ + uint8_t vector = X86_CPU(cs)->env.xen_vcpu_callback_vector; + struct kvm_xen_vcpu_attr xva; + + xva.type = KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR; + xva.u.vector = vector; + + trace_kvm_xen_set_vcpu_callback(cs->cpu_index, vector); + + return kvm_vcpu_ioctl(cs, KVM_XEN_HVM_SET_ATTR, &xva); +} + +static void do_set_vcpu_callback_vector(CPUState *cs, run_on_cpu_data data) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + + env->xen_vcpu_callback_vector = data.host_int; + + if (kvm_xen_has_cap(EVTCHN_SEND)) { + kvm_xen_set_vcpu_callback_vector(cs); + } +} + +static int set_vcpu_info(CPUState *cs, uint64_t gpa) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + MemoryRegionSection mrs = { .mr = NULL }; + void *vcpu_info_hva = NULL; + int ret; + + ret = kvm_xen_set_vcpu_attr(cs, KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO, gpa); + if (ret || gpa == INVALID_GPA) { + goto out; + } + + mrs = memory_region_find(get_system_memory(), gpa, + sizeof(struct vcpu_info)); + if (mrs.mr && mrs.mr->ram_block && + !int128_lt(mrs.size, int128_make64(sizeof(struct vcpu_info)))) { + vcpu_info_hva = qemu_map_ram_ptr(mrs.mr->ram_block, + mrs.offset_within_region); + } + if (!vcpu_info_hva) { + if (mrs.mr) { + memory_region_unref(mrs.mr); + mrs.mr = NULL; + } + ret = -EINVAL; + } + + out: + if (env->xen_vcpu_info_mr) { + memory_region_unref(env->xen_vcpu_info_mr); + } + env->xen_vcpu_info_hva = vcpu_info_hva; + env->xen_vcpu_info_mr = mrs.mr; + return ret; +} + +static void do_set_vcpu_info_default_gpa(CPUState *cs, run_on_cpu_data data) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + + env->xen_vcpu_info_default_gpa = data.host_ulong; + + /* Changing the default does nothing if a vcpu_info was explicitly set. */ + if (env->xen_vcpu_info_gpa == INVALID_GPA) { + set_vcpu_info(cs, env->xen_vcpu_info_default_gpa); + } +} + +static void do_set_vcpu_info_gpa(CPUState *cs, run_on_cpu_data data) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + + env->xen_vcpu_info_gpa = data.host_ulong; + + set_vcpu_info(cs, env->xen_vcpu_info_gpa); +} + +void *kvm_xen_get_vcpu_info_hva(uint32_t vcpu_id) +{ + CPUState *cs = qemu_get_cpu(vcpu_id); + if (!cs) { + return NULL; + } + + return X86_CPU(cs)->env.xen_vcpu_info_hva; +} + +void kvm_xen_maybe_deassert_callback(CPUState *cs) +{ + CPUX86State *env = &X86_CPU(cs)->env; + struct vcpu_info *vi = env->xen_vcpu_info_hva; + if (!vi) { + return; + } + + /* If the evtchn_upcall_pending flag is cleared, turn the GSI off. */ + if (!vi->evtchn_upcall_pending) { + qemu_mutex_lock_iothread(); + /* + * Check again now we have the lock, because it may have been + * asserted in the interim. And we don't want to take the lock + * every time because this is a fast path. + */ + if (!vi->evtchn_upcall_pending) { + X86_CPU(cs)->env.xen_callback_asserted = false; + xen_evtchn_set_callback_level(0); + } + qemu_mutex_unlock_iothread(); + } +} + +void kvm_xen_set_callback_asserted(void) +{ + CPUState *cs = qemu_get_cpu(0); + + if (cs) { + X86_CPU(cs)->env.xen_callback_asserted = true; + } +} + +void kvm_xen_inject_vcpu_callback_vector(uint32_t vcpu_id, int type) +{ + CPUState *cs = qemu_get_cpu(vcpu_id); + uint8_t vector; + + if (!cs) { + return; + } + + vector = X86_CPU(cs)->env.xen_vcpu_callback_vector; + if (vector) { + /* + * The per-vCPU callback vector injected via lapic. Just + * deliver it as an MSI. + */ + MSIMessage msg = { + .address = APIC_DEFAULT_ADDRESS | X86_CPU(cs)->apic_id, + .data = vector | (1UL << MSI_DATA_LEVEL_SHIFT), + }; + kvm_irqchip_send_msi(kvm_state, msg); + return; + } + + switch (type) { + case HVM_PARAM_CALLBACK_TYPE_VECTOR: + /* + * If the evtchn_upcall_pending field in the vcpu_info is set, then + * KVM will automatically deliver the vector on entering the vCPU + * so all we have to do is kick it out. + */ + qemu_cpu_kick(cs); + break; + + case HVM_PARAM_CALLBACK_TYPE_GSI: + case HVM_PARAM_CALLBACK_TYPE_PCI_INTX: + if (vcpu_id == 0) { + xen_evtchn_set_callback_level(1); + } + break; + } +} + +static int kvm_xen_set_vcpu_timer(CPUState *cs) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + + struct kvm_xen_vcpu_attr va = { + .type = KVM_XEN_VCPU_ATTR_TYPE_TIMER, + .u.timer.port = env->xen_virq[VIRQ_TIMER], + .u.timer.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL, + .u.timer.expires_ns = env->xen_singleshot_timer_ns, + }; + + return kvm_vcpu_ioctl(cs, KVM_XEN_VCPU_SET_ATTR, &va); +} + +static void do_set_vcpu_timer_virq(CPUState *cs, run_on_cpu_data data) +{ + kvm_xen_set_vcpu_timer(cs); +} + +int kvm_xen_set_vcpu_virq(uint32_t vcpu_id, uint16_t virq, uint16_t port) +{ + CPUState *cs = qemu_get_cpu(vcpu_id); + + if (!cs) { + return -ENOENT; + } + + /* cpu.h doesn't include the actual Xen header. */ + qemu_build_assert(NR_VIRQS == XEN_NR_VIRQS); + + if (virq >= NR_VIRQS) { + return -EINVAL; + } + + if (port && X86_CPU(cs)->env.xen_virq[virq]) { + return -EEXIST; + } + + X86_CPU(cs)->env.xen_virq[virq] = port; + if (virq == VIRQ_TIMER && kvm_xen_has_cap(EVTCHN_SEND)) { + async_run_on_cpu(cs, do_set_vcpu_timer_virq, + RUN_ON_CPU_HOST_INT(port)); + } + return 0; +} + +static void do_set_vcpu_time_info_gpa(CPUState *cs, run_on_cpu_data data) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + + env->xen_vcpu_time_info_gpa = data.host_ulong; + + kvm_xen_set_vcpu_attr(cs, KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO, + env->xen_vcpu_time_info_gpa); +} + +static void do_set_vcpu_runstate_gpa(CPUState *cs, run_on_cpu_data data) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + + env->xen_vcpu_runstate_gpa = data.host_ulong; + + kvm_xen_set_vcpu_attr(cs, KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADDR, + env->xen_vcpu_runstate_gpa); +} + +static void do_vcpu_soft_reset(CPUState *cs, run_on_cpu_data data) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + + env->xen_vcpu_info_gpa = INVALID_GPA; + env->xen_vcpu_info_default_gpa = INVALID_GPA; + env->xen_vcpu_time_info_gpa = INVALID_GPA; + env->xen_vcpu_runstate_gpa = INVALID_GPA; + env->xen_vcpu_callback_vector = 0; + env->xen_singleshot_timer_ns = 0; + memset(env->xen_virq, 0, sizeof(env->xen_virq)); + + set_vcpu_info(cs, INVALID_GPA); + kvm_xen_set_vcpu_attr(cs, KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO, + INVALID_GPA); + kvm_xen_set_vcpu_attr(cs, KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADDR, + INVALID_GPA); + if (kvm_xen_has_cap(EVTCHN_SEND)) { + kvm_xen_set_vcpu_callback_vector(cs); + kvm_xen_set_vcpu_timer(cs); + } + +} + +static int xen_set_shared_info(uint64_t gfn) +{ + uint64_t gpa = gfn << TARGET_PAGE_BITS; + int i, err; + + QEMU_IOTHREAD_LOCK_GUARD(); + + /* + * The xen_overlay device tells KVM about it too, since it had to + * do that on migration load anyway (unless we're going to jump + * through lots of hoops to maintain the fiction that this isn't + * KVM-specific. + */ + err = xen_overlay_map_shinfo_page(gpa); + if (err) { + return err; + } + + trace_kvm_xen_set_shared_info(gfn); + + for (i = 0; i < XEN_LEGACY_MAX_VCPUS; i++) { + CPUState *cpu = qemu_get_cpu(i); + if (cpu) { + async_run_on_cpu(cpu, do_set_vcpu_info_default_gpa, + RUN_ON_CPU_HOST_ULONG(gpa)); + } + gpa += sizeof(vcpu_info_t); + } + + return err; +} + +static int add_to_physmap_one(uint32_t space, uint64_t idx, uint64_t gfn) +{ + switch (space) { + case XENMAPSPACE_shared_info: + if (idx > 0) { + return -EINVAL; + } + return xen_set_shared_info(gfn); + + case XENMAPSPACE_grant_table: + return xen_gnttab_map_page(idx, gfn); + + case XENMAPSPACE_gmfn: + case XENMAPSPACE_gmfn_range: + return -ENOTSUP; + + case XENMAPSPACE_gmfn_foreign: + case XENMAPSPACE_dev_mmio: + return -EPERM; + + default: + return -EINVAL; + } +} + +static int do_add_to_physmap(struct kvm_xen_exit *exit, X86CPU *cpu, + uint64_t arg) +{ + struct xen_add_to_physmap xatp; + CPUState *cs = CPU(cpu); + + if (hypercall_compat32(exit->u.hcall.longmode)) { + struct compat_xen_add_to_physmap xatp32; + + qemu_build_assert(sizeof(struct compat_xen_add_to_physmap) == 16); + if (kvm_copy_from_gva(cs, arg, &xatp32, sizeof(xatp32))) { + return -EFAULT; + } + xatp.domid = xatp32.domid; + xatp.size = xatp32.size; + xatp.space = xatp32.space; + xatp.idx = xatp32.idx; + xatp.gpfn = xatp32.gpfn; + } else { + if (kvm_copy_from_gva(cs, arg, &xatp, sizeof(xatp))) { + return -EFAULT; + } + } + + if (xatp.domid != DOMID_SELF && xatp.domid != xen_domid) { + return -ESRCH; + } + + return add_to_physmap_one(xatp.space, xatp.idx, xatp.gpfn); +} + +static int do_add_to_physmap_batch(struct kvm_xen_exit *exit, X86CPU *cpu, + uint64_t arg) +{ + struct xen_add_to_physmap_batch xatpb; + unsigned long idxs_gva, gpfns_gva, errs_gva; + CPUState *cs = CPU(cpu); + size_t op_sz; + + if (hypercall_compat32(exit->u.hcall.longmode)) { + struct compat_xen_add_to_physmap_batch xatpb32; + + qemu_build_assert(sizeof(struct compat_xen_add_to_physmap_batch) == 20); + if (kvm_copy_from_gva(cs, arg, &xatpb32, sizeof(xatpb32))) { + return -EFAULT; + } + xatpb.domid = xatpb32.domid; + xatpb.space = xatpb32.space; + xatpb.size = xatpb32.size; + + idxs_gva = xatpb32.idxs.c; + gpfns_gva = xatpb32.gpfns.c; + errs_gva = xatpb32.errs.c; + op_sz = sizeof(uint32_t); + } else { + if (kvm_copy_from_gva(cs, arg, &xatpb, sizeof(xatpb))) { + return -EFAULT; + } + op_sz = sizeof(unsigned long); + idxs_gva = (unsigned long)xatpb.idxs.p; + gpfns_gva = (unsigned long)xatpb.gpfns.p; + errs_gva = (unsigned long)xatpb.errs.p; + } + + if (xatpb.domid != DOMID_SELF && xatpb.domid != xen_domid) { + return -ESRCH; + } + + /* Explicitly invalid for the batch op. Not that we implement it anyway. */ + if (xatpb.space == XENMAPSPACE_gmfn_range) { + return -EINVAL; + } + + while (xatpb.size--) { + unsigned long idx = 0; + unsigned long gpfn = 0; + int err; + + /* For 32-bit compat this only copies the low 32 bits of each */ + if (kvm_copy_from_gva(cs, idxs_gva, &idx, op_sz) || + kvm_copy_from_gva(cs, gpfns_gva, &gpfn, op_sz)) { + return -EFAULT; + } + idxs_gva += op_sz; + gpfns_gva += op_sz; + + err = add_to_physmap_one(xatpb.space, idx, gpfn); + + if (kvm_copy_to_gva(cs, errs_gva, &err, sizeof(err))) { + return -EFAULT; + } + errs_gva += sizeof(err); + } + return 0; +} + +static bool kvm_xen_hcall_memory_op(struct kvm_xen_exit *exit, X86CPU *cpu, + int cmd, uint64_t arg) +{ + int err; + + switch (cmd) { + case XENMEM_add_to_physmap: + err = do_add_to_physmap(exit, cpu, arg); + break; + + case XENMEM_add_to_physmap_batch: + err = do_add_to_physmap_batch(exit, cpu, arg); + break; + + default: + return false; + } + + exit->u.hcall.result = err; + return true; +} + +static bool handle_set_param(struct kvm_xen_exit *exit, X86CPU *cpu, + uint64_t arg) +{ + CPUState *cs = CPU(cpu); + struct xen_hvm_param hp; + int err = 0; + + /* No need for 32/64 compat handling */ + qemu_build_assert(sizeof(hp) == 16); + + if (kvm_copy_from_gva(cs, arg, &hp, sizeof(hp))) { + err = -EFAULT; + goto out; + } + + if (hp.domid != DOMID_SELF && hp.domid != xen_domid) { + err = -ESRCH; + goto out; + } + + switch (hp.index) { + case HVM_PARAM_CALLBACK_IRQ: + qemu_mutex_lock_iothread(); + err = xen_evtchn_set_callback_param(hp.value); + qemu_mutex_unlock_iothread(); + xen_set_long_mode(exit->u.hcall.longmode); + break; + default: + return false; + } + +out: + exit->u.hcall.result = err; + return true; +} + +static bool handle_get_param(struct kvm_xen_exit *exit, X86CPU *cpu, + uint64_t arg) +{ + CPUState *cs = CPU(cpu); + struct xen_hvm_param hp; + int err = 0; + + /* No need for 32/64 compat handling */ + qemu_build_assert(sizeof(hp) == 16); + + if (kvm_copy_from_gva(cs, arg, &hp, sizeof(hp))) { + err = -EFAULT; + goto out; + } + + if (hp.domid != DOMID_SELF && hp.domid != xen_domid) { + err = -ESRCH; + goto out; + } + + switch (hp.index) { + case HVM_PARAM_STORE_PFN: + hp.value = XEN_SPECIAL_PFN(XENSTORE); + break; + case HVM_PARAM_STORE_EVTCHN: + hp.value = xen_xenstore_get_port(); + break; + default: + return false; + } + + if (kvm_copy_to_gva(cs, arg, &hp, sizeof(hp))) { + err = -EFAULT; + } +out: + exit->u.hcall.result = err; + return true; +} + +static int kvm_xen_hcall_evtchn_upcall_vector(struct kvm_xen_exit *exit, + X86CPU *cpu, uint64_t arg) +{ + struct xen_hvm_evtchn_upcall_vector up; + CPUState *target_cs; + + /* No need for 32/64 compat handling */ + qemu_build_assert(sizeof(up) == 8); + + if (kvm_copy_from_gva(CPU(cpu), arg, &up, sizeof(up))) { + return -EFAULT; + } + + if (up.vector < 0x10) { + return -EINVAL; + } + + target_cs = qemu_get_cpu(up.vcpu); + if (!target_cs) { + return -EINVAL; + } + + async_run_on_cpu(target_cs, do_set_vcpu_callback_vector, + RUN_ON_CPU_HOST_INT(up.vector)); + return 0; +} + +static bool kvm_xen_hcall_hvm_op(struct kvm_xen_exit *exit, X86CPU *cpu, + int cmd, uint64_t arg) +{ + int ret = -ENOSYS; + switch (cmd) { + case HVMOP_set_evtchn_upcall_vector: + ret = kvm_xen_hcall_evtchn_upcall_vector(exit, cpu, + exit->u.hcall.params[0]); + break; + + case HVMOP_pagetable_dying: + ret = -ENOSYS; + break; + + case HVMOP_set_param: + return handle_set_param(exit, cpu, arg); + + case HVMOP_get_param: + return handle_get_param(exit, cpu, arg); + + default: + return false; + } + + exit->u.hcall.result = ret; + return true; +} + +static int vcpuop_register_vcpu_info(CPUState *cs, CPUState *target, + uint64_t arg) +{ + struct vcpu_register_vcpu_info rvi; + uint64_t gpa; + + /* No need for 32/64 compat handling */ + qemu_build_assert(sizeof(rvi) == 16); + qemu_build_assert(sizeof(struct vcpu_info) == 64); + + if (!target) { + return -ENOENT; + } + + if (kvm_copy_from_gva(cs, arg, &rvi, sizeof(rvi))) { + return -EFAULT; + } + + if (rvi.offset > TARGET_PAGE_SIZE - sizeof(struct vcpu_info)) { + return -EINVAL; + } + + gpa = ((rvi.mfn << TARGET_PAGE_BITS) + rvi.offset); + async_run_on_cpu(target, do_set_vcpu_info_gpa, RUN_ON_CPU_HOST_ULONG(gpa)); + return 0; +} + +static int vcpuop_register_vcpu_time_info(CPUState *cs, CPUState *target, + uint64_t arg) +{ + struct vcpu_register_time_memory_area tma; + uint64_t gpa; + size_t len; + + /* No need for 32/64 compat handling */ + qemu_build_assert(sizeof(tma) == 8); + qemu_build_assert(sizeof(struct vcpu_time_info) == 32); + + if (!target) { + return -ENOENT; + } + + if (kvm_copy_from_gva(cs, arg, &tma, sizeof(tma))) { + return -EFAULT; + } + + /* + * Xen actually uses the GVA and does the translation through the guest + * page tables each time. But Linux/KVM uses the GPA, on the assumption + * that guests only ever use *global* addresses (kernel virtual addresses) + * for it. If Linux is changed to redo the GVA→GPA translation each time, + * it will offer a new vCPU attribute for that, and we'll use it instead. + */ + if (!kvm_gva_to_gpa(cs, tma.addr.p, &gpa, &len, false) || + len < sizeof(struct vcpu_time_info)) { + return -EFAULT; + } + + async_run_on_cpu(target, do_set_vcpu_time_info_gpa, + RUN_ON_CPU_HOST_ULONG(gpa)); + return 0; +} + +static int vcpuop_register_runstate_info(CPUState *cs, CPUState *target, + uint64_t arg) +{ + struct vcpu_register_runstate_memory_area rma; + uint64_t gpa; + size_t len; + + /* No need for 32/64 compat handling */ + qemu_build_assert(sizeof(rma) == 8); + /* The runstate area actually does change size, but Linux copes. */ + + if (!target) { + return -ENOENT; + } + + if (kvm_copy_from_gva(cs, arg, &rma, sizeof(rma))) { + return -EFAULT; + } + + /* As with vcpu_time_info, Xen actually uses the GVA but KVM doesn't. */ + if (!kvm_gva_to_gpa(cs, rma.addr.p, &gpa, &len, false)) { + return -EFAULT; + } + + async_run_on_cpu(target, do_set_vcpu_runstate_gpa, + RUN_ON_CPU_HOST_ULONG(gpa)); + return 0; +} + +static uint64_t kvm_get_current_ns(void) +{ + struct kvm_clock_data data; + int ret; + + ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data); + if (ret < 0) { + fprintf(stderr, "KVM_GET_CLOCK failed: %s\n", strerror(ret)); + abort(); + } + + return data.clock; +} + +static void xen_vcpu_singleshot_timer_event(void *opaque) +{ + CPUState *cpu = opaque; + CPUX86State *env = &X86_CPU(cpu)->env; + uint16_t port = env->xen_virq[VIRQ_TIMER]; + + if (likely(port)) { + xen_evtchn_set_port(port); + } + + qemu_mutex_lock(&env->xen_timers_lock); + env->xen_singleshot_timer_ns = 0; + qemu_mutex_unlock(&env->xen_timers_lock); +} + +static void xen_vcpu_periodic_timer_event(void *opaque) +{ + CPUState *cpu = opaque; + CPUX86State *env = &X86_CPU(cpu)->env; + uint16_t port = env->xen_virq[VIRQ_TIMER]; + int64_t qemu_now; + + if (likely(port)) { + xen_evtchn_set_port(port); + } + + qemu_mutex_lock(&env->xen_timers_lock); + + qemu_now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL); + timer_mod_ns(env->xen_periodic_timer, + qemu_now + env->xen_periodic_timer_period); + + qemu_mutex_unlock(&env->xen_timers_lock); +} + +static int do_set_periodic_timer(CPUState *target, uint64_t period_ns) +{ + CPUX86State *tenv = &X86_CPU(target)->env; + int64_t qemu_now; + + timer_del(tenv->xen_periodic_timer); + + qemu_mutex_lock(&tenv->xen_timers_lock); + + qemu_now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL); + timer_mod_ns(tenv->xen_periodic_timer, qemu_now + period_ns); + tenv->xen_periodic_timer_period = period_ns; + + qemu_mutex_unlock(&tenv->xen_timers_lock); + return 0; +} + +#define MILLISECS(_ms) ((int64_t)((_ms) * 1000000ULL)) +#define MICROSECS(_us) ((int64_t)((_us) * 1000ULL)) +#define STIME_MAX ((time_t)((int64_t)~0ull >> 1)) +/* Chosen so (NOW() + delta) wont overflow without an uptime of 200 years */ +#define STIME_DELTA_MAX ((int64_t)((uint64_t)~0ull >> 2)) + +static int vcpuop_set_periodic_timer(CPUState *cs, CPUState *target, + uint64_t arg) +{ + struct vcpu_set_periodic_timer spt; + + qemu_build_assert(sizeof(spt) == 8); + if (kvm_copy_from_gva(cs, arg, &spt, sizeof(spt))) { + return -EFAULT; + } + + if (spt.period_ns < MILLISECS(1) || spt.period_ns > STIME_DELTA_MAX) { + return -EINVAL; + } + + return do_set_periodic_timer(target, spt.period_ns); +} + +static int vcpuop_stop_periodic_timer(CPUState *target) +{ + CPUX86State *tenv = &X86_CPU(target)->env; + + qemu_mutex_lock(&tenv->xen_timers_lock); + + timer_del(tenv->xen_periodic_timer); + tenv->xen_periodic_timer_period = 0; + + qemu_mutex_unlock(&tenv->xen_timers_lock); + return 0; +} + +static int do_set_singleshot_timer(CPUState *cs, uint64_t timeout_abs, + bool future, bool linux_wa) +{ + CPUX86State *env = &X86_CPU(cs)->env; + int64_t now = kvm_get_current_ns(); + int64_t qemu_now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL); + int64_t delta = timeout_abs - now; + + if (future && timeout_abs < now) { + return -ETIME; + } + + if (linux_wa && unlikely((int64_t)timeout_abs < 0 || + (delta > 0 && (uint32_t)(delta >> 50) != 0))) { + /* + * Xen has a 'Linux workaround' in do_set_timer_op() which checks + * for negative absolute timeout values (caused by integer + * overflow), and for values about 13 days in the future (2^50ns) + * which would be caused by jiffies overflow. For those cases, it + * sets the timeout 100ms in the future (not *too* soon, since if + * a guest really did set a long timeout on purpose we don't want + * to keep churning CPU time by waking it up). + */ + delta = (100 * SCALE_MS); + timeout_abs = now + delta; + } + + qemu_mutex_lock(&env->xen_timers_lock); + + timer_mod_ns(env->xen_singleshot_timer, qemu_now + delta); + env->xen_singleshot_timer_ns = now + delta; + + qemu_mutex_unlock(&env->xen_timers_lock); + return 0; +} + +static int vcpuop_set_singleshot_timer(CPUState *cs, uint64_t arg) +{ + struct vcpu_set_singleshot_timer sst = { 0 }; + + /* + * The struct is a uint64_t followed by a uint32_t. On 32-bit that + * makes it 12 bytes. On 64-bit it gets padded to 16. The parts + * that get used are identical, and there's four bytes of padding + * unused at the end. For true Xen compatibility we should attempt + * to copy the full 16 bytes from 64-bit guests, and return -EFAULT + * if we can't get the padding too. But that's daft. Just copy what + * we need. + */ + qemu_build_assert(offsetof(struct vcpu_set_singleshot_timer, flags) == 8); + qemu_build_assert(sizeof(sst) >= 12); + + if (kvm_copy_from_gva(cs, arg, &sst, 12)) { + return -EFAULT; + } + + return do_set_singleshot_timer(cs, sst.timeout_abs_ns, + !!(sst.flags & VCPU_SSHOTTMR_future), + false); +} + +static int vcpuop_stop_singleshot_timer(CPUState *cs) +{ + CPUX86State *env = &X86_CPU(cs)->env; + + qemu_mutex_lock(&env->xen_timers_lock); + + timer_del(env->xen_singleshot_timer); + env->xen_singleshot_timer_ns = 0; + + qemu_mutex_unlock(&env->xen_timers_lock); + return 0; +} + +static bool kvm_xen_hcall_set_timer_op(struct kvm_xen_exit *exit, X86CPU *cpu, + uint64_t timeout) +{ + int err; + + if (unlikely(timeout == 0)) { + err = vcpuop_stop_singleshot_timer(CPU(cpu)); + } else { + err = do_set_singleshot_timer(CPU(cpu), timeout, false, true); + } + exit->u.hcall.result = err; + return true; +} + +static bool kvm_xen_hcall_vcpu_op(struct kvm_xen_exit *exit, X86CPU *cpu, + int cmd, int vcpu_id, uint64_t arg) +{ + CPUState *cs = CPU(cpu); + CPUState *dest = cs->cpu_index == vcpu_id ? cs : qemu_get_cpu(vcpu_id); + int err; + + if (!dest) { + err = -ENOENT; + goto out; + } + + switch (cmd) { + case VCPUOP_register_runstate_memory_area: + err = vcpuop_register_runstate_info(cs, dest, arg); + break; + case VCPUOP_register_vcpu_time_memory_area: + err = vcpuop_register_vcpu_time_info(cs, dest, arg); + break; + case VCPUOP_register_vcpu_info: + err = vcpuop_register_vcpu_info(cs, dest, arg); + break; + case VCPUOP_set_singleshot_timer: { + if (cs->cpu_index == vcpu_id) { + err = vcpuop_set_singleshot_timer(dest, arg); + } else { + err = -EINVAL; + } + break; + } + case VCPUOP_stop_singleshot_timer: + if (cs->cpu_index == vcpu_id) { + err = vcpuop_stop_singleshot_timer(dest); + } else { + err = -EINVAL; + } + break; + case VCPUOP_set_periodic_timer: { + err = vcpuop_set_periodic_timer(cs, dest, arg); + break; + } + case VCPUOP_stop_periodic_timer: + err = vcpuop_stop_periodic_timer(dest); + break; + + default: + return false; + } + + out: + exit->u.hcall.result = err; + return true; +} + +static bool kvm_xen_hcall_evtchn_op(struct kvm_xen_exit *exit, X86CPU *cpu, + int cmd, uint64_t arg) +{ + CPUState *cs = CPU(cpu); + int err = -ENOSYS; + + switch (cmd) { + case EVTCHNOP_init_control: + case EVTCHNOP_expand_array: + case EVTCHNOP_set_priority: + /* We do not support FIFO channels at this point */ + err = -ENOSYS; + break; + + case EVTCHNOP_status: { + struct evtchn_status status; + + qemu_build_assert(sizeof(status) == 24); + if (kvm_copy_from_gva(cs, arg, &status, sizeof(status))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_status_op(&status); + if (!err && kvm_copy_to_gva(cs, arg, &status, sizeof(status))) { + err = -EFAULT; + } + break; + } + case EVTCHNOP_close: { + struct evtchn_close close; + + qemu_build_assert(sizeof(close) == 4); + if (kvm_copy_from_gva(cs, arg, &close, sizeof(close))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_close_op(&close); + break; + } + case EVTCHNOP_unmask: { + struct evtchn_unmask unmask; + + qemu_build_assert(sizeof(unmask) == 4); + if (kvm_copy_from_gva(cs, arg, &unmask, sizeof(unmask))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_unmask_op(&unmask); + break; + } + case EVTCHNOP_bind_virq: { + struct evtchn_bind_virq virq; + + qemu_build_assert(sizeof(virq) == 12); + if (kvm_copy_from_gva(cs, arg, &virq, sizeof(virq))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_bind_virq_op(&virq); + if (!err && kvm_copy_to_gva(cs, arg, &virq, sizeof(virq))) { + err = -EFAULT; + } + break; + } + case EVTCHNOP_bind_pirq: { + struct evtchn_bind_pirq pirq; + + qemu_build_assert(sizeof(pirq) == 12); + if (kvm_copy_from_gva(cs, arg, &pirq, sizeof(pirq))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_bind_pirq_op(&pirq); + if (!err && kvm_copy_to_gva(cs, arg, &pirq, sizeof(pirq))) { + err = -EFAULT; + } + break; + } + case EVTCHNOP_bind_ipi: { + struct evtchn_bind_ipi ipi; + + qemu_build_assert(sizeof(ipi) == 8); + if (kvm_copy_from_gva(cs, arg, &ipi, sizeof(ipi))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_bind_ipi_op(&ipi); + if (!err && kvm_copy_to_gva(cs, arg, &ipi, sizeof(ipi))) { + err = -EFAULT; + } + break; + } + case EVTCHNOP_send: { + struct evtchn_send send; + + qemu_build_assert(sizeof(send) == 4); + if (kvm_copy_from_gva(cs, arg, &send, sizeof(send))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_send_op(&send); + break; + } + case EVTCHNOP_alloc_unbound: { + struct evtchn_alloc_unbound alloc; + + qemu_build_assert(sizeof(alloc) == 8); + if (kvm_copy_from_gva(cs, arg, &alloc, sizeof(alloc))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_alloc_unbound_op(&alloc); + if (!err && kvm_copy_to_gva(cs, arg, &alloc, sizeof(alloc))) { + err = -EFAULT; + } + break; + } + case EVTCHNOP_bind_interdomain: { + struct evtchn_bind_interdomain interdomain; + + qemu_build_assert(sizeof(interdomain) == 12); + if (kvm_copy_from_gva(cs, arg, &interdomain, sizeof(interdomain))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_bind_interdomain_op(&interdomain); + if (!err && + kvm_copy_to_gva(cs, arg, &interdomain, sizeof(interdomain))) { + err = -EFAULT; + } + break; + } + case EVTCHNOP_bind_vcpu: { + struct evtchn_bind_vcpu vcpu; + + qemu_build_assert(sizeof(vcpu) == 8); + if (kvm_copy_from_gva(cs, arg, &vcpu, sizeof(vcpu))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_bind_vcpu_op(&vcpu); + break; + } + case EVTCHNOP_reset: { + struct evtchn_reset reset; + + qemu_build_assert(sizeof(reset) == 2); + if (kvm_copy_from_gva(cs, arg, &reset, sizeof(reset))) { + err = -EFAULT; + break; + } + + err = xen_evtchn_reset_op(&reset); + break; + } + default: + return false; + } + + exit->u.hcall.result = err; + return true; +} + +int kvm_xen_soft_reset(void) +{ + CPUState *cpu; + int err; + + assert(qemu_mutex_iothread_locked()); + + trace_kvm_xen_soft_reset(); + + err = xen_evtchn_soft_reset(); + if (err) { + return err; + } + + /* + * Zero is the reset/startup state for HVM_PARAM_CALLBACK_IRQ. Strictly, + * it maps to HVM_PARAM_CALLBACK_TYPE_GSI with GSI#0, but Xen refuses to + * to deliver to the timer interrupt and treats that as 'disabled'. + */ + err = xen_evtchn_set_callback_param(0); + if (err) { + return err; + } + + CPU_FOREACH(cpu) { + async_run_on_cpu(cpu, do_vcpu_soft_reset, RUN_ON_CPU_NULL); + } + + err = xen_overlay_map_shinfo_page(INVALID_GFN); + if (err) { + return err; + } + + err = xen_xenstore_reset(); + if (err) { + return err; + } + + return 0; +} + +static int schedop_shutdown(CPUState *cs, uint64_t arg) +{ + struct sched_shutdown shutdown; + int ret = 0; + + /* No need for 32/64 compat handling */ + qemu_build_assert(sizeof(shutdown) == 4); + + if (kvm_copy_from_gva(cs, arg, &shutdown, sizeof(shutdown))) { + return -EFAULT; + } + + switch (shutdown.reason) { + case SHUTDOWN_crash: + cpu_dump_state(cs, stderr, CPU_DUMP_CODE); + qemu_system_guest_panicked(NULL); + break; + + case SHUTDOWN_reboot: + qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); + break; + + case SHUTDOWN_poweroff: + qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN); + break; + + case SHUTDOWN_soft_reset: + qemu_mutex_lock_iothread(); + ret = kvm_xen_soft_reset(); + qemu_mutex_unlock_iothread(); + break; + + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static bool kvm_xen_hcall_sched_op(struct kvm_xen_exit *exit, X86CPU *cpu, + int cmd, uint64_t arg) +{ + CPUState *cs = CPU(cpu); + int err = -ENOSYS; + + switch (cmd) { + case SCHEDOP_shutdown: + err = schedop_shutdown(cs, arg); + break; + + case SCHEDOP_poll: + /* + * Linux will panic if this doesn't work. Just yield; it's not + * worth overthinking it because with event channel handling + * in KVM, the kernel will intercept this and it will never + * reach QEMU anyway. The semantics of the hypercall explicltly + * permit spurious wakeups. + */ + case SCHEDOP_yield: + sched_yield(); + err = 0; + break; + + default: + return false; + } + + exit->u.hcall.result = err; + return true; +} + +static bool kvm_xen_hcall_gnttab_op(struct kvm_xen_exit *exit, X86CPU *cpu, + int cmd, uint64_t arg, int count) +{ + CPUState *cs = CPU(cpu); + int err; + + switch (cmd) { + case GNTTABOP_set_version: { + struct gnttab_set_version set; + + qemu_build_assert(sizeof(set) == 4); + if (kvm_copy_from_gva(cs, arg, &set, sizeof(set))) { + err = -EFAULT; + break; + } + + err = xen_gnttab_set_version_op(&set); + if (!err && kvm_copy_to_gva(cs, arg, &set, sizeof(set))) { + err = -EFAULT; + } + break; + } + case GNTTABOP_get_version: { + struct gnttab_get_version get; + + qemu_build_assert(sizeof(get) == 8); + if (kvm_copy_from_gva(cs, arg, &get, sizeof(get))) { + err = -EFAULT; + break; + } + + err = xen_gnttab_get_version_op(&get); + if (!err && kvm_copy_to_gva(cs, arg, &get, sizeof(get))) { + err = -EFAULT; + } + break; + } + case GNTTABOP_query_size: { + struct gnttab_query_size size; + + qemu_build_assert(sizeof(size) == 16); + if (kvm_copy_from_gva(cs, arg, &size, sizeof(size))) { + err = -EFAULT; + break; + } + + err = xen_gnttab_query_size_op(&size); + if (!err && kvm_copy_to_gva(cs, arg, &size, sizeof(size))) { + err = -EFAULT; + } + break; + } + case GNTTABOP_setup_table: + case GNTTABOP_copy: + case GNTTABOP_map_grant_ref: + case GNTTABOP_unmap_grant_ref: + case GNTTABOP_swap_grant_ref: + return false; + + default: + /* Xen explicitly returns -ENOSYS to HVM guests for all others */ + err = -ENOSYS; + break; + } + + exit->u.hcall.result = err; + return true; +} + +static bool kvm_xen_hcall_physdev_op(struct kvm_xen_exit *exit, X86CPU *cpu, + int cmd, uint64_t arg) +{ + CPUState *cs = CPU(cpu); + int err; + + switch (cmd) { + case PHYSDEVOP_map_pirq: { + struct physdev_map_pirq map; + + if (hypercall_compat32(exit->u.hcall.longmode)) { + struct compat_physdev_map_pirq *map32 = (void *)↦ + + if (kvm_copy_from_gva(cs, arg, map32, sizeof(*map32))) { + return -EFAULT; + } + + /* + * The only thing that's different is the alignment of the + * uint64_t table_base at the end, which gets padding to make + * it 64-bit aligned in the 64-bit version. + */ + qemu_build_assert(sizeof(*map32) == 36); + qemu_build_assert(offsetof(struct physdev_map_pirq, entry_nr) == + offsetof(struct compat_physdev_map_pirq, entry_nr)); + memmove(&map.table_base, &map32->table_base, sizeof(map.table_base)); + } else { + if (kvm_copy_from_gva(cs, arg, &map, sizeof(map))) { + err = -EFAULT; + break; + } + } + err = xen_physdev_map_pirq(&map); + /* + * Since table_base is an IN parameter and won't be changed, just + * copy the size of the compat structure back to the guest. + */ + if (!err && kvm_copy_to_gva(cs, arg, &map, + sizeof(struct compat_physdev_map_pirq))) { + err = -EFAULT; + } + break; + } + case PHYSDEVOP_unmap_pirq: { + struct physdev_unmap_pirq unmap; + + qemu_build_assert(sizeof(unmap) == 8); + if (kvm_copy_from_gva(cs, arg, &unmap, sizeof(unmap))) { + err = -EFAULT; + break; + } + + err = xen_physdev_unmap_pirq(&unmap); + if (!err && kvm_copy_to_gva(cs, arg, &unmap, sizeof(unmap))) { + err = -EFAULT; + } + break; + } + case PHYSDEVOP_eoi: { + struct physdev_eoi eoi; + + qemu_build_assert(sizeof(eoi) == 4); + if (kvm_copy_from_gva(cs, arg, &eoi, sizeof(eoi))) { + err = -EFAULT; + break; + } + + err = xen_physdev_eoi_pirq(&eoi); + if (!err && kvm_copy_to_gva(cs, arg, &eoi, sizeof(eoi))) { + err = -EFAULT; + } + break; + } + case PHYSDEVOP_irq_status_query: { + struct physdev_irq_status_query query; + + qemu_build_assert(sizeof(query) == 8); + if (kvm_copy_from_gva(cs, arg, &query, sizeof(query))) { + err = -EFAULT; + break; + } + + err = xen_physdev_query_pirq(&query); + if (!err && kvm_copy_to_gva(cs, arg, &query, sizeof(query))) { + err = -EFAULT; + } + break; + } + case PHYSDEVOP_get_free_pirq: { + struct physdev_get_free_pirq get; + + qemu_build_assert(sizeof(get) == 8); + if (kvm_copy_from_gva(cs, arg, &get, sizeof(get))) { + err = -EFAULT; + break; + } + + err = xen_physdev_get_free_pirq(&get); + if (!err && kvm_copy_to_gva(cs, arg, &get, sizeof(get))) { + err = -EFAULT; + } + break; + } + case PHYSDEVOP_pirq_eoi_gmfn_v2: /* FreeBSD 13 makes this hypercall */ + err = -ENOSYS; + break; + + default: + return false; + } + + exit->u.hcall.result = err; + return true; +} + +static bool do_kvm_xen_handle_exit(X86CPU *cpu, struct kvm_xen_exit *exit) +{ + uint16_t code = exit->u.hcall.input; + + if (exit->u.hcall.cpl > 0) { + exit->u.hcall.result = -EPERM; + return true; + } + + switch (code) { + case __HYPERVISOR_set_timer_op: + if (exit->u.hcall.longmode) { + return kvm_xen_hcall_set_timer_op(exit, cpu, + exit->u.hcall.params[0]); + } else { + /* In 32-bit mode, the 64-bit timer value is in two args. */ + uint64_t val = ((uint64_t)exit->u.hcall.params[1]) << 32 | + (uint32_t)exit->u.hcall.params[0]; + return kvm_xen_hcall_set_timer_op(exit, cpu, val); + } + case __HYPERVISOR_grant_table_op: + return kvm_xen_hcall_gnttab_op(exit, cpu, exit->u.hcall.params[0], + exit->u.hcall.params[1], + exit->u.hcall.params[2]); + case __HYPERVISOR_sched_op: + return kvm_xen_hcall_sched_op(exit, cpu, exit->u.hcall.params[0], + exit->u.hcall.params[1]); + case __HYPERVISOR_event_channel_op: + return kvm_xen_hcall_evtchn_op(exit, cpu, exit->u.hcall.params[0], + exit->u.hcall.params[1]); + case __HYPERVISOR_vcpu_op: + return kvm_xen_hcall_vcpu_op(exit, cpu, + exit->u.hcall.params[0], + exit->u.hcall.params[1], + exit->u.hcall.params[2]); + case __HYPERVISOR_hvm_op: + return kvm_xen_hcall_hvm_op(exit, cpu, exit->u.hcall.params[0], + exit->u.hcall.params[1]); + case __HYPERVISOR_memory_op: + return kvm_xen_hcall_memory_op(exit, cpu, exit->u.hcall.params[0], + exit->u.hcall.params[1]); + case __HYPERVISOR_physdev_op: + return kvm_xen_hcall_physdev_op(exit, cpu, exit->u.hcall.params[0], + exit->u.hcall.params[1]); + case __HYPERVISOR_xen_version: + return kvm_xen_hcall_xen_version(exit, cpu, exit->u.hcall.params[0], + exit->u.hcall.params[1]); + default: + return false; + } +} + +int kvm_xen_handle_exit(X86CPU *cpu, struct kvm_xen_exit *exit) +{ + if (exit->type != KVM_EXIT_XEN_HCALL) { + return -1; + } + + /* + * The kernel latches the guest 32/64 mode when the MSR is used to fill + * the hypercall page. So if we see a hypercall in a mode that doesn't + * match our own idea of the guest mode, fetch the kernel's idea of the + * "long mode" to remain in sync. + */ + if (exit->u.hcall.longmode != xen_is_long_mode()) { + xen_sync_long_mode(); + } + + if (!do_kvm_xen_handle_exit(cpu, exit)) { + /* + * Some hypercalls will be deliberately "implemented" by returning + * -ENOSYS. This case is for hypercalls which are unexpected. + */ + exit->u.hcall.result = -ENOSYS; + qemu_log_mask(LOG_UNIMP, "Unimplemented Xen hypercall %" + PRId64 " (0x%" PRIx64 " 0x%" PRIx64 " 0x%" PRIx64 ")\n", + (uint64_t)exit->u.hcall.input, + (uint64_t)exit->u.hcall.params[0], + (uint64_t)exit->u.hcall.params[1], + (uint64_t)exit->u.hcall.params[2]); + } + + trace_kvm_xen_hypercall(CPU(cpu)->cpu_index, exit->u.hcall.cpl, + exit->u.hcall.input, exit->u.hcall.params[0], + exit->u.hcall.params[1], exit->u.hcall.params[2], + exit->u.hcall.result); + return 0; +} + +uint16_t kvm_xen_get_gnttab_max_frames(void) +{ + KVMState *s = KVM_STATE(current_accel()); + return s->xen_gnttab_max_frames; +} + +uint16_t kvm_xen_get_evtchn_max_pirq(void) +{ + KVMState *s = KVM_STATE(current_accel()); + return s->xen_evtchn_max_pirq; +} + +int kvm_put_xen_state(CPUState *cs) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + uint64_t gpa; + int ret; + + gpa = env->xen_vcpu_info_gpa; + if (gpa == INVALID_GPA) { + gpa = env->xen_vcpu_info_default_gpa; + } + + if (gpa != INVALID_GPA) { + ret = set_vcpu_info(cs, gpa); + if (ret < 0) { + return ret; + } + } + + gpa = env->xen_vcpu_time_info_gpa; + if (gpa != INVALID_GPA) { + ret = kvm_xen_set_vcpu_attr(cs, KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO, + gpa); + if (ret < 0) { + return ret; + } + } + + gpa = env->xen_vcpu_runstate_gpa; + if (gpa != INVALID_GPA) { + ret = kvm_xen_set_vcpu_attr(cs, KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADDR, + gpa); + if (ret < 0) { + return ret; + } + } + + if (env->xen_periodic_timer_period) { + ret = do_set_periodic_timer(cs, env->xen_periodic_timer_period); + if (ret < 0) { + return ret; + } + } + + if (!kvm_xen_has_cap(EVTCHN_SEND)) { + /* + * If the kernel has EVTCHN_SEND support then it handles timers too, + * so the timer will be restored by kvm_xen_set_vcpu_timer() below. + */ + if (env->xen_singleshot_timer_ns) { + ret = do_set_singleshot_timer(cs, env->xen_singleshot_timer_ns, + false, false); + if (ret < 0) { + return ret; + } + } + return 0; + } + + if (env->xen_vcpu_callback_vector) { + ret = kvm_xen_set_vcpu_callback_vector(cs); + if (ret < 0) { + return ret; + } + } + + if (env->xen_virq[VIRQ_TIMER]) { + ret = kvm_xen_set_vcpu_timer(cs); + if (ret < 0) { + return ret; + } + } + return 0; +} + +int kvm_get_xen_state(CPUState *cs) +{ + X86CPU *cpu = X86_CPU(cs); + CPUX86State *env = &cpu->env; + uint64_t gpa; + int ret; + + /* + * The kernel does not mark vcpu_info as dirty when it delivers interrupts + * to it. It's up to userspace to *assume* that any page shared thus is + * always considered dirty. The shared_info page is different since it's + * an overlay and migrated separately anyway. + */ + gpa = env->xen_vcpu_info_gpa; + if (gpa == INVALID_GPA) { + gpa = env->xen_vcpu_info_default_gpa; + } + if (gpa != INVALID_GPA) { + MemoryRegionSection mrs = memory_region_find(get_system_memory(), + gpa, + sizeof(struct vcpu_info)); + if (mrs.mr && + !int128_lt(mrs.size, int128_make64(sizeof(struct vcpu_info)))) { + memory_region_set_dirty(mrs.mr, mrs.offset_within_region, + sizeof(struct vcpu_info)); + } + } + + if (!kvm_xen_has_cap(EVTCHN_SEND)) { + return 0; + } + + /* + * If the kernel is accelerating timers, read out the current value of the + * singleshot timer deadline. + */ + if (env->xen_virq[VIRQ_TIMER]) { + struct kvm_xen_vcpu_attr va = { + .type = KVM_XEN_VCPU_ATTR_TYPE_TIMER, + }; + ret = kvm_vcpu_ioctl(cs, KVM_XEN_VCPU_GET_ATTR, &va); + if (ret < 0) { + return ret; + } + env->xen_singleshot_timer_ns = va.u.timer.expires_ns; + } + + return 0; +} diff --git a/target/i386/kvm/xen-emu.h b/target/i386/kvm/xen-emu.h new file mode 100644 index 0000000000..fe85e0b195 --- /dev/null +++ b/target/i386/kvm/xen-emu.h @@ -0,0 +1,33 @@ +/* + * Xen HVM emulation support in KVM + * + * Copyright © 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright © 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ + +#ifndef QEMU_I386_KVM_XEN_EMU_H +#define QEMU_I386_KVM_XEN_EMU_H + +#define XEN_HYPERCALL_MSR 0x40000000 +#define XEN_HYPERCALL_MSR_HYPERV 0x40000200 + +#define XEN_CPUID_SIGNATURE 0 +#define XEN_CPUID_VENDOR 1 +#define XEN_CPUID_HVM_MSR 2 +#define XEN_CPUID_TIME 3 +#define XEN_CPUID_HVM 4 + +#define XEN_VERSION(maj, min) ((maj) << 16 | (min)) + +int kvm_xen_init(KVMState *s, uint32_t hypercall_msr); +int kvm_xen_init_vcpu(CPUState *cs); +int kvm_xen_handle_exit(X86CPU *cpu, struct kvm_xen_exit *exit); +int kvm_put_xen_state(CPUState *cs); +int kvm_get_xen_state(CPUState *cs); +void kvm_xen_maybe_deassert_callback(CPUState *cs); + +#endif /* QEMU_I386_KVM_XEN_EMU_H */ diff --git a/target/i386/machine.c b/target/i386/machine.c index 310b125235..c7ac8084b2 100644 --- a/target/i386/machine.c +++ b/target/i386/machine.c @@ -6,8 +6,10 @@ #include "kvm/hyperv.h" #include "hw/i386/x86.h" #include "kvm/kvm_i386.h" +#include "hw/xen/xen.h" #include "sysemu/kvm.h" +#include "sysemu/kvm_xen.h" #include "sysemu/tcg.h" #include "qemu/error-report.h" @@ -1257,6 +1259,28 @@ static const VMStateDescription vmstate_nested_state = { } }; +static bool xen_vcpu_needed(void *opaque) +{ + return (xen_mode == XEN_EMULATE); +} + +static const VMStateDescription vmstate_xen_vcpu = { + .name = "cpu/xen_vcpu", + .version_id = 1, + .minimum_version_id = 1, + .needed = xen_vcpu_needed, + .fields = (VMStateField[]) { + VMSTATE_UINT64(env.xen_vcpu_info_gpa, X86CPU), + VMSTATE_UINT64(env.xen_vcpu_info_default_gpa, X86CPU), + VMSTATE_UINT64(env.xen_vcpu_time_info_gpa, X86CPU), + VMSTATE_UINT64(env.xen_vcpu_runstate_gpa, X86CPU), + VMSTATE_UINT8(env.xen_vcpu_callback_vector, X86CPU), + VMSTATE_UINT16_ARRAY(env.xen_virq, X86CPU, XEN_NR_VIRQS), + VMSTATE_UINT64(env.xen_singleshot_timer_ns, X86CPU), + VMSTATE_UINT64(env.xen_periodic_timer_period, X86CPU), + VMSTATE_END_OF_LIST() + } +}; #endif static bool mcg_ext_ctl_needed(void *opaque) @@ -1716,6 +1740,7 @@ const VMStateDescription vmstate_x86_cpu = { #endif #ifdef CONFIG_KVM &vmstate_nested_state, + &vmstate_xen_vcpu, #endif &vmstate_msr_tsx_ctrl, &vmstate_msr_intel_sgx, diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c index e5efe4ebd7..97e6579f6a 100644 --- a/target/loongarch/cpu.c +++ b/target/loongarch/cpu.c @@ -12,7 +12,6 @@ #include "qemu/module.h" #include "sysemu/qtest.h" #include "exec/exec-all.h" -#include "qapi/qapi-commands-machine-target.h" #include "cpu.h" #include "internals.h" #include "fpu/softfloat-helpers.h" @@ -547,6 +546,8 @@ static void loongarch_qemu_write(void *opaque, hwaddr addr, static uint64_t loongarch_qemu_read(void *opaque, hwaddr addr, unsigned size) { switch (addr) { + case VERSION_REG: + return 0x11ULL; case FEATURE_REG: return 1ULL << IOCSRF_MSI | 1ULL << IOCSRF_EXTIOI | 1ULL << IOCSRF_CSRIPI; @@ -750,29 +751,3 @@ static const TypeInfo loongarch_cpu_type_infos[] = { }; DEFINE_TYPES(loongarch_cpu_type_infos) - -static void loongarch_cpu_add_definition(gpointer data, gpointer user_data) -{ - ObjectClass *oc = data; - CpuDefinitionInfoList **cpu_list = user_data; - CpuDefinitionInfo *info = g_new0(CpuDefinitionInfo, 1); - const char *typename = object_class_get_name(oc); - - info->name = g_strndup(typename, - strlen(typename) - strlen("-" TYPE_LOONGARCH_CPU)); - info->q_typename = g_strdup(typename); - - QAPI_LIST_PREPEND(*cpu_list, info); -} - -CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp) -{ - CpuDefinitionInfoList *cpu_list = NULL; - GSList *list; - - list = object_class_get_list(TYPE_LOONGARCH_CPU, false); - g_slist_foreach(list, loongarch_cpu_add_definition, &cpu_list); - g_slist_free(list); - - return cpu_list; -} diff --git a/target/loongarch/cpu.h b/target/loongarch/cpu.h index d60693fafe..e11c875188 100644 --- a/target/loongarch/cpu.h +++ b/target/loongarch/cpu.h @@ -28,6 +28,7 @@ #define IOCSRF_GMOD 9 #define IOCSRF_VM 11 +#define VERSION_REG 0x0 #define FEATURE_REG 0x8 #define VENDOR_REG 0x10 #define CPUNAME_REG 0x20 diff --git a/target/loongarch/loongarch-qmp-cmds.c b/target/loongarch/loongarch-qmp-cmds.c new file mode 100644 index 0000000000..6c25957881 --- /dev/null +++ b/target/loongarch/loongarch-qmp-cmds.c @@ -0,0 +1,37 @@ +/* + * QEMU LoongArch CPU (monitor definitions) + * + * SPDX-FileCopyrightText: 2021 Loongson Technology Corporation Limited + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "qapi/qapi-commands-machine-target.h" +#include "cpu.h" + +static void loongarch_cpu_add_definition(gpointer data, gpointer user_data) +{ + ObjectClass *oc = data; + CpuDefinitionInfoList **cpu_list = user_data; + CpuDefinitionInfo *info = g_new0(CpuDefinitionInfo, 1); + const char *typename = object_class_get_name(oc); + + info->name = g_strndup(typename, + strlen(typename) - strlen("-" TYPE_LOONGARCH_CPU)); + info->q_typename = g_strdup(typename); + + QAPI_LIST_PREPEND(*cpu_list, info); +} + +CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp) +{ + CpuDefinitionInfoList *cpu_list = NULL; + GSList *list; + + list = object_class_get_list(TYPE_LOONGARCH_CPU, false); + g_slist_foreach(list, loongarch_cpu_add_definition, &cpu_list); + g_slist_free(list); + + return cpu_list; +} diff --git a/target/loongarch/meson.build b/target/loongarch/meson.build index 690633969f..9293a8ab78 100644 --- a/target/loongarch/meson.build +++ b/target/loongarch/meson.build @@ -16,6 +16,7 @@ loongarch_tcg_ss.add(zlib) loongarch_softmmu_ss = ss.source_set() loongarch_softmmu_ss.add(files( + 'loongarch-qmp-cmds.c', 'machine.c', 'tlb_helper.c', 'constant_timer.c', diff --git a/target/ppc/cpu-qom.h b/target/ppc/cpu-qom.h index 0fbd8b7246..9666f54f65 100644 --- a/target/ppc/cpu-qom.h +++ b/target/ppc/cpu-qom.h @@ -31,6 +31,8 @@ OBJECT_DECLARE_CPU_TYPE(PowerPCCPU, PowerPCCPUClass, POWERPC_CPU) +ObjectClass *ppc_cpu_class_by_name(const char *name); + typedef struct CPUArchState CPUPPCState; typedef struct ppc_tb_t ppc_tb_t; typedef struct ppc_dcr_t ppc_dcr_t; diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c index abee71d407..d62ffe8a6f 100644 --- a/target/ppc/cpu_init.c +++ b/target/ppc/cpu_init.c @@ -40,7 +40,6 @@ #include "qemu/cutils.h" #include "disas/capstone.h" #include "fpu/softfloat.h" -#include "qapi/qapi-commands-machine-target.h" #include "helper_regs.h" #include "internal.h" @@ -6841,7 +6840,7 @@ static const char *ppc_cpu_lookup_alias(const char *alias) return NULL; } -static ObjectClass *ppc_cpu_class_by_name(const char *name) +ObjectClass *ppc_cpu_class_by_name(const char *name) { char *cpu_model, *typename; ObjectClass *oc; @@ -6981,51 +6980,6 @@ void ppc_cpu_list(void) #endif } -static void ppc_cpu_defs_entry(gpointer data, gpointer user_data) -{ - ObjectClass *oc = data; - CpuDefinitionInfoList **first = user_data; - const char *typename; - CpuDefinitionInfo *info; - - typename = object_class_get_name(oc); - info = g_malloc0(sizeof(*info)); - info->name = g_strndup(typename, - strlen(typename) - strlen(POWERPC_CPU_TYPE_SUFFIX)); - - QAPI_LIST_PREPEND(*first, info); -} - -CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp) -{ - CpuDefinitionInfoList *cpu_list = NULL; - GSList *list; - int i; - - list = object_class_get_list(TYPE_POWERPC_CPU, false); - g_slist_foreach(list, ppc_cpu_defs_entry, &cpu_list); - g_slist_free(list); - - for (i = 0; ppc_cpu_aliases[i].alias != NULL; i++) { - PowerPCCPUAlias *alias = &ppc_cpu_aliases[i]; - ObjectClass *oc; - CpuDefinitionInfo *info; - - oc = ppc_cpu_class_by_name(alias->model); - if (oc == NULL) { - continue; - } - - info = g_malloc0(sizeof(*info)); - info->name = g_strdup(alias->alias); - info->q_typename = g_strdup(object_class_get_name(oc)); - - QAPI_LIST_PREPEND(cpu_list, info); - } - - return cpu_list; -} - static void ppc_cpu_set_pc(CPUState *cs, vaddr value) { PowerPCCPU *cpu = POWERPC_CPU(cs); diff --git a/target/ppc/meson.build b/target/ppc/meson.build index 79beaff147..7929de8360 100644 --- a/target/ppc/meson.build +++ b/target/ppc/meson.build @@ -39,7 +39,7 @@ ppc_softmmu_ss.add(files( 'machine.c', 'mmu-hash32.c', 'mmu_common.c', - 'monitor.c', + 'ppc-qmp-cmds.c', )) ppc_softmmu_ss.add(when: 'CONFIG_TCG', if_true: files( 'mmu_helper.c', diff --git a/target/ppc/monitor.c b/target/ppc/ppc-qmp-cmds.c index 8250b1304e..36e5b5eff8 100644 --- a/target/ppc/monitor.c +++ b/target/ppc/ppc-qmp-cmds.c @@ -1,5 +1,5 @@ /* - * QEMU monitor + * QEMU PPC (monitor definitions) * * Copyright (c) 2003-2004 Fabrice Bellard * @@ -28,6 +28,9 @@ #include "qemu/ctype.h" #include "monitor/hmp-target.h" #include "monitor/hmp.h" +#include "qapi/qapi-commands-machine-target.h" +#include "cpu-models.h" +#include "cpu-qom.h" static target_long monitor_get_ccr(Monitor *mon, const struct MonitorDef *md, int val) @@ -172,3 +175,48 @@ int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval) return -EINVAL; } + +static void ppc_cpu_defs_entry(gpointer data, gpointer user_data) +{ + ObjectClass *oc = data; + CpuDefinitionInfoList **first = user_data; + const char *typename; + CpuDefinitionInfo *info; + + typename = object_class_get_name(oc); + info = g_malloc0(sizeof(*info)); + info->name = g_strndup(typename, + strlen(typename) - strlen(POWERPC_CPU_TYPE_SUFFIX)); + + QAPI_LIST_PREPEND(*first, info); +} + +CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp) +{ + CpuDefinitionInfoList *cpu_list = NULL; + GSList *list; + int i; + + list = object_class_get_list(TYPE_POWERPC_CPU, false); + g_slist_foreach(list, ppc_cpu_defs_entry, &cpu_list); + g_slist_free(list); + + for (i = 0; ppc_cpu_aliases[i].alias != NULL; i++) { + PowerPCCPUAlias *alias = &ppc_cpu_aliases[i]; + ObjectClass *oc; + CpuDefinitionInfo *info; + + oc = ppc_cpu_class_by_name(alias->model); + if (oc == NULL) { + continue; + } + + info = g_malloc0(sizeof(*info)); + info->name = g_strdup(alias->alias); + info->q_typename = g_strdup(object_class_get_name(oc)); + + QAPI_LIST_PREPEND(cpu_list, info); + } + + return cpu_list; +} diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index 9eb748a283..5bc0005cc7 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -75,6 +75,7 @@ struct isa_ext_data { static const struct isa_ext_data isa_edata_arr[] = { ISA_EXT_DATA_ENTRY(h, false, PRIV_VERSION_1_12_0, ext_h), ISA_EXT_DATA_ENTRY(v, false, PRIV_VERSION_1_10_0, ext_v), + ISA_EXT_DATA_ENTRY(zicond, true, PRIV_VERSION_1_12_0, ext_zicond), ISA_EXT_DATA_ENTRY(zicsr, true, PRIV_VERSION_1_10_0, ext_icsr), ISA_EXT_DATA_ENTRY(zifencei, true, PRIV_VERSION_1_10_0, ext_ifencei), ISA_EXT_DATA_ENTRY(zihintpause, true, PRIV_VERSION_1_10_0, ext_zihintpause), @@ -102,12 +103,16 @@ static const struct isa_ext_data isa_edata_arr[] = { ISA_EXT_DATA_ENTRY(zkt, true, PRIV_VERSION_1_12_0, ext_zkt), ISA_EXT_DATA_ENTRY(zve32f, true, PRIV_VERSION_1_12_0, ext_zve32f), ISA_EXT_DATA_ENTRY(zve64f, true, PRIV_VERSION_1_12_0, ext_zve64f), + ISA_EXT_DATA_ENTRY(zve64d, true, PRIV_VERSION_1_12_0, ext_zve64d), + ISA_EXT_DATA_ENTRY(zvfh, true, PRIV_VERSION_1_12_0, ext_zvfh), + ISA_EXT_DATA_ENTRY(zvfhmin, true, PRIV_VERSION_1_12_0, ext_zvfhmin), ISA_EXT_DATA_ENTRY(zhinx, true, PRIV_VERSION_1_12_0, ext_zhinx), ISA_EXT_DATA_ENTRY(zhinxmin, true, PRIV_VERSION_1_12_0, ext_zhinxmin), ISA_EXT_DATA_ENTRY(smaia, true, PRIV_VERSION_1_12_0, ext_smaia), ISA_EXT_DATA_ENTRY(ssaia, true, PRIV_VERSION_1_12_0, ext_ssaia), ISA_EXT_DATA_ENTRY(sscofpmf, true, PRIV_VERSION_1_12_0, ext_sscofpmf), ISA_EXT_DATA_ENTRY(sstc, true, PRIV_VERSION_1_12_0, ext_sstc), + ISA_EXT_DATA_ENTRY(svadu, true, PRIV_VERSION_1_12_0, ext_svadu), ISA_EXT_DATA_ENTRY(svinval, true, PRIV_VERSION_1_12_0, ext_svinval), ISA_EXT_DATA_ENTRY(svnapot, true, PRIV_VERSION_1_12_0, ext_svnapot), ISA_EXT_DATA_ENTRY(svpbmt, true, PRIV_VERSION_1_12_0, ext_svpbmt), @@ -616,6 +621,11 @@ static void riscv_cpu_reset_hold(Object *obj) env->bins = 0; env->two_stage_lookup = false; + env->menvcfg = (cpu->cfg.ext_svpbmt ? MENVCFG_PBMTE : 0) | + (cpu->cfg.ext_svadu ? MENVCFG_HADE : 0); + env->henvcfg = (cpu->cfg.ext_svpbmt ? HENVCFG_PBMTE : 0) | + (cpu->cfg.ext_svadu ? HENVCFG_HADE : 0); + /* Initialized default priorities of local interrupts. */ for (i = 0; i < ARRAY_SIZE(env->miprio); i++) { iprio = riscv_cpu_default_priority(i); @@ -640,7 +650,7 @@ static void riscv_cpu_reset_hold(Object *obj) set_default_nan_mode(1, &env->fp_status); #ifndef CONFIG_USER_ONLY - if (riscv_feature(env, RISCV_FEATURE_DEBUG)) { + if (cpu->cfg.debug) { riscv_trigger_init(env); } @@ -732,7 +742,11 @@ static void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) return; } - if ((cpu->cfg.ext_zfh || cpu->cfg.ext_zfhmin) && !cpu->cfg.ext_f) { + if (cpu->cfg.ext_zfh) { + cpu->cfg.ext_zfhmin = true; + } + + if (cpu->cfg.ext_zfhmin && !cpu->cfg.ext_f) { error_setg(errp, "Zfh/Zfhmin extensions require F extension"); return; } @@ -742,19 +756,51 @@ static void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) return; } - if (cpu->cfg.ext_v && !cpu->cfg.ext_d) { - error_setg(errp, "V extension requires D extension"); + /* The V vector extension depends on the Zve64d extension */ + if (cpu->cfg.ext_v) { + cpu->cfg.ext_zve64d = true; + } + + /* The Zve64d extension depends on the Zve64f extension */ + if (cpu->cfg.ext_zve64d) { + cpu->cfg.ext_zve64f = true; + } + + /* The Zve64f extension depends on the Zve32f extension */ + if (cpu->cfg.ext_zve64f) { + cpu->cfg.ext_zve32f = true; + } + + if (cpu->cfg.ext_zve64d && !cpu->cfg.ext_d) { + error_setg(errp, "Zve64d/V extensions require D extension"); return; } - if ((cpu->cfg.ext_zve32f || cpu->cfg.ext_zve64f) && !cpu->cfg.ext_f) { + if (cpu->cfg.ext_zve32f && !cpu->cfg.ext_f) { error_setg(errp, "Zve32f/Zve64f extensions require F extension"); return; } + if (cpu->cfg.ext_zvfh) { + cpu->cfg.ext_zvfhmin = true; + } + + if (cpu->cfg.ext_zvfhmin && !cpu->cfg.ext_zve32f) { + error_setg(errp, "Zvfh/Zvfhmin extensions require Zve32f extension"); + return; + } + + if (cpu->cfg.ext_zvfh && !cpu->cfg.ext_zfhmin) { + error_setg(errp, "Zvfh extensions requires Zfhmin extension"); + return; + } + /* Set the ISA extensions, checks should have happened above */ - if (cpu->cfg.ext_zdinx || cpu->cfg.ext_zhinx || - cpu->cfg.ext_zhinxmin) { + if (cpu->cfg.ext_zhinx) { + cpu->cfg.ext_zhinxmin = true; + } + + if (cpu->cfg.ext_zdinx || cpu->cfg.ext_zhinxmin) { cpu->cfg.ext_zfinx = true; } @@ -765,7 +811,7 @@ static void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) } if (cpu->cfg.ext_f) { error_setg(errp, - "Zfinx cannot be supported together with F extension"); + "Zfinx cannot be supported together with F extension"); return; } } @@ -828,40 +874,40 @@ static void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) ext |= RVV; if (!is_power_of_2(cpu->cfg.vlen)) { error_setg(errp, - "Vector extension VLEN must be power of 2"); + "Vector extension VLEN must be power of 2"); return; } if (cpu->cfg.vlen > RV_VLEN_MAX || cpu->cfg.vlen < 128) { error_setg(errp, - "Vector extension implementation only supports VLEN " - "in the range [128, %d]", RV_VLEN_MAX); + "Vector extension implementation only supports VLEN " + "in the range [128, %d]", RV_VLEN_MAX); return; } if (!is_power_of_2(cpu->cfg.elen)) { error_setg(errp, - "Vector extension ELEN must be power of 2"); + "Vector extension ELEN must be power of 2"); return; } - if (cpu->cfg.elen > 64 || cpu->cfg.elen < 8) { - error_setg(errp, - "Vector extension implementation only supports ELEN " - "in the range [8, 64]"); - return; - } - if (cpu->cfg.vext_spec) { - if (!g_strcmp0(cpu->cfg.vext_spec, "v1.0")) { - vext_version = VEXT_VERSION_1_00_0; - } else { + if (cpu->cfg.elen > 64 || cpu->cfg.elen < 8) { error_setg(errp, - "Unsupported vector spec version '%s'", - cpu->cfg.vext_spec); + "Vector extension implementation only supports ELEN " + "in the range [8, 64]"); return; } - } else { - qemu_log("vector version is not specified, " - "use the default value v1.0\n"); - } - set_vext_version(env, vext_version); + if (cpu->cfg.vext_spec) { + if (!g_strcmp0(cpu->cfg.vext_spec, "v1.0")) { + vext_version = VEXT_VERSION_1_00_0; + } else { + error_setg(errp, + "Unsupported vector spec version '%s'", + cpu->cfg.vext_spec); + return; + } + } else { + qemu_log("vector version is not specified, " + "use the default value v1.0\n"); + } + set_vext_version(env, vext_version); } if (cpu->cfg.ext_j) { ext |= RVJ; @@ -922,24 +968,13 @@ static void riscv_cpu_realize(DeviceState *dev, Error **errp) } } - if (cpu->cfg.mmu) { - riscv_set_feature(env, RISCV_FEATURE_MMU); - } - - if (cpu->cfg.pmp) { - riscv_set_feature(env, RISCV_FEATURE_PMP); - + if (cpu->cfg.epmp && !cpu->cfg.pmp) { /* * Enhanced PMP should only be available * on harts with PMP support */ - if (cpu->cfg.epmp) { - riscv_set_feature(env, RISCV_FEATURE_EPMP); - } - } - - if (cpu->cfg.debug) { - riscv_set_feature(env, RISCV_FEATURE_DEBUG); + error_setg(errp, "Invalid configuration: EPMP requires PMP support"); + return; } @@ -1093,6 +1128,7 @@ static Property riscv_cpu_extensions[] = { DEFINE_PROP_BOOL("Zfhmin", RISCVCPU, cfg.ext_zfhmin, false), DEFINE_PROP_BOOL("Zve32f", RISCVCPU, cfg.ext_zve32f, false), DEFINE_PROP_BOOL("Zve64f", RISCVCPU, cfg.ext_zve64f, false), + DEFINE_PROP_BOOL("Zve64d", RISCVCPU, cfg.ext_zve64d, false), DEFINE_PROP_BOOL("mmu", RISCVCPU, cfg.mmu, true), DEFINE_PROP_BOOL("pmp", RISCVCPU, cfg.pmp, true), DEFINE_PROP_BOOL("sstc", RISCVCPU, cfg.ext_sstc, true), @@ -1102,6 +1138,8 @@ static Property riscv_cpu_extensions[] = { DEFINE_PROP_UINT16("vlen", RISCVCPU, cfg.vlen, 128), DEFINE_PROP_UINT16("elen", RISCVCPU, cfg.elen, 64), + DEFINE_PROP_BOOL("svadu", RISCVCPU, cfg.ext_svadu, true), + DEFINE_PROP_BOOL("svinval", RISCVCPU, cfg.ext_svinval, false), DEFINE_PROP_BOOL("svnapot", RISCVCPU, cfg.ext_svnapot, false), DEFINE_PROP_BOOL("svpbmt", RISCVCPU, cfg.ext_svpbmt, false), @@ -1146,12 +1184,16 @@ static Property riscv_cpu_extensions[] = { DEFINE_PROP_BOOL("xventanacondops", RISCVCPU, cfg.ext_XVentanaCondOps, false), /* These are experimental so mark with 'x-' */ + DEFINE_PROP_BOOL("x-zicond", RISCVCPU, cfg.ext_zicond, false), DEFINE_PROP_BOOL("x-j", RISCVCPU, cfg.ext_j, false), /* ePMP 0.9.3 */ DEFINE_PROP_BOOL("x-epmp", RISCVCPU, cfg.epmp, false), DEFINE_PROP_BOOL("x-smaia", RISCVCPU, cfg.ext_smaia, false), DEFINE_PROP_BOOL("x-ssaia", RISCVCPU, cfg.ext_ssaia, false), + DEFINE_PROP_BOOL("x-zvfh", RISCVCPU, cfg.ext_zvfh, false), + DEFINE_PROP_BOOL("x-zvfhmin", RISCVCPU, cfg.ext_zvfhmin, false), + DEFINE_PROP_END_OF_LIST(), }; @@ -1213,6 +1255,12 @@ static Property riscv_cpu_properties[] = { DEFINE_PROP_BOOL("rvv_ta_all_1s", RISCVCPU, cfg.rvv_ta_all_1s, false), DEFINE_PROP_BOOL("rvv_ma_all_1s", RISCVCPU, cfg.rvv_ma_all_1s, false), + + /* + * write_misa() is marked as experimental for now so mark + * it with -x and default to 'false'. + */ + DEFINE_PROP_BOOL("x-misa-w", RISCVCPU, cfg.misa_w, false), DEFINE_PROP_END_OF_LIST(), }; diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index 31537fc05f..665b4c60b0 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -81,17 +81,6 @@ #define RVH RV('H') #define RVJ RV('J') -/* S extension denotes that Supervisor mode exists, however it is possible - to have a core that support S mode but does not have an MMU and there - is currently no bit in misa to indicate whether an MMU exists or not - so a cpu features bitfield is required, likewise for optional PMP support */ -enum { - RISCV_FEATURE_MMU, - RISCV_FEATURE_PMP, - RISCV_FEATURE_EPMP, - RISCV_FEATURE_MISA, - RISCV_FEATURE_DEBUG -}; /* Privileged specification version */ enum { @@ -186,8 +175,6 @@ struct CPUArchState { /* 128-bit helpers upper part return value */ target_ulong retxh; - uint32_t features; - #ifdef CONFIG_USER_ONLY uint32_t elf_flags; #endif @@ -447,9 +434,11 @@ struct RISCVCPUConfig { bool ext_zkt; bool ext_ifencei; bool ext_icsr; + bool ext_zicond; bool ext_zihintpause; bool ext_smstateen; bool ext_sstc; + bool ext_svadu; bool ext_svinval; bool ext_svnapot; bool ext_svpbmt; @@ -462,7 +451,10 @@ struct RISCVCPUConfig { bool ext_zhinxmin; bool ext_zve32f; bool ext_zve64f; + bool ext_zve64d; bool ext_zmmul; + bool ext_zvfh; + bool ext_zvfhmin; bool ext_smaia; bool ext_ssaia; bool ext_sscofpmf; @@ -498,6 +490,7 @@ struct RISCVCPUConfig { bool pmp; bool epmp; bool debug; + bool misa_w; bool short_isa_string; }; @@ -535,16 +528,6 @@ static inline int riscv_has_ext(CPURISCVState *env, target_ulong ext) return (env->misa_ext & ext) != 0; } -static inline bool riscv_feature(CPURISCVState *env, int feature) -{ - return env->features & (1ULL << feature); -} - -static inline void riscv_set_feature(CPURISCVState *env, int feature) -{ - env->features |= (1ULL << feature); -} - #include "cpu_user.h" extern const char * const riscv_int_regnames[]; @@ -654,6 +637,11 @@ static inline RISCVMXL riscv_cpu_mxl(CPURISCVState *env) #endif #define riscv_cpu_mxl_bits(env) (1UL << (4 + riscv_cpu_mxl(env))) +static inline const RISCVCPUConfig *riscv_cpu_cfg(CPURISCVState *env) +{ + return &env_archcpu(env)->cfg; +} + #if defined(TARGET_RISCV32) #define cpu_recompute_xl(env) ((void)(env), MXL_RV32) #else diff --git a/target/riscv/cpu_bits.h b/target/riscv/cpu_bits.h index 8b0d7e20ea..fca7ef0cef 100644 --- a/target/riscv/cpu_bits.h +++ b/target/riscv/cpu_bits.h @@ -747,10 +747,12 @@ typedef enum RISCVException { #define MENVCFG_CBIE (3UL << 4) #define MENVCFG_CBCFE BIT(6) #define MENVCFG_CBZE BIT(7) +#define MENVCFG_HADE (1ULL << 61) #define MENVCFG_PBMTE (1ULL << 62) #define MENVCFG_STCE (1ULL << 63) /* For RV32 */ +#define MENVCFGH_HADE BIT(29) #define MENVCFGH_PBMTE BIT(30) #define MENVCFGH_STCE BIT(31) @@ -763,10 +765,12 @@ typedef enum RISCVException { #define HENVCFG_CBIE MENVCFG_CBIE #define HENVCFG_CBCFE MENVCFG_CBCFE #define HENVCFG_CBZE MENVCFG_CBZE +#define HENVCFG_HADE MENVCFG_HADE #define HENVCFG_PBMTE MENVCFG_PBMTE #define HENVCFG_STCE MENVCFG_STCE /* For RV32 */ +#define HENVCFGH_HADE MENVCFGH_HADE #define HENVCFGH_PBMTE MENVCFGH_PBMTE #define HENVCFGH_STCE MENVCFGH_STCE diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c index 3a9472a2ff..f88c503cf4 100644 --- a/target/riscv/cpu_helper.c +++ b/target/riscv/cpu_helper.c @@ -51,7 +51,7 @@ void cpu_get_tb_cpu_state(CPURISCVState *env, target_ulong *pc, *pc = env->xl == MXL_RV32 ? env->pc & UINT32_MAX : env->pc; *cs_base = 0; - if (riscv_has_ext(env, RVV) || cpu->cfg.ext_zve32f || cpu->cfg.ext_zve64f) { + if (cpu->cfg.ext_zve32f) { /* * If env->vl equals to VLMAX, we can use generic vector operation * expanders (GVEC) to accerlate the vector operations. @@ -105,7 +105,7 @@ void cpu_get_tb_cpu_state(CPURISCVState *env, target_ulong *pc, flags = FIELD_DP32(flags, TB_FLAGS, MSTATUS_HS_VS, get_field(env->mstatus_hs, MSTATUS_VS)); } - if (riscv_feature(env, RISCV_FEATURE_DEBUG) && !icount_enabled()) { + if (cpu->cfg.debug && !icount_enabled()) { flags = FIELD_DP32(flags, TB_FLAGS, ITRIGGER, env->itrigger_enabled); } #endif @@ -706,7 +706,7 @@ static int get_physical_address_pmp(CPURISCVState *env, int *prot, pmp_priv_t pmp_priv; int pmp_index = -1; - if (!riscv_feature(env, RISCV_FEATURE_PMP)) { + if (!riscv_cpu_cfg(env)->pmp) { *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; return TRANSLATE_SUCCESS; } @@ -796,7 +796,7 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical, mode = PRV_U; } - if (mode == PRV_M || !riscv_feature(env, RISCV_FEATURE_MMU)) { + if (mode == PRV_M || !riscv_cpu_cfg(env)->mmu) { *physical = addr; *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; return TRANSLATE_SUCCESS; @@ -936,9 +936,17 @@ restart: return TRANSLATE_FAIL; } + bool pbmte = env->menvcfg & MENVCFG_PBMTE; + bool hade = env->menvcfg & MENVCFG_HADE; + + if (first_stage && two_stage && riscv_cpu_virt_enabled(env)) { + pbmte = pbmte && (env->henvcfg & HENVCFG_PBMTE); + hade = hade && (env->henvcfg & HENVCFG_HADE); + } + if (riscv_cpu_sxl(env) == MXL_RV32) { ppn = pte >> PTE_PPN_SHIFT; - } else if (cpu->cfg.ext_svpbmt || cpu->cfg.ext_svnapot) { + } else if (pbmte || cpu->cfg.ext_svnapot) { ppn = (pte & (target_ulong)PTE_PPN_MASK) >> PTE_PPN_SHIFT; } else { ppn = pte >> PTE_PPN_SHIFT; @@ -950,7 +958,7 @@ restart: if (!(pte & PTE_V)) { /* Invalid PTE */ return TRANSLATE_FAIL; - } else if (!cpu->cfg.ext_svpbmt && (pte & PTE_PBMT)) { + } else if (!pbmte && (pte & PTE_PBMT)) { return TRANSLATE_FAIL; } else if (!(pte & (PTE_R | PTE_W | PTE_X))) { /* Inner PTE, continue walking */ @@ -992,6 +1000,10 @@ restart: /* Page table updates need to be atomic with MTTCG enabled */ if (updated_pte != pte) { + if (!hade) { + return TRANSLATE_FAIL; + } + /* * - if accessed or dirty bits need updating, and the PTE is * in RAM, then we do so atomically with a compare and swap. diff --git a/target/riscv/csr.c b/target/riscv/csr.c index 1b0a0c1693..3106f96212 100644 --- a/target/riscv/csr.c +++ b/target/riscv/csr.c @@ -46,10 +46,8 @@ static RISCVException smstateen_acc_ok(CPURISCVState *env, int index, uint64_t bit) { bool virt = riscv_cpu_virt_enabled(env); - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); - if (env->priv == PRV_M || !cpu->cfg.ext_smstateen) { + if (env->priv == PRV_M || !riscv_cpu_cfg(env)->ext_smstateen) { return RISCV_EXCP_NONE; } @@ -81,7 +79,7 @@ static RISCVException fs(CPURISCVState *env, int csrno) { #if !defined(CONFIG_USER_ONLY) if (!env->debugger && !riscv_cpu_fp_enabled(env) && - !RISCV_CPU(env_cpu(env))->cfg.ext_zfinx) { + !riscv_cpu_cfg(env)->ext_zfinx) { return RISCV_EXCP_ILLEGAL_INST; } #endif @@ -90,11 +88,9 @@ static RISCVException fs(CPURISCVState *env, int csrno) static RISCVException vs(CPURISCVState *env, int csrno) { - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); + RISCVCPU *cpu = env_archcpu(env); - if (env->misa_ext & RVV || - cpu->cfg.ext_zve32f || cpu->cfg.ext_zve64f) { + if (cpu->cfg.ext_zve32f) { #if !defined(CONFIG_USER_ONLY) if (!env->debugger && !riscv_cpu_vector_enabled(env)) { return RISCV_EXCP_ILLEGAL_INST; @@ -108,8 +104,7 @@ static RISCVException vs(CPURISCVState *env, int csrno) static RISCVException ctr(CPURISCVState *env, int csrno) { #if !defined(CONFIG_USER_ONLY) - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); + RISCVCPU *cpu = env_archcpu(env); int ctr_index; target_ulong ctr_mask; int base_csrno = CSR_CYCLE; @@ -134,6 +129,10 @@ static RISCVException ctr(CPURISCVState *env, int csrno) skip_ext_pmu_check: + if (env->debugger) { + return RISCV_EXCP_NONE; + } + if (env->priv < PRV_M && !get_field(env->mcounteren, ctr_mask)) { return RISCV_EXCP_ILLEGAL_INST; } @@ -166,8 +165,7 @@ static RISCVException ctr32(CPURISCVState *env, int csrno) #if !defined(CONFIG_USER_ONLY) static RISCVException mctr(CPURISCVState *env, int csrno) { - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); + int pmu_num = riscv_cpu_cfg(env)->pmu_num; int ctr_index; int base_csrno = CSR_MHPMCOUNTER3; @@ -176,7 +174,7 @@ static RISCVException mctr(CPURISCVState *env, int csrno) base_csrno += 0x80; } ctr_index = csrno - base_csrno; - if (!cpu->cfg.pmu_num || ctr_index >= cpu->cfg.pmu_num) { + if (!pmu_num || ctr_index >= pmu_num) { /* The PMU is not enabled or counter is out of range*/ return RISCV_EXCP_ILLEGAL_INST; } @@ -195,8 +193,7 @@ static RISCVException mctr32(CPURISCVState *env, int csrno) static RISCVException sscofpmf(CPURISCVState *env, int csrno) { - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); + RISCVCPU *cpu = env_archcpu(env); if (!cpu->cfg.ext_sscofpmf) { return RISCV_EXCP_ILLEGAL_INST; @@ -222,9 +219,7 @@ static RISCVException any32(CPURISCVState *env, int csrno) static int aia_any(CPURISCVState *env, int csrno) { - RISCVCPU *cpu = env_archcpu(env); - - if (!cpu->cfg.ext_smaia) { + if (!riscv_cpu_cfg(env)->ext_smaia) { return RISCV_EXCP_ILLEGAL_INST; } @@ -233,9 +228,7 @@ static int aia_any(CPURISCVState *env, int csrno) static int aia_any32(CPURISCVState *env, int csrno) { - RISCVCPU *cpu = env_archcpu(env); - - if (!cpu->cfg.ext_smaia) { + if (!riscv_cpu_cfg(env)->ext_smaia) { return RISCV_EXCP_ILLEGAL_INST; } @@ -262,9 +255,7 @@ static int smode32(CPURISCVState *env, int csrno) static int aia_smode(CPURISCVState *env, int csrno) { - RISCVCPU *cpu = env_archcpu(env); - - if (!cpu->cfg.ext_ssaia) { + if (!riscv_cpu_cfg(env)->ext_ssaia) { return RISCV_EXCP_ILLEGAL_INST; } @@ -273,9 +264,7 @@ static int aia_smode(CPURISCVState *env, int csrno) static int aia_smode32(CPURISCVState *env, int csrno) { - RISCVCPU *cpu = env_archcpu(env); - - if (!cpu->cfg.ext_ssaia) { + if (!riscv_cpu_cfg(env)->ext_ssaia) { return RISCV_EXCP_ILLEGAL_INST; } @@ -321,8 +310,7 @@ static RISCVException umode32(CPURISCVState *env, int csrno) static RISCVException mstateen(CPURISCVState *env, int csrno) { - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); + RISCVCPU *cpu = env_archcpu(env); if (!cpu->cfg.ext_smstateen) { return RISCV_EXCP_ILLEGAL_INST; @@ -333,20 +321,28 @@ static RISCVException mstateen(CPURISCVState *env, int csrno) static RISCVException hstateen_pred(CPURISCVState *env, int csrno, int base) { - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); + RISCVCPU *cpu = env_archcpu(env); if (!cpu->cfg.ext_smstateen) { return RISCV_EXCP_ILLEGAL_INST; } + RISCVException ret = hmode(env, csrno); + if (ret != RISCV_EXCP_NONE) { + return ret; + } + + if (env->debugger) { + return RISCV_EXCP_NONE; + } + if (env->priv < PRV_M) { if (!(env->mstateen[csrno - base] & SMSTATEEN_STATEEN)) { return RISCV_EXCP_ILLEGAL_INST; } } - return hmode(env, csrno); + return RISCV_EXCP_NONE; } static RISCVException hstateen(CPURISCVState *env, int csrno) @@ -363,13 +359,20 @@ static RISCVException sstateen(CPURISCVState *env, int csrno) { bool virt = riscv_cpu_virt_enabled(env); int index = csrno - CSR_SSTATEEN0; - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); - if (!cpu->cfg.ext_smstateen) { + if (!riscv_cpu_cfg(env)->ext_smstateen) { return RISCV_EXCP_ILLEGAL_INST; } + RISCVException ret = smode(env, csrno); + if (ret != RISCV_EXCP_NONE) { + return ret; + } + + if (env->debugger) { + return RISCV_EXCP_NONE; + } + if (env->priv < PRV_M) { if (!(env->mstateen[index] & SMSTATEEN_STATEEN)) { return RISCV_EXCP_ILLEGAL_INST; @@ -382,7 +385,61 @@ static RISCVException sstateen(CPURISCVState *env, int csrno) } } - return smode(env, csrno); + return RISCV_EXCP_NONE; +} + +static RISCVException sstc(CPURISCVState *env, int csrno) +{ + RISCVCPU *cpu = env_archcpu(env); + bool hmode_check = false; + + if (!cpu->cfg.ext_sstc || !env->rdtime_fn) { + return RISCV_EXCP_ILLEGAL_INST; + } + + if ((csrno == CSR_VSTIMECMP) || (csrno == CSR_VSTIMECMPH)) { + hmode_check = true; + } + + RISCVException ret = hmode_check ? hmode(env, csrno) : smode(env, csrno); + if (ret != RISCV_EXCP_NONE) { + return ret; + } + + if (env->debugger) { + return RISCV_EXCP_NONE; + } + + if (env->priv == PRV_M) { + return RISCV_EXCP_NONE; + } + + /* + * No need of separate function for rv32 as menvcfg stores both menvcfg + * menvcfgh for RV32. + */ + if (!(get_field(env->mcounteren, COUNTEREN_TM) && + get_field(env->menvcfg, MENVCFG_STCE))) { + return RISCV_EXCP_ILLEGAL_INST; + } + + if (riscv_cpu_virt_enabled(env)) { + if (!(get_field(env->hcounteren, COUNTEREN_TM) && + get_field(env->henvcfg, HENVCFG_STCE))) { + return RISCV_EXCP_VIRT_INSTRUCTION_FAULT; + } + } + + return RISCV_EXCP_NONE; +} + +static RISCVException sstc_32(CPURISCVState *env, int csrno) +{ + if (riscv_cpu_mxl(env) != MXL_RV32) { + return RISCV_EXCP_ILLEGAL_INST; + } + + return sstc(env, csrno); } /* Checks if PointerMasking registers could be accessed */ @@ -397,9 +454,7 @@ static RISCVException pointer_masking(CPURISCVState *env, int csrno) static int aia_hmode(CPURISCVState *env, int csrno) { - RISCVCPU *cpu = env_archcpu(env); - - if (!cpu->cfg.ext_ssaia) { + if (!riscv_cpu_cfg(env)->ext_ssaia) { return RISCV_EXCP_ILLEGAL_INST; } @@ -408,9 +463,7 @@ static int aia_hmode(CPURISCVState *env, int csrno) static int aia_hmode32(CPURISCVState *env, int csrno) { - RISCVCPU *cpu = env_archcpu(env); - - if (!cpu->cfg.ext_ssaia) { + if (!riscv_cpu_cfg(env)->ext_ssaia) { return RISCV_EXCP_ILLEGAL_INST; } @@ -419,7 +472,16 @@ static int aia_hmode32(CPURISCVState *env, int csrno) static RISCVException pmp(CPURISCVState *env, int csrno) { - if (riscv_feature(env, RISCV_FEATURE_PMP)) { + if (riscv_cpu_cfg(env)->pmp) { + if (csrno <= CSR_PMPCFG3) { + uint32_t reg_index = csrno - CSR_PMPCFG0; + + /* TODO: RV128 restriction check */ + if ((reg_index & 1) && (riscv_cpu_mxl(env) == MXL_RV64)) { + return RISCV_EXCP_ILLEGAL_INST; + } + } + return RISCV_EXCP_NONE; } @@ -428,7 +490,7 @@ static RISCVException pmp(CPURISCVState *env, int csrno) static RISCVException epmp(CPURISCVState *env, int csrno) { - if (env->priv == PRV_M && riscv_feature(env, RISCV_FEATURE_EPMP)) { + if (riscv_cpu_cfg(env)->epmp) { return RISCV_EXCP_NONE; } @@ -437,7 +499,7 @@ static RISCVException epmp(CPURISCVState *env, int csrno) static RISCVException debug(CPURISCVState *env, int csrno) { - if (riscv_feature(env, RISCV_FEATURE_DEBUG)) { + if (riscv_cpu_cfg(env)->debug) { return RISCV_EXCP_NONE; } @@ -447,13 +509,15 @@ static RISCVException debug(CPURISCVState *env, int csrno) static RISCVException seed(CPURISCVState *env, int csrno) { - RISCVCPU *cpu = env_archcpu(env); - - if (!cpu->cfg.ext_zkr) { + if (!riscv_cpu_cfg(env)->ext_zkr) { return RISCV_EXCP_ILLEGAL_INST; } #if !defined(CONFIG_USER_ONLY) + if (env->debugger) { + return RISCV_EXCP_NONE; + } + /* * With a CSR read-write instruction: * 1) The seed CSR is always available in machine mode as normal. @@ -572,7 +636,7 @@ static RISCVException read_vl(CPURISCVState *env, int csrno, static int read_vlenb(CPURISCVState *env, int csrno, target_ulong *val) { - *val = env_archcpu(env)->cfg.vlen >> 3; + *val = riscv_cpu_cfg(env)->vlen >> 3; return RISCV_EXCP_NONE; } @@ -627,7 +691,7 @@ static RISCVException write_vstart(CPURISCVState *env, int csrno, * The vstart CSR is defined to have only enough writable bits * to hold the largest element index, i.e. lg2(VLEN) bits. */ - env->vstart = val & ~(~0ULL << ctzl(env_archcpu(env)->cfg.vlen)); + env->vstart = val & ~(~0ULL << ctzl(riscv_cpu_cfg(env)->vlen)); return RISCV_EXCP_NONE; } @@ -916,54 +980,8 @@ static RISCVException read_timeh(CPURISCVState *env, int csrno, return RISCV_EXCP_NONE; } -static RISCVException sstc(CPURISCVState *env, int csrno) -{ - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); - bool hmode_check = false; - - if (!cpu->cfg.ext_sstc || !env->rdtime_fn) { - return RISCV_EXCP_ILLEGAL_INST; - } - - if (env->priv == PRV_M) { - return RISCV_EXCP_NONE; - } - - /* - * No need of separate function for rv32 as menvcfg stores both menvcfg - * menvcfgh for RV32. - */ - if (!(get_field(env->mcounteren, COUNTEREN_TM) && - get_field(env->menvcfg, MENVCFG_STCE))) { - return RISCV_EXCP_ILLEGAL_INST; - } - - if (riscv_cpu_virt_enabled(env)) { - if (!(get_field(env->hcounteren, COUNTEREN_TM) && - get_field(env->henvcfg, HENVCFG_STCE))) { - return RISCV_EXCP_VIRT_INSTRUCTION_FAULT; - } - } - - if ((csrno == CSR_VSTIMECMP) || (csrno == CSR_VSTIMECMPH)) { - hmode_check = true; - } - - return hmode_check ? hmode(env, csrno) : smode(env, csrno); -} - -static RISCVException sstc_32(CPURISCVState *env, int csrno) -{ - if (riscv_cpu_mxl(env) != MXL_RV32) { - return RISCV_EXCP_ILLEGAL_INST; - } - - return sstc(env, csrno); -} - static RISCVException read_vstimecmp(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { *val = env->vstimecmp; @@ -971,7 +989,7 @@ static RISCVException read_vstimecmp(CPURISCVState *env, int csrno, } static RISCVException read_vstimecmph(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { *val = env->vstimecmp >> 32; @@ -979,7 +997,7 @@ static RISCVException read_vstimecmph(CPURISCVState *env, int csrno, } static RISCVException write_vstimecmp(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { RISCVCPU *cpu = env_archcpu(env); @@ -996,7 +1014,7 @@ static RISCVException write_vstimecmp(CPURISCVState *env, int csrno, } static RISCVException write_vstimecmph(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { RISCVCPU *cpu = env_archcpu(env); @@ -1020,7 +1038,7 @@ static RISCVException read_stimecmp(CPURISCVState *env, int csrno, } static RISCVException read_stimecmph(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { if (riscv_cpu_virt_enabled(env)) { *val = env->vstimecmp >> 32; @@ -1032,7 +1050,7 @@ static RISCVException read_stimecmph(CPURISCVState *env, int csrno, } static RISCVException write_stimecmp(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { RISCVCPU *cpu = env_archcpu(env); @@ -1055,7 +1073,7 @@ static RISCVException write_stimecmp(CPURISCVState *env, int csrno, } static RISCVException write_stimecmph(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { RISCVCPU *cpu = env_archcpu(env); @@ -1152,8 +1170,7 @@ static RISCVException write_ignore(CPURISCVState *env, int csrno, static RISCVException read_mvendorid(CPURISCVState *env, int csrno, target_ulong *val) { - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); + RISCVCPU *cpu = env_archcpu(env); *val = cpu->cfg.mvendorid; return RISCV_EXCP_NONE; @@ -1162,8 +1179,7 @@ static RISCVException read_mvendorid(CPURISCVState *env, int csrno, static RISCVException read_marchid(CPURISCVState *env, int csrno, target_ulong *val) { - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); + RISCVCPU *cpu = env_archcpu(env); *val = cpu->cfg.marchid; return RISCV_EXCP_NONE; @@ -1172,8 +1188,7 @@ static RISCVException read_marchid(CPURISCVState *env, int csrno, static RISCVException read_mimpid(CPURISCVState *env, int csrno, target_ulong *val) { - CPUState *cs = env_cpu(env); - RISCVCPU *cpu = RISCV_CPU(cs); + RISCVCPU *cpu = env_archcpu(env); *val = cpu->cfg.mimpid; return RISCV_EXCP_NONE; @@ -1329,7 +1344,7 @@ static RISCVException read_misa(CPURISCVState *env, int csrno, static RISCVException write_misa(CPURISCVState *env, int csrno, target_ulong val) { - if (!riscv_feature(env, RISCV_FEATURE_MISA)) { + if (!riscv_cpu_cfg(env)->misa_w) { /* drop write to misa */ return RISCV_EXCP_NONE; } @@ -1342,7 +1357,8 @@ static RISCVException write_misa(CPURISCVState *env, int csrno, /* 'E' excludes all other extensions */ if (val & RVE) { - /* when we support 'E' we can do "val = RVE;" however + /* + * when we support 'E' we can do "val = RVE;" however * for now we just drop writes if 'E' is present. */ return RISCV_EXCP_NONE; @@ -1356,15 +1372,13 @@ static RISCVException write_misa(CPURISCVState *env, int csrno, /* Mask extensions that are not supported by this hart */ val &= env->misa_ext_mask; - /* Mask extensions that are not supported by QEMU */ - val &= (RVI | RVE | RVM | RVA | RVF | RVD | RVC | RVS | RVU | RVV); - /* 'D' depends on 'F', so clear 'D' if 'F' is not present */ if ((val & RVD) && !(val & RVF)) { val &= ~RVD; } - /* Suppress 'C' if next instruction is not aligned + /* + * Suppress 'C' if next instruction is not aligned * TODO: this should check next_pc */ if ((val & RVC) && (GETPC() & ~3) != 0) { @@ -1833,28 +1847,28 @@ static RISCVException write_mscratch(CPURISCVState *env, int csrno, } static RISCVException read_mepc(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { *val = env->mepc; return RISCV_EXCP_NONE; } static RISCVException write_mepc(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { env->mepc = val; return RISCV_EXCP_NONE; } static RISCVException read_mcause(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { *val = env->mcause; return RISCV_EXCP_NONE; } static RISCVException write_mcause(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { env->mcause = val; return RISCV_EXCP_NONE; @@ -1876,19 +1890,22 @@ static RISCVException write_mtval(CPURISCVState *env, int csrno, /* Execution environment configuration setup */ static RISCVException read_menvcfg(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { *val = env->menvcfg; return RISCV_EXCP_NONE; } static RISCVException write_menvcfg(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { + RISCVCPUConfig *cfg = &env_archcpu(env)->cfg; uint64_t mask = MENVCFG_FIOM | MENVCFG_CBIE | MENVCFG_CBCFE | MENVCFG_CBZE; if (riscv_cpu_mxl(env) == MXL_RV64) { - mask |= MENVCFG_PBMTE | MENVCFG_STCE; + mask |= (cfg->ext_svpbmt ? MENVCFG_PBMTE : 0) | + (cfg->ext_sstc ? MENVCFG_STCE : 0) | + (cfg->ext_svadu ? MENVCFG_HADE : 0); } env->menvcfg = (env->menvcfg & ~mask) | (val & mask); @@ -1896,16 +1913,19 @@ static RISCVException write_menvcfg(CPURISCVState *env, int csrno, } static RISCVException read_menvcfgh(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { *val = env->menvcfg >> 32; return RISCV_EXCP_NONE; } static RISCVException write_menvcfgh(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { - uint64_t mask = MENVCFG_PBMTE | MENVCFG_STCE; + RISCVCPUConfig *cfg = &env_archcpu(env)->cfg; + uint64_t mask = (cfg->ext_svpbmt ? MENVCFG_PBMTE : 0) | + (cfg->ext_sstc ? MENVCFG_STCE : 0) | + (cfg->ext_svadu ? MENVCFG_HADE : 0); uint64_t valh = (uint64_t)val << 32; env->menvcfg = (env->menvcfg & ~mask) | (valh & mask); @@ -1914,7 +1934,7 @@ static RISCVException write_menvcfgh(CPURISCVState *env, int csrno, } static RISCVException read_senvcfg(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { RISCVException ret; @@ -1928,7 +1948,7 @@ static RISCVException read_senvcfg(CPURISCVState *env, int csrno, } static RISCVException write_senvcfg(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { uint64_t mask = SENVCFG_FIOM | SENVCFG_CBIE | SENVCFG_CBCFE | SENVCFG_CBZE; RISCVException ret; @@ -1943,7 +1963,7 @@ static RISCVException write_senvcfg(CPURISCVState *env, int csrno, } static RISCVException read_henvcfg(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { RISCVException ret; @@ -1952,12 +1972,18 @@ static RISCVException read_henvcfg(CPURISCVState *env, int csrno, return ret; } - *val = env->henvcfg; + /* + * henvcfg.pbmte is read_only 0 when menvcfg.pbmte = 0 + * henvcfg.stce is read_only 0 when menvcfg.stce = 0 + * henvcfg.hade is read_only 0 when menvcfg.hade = 0 + */ + *val = env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_HADE) | + env->menvcfg); return RISCV_EXCP_NONE; } static RISCVException write_henvcfg(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { uint64_t mask = HENVCFG_FIOM | HENVCFG_CBIE | HENVCFG_CBCFE | HENVCFG_CBZE; RISCVException ret; @@ -1968,7 +1994,7 @@ static RISCVException write_henvcfg(CPURISCVState *env, int csrno, } if (riscv_cpu_mxl(env) == MXL_RV64) { - mask |= HENVCFG_PBMTE | HENVCFG_STCE; + mask |= env->menvcfg & (HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_HADE); } env->henvcfg = (env->henvcfg & ~mask) | (val & mask); @@ -1977,7 +2003,7 @@ static RISCVException write_henvcfg(CPURISCVState *env, int csrno, } static RISCVException read_henvcfgh(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { RISCVException ret; @@ -1986,14 +2012,16 @@ static RISCVException read_henvcfgh(CPURISCVState *env, int csrno, return ret; } - *val = env->henvcfg >> 32; + *val = (env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_HADE) | + env->menvcfg)) >> 32; return RISCV_EXCP_NONE; } static RISCVException write_henvcfgh(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { - uint64_t mask = HENVCFG_PBMTE | HENVCFG_STCE; + uint64_t mask = env->menvcfg & (HENVCFG_PBMTE | HENVCFG_STCE | + HENVCFG_HADE); uint64_t valh = (uint64_t)val << 32; RISCVException ret; @@ -2034,13 +2062,13 @@ static RISCVException write_mstateen0(CPURISCVState *env, int csrno, } static RISCVException write_mstateen_1_3(CPURISCVState *env, int csrno, - target_ulong new_val) + target_ulong new_val) { return write_mstateen(env, csrno, SMSTATEEN_STATEEN, new_val); } static RISCVException read_mstateenh(CPURISCVState *env, int csrno, - target_ulong *val) + target_ulong *val) { *val = env->mstateen[csrno - CSR_MSTATEEN0H] >> 32; @@ -2061,7 +2089,7 @@ static RISCVException write_mstateenh(CPURISCVState *env, int csrno, } static RISCVException write_mstateen0h(CPURISCVState *env, int csrno, - target_ulong new_val) + target_ulong new_val) { uint64_t wr_mask = SMSTATEEN_STATEEN | SMSTATEEN0_HSENVCFG; @@ -2069,7 +2097,7 @@ static RISCVException write_mstateen0h(CPURISCVState *env, int csrno, } static RISCVException write_mstateenh_1_3(CPURISCVState *env, int csrno, - target_ulong new_val) + target_ulong new_val) { return write_mstateenh(env, csrno, SMSTATEEN_STATEEN, new_val); } @@ -2106,7 +2134,7 @@ static RISCVException write_hstateen0(CPURISCVState *env, int csrno, } static RISCVException write_hstateen_1_3(CPURISCVState *env, int csrno, - target_ulong new_val) + target_ulong new_val) { return write_hstateen(env, csrno, SMSTATEEN_STATEEN, new_val); } @@ -2145,7 +2173,7 @@ static RISCVException write_hstateen0h(CPURISCVState *env, int csrno, } static RISCVException write_hstateenh_1_3(CPURISCVState *env, int csrno, - target_ulong new_val) + target_ulong new_val) { return write_hstateenh(env, csrno, SMSTATEEN_STATEEN, new_val); } @@ -2624,7 +2652,7 @@ static RISCVException rmw_siph(CPURISCVState *env, int csrno, static RISCVException read_satp(CPURISCVState *env, int csrno, target_ulong *val) { - if (!riscv_feature(env, RISCV_FEATURE_MMU)) { + if (!riscv_cpu_cfg(env)->mmu) { *val = 0; return RISCV_EXCP_NONE; } @@ -2643,7 +2671,7 @@ static RISCVException write_satp(CPURISCVState *env, int csrno, { target_ulong vm, mask; - if (!riscv_feature(env, RISCV_FEATURE_MMU)) { + if (!riscv_cpu_cfg(env)->mmu) { return RISCV_EXCP_NONE; } @@ -3338,30 +3366,18 @@ static RISCVException read_mseccfg(CPURISCVState *env, int csrno, } static RISCVException write_mseccfg(CPURISCVState *env, int csrno, - target_ulong val) + target_ulong val) { mseccfg_csr_write(env, val); return RISCV_EXCP_NONE; } -static bool check_pmp_reg_index(CPURISCVState *env, uint32_t reg_index) -{ - /* TODO: RV128 restriction check */ - if ((reg_index & 1) && (riscv_cpu_mxl(env) == MXL_RV64)) { - return false; - } - return true; -} - static RISCVException read_pmpcfg(CPURISCVState *env, int csrno, target_ulong *val) { uint32_t reg_index = csrno - CSR_PMPCFG0; - if (!check_pmp_reg_index(env, reg_index)) { - return RISCV_EXCP_ILLEGAL_INST; - } - *val = pmpcfg_csr_read(env, csrno - CSR_PMPCFG0); + *val = pmpcfg_csr_read(env, reg_index); return RISCV_EXCP_NONE; } @@ -3370,10 +3386,7 @@ static RISCVException write_pmpcfg(CPURISCVState *env, int csrno, { uint32_t reg_index = csrno - CSR_PMPCFG0; - if (!check_pmp_reg_index(env, reg_index)) { - return RISCV_EXCP_ILLEGAL_INST; - } - pmpcfg_csr_write(env, csrno - CSR_PMPCFG0, val); + pmpcfg_csr_write(env, reg_index, val); return RISCV_EXCP_NONE; } @@ -3776,27 +3789,32 @@ static inline RISCVException riscv_csrrw_check(CPURISCVState *env, RISCVCPU *cpu) { /* check privileges and return RISCV_EXCP_ILLEGAL_INST if check fails */ - int read_only = get_field(csrno, 0xC00) == 3; + bool read_only = get_field(csrno, 0xC00) == 3; int csr_min_priv = csr_ops[csrno].min_priv_ver; - /* ensure the CSR extension is enabled. */ + /* ensure the CSR extension is enabled */ if (!cpu->cfg.ext_icsr) { return RISCV_EXCP_ILLEGAL_INST; } + /* privileged spec version check */ if (env->priv_ver < csr_min_priv) { return RISCV_EXCP_ILLEGAL_INST; } - /* check predicate */ - if (!csr_ops[csrno].predicate) { - return RISCV_EXCP_ILLEGAL_INST; - } - + /* read / write check */ if (write_mask && read_only) { return RISCV_EXCP_ILLEGAL_INST; } + /* + * The predicate() not only does existence check but also does some + * access control check which triggers for example virtual instruction + * exception in some cases. When writing read-only CSRs in those cases + * illegal instruction exception should be triggered instead of virtual + * instruction exception. Hence this comes after the read / write check. + */ + g_assert(csr_ops[csrno].predicate != NULL); RISCVException ret = csr_ops[csrno].predicate(env, csrno); if (ret != RISCV_EXCP_NONE) { return ret; diff --git a/target/riscv/gdbstub.c b/target/riscv/gdbstub.c index 6e7bbdbd5e..6048541606 100644 --- a/target/riscv/gdbstub.c +++ b/target/riscv/gdbstub.c @@ -127,40 +127,6 @@ static int riscv_gdb_set_fpu(CPURISCVState *env, uint8_t *mem_buf, int n) return 0; } -/* - * Convert register index number passed by GDB to the correspond - * vector CSR number. Vector CSRs are defined after vector registers - * in dynamic generated riscv-vector.xml, thus the starting register index - * of vector CSRs is 32. - * Return 0 if register index number is out of range. - */ -static int riscv_gdb_vector_csrno(int num_regs) -{ - /* - * The order of vector CSRs in the switch case - * should match with the order defined in csr_ops[]. - */ - switch (num_regs) { - case 32: - return CSR_VSTART; - case 33: - return CSR_VXSAT; - case 34: - return CSR_VXRM; - case 35: - return CSR_VCSR; - case 36: - return CSR_VL; - case 37: - return CSR_VTYPE; - case 38: - return CSR_VLENB; - default: - /* Unknown register. */ - return 0; - } -} - static int riscv_gdb_get_vector(CPURISCVState *env, GByteArray *buf, int n) { uint16_t vlenb = env_archcpu(env)->cfg.vlen >> 3; @@ -174,19 +140,6 @@ static int riscv_gdb_get_vector(CPURISCVState *env, GByteArray *buf, int n) return cnt; } - int csrno = riscv_gdb_vector_csrno(n); - - if (!csrno) { - return 0; - } - - target_ulong val = 0; - int result = riscv_csrrw_debug(env, csrno, &val, 0, 0); - - if (result == RISCV_EXCP_NONE) { - return gdb_get_regl(buf, val); - } - return 0; } @@ -201,19 +154,6 @@ static int riscv_gdb_set_vector(CPURISCVState *env, uint8_t *mem_buf, int n) return vlenb; } - int csrno = riscv_gdb_vector_csrno(n); - - if (!csrno) { - return 0; - } - - target_ulong val = ldtul_p(mem_buf); - int result = riscv_csrrw_debug(env, csrno, NULL, val, -1); - - if (result == RISCV_EXCP_NONE) { - return sizeof(target_ulong); - } - return 0; } @@ -280,6 +220,10 @@ static int riscv_gen_dynamic_csr_xml(CPUState *cs, int base_reg) int bitsize = 16 << env->misa_mxl_max; int i; +#if !defined(CONFIG_USER_ONLY) + env->debugger = true; +#endif + /* Until gdb knows about 128-bit registers */ if (bitsize > 64) { bitsize = 64; @@ -290,6 +234,9 @@ static int riscv_gen_dynamic_csr_xml(CPUState *cs, int base_reg) g_string_append_printf(s, "<feature name=\"org.gnu.gdb.riscv.csr\">"); for (i = 0; i < CSR_TABLE_SIZE; i++) { + if (env->priv_ver < csr_ops[i].min_priv_ver) { + continue; + } predicate = csr_ops[i].predicate; if (predicate && (predicate(env, i) == RISCV_EXCP_NONE)) { if (csr_ops[i].name) { @@ -305,6 +252,11 @@ static int riscv_gen_dynamic_csr_xml(CPUState *cs, int base_reg) g_string_append_printf(s, "</feature>"); cpu->dyn_csr_xml = g_string_free(s, false); + +#if !defined(CONFIG_USER_ONLY) + env->debugger = false; +#endif + return CSR_TABLE_SIZE; } @@ -349,21 +301,6 @@ static int ricsv_gen_dynamic_vector_xml(CPUState *cs, int base_reg) num_regs++; } - /* Define vector CSRs */ - const char *vector_csrs[7] = { - "vstart", "vxsat", "vxrm", "vcsr", - "vl", "vtype", "vlenb" - }; - - for (i = 0; i < 7; i++) { - g_string_append_printf(s, - "<reg name=\"%s\" bitsize=\"%d\"" - " regnum=\"%d\" group=\"vector\"" - " type=\"int\"/>", - vector_csrs[i], TARGET_LONG_BITS, base_reg++); - num_regs++; - } - g_string_append_printf(s, "</feature>"); cpu->dyn_vreg_xml = g_string_free(s, false); @@ -382,9 +319,9 @@ void riscv_cpu_register_gdb_regs_for_features(CPUState *cs) 32, "riscv-32bit-fpu.xml", 0); } if (env->misa_ext & RVV) { + int base_reg = cs->gdb_num_regs; gdb_register_coprocessor(cs, riscv_gdb_get_vector, riscv_gdb_set_vector, - ricsv_gen_dynamic_vector_xml(cs, - cs->gdb_num_regs), + ricsv_gen_dynamic_vector_xml(cs, base_reg), "riscv-vector.xml", 0); } switch (env->misa_mxl_max) { @@ -403,7 +340,10 @@ void riscv_cpu_register_gdb_regs_for_features(CPUState *cs) g_assert_not_reached(); } - gdb_register_coprocessor(cs, riscv_gdb_get_csr, riscv_gdb_set_csr, - riscv_gen_dynamic_csr_xml(cs, cs->gdb_num_regs), - "riscv-csr.xml", 0); + if (cpu->cfg.ext_icsr) { + int base_reg = cs->gdb_num_regs; + gdb_register_coprocessor(cs, riscv_gdb_get_csr, riscv_gdb_set_csr, + riscv_gen_dynamic_csr_xml(cs, base_reg), + "riscv-csr.xml", 0); + } } diff --git a/target/riscv/insn32.decode b/target/riscv/insn32.decode index b7e7613ea2..fb537e922e 100644 --- a/target/riscv/insn32.decode +++ b/target/riscv/insn32.decode @@ -890,3 +890,7 @@ sm3p1 00 01000 01001 ..... 001 ..... 0010011 @r2 # *** RV32 Zksed Standard Extension *** sm4ed .. 11000 ..... ..... 000 ..... 0110011 @k_aes sm4ks .. 11010 ..... ..... 000 ..... 0110011 @k_aes + +# *** RV32 Zicond Standard Extension *** +czero_eqz 0000111 ..... ..... 101 ..... 0110011 @r +czero_nez 0000111 ..... ..... 111 ..... 0110011 @r diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index bbb5c3a7b5..fc0d0d60e8 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -40,10 +40,11 @@ static bool require_rvf(DisasContext *s) switch (s->sew) { case MO_16: + return s->cfg_ptr->ext_zvfh; case MO_32: - return has_ext(s, RVF); + return s->cfg_ptr->ext_zve32f; case MO_64: - return has_ext(s, RVD); + return s->cfg_ptr->ext_zve64d; default: return false; } @@ -57,57 +58,32 @@ static bool require_scale_rvf(DisasContext *s) switch (s->sew) { case MO_8: + return s->cfg_ptr->ext_zvfh; case MO_16: - return has_ext(s, RVF); + return s->cfg_ptr->ext_zve32f; case MO_32: - return has_ext(s, RVD); + return s->cfg_ptr->ext_zve64d; default: return false; } } -static bool require_zve32f(DisasContext *s) +static bool require_scale_rvfmin(DisasContext *s) { - /* RVV + Zve32f = RVV. */ - if (has_ext(s, RVV)) { - return true; - } - - /* Zve32f doesn't support FP64. (Section 18.2) */ - return s->cfg_ptr->ext_zve32f ? s->sew <= MO_32 : true; -} - -static bool require_scale_zve32f(DisasContext *s) -{ - /* RVV + Zve32f = RVV. */ - if (has_ext(s, RVV)) { - return true; - } - - /* Zve32f doesn't support FP64. (Section 18.2) */ - return s->cfg_ptr->ext_zve64f ? s->sew <= MO_16 : true; -} - -static bool require_zve64f(DisasContext *s) -{ - /* RVV + Zve64f = RVV. */ - if (has_ext(s, RVV)) { - return true; + if (s->mstatus_fs == 0) { + return false; } - /* Zve64f doesn't support FP64. (Section 18.2) */ - return s->cfg_ptr->ext_zve64f ? s->sew <= MO_32 : true; -} - -static bool require_scale_zve64f(DisasContext *s) -{ - /* RVV + Zve64f = RVV. */ - if (has_ext(s, RVV)) { - return true; + switch (s->sew) { + case MO_8: + return s->cfg_ptr->ext_zvfhmin; + case MO_16: + return s->cfg_ptr->ext_zve32f; + case MO_32: + return s->cfg_ptr->ext_zve64d; + default: + return false; } - - /* Zve64f doesn't support FP64. (Section 18.2) */ - return s->cfg_ptr->ext_zve64f ? s->sew <= MO_16 : true; } /* Destination vector register group cannot overlap source mask register. */ @@ -173,9 +149,7 @@ static bool do_vsetvl(DisasContext *s, int rd, int rs1, TCGv s2) { TCGv s1, dst; - if (!require_rvv(s) || - !(has_ext(s, RVV) || s->cfg_ptr->ext_zve32f || - s->cfg_ptr->ext_zve64f)) { + if (!require_rvv(s) || !s->cfg_ptr->ext_zve32f) { return false; } @@ -210,9 +184,7 @@ static bool do_vsetivli(DisasContext *s, int rd, TCGv s1, TCGv s2) { TCGv dst; - if (!require_rvv(s) || - !(has_ext(s, RVV) || s->cfg_ptr->ext_zve32f || - s->cfg_ptr->ext_zve64f)) { + if (!require_rvv(s) || !s->cfg_ptr->ext_zve32f) { return false; } @@ -315,13 +287,12 @@ static bool vext_check_st_index(DisasContext *s, int vd, int vs2, int nf, require_nf(vd, nf, s->lmul); /* - * All Zve* extensions support all vector load and store instructions, - * except Zve64* extensions do not support EEW=64 for index values - * when XLEN=32. (Section 18.2) + * V extension supports all vector load and store instructions, + * except V extension does not support EEW=64 for index values + * when XLEN=32. (Section 18.3) */ if (get_xl(s) == MXL_RV32) { - ret &= (!has_ext(s, RVV) && - s->cfg_ptr->ext_zve64f ? eew != MO_64 : true); + ret &= (eew != MO_64); } return ret; @@ -2027,8 +1998,7 @@ static bool vmulh_vv_check(DisasContext *s, arg_rmrr *a) * are not included for EEW=64 in Zve64*. (Section 18.2) */ return opivv_check(s, a) && - (!has_ext(s, RVV) && - s->cfg_ptr->ext_zve64f ? s->sew != MO_64 : true); + (!has_ext(s, RVV) ? s->sew != MO_64 : true); } static bool vmulh_vx_check(DisasContext *s, arg_rmrr *a) @@ -2041,8 +2011,7 @@ static bool vmulh_vx_check(DisasContext *s, arg_rmrr *a) * are not included for EEW=64 in Zve64*. (Section 18.2) */ return opivx_check(s, a) && - (!has_ext(s, RVV) && - s->cfg_ptr->ext_zve64f ? s->sew != MO_64 : true); + (!has_ext(s, RVV) ? s->sew != MO_64 : true); } GEN_OPIVV_GVEC_TRANS(vmul_vv, mul) @@ -2259,8 +2228,7 @@ static bool vsmul_vv_check(DisasContext *s, arg_rmrr *a) * for EEW=64 in Zve64*. (Section 18.2) */ return opivv_check(s, a) && - (!has_ext(s, RVV) && - s->cfg_ptr->ext_zve64f ? s->sew != MO_64 : true); + (!has_ext(s, RVV) ? s->sew != MO_64 : true); } static bool vsmul_vx_check(DisasContext *s, arg_rmrr *a) @@ -2271,8 +2239,7 @@ static bool vsmul_vx_check(DisasContext *s, arg_rmrr *a) * for EEW=64 in Zve64*. (Section 18.2) */ return opivx_check(s, a) && - (!has_ext(s, RVV) && - s->cfg_ptr->ext_zve64f ? s->sew != MO_64 : true); + (!has_ext(s, RVV) ? s->sew != MO_64 : true); } GEN_OPIVV_TRANS(vsmul_vv, vsmul_vv_check) @@ -2335,9 +2302,7 @@ static bool opfvv_check(DisasContext *s, arg_rmrr *a) return require_rvv(s) && require_rvf(s) && vext_check_isa_ill(s) && - vext_check_sss(s, a->rd, a->rs1, a->rs2, a->vm) && - require_zve32f(s) && - require_zve64f(s); + vext_check_sss(s, a->rd, a->rs1, a->rs2, a->vm); } /* OPFVV without GVEC IR */ @@ -2425,9 +2390,7 @@ static bool opfvf_check(DisasContext *s, arg_rmrr *a) return require_rvv(s) && require_rvf(s) && vext_check_isa_ill(s) && - vext_check_ss(s, a->rd, a->rs2, a->vm) && - require_zve32f(s) && - require_zve64f(s); + vext_check_ss(s, a->rd, a->rs2, a->vm); } /* OPFVF without GVEC IR */ @@ -2465,9 +2428,7 @@ static bool opfvv_widen_check(DisasContext *s, arg_rmrr *a) require_scale_rvf(s) && (s->sew != MO_8) && vext_check_isa_ill(s) && - vext_check_dss(s, a->rd, a->rs1, a->rs2, a->vm) && - require_scale_zve32f(s) && - require_scale_zve64f(s); + vext_check_dss(s, a->rd, a->rs1, a->rs2, a->vm); } /* OPFVV with WIDEN */ @@ -2510,9 +2471,7 @@ static bool opfvf_widen_check(DisasContext *s, arg_rmrr *a) require_scale_rvf(s) && (s->sew != MO_8) && vext_check_isa_ill(s) && - vext_check_ds(s, a->rd, a->rs2, a->vm) && - require_scale_zve32f(s) && - require_scale_zve64f(s); + vext_check_ds(s, a->rd, a->rs2, a->vm); } /* OPFVF with WIDEN */ @@ -2544,9 +2503,7 @@ static bool opfwv_widen_check(DisasContext *s, arg_rmrr *a) require_scale_rvf(s) && (s->sew != MO_8) && vext_check_isa_ill(s) && - vext_check_dds(s, a->rd, a->rs1, a->rs2, a->vm) && - require_scale_zve32f(s) && - require_scale_zve64f(s); + vext_check_dds(s, a->rd, a->rs1, a->rs2, a->vm); } /* WIDEN OPFVV with WIDEN */ @@ -2589,9 +2546,7 @@ static bool opfwf_widen_check(DisasContext *s, arg_rmrr *a) require_scale_rvf(s) && (s->sew != MO_8) && vext_check_isa_ill(s) && - vext_check_dd(s, a->rd, a->rs2, a->vm) && - require_scale_zve32f(s) && - require_scale_zve64f(s); + vext_check_dd(s, a->rd, a->rs2, a->vm); } /* WIDEN OPFVF with WIDEN */ @@ -2668,9 +2623,7 @@ static bool opfv_check(DisasContext *s, arg_rmr *a) require_rvf(s) && vext_check_isa_ill(s) && /* OPFV instructions ignore vs1 check */ - vext_check_ss(s, a->rd, a->rs2, a->vm) && - require_zve32f(s) && - require_zve64f(s); + vext_check_ss(s, a->rd, a->rs2, a->vm); } static bool do_opfv(DisasContext *s, arg_rmr *a, @@ -2735,9 +2688,7 @@ static bool opfvv_cmp_check(DisasContext *s, arg_rmrr *a) return require_rvv(s) && require_rvf(s) && vext_check_isa_ill(s) && - vext_check_mss(s, a->rd, a->rs1, a->rs2) && - require_zve32f(s) && - require_zve64f(s); + vext_check_mss(s, a->rd, a->rs1, a->rs2); } GEN_OPFVV_TRANS(vmfeq_vv, opfvv_cmp_check) @@ -2750,9 +2701,7 @@ static bool opfvf_cmp_check(DisasContext *s, arg_rmrr *a) return require_rvv(s) && require_rvf(s) && vext_check_isa_ill(s) && - vext_check_ms(s, a->rd, a->rs2) && - require_zve32f(s) && - require_zve64f(s); + vext_check_ms(s, a->rd, a->rs2); } GEN_OPFVF_TRANS(vmfeq_vf, opfvf_cmp_check) @@ -2773,9 +2722,7 @@ static bool trans_vfmv_v_f(DisasContext *s, arg_vfmv_v_f *a) if (require_rvv(s) && require_rvf(s) && vext_check_isa_ill(s) && - require_align(a->rd, s->lmul) && - require_zve32f(s) && - require_zve64f(s)) { + require_align(a->rd, s->lmul)) { gen_set_rm(s, RISCV_FRM_DYN); TCGv_i64 t1; @@ -2860,18 +2807,14 @@ static bool opfv_widen_check(DisasContext *s, arg_rmr *a) static bool opxfv_widen_check(DisasContext *s, arg_rmr *a) { return opfv_widen_check(s, a) && - require_rvf(s) && - require_zve32f(s) && - require_zve64f(s); + require_rvf(s); } static bool opffv_widen_check(DisasContext *s, arg_rmr *a) { return opfv_widen_check(s, a) && - require_scale_rvf(s) && - (s->sew != MO_8) && - require_scale_zve32f(s) && - require_scale_zve64f(s); + require_scale_rvfmin(s) && + (s->sew != MO_8); } #define GEN_OPFV_WIDEN_TRANS(NAME, CHECK, HELPER, FRM) \ @@ -2922,9 +2865,7 @@ static bool opfxv_widen_check(DisasContext *s, arg_rmr *a) require_scale_rvf(s) && vext_check_isa_ill(s) && /* OPFV widening instructions ignore vs1 check */ - vext_check_ds(s, a->rd, a->rs2, a->vm) && - require_scale_zve32f(s) && - require_scale_zve64f(s); + vext_check_ds(s, a->rd, a->rs2, a->vm); } #define GEN_OPFXV_WIDEN_TRANS(NAME) \ @@ -2979,18 +2920,21 @@ static bool opfxv_narrow_check(DisasContext *s, arg_rmr *a) { return opfv_narrow_check(s, a) && require_rvf(s) && - (s->sew != MO_64) && - require_zve32f(s) && - require_zve64f(s); + (s->sew != MO_64); } static bool opffv_narrow_check(DisasContext *s, arg_rmr *a) { return opfv_narrow_check(s, a) && + require_scale_rvfmin(s) && + (s->sew != MO_8); +} + +static bool opffv_rod_narrow_check(DisasContext *s, arg_rmr *a) +{ + return opfv_narrow_check(s, a) && require_scale_rvf(s) && - (s->sew != MO_8) && - require_scale_zve32f(s) && - require_scale_zve64f(s); + (s->sew != MO_8); } #define GEN_OPFV_NARROW_TRANS(NAME, CHECK, HELPER, FRM) \ @@ -3030,7 +2974,7 @@ GEN_OPFV_NARROW_TRANS(vfncvt_f_x_w, opfxv_narrow_check, vfncvt_f_x_w, GEN_OPFV_NARROW_TRANS(vfncvt_f_f_w, opffv_narrow_check, vfncvt_f_f_w, RISCV_FRM_DYN) /* Reuse the helper function from vfncvt.f.f.w */ -GEN_OPFV_NARROW_TRANS(vfncvt_rod_f_f_w, opffv_narrow_check, vfncvt_f_f_w, +GEN_OPFV_NARROW_TRANS(vfncvt_rod_f_f_w, opffv_rod_narrow_check, vfncvt_f_f_w, RISCV_FRM_ROD) static bool opxfv_narrow_check(DisasContext *s, arg_rmr *a) @@ -3039,9 +2983,7 @@ static bool opxfv_narrow_check(DisasContext *s, arg_rmr *a) require_scale_rvf(s) && vext_check_isa_ill(s) && /* OPFV narrowing instructions ignore vs1 check */ - vext_check_sd(s, a->rd, a->rs2, a->vm) && - require_scale_zve32f(s) && - require_scale_zve64f(s); + vext_check_sd(s, a->rd, a->rs2, a->vm); } #define GEN_OPXFV_NARROW_TRANS(NAME, HELPER, FRM) \ @@ -3115,9 +3057,7 @@ GEN_OPIVV_WIDEN_TRANS(vwredsumu_vs, reduction_widen_check) static bool freduction_check(DisasContext *s, arg_rmrr *a) { return reduction_check(s, a) && - require_rvf(s) && - require_zve32f(s) && - require_zve64f(s); + require_rvf(s); } GEN_OPFVV_TRANS(vfredusum_vs, freduction_check) @@ -3544,9 +3484,7 @@ static bool trans_vfmv_f_s(DisasContext *s, arg_vfmv_f_s *a) { if (require_rvv(s) && require_rvf(s) && - vext_check_isa_ill(s) && - require_zve32f(s) && - require_zve64f(s)) { + vext_check_isa_ill(s)) { gen_set_rm(s, RISCV_FRM_DYN); unsigned int ofs = (8 << s->sew); @@ -3572,9 +3510,7 @@ static bool trans_vfmv_s_f(DisasContext *s, arg_vfmv_s_f *a) { if (require_rvv(s) && require_rvf(s) && - vext_check_isa_ill(s) && - require_zve32f(s) && - require_zve64f(s)) { + vext_check_isa_ill(s)) { gen_set_rm(s, RISCV_FRM_DYN); /* The instructions ignore LMUL and vector register group. */ @@ -3625,17 +3561,13 @@ GEN_OPIVI_TRANS(vslidedown_vi, IMM_ZX, vslidedown_vx, slidedown_check) static bool fslideup_check(DisasContext *s, arg_rmrr *a) { return slideup_check(s, a) && - require_rvf(s) && - require_zve32f(s) && - require_zve64f(s); + require_rvf(s); } static bool fslidedown_check(DisasContext *s, arg_rmrr *a) { return slidedown_check(s, a) && - require_rvf(s) && - require_zve32f(s) && - require_zve64f(s); + require_rvf(s); } GEN_OPFVF_TRANS(vfslide1up_vf, fslideup_check) diff --git a/target/riscv/insn_trans/trans_rvzfh.c.inc b/target/riscv/insn_trans/trans_rvzfh.c.inc index 2ad5716312..85fc1aa822 100644 --- a/target/riscv/insn_trans/trans_rvzfh.c.inc +++ b/target/riscv/insn_trans/trans_rvzfh.c.inc @@ -28,15 +28,14 @@ } \ } while (0) -#define REQUIRE_ZFH_OR_ZFHMIN(ctx) do { \ - if (!(ctx->cfg_ptr->ext_zfh || ctx->cfg_ptr->ext_zfhmin)) { \ +#define REQUIRE_ZFHMIN(ctx) do { \ + if (!ctx->cfg_ptr->ext_zfhmin) { \ return false; \ } \ } while (0) -#define REQUIRE_ZFH_OR_ZFHMIN_OR_ZHINX_OR_ZHINXMIN(ctx) do { \ - if (!(ctx->cfg_ptr->ext_zfh || ctx->cfg_ptr->ext_zfhmin || \ - ctx->cfg_ptr->ext_zhinx || ctx->cfg_ptr->ext_zhinxmin)) { \ +#define REQUIRE_ZFHMIN_OR_ZHINXMIN(ctx) do { \ + if (!(ctx->cfg_ptr->ext_zfhmin || ctx->cfg_ptr->ext_zhinxmin)) { \ return false; \ } \ } while (0) @@ -47,7 +46,7 @@ static bool trans_flh(DisasContext *ctx, arg_flh *a) TCGv t0; REQUIRE_FPU; - REQUIRE_ZFH_OR_ZFHMIN(ctx); + REQUIRE_ZFHMIN(ctx); decode_save_opc(ctx); t0 = get_gpr(ctx, a->rs1, EXT_NONE); @@ -70,7 +69,7 @@ static bool trans_fsh(DisasContext *ctx, arg_fsh *a) TCGv t0; REQUIRE_FPU; - REQUIRE_ZFH_OR_ZFHMIN(ctx); + REQUIRE_ZFHMIN(ctx); decode_save_opc(ctx); t0 = get_gpr(ctx, a->rs1, EXT_NONE); @@ -401,7 +400,7 @@ static bool trans_fmax_h(DisasContext *ctx, arg_fmax_h *a) static bool trans_fcvt_s_h(DisasContext *ctx, arg_fcvt_s_h *a) { REQUIRE_FPU; - REQUIRE_ZFH_OR_ZFHMIN_OR_ZHINX_OR_ZHINXMIN(ctx); + REQUIRE_ZFHMIN_OR_ZHINXMIN(ctx); TCGv_i64 dest = dest_fpr(ctx, a->rd); TCGv_i64 src1 = get_fpr_hs(ctx, a->rs1); @@ -418,7 +417,7 @@ static bool trans_fcvt_s_h(DisasContext *ctx, arg_fcvt_s_h *a) static bool trans_fcvt_d_h(DisasContext *ctx, arg_fcvt_d_h *a) { REQUIRE_FPU; - REQUIRE_ZFH_OR_ZFHMIN_OR_ZHINX_OR_ZHINXMIN(ctx); + REQUIRE_ZFHMIN_OR_ZHINXMIN(ctx); REQUIRE_ZDINX_OR_D(ctx); TCGv_i64 dest = dest_fpr(ctx, a->rd); @@ -436,7 +435,7 @@ static bool trans_fcvt_d_h(DisasContext *ctx, arg_fcvt_d_h *a) static bool trans_fcvt_h_s(DisasContext *ctx, arg_fcvt_h_s *a) { REQUIRE_FPU; - REQUIRE_ZFH_OR_ZFHMIN_OR_ZHINX_OR_ZHINXMIN(ctx); + REQUIRE_ZFHMIN_OR_ZHINXMIN(ctx); TCGv_i64 dest = dest_fpr(ctx, a->rd); TCGv_i64 src1 = get_fpr_hs(ctx, a->rs1); @@ -452,7 +451,7 @@ static bool trans_fcvt_h_s(DisasContext *ctx, arg_fcvt_h_s *a) static bool trans_fcvt_h_d(DisasContext *ctx, arg_fcvt_h_d *a) { REQUIRE_FPU; - REQUIRE_ZFH_OR_ZFHMIN_OR_ZHINX_OR_ZHINXMIN(ctx); + REQUIRE_ZFHMIN_OR_ZHINXMIN(ctx); REQUIRE_ZDINX_OR_D(ctx); TCGv_i64 dest = dest_fpr(ctx, a->rd); @@ -585,7 +584,7 @@ static bool trans_fcvt_h_wu(DisasContext *ctx, arg_fcvt_h_wu *a) static bool trans_fmv_x_h(DisasContext *ctx, arg_fmv_x_h *a) { REQUIRE_FPU; - REQUIRE_ZFH_OR_ZFHMIN(ctx); + REQUIRE_ZFHMIN(ctx); TCGv dest = dest_gpr(ctx, a->rd); @@ -605,7 +604,7 @@ static bool trans_fmv_x_h(DisasContext *ctx, arg_fmv_x_h *a) static bool trans_fmv_h_x(DisasContext *ctx, arg_fmv_h_x *a) { REQUIRE_FPU; - REQUIRE_ZFH_OR_ZFHMIN(ctx); + REQUIRE_ZFHMIN(ctx); TCGv t0 = get_gpr(ctx, a->rs1, EXT_ZERO); diff --git a/target/riscv/insn_trans/trans_rvzicond.c.inc b/target/riscv/insn_trans/trans_rvzicond.c.inc new file mode 100644 index 0000000000..645260164e --- /dev/null +++ b/target/riscv/insn_trans/trans_rvzicond.c.inc @@ -0,0 +1,49 @@ +/* + * RISC-V translation routines for the Zicond Standard Extension. + * + * Copyright (c) 2020-2023 PLCT Lab + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2 or later, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#define REQUIRE_ZICOND(ctx) do { \ + if (!ctx->cfg_ptr->ext_zicond) { \ + return false; \ + } \ +} while (0) + +static bool trans_czero_eqz(DisasContext *ctx, arg_czero_eqz *a) +{ + REQUIRE_ZICOND(ctx); + + TCGv dest = dest_gpr(ctx, a->rd); + TCGv src1 = get_gpr(ctx, a->rs1, EXT_NONE); + TCGv src2 = get_gpr(ctx, a->rs2, EXT_NONE); + + tcg_gen_movcond_tl(TCG_COND_EQ, dest, src2, ctx->zero, ctx->zero, src1); + gen_set_gpr(ctx, a->rd, dest); + return true; +} + +static bool trans_czero_nez(DisasContext *ctx, arg_czero_nez *a) +{ + REQUIRE_ZICOND(ctx); + + TCGv dest = dest_gpr(ctx, a->rd); + TCGv src1 = get_gpr(ctx, a->rs1, EXT_NONE); + TCGv src2 = get_gpr(ctx, a->rs2, EXT_NONE); + + tcg_gen_movcond_tl(TCG_COND_NE, dest, src2, ctx->zero, ctx->zero, src1); + gen_set_gpr(ctx, a->rd, dest); + return true; +} diff --git a/target/riscv/insn_trans/trans_xthead.c.inc b/target/riscv/insn_trans/trans_xthead.c.inc index be87c34f56..cf1731b08d 100644 --- a/target/riscv/insn_trans/trans_xthead.c.inc +++ b/target/riscv/insn_trans/trans_xthead.c.inc @@ -980,10 +980,6 @@ static bool trans_th_lwud(DisasContext *ctx, arg_th_pair *a) static bool gen_storepair_tl(DisasContext *ctx, arg_th_pair *a, MemOp memop, int shamt) { - if (a->rs == a->rd1 || a->rs == a->rd2 || a->rd1 == a->rd2) { - return false; - } - TCGv data1 = get_gpr(ctx, a->rd1, EXT_NONE); TCGv data2 = get_gpr(ctx, a->rd2, EXT_NONE); TCGv addr1 = tcg_temp_new(); diff --git a/target/riscv/machine.c b/target/riscv/machine.c index c6ce318cce..9c455931d8 100644 --- a/target/riscv/machine.c +++ b/target/riscv/machine.c @@ -27,9 +27,8 @@ static bool pmp_needed(void *opaque) { RISCVCPU *cpu = opaque; - CPURISCVState *env = &cpu->env; - return riscv_feature(env, RISCV_FEATURE_PMP); + return cpu->cfg.pmp; } static int pmp_post_load(void *opaque, int version_id) @@ -226,9 +225,8 @@ static const VMStateDescription vmstate_kvmtimer = { static bool debug_needed(void *opaque) { RISCVCPU *cpu = opaque; - CPURISCVState *env = &cpu->env; - return riscv_feature(env, RISCV_FEATURE_DEBUG); + return cpu->cfg.debug; } static int debug_post_load(void *opaque, int version_id) @@ -333,8 +331,8 @@ static const VMStateDescription vmstate_pmu_ctr_state = { const VMStateDescription vmstate_riscv_cpu = { .name = "cpu", - .version_id = 6, - .minimum_version_id = 6, + .version_id = 7, + .minimum_version_id = 7, .post_load = riscv_cpu_post_load, .fields = (VMStateField[]) { VMSTATE_UINTTL_ARRAY(env.gpr, RISCVCPU, 32), @@ -353,7 +351,6 @@ const VMStateDescription vmstate_riscv_cpu = { VMSTATE_UINT32(env.misa_ext, RISCVCPU), VMSTATE_UINT32(env.misa_mxl_max, RISCVCPU), VMSTATE_UINT32(env.misa_ext_mask, RISCVCPU), - VMSTATE_UINT32(env.features, RISCVCPU), VMSTATE_UINTTL(env.priv, RISCVCPU), VMSTATE_UINTTL(env.virt, RISCVCPU), VMSTATE_UINT64(env.resetvec, RISCVCPU), diff --git a/target/riscv/monitor.c b/target/riscv/monitor.c index 236f93b9f5..f36ddfa967 100644 --- a/target/riscv/monitor.c +++ b/target/riscv/monitor.c @@ -218,7 +218,7 @@ void hmp_info_mem(Monitor *mon, const QDict *qdict) return; } - if (!riscv_feature(env, RISCV_FEATURE_MMU)) { + if (!riscv_cpu_cfg(env)->mmu) { monitor_printf(mon, "S-mode MMU unavailable\n"); return; } diff --git a/target/riscv/op_helper.c b/target/riscv/op_helper.c index 48f918b71b..9c0b91c88f 100644 --- a/target/riscv/op_helper.c +++ b/target/riscv/op_helper.c @@ -195,7 +195,7 @@ target_ulong helper_mret(CPURISCVState *env) uint64_t mstatus = env->mstatus; target_ulong prev_priv = get_field(mstatus, MSTATUS_MPP); - if (riscv_feature(env, RISCV_FEATURE_PMP) && + if (riscv_cpu_cfg(env)->pmp && !pmp_get_num_rules(env) && (prev_priv != PRV_M)) { riscv_raise_exception(env, RISCV_EXCP_INST_ACCESS_FAULT, GETPC()); } diff --git a/target/riscv/pmp.c b/target/riscv/pmp.c index 4bc4113531..a08cd95658 100644 --- a/target/riscv/pmp.c +++ b/target/riscv/pmp.c @@ -88,7 +88,7 @@ static void pmp_write_cfg(CPURISCVState *env, uint32_t pmp_index, uint8_t val) if (pmp_index < MAX_RISCV_PMPS) { bool locked = true; - if (riscv_feature(env, RISCV_FEATURE_EPMP)) { + if (riscv_cpu_cfg(env)->epmp) { /* mseccfg.RLB is set */ if (MSECCFG_RLB_ISSET(env)) { locked = false; @@ -239,7 +239,7 @@ static bool pmp_hart_has_privs_default(CPURISCVState *env, target_ulong addr, { bool ret; - if (riscv_feature(env, RISCV_FEATURE_EPMP)) { + if (riscv_cpu_cfg(env)->epmp) { if (MSECCFG_MMWP_ISSET(env)) { /* * The Machine Mode Whitelist Policy (mseccfg.MMWP) is set @@ -265,7 +265,7 @@ static bool pmp_hart_has_privs_default(CPURISCVState *env, target_ulong addr, } } - if ((!riscv_feature(env, RISCV_FEATURE_PMP)) || (mode == PRV_M)) { + if (!riscv_cpu_cfg(env)->pmp || (mode == PRV_M)) { /* * Privileged spec v1.10 states if HW doesn't implement any PMP entry * or no PMP entry matches an M-Mode access, the access succeeds. @@ -315,7 +315,7 @@ int pmp_hart_has_privs(CPURISCVState *env, target_ulong addr, } if (size == 0) { - if (riscv_feature(env, RISCV_FEATURE_MMU)) { + if (riscv_cpu_cfg(env)->mmu) { /* * If size is unknown (0), assume that all bytes * from addr to the end of the page will be accessed. diff --git a/target/riscv/translate.c b/target/riscv/translate.c index f9d5d1097e..a8d516ca3e 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -1103,6 +1103,7 @@ static uint32_t opcode_at(DisasContextBase *dcbase, target_ulong pc) #include "insn_trans/trans_rvh.c.inc" #include "insn_trans/trans_rvv.c.inc" #include "insn_trans/trans_rvb.c.inc" +#include "insn_trans/trans_rvzicond.c.inc" #include "insn_trans/trans_rvzawrs.c.inc" #include "insn_trans/trans_rvzfh.c.inc" #include "insn_trans/trans_rvk.c.inc" @@ -1261,7 +1262,7 @@ static void riscv_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) uint16_t next_insn = cpu_lduw_code(env, ctx->base.pc_next); int len = insn_len(next_insn); - if (!is_same_page(&ctx->base, ctx->base.pc_next + len)) { + if (!is_same_page(&ctx->base, ctx->base.pc_next + len - 1)) { ctx->base.is_jmp = DISAS_TOO_MANY; } } diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 3073c54871..2423affe37 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -267,6 +267,28 @@ GEN_VEXT_ST_ELEM(ste_h, int16_t, H2, stw) GEN_VEXT_ST_ELEM(ste_w, int32_t, H4, stl) GEN_VEXT_ST_ELEM(ste_d, int64_t, H8, stq) +static void vext_set_tail_elems_1s(CPURISCVState *env, target_ulong vl, + void *vd, uint32_t desc, uint32_t nf, + uint32_t esz, uint32_t max_elems) +{ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); + uint32_t vlenb = riscv_cpu_cfg(env)->vlen >> 3; + uint32_t vta = vext_vta(desc); + uint32_t registers_used; + int k; + + for (k = 0; k < nf; ++k) { + vext_set_elems_1s(vd, vta, (k * max_elems + vl) * esz, + (k * max_elems + max_elems) * esz); + } + + if (nf * max_elems % total_elems != 0) { + registers_used = ((nf * max_elems) * esz + (vlenb - 1)) / vlenb; + vext_set_elems_1s(vd, vta, (nf * max_elems) * esz, + registers_used * vlenb); + } +} + /* *** stride: access vector element from strided memory */ @@ -281,8 +303,6 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, uint32_t nf = vext_nf(desc); uint32_t max_elems = vext_max_elems(desc, log2_esz); uint32_t esz = 1 << log2_esz; - uint32_t total_elems = vext_get_total_elems(env, desc, esz); - uint32_t vta = vext_vta(desc); uint32_t vma = vext_vma(desc); for (i = env->vstart; i < env->vl; i++, env->vstart++) { @@ -301,18 +321,8 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, } } env->vstart = 0; - /* set tail elements to 1s */ - for (k = 0; k < nf; ++k) { - vext_set_elems_1s(vd, vta, (k * max_elems + env->vl) * esz, - (k * max_elems + max_elems) * esz); - } - if (nf * max_elems % total_elems != 0) { - uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; - uint32_t registers_used = - ((nf * max_elems) * esz + (vlenb - 1)) / vlenb; - vext_set_elems_1s(vd, vta, (nf * max_elems) * esz, - registers_used * vlenb); - } + + vext_set_tail_elems_1s(env, env->vl, vd, desc, nf, esz, max_elems); } #define GEN_VEXT_LD_STRIDE(NAME, ETYPE, LOAD_FN) \ @@ -359,8 +369,6 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, uint32_t nf = vext_nf(desc); uint32_t max_elems = vext_max_elems(desc, log2_esz); uint32_t esz = 1 << log2_esz; - uint32_t total_elems = vext_get_total_elems(env, desc, esz); - uint32_t vta = vext_vta(desc); /* load bytes from guest memory */ for (i = env->vstart; i < evl; i++, env->vstart++) { @@ -372,18 +380,8 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, } } env->vstart = 0; - /* set tail elements to 1s */ - for (k = 0; k < nf; ++k) { - vext_set_elems_1s(vd, vta, (k * max_elems + evl) * esz, - (k * max_elems + max_elems) * esz); - } - if (nf * max_elems % total_elems != 0) { - uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; - uint32_t registers_used = - ((nf * max_elems) * esz + (vlenb - 1)) / vlenb; - vext_set_elems_1s(vd, vta, (nf * max_elems) * esz, - registers_used * vlenb); - } + + vext_set_tail_elems_1s(env, evl, vd, desc, nf, esz, max_elems); } /* @@ -484,8 +482,6 @@ vext_ldst_index(void *vd, void *v0, target_ulong base, uint32_t vm = vext_vm(desc); uint32_t max_elems = vext_max_elems(desc, log2_esz); uint32_t esz = 1 << log2_esz; - uint32_t total_elems = vext_get_total_elems(env, desc, esz); - uint32_t vta = vext_vta(desc); uint32_t vma = vext_vma(desc); /* load bytes from guest memory */ @@ -505,18 +501,8 @@ vext_ldst_index(void *vd, void *v0, target_ulong base, } } env->vstart = 0; - /* set tail elements to 1s */ - for (k = 0; k < nf; ++k) { - vext_set_elems_1s(vd, vta, (k * max_elems + env->vl) * esz, - (k * max_elems + max_elems) * esz); - } - if (nf * max_elems % total_elems != 0) { - uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; - uint32_t registers_used = - ((nf * max_elems) * esz + (vlenb - 1)) / vlenb; - vext_set_elems_1s(vd, vta, (nf * max_elems) * esz, - registers_used * vlenb); - } + + vext_set_tail_elems_1s(env, env->vl, vd, desc, nf, esz, max_elems); } #define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN) \ @@ -585,8 +571,6 @@ vext_ldff(void *vd, void *v0, target_ulong base, uint32_t vm = vext_vm(desc); uint32_t max_elems = vext_max_elems(desc, log2_esz); uint32_t esz = 1 << log2_esz; - uint32_t total_elems = vext_get_total_elems(env, desc, esz); - uint32_t vta = vext_vta(desc); uint32_t vma = vext_vma(desc); target_ulong addr, offset, remain; @@ -647,18 +631,8 @@ ProbeSuccess: } } env->vstart = 0; - /* set tail elements to 1s */ - for (k = 0; k < nf; ++k) { - vext_set_elems_1s(vd, vta, (k * max_elems + env->vl) * esz, - (k * max_elems + max_elems) * esz); - } - if (nf * max_elems % total_elems != 0) { - uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; - uint32_t registers_used = - ((nf * max_elems) * esz + (vlenb - 1)) / vlenb; - vext_set_elems_1s(vd, vta, (nf * max_elems) * esz, - registers_used * vlenb); - } + + vext_set_tail_elems_1s(env, env->vl, vd, desc, nf, esz, max_elems); } #define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN) \ @@ -697,7 +671,7 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, { uint32_t i, k, off, pos; uint32_t nf = vext_nf(desc); - uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; + uint32_t vlenb = riscv_cpu_cfg(env)->vlen >> 3; uint32_t max_elems = vlenb >> log2_esz; k = env->vstart / max_elems; @@ -1167,7 +1141,7 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ { \ uint32_t vl = env->vl; \ uint32_t vm = vext_vm(desc); \ - uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ @@ -1203,7 +1177,7 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ { \ uint32_t vl = env->vl; \ uint32_t vm = vext_vm(desc); \ - uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ @@ -1402,7 +1376,7 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ - uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ @@ -1465,7 +1439,7 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ - uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ @@ -4178,7 +4152,7 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ - uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ @@ -4216,7 +4190,7 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ - uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ @@ -4747,7 +4721,7 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ uint32_t desc) \ { \ uint32_t vl = env->vl; \ - uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ int a, b; \ @@ -4834,7 +4808,7 @@ static void vmsetm(void *vd, void *v0, void *vs2, CPURISCVState *env, { uint32_t vm = vext_vm(desc); uint32_t vl = env->vl; - uint32_t total_elems = env_archcpu(env)->cfg.vlen; + uint32_t total_elems = riscv_cpu_cfg(env)->vlen; uint32_t vta_all_1s = vext_vta_all_1s(desc); uint32_t vma = vext_vma(desc); int i; |