diff options
93 files changed, 3651 insertions, 1829 deletions
diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index 79bbc8585b..f56df59c94 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -189,6 +189,8 @@ build-previous-qemu: TARGETS: x86_64-softmmu aarch64-softmmu before_script: - export QEMU_PREV_VERSION="$(sed 's/\([0-9.]*\)\.[0-9]*/v\1.0/' VERSION)" + - git remote add upstream https://gitlab.com/qemu-project/qemu + - git fetch upstream $QEMU_PREV_VERSION - git checkout $QEMU_PREV_VERSION after_script: - mv build build-previous @@ -217,9 +219,10 @@ build-previous-qemu: - QTEST_QEMU_BINARY_DST=./qemu-system-${TARGET} QTEST_QEMU_BINARY=../build/qemu-system-${TARGET} ./tests/qtest/migration-test -# This job is disabled until we release 9.0. The existing -# migration-test in 8.2 is broken on aarch64. The fix was already -# commited, but it will only take effect once 9.0 is out. +# This job needs to be disabled until we can have an aarch64 CPU model that +# will both (1) support both KVM and TCG, and (2) provide a stable ABI. +# Currently only "-cpu max" can provide (1), however it doesn't guarantee +# (2). Mark this test skipped until later. migration-compat-aarch64: extends: .migration-compat-common variables: diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 49e755ec4a..a8cecd040e 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -1119,6 +1119,11 @@ int kvm_vm_check_extension(KVMState *s, unsigned int extension) return ret; } +/* + * We track the poisoned pages to be able to: + * - replace them on VM reset + * - block a migration for a VM with a poisoned page + */ typedef struct HWPoisonPage { ram_addr_t ram_addr; QLIST_ENTRY(HWPoisonPage) list; @@ -1152,6 +1157,11 @@ void kvm_hwpoison_page_add(ram_addr_t ram_addr) QLIST_INSERT_HEAD(&hwpoison_page_list, page, list); } +bool kvm_hwpoisoned_mem(void) +{ + return !QLIST_EMPTY(&hwpoison_page_list); +} + static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size) { #if HOST_BIG_ENDIAN != TARGET_BIG_ENDIAN diff --git a/accel/stubs/kvm-stub.c b/accel/stubs/kvm-stub.c index 1b37d9a302..ca38172884 100644 --- a/accel/stubs/kvm-stub.c +++ b/accel/stubs/kvm-stub.c @@ -124,3 +124,8 @@ uint32_t kvm_dirty_ring_size(void) { return 0; } + +bool kvm_hwpoisoned_mem(void) +{ + return false; +} diff --git a/block/blkio.c b/block/blkio.c index bc2f21784c..882e1c297b 100644 --- a/block/blkio.c +++ b/block/blkio.c @@ -89,6 +89,9 @@ static int blkio_resize_bounce_pool(BDRVBlkioState *s, int64_t bytes) /* Pad size to reduce frequency of resize calls */ bytes += 128 * 1024; + /* Align the pool size to avoid blkio_alloc_mem_region() failure */ + bytes = QEMU_ALIGN_UP(bytes, s->mem_region_alignment); + WITH_QEMU_LOCK_GUARD(&s->blkio_lock) { int ret; diff --git a/block/block-backend.c b/block/block-backend.c index 209eb07528..9c4de79e6b 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -44,7 +44,7 @@ struct BlockBackend { char *name; int refcnt; BdrvChild *root; - AioContext *ctx; + AioContext *ctx; /* access with atomic operations only */ DriveInfo *legacy_dinfo; /* null unless created by drive_new() */ QTAILQ_ENTRY(BlockBackend) link; /* for block_backends */ QTAILQ_ENTRY(BlockBackend) monitor_link; /* for monitor_block_backends */ @@ -2414,22 +2414,22 @@ void blk_op_unblock_all(BlockBackend *blk, Error *reason) } } +/** + * Return BB's current AioContext. Note that this context may change + * concurrently at any time, with one exception: If the BB has a root node + * attached, its context will only change through bdrv_try_change_aio_context(), + * which creates a drained section. Therefore, incrementing such a BB's + * in-flight counter will prevent its context from changing. + */ AioContext *blk_get_aio_context(BlockBackend *blk) { - BlockDriverState *bs; IO_CODE(); if (!blk) { return qemu_get_aio_context(); } - bs = blk_bs(blk); - if (bs) { - AioContext *ctx = bdrv_get_aio_context(blk_bs(blk)); - assert(ctx == blk->ctx); - } - - return blk->ctx; + return qatomic_read(&blk->ctx); } int blk_set_aio_context(BlockBackend *blk, AioContext *new_context, @@ -2442,7 +2442,7 @@ int blk_set_aio_context(BlockBackend *blk, AioContext *new_context, GLOBAL_STATE_CODE(); if (!bs) { - blk->ctx = new_context; + qatomic_set(&blk->ctx, new_context); return 0; } @@ -2471,7 +2471,7 @@ static void blk_root_set_aio_ctx_commit(void *opaque) AioContext *new_context = s->new_ctx; ThrottleGroupMember *tgm = &blk->public.throttle_group_member; - blk->ctx = new_context; + qatomic_set(&blk->ctx, new_context); if (tgm->throttle_state) { throttle_group_detach_aio_context(tgm); throttle_group_attach_aio_context(tgm, new_context); diff --git a/docs/devel/tcg-ops.rst b/docs/devel/tcg-ops.rst index 8ae59ea02b..d46b625e0e 100644 --- a/docs/devel/tcg-ops.rst +++ b/docs/devel/tcg-ops.rst @@ -253,6 +253,8 @@ Jumps/Labels | ``TCG_COND_GEU /* unsigned */`` | ``TCG_COND_LEU /* unsigned */`` | ``TCG_COND_GTU /* unsigned */`` + | ``TCG_COND_TSTEQ /* t1 & t2 == 0 */`` + | ``TCG_COND_TSTNE /* t1 & t2 != 0 */`` Arithmetic ---------- diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index 227d83569f..738cb2ac36 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -37,6 +37,8 @@ #include "hw/virtio/virtio-blk-common.h" #include "qemu/coroutine.h" +static void virtio_blk_ioeventfd_attach(VirtIOBlock *s); + static void virtio_blk_init_request(VirtIOBlock *s, VirtQueue *vq, VirtIOBlockReq *req) { @@ -64,7 +66,7 @@ static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status) iov_discard_undo(&req->inhdr_undo); iov_discard_undo(&req->outhdr_undo); virtqueue_push(req->vq, &req->elem, req->in_len); - if (s->ioeventfd_started && !s->ioeventfd_disabled) { + if (qemu_in_iothread()) { virtio_notify_irqfd(vdev, req->vq); } else { virtio_notify(vdev, req->vq); @@ -661,6 +663,9 @@ static void virtio_blk_zone_report_complete(void *opaque, int ret) int64_t zrp_size, n, j = 0; int64_t nz = data->zone_report_data.nr_zones; int8_t err_status = VIRTIO_BLK_S_OK; + struct virtio_blk_zone_report zrp_hdr = (struct virtio_blk_zone_report) { + .nr_zones = cpu_to_le64(nz), + }; trace_virtio_blk_zone_report_complete(vdev, req, nz, ret); if (ret) { @@ -668,9 +673,6 @@ static void virtio_blk_zone_report_complete(void *opaque, int ret) goto out; } - struct virtio_blk_zone_report zrp_hdr = (struct virtio_blk_zone_report) { - .nr_zones = cpu_to_le64(nz), - }; zrp_size = sizeof(struct virtio_blk_zone_report) + sizeof(struct virtio_blk_zone_descriptor) * nz; n = iov_from_buf(in_iov, in_num, 0, &zrp_hdr, sizeof(zrp_hdr)); @@ -898,13 +900,14 @@ static int virtio_blk_handle_zone_append(VirtIOBlockReq *req, int64_t offset = virtio_ldq_p(vdev, &req->out.sector) << BDRV_SECTOR_BITS; int64_t len = iov_size(out_iov, out_num); + ZoneCmdData *data; trace_virtio_blk_handle_zone_append(vdev, req, offset >> BDRV_SECTOR_BITS); if (!check_zoned_request(s, offset, len, true, &err_status)) { goto out; } - ZoneCmdData *data = g_malloc(sizeof(ZoneCmdData)); + data = g_malloc(sizeof(ZoneCmdData)); data->req = req; data->in_iov = in_iov; data->in_num = in_num; @@ -1191,14 +1194,15 @@ static void virtio_blk_dma_restart_cb(void *opaque, bool running, { VirtIOBlock *s = opaque; uint16_t num_queues = s->conf.num_queues; + g_autofree VirtIOBlockReq **vq_rq = NULL; + VirtIOBlockReq *rq; if (!running) { return; } /* Split the device-wide s->rq request list into per-vq request lists */ - g_autofree VirtIOBlockReq **vq_rq = g_new0(VirtIOBlockReq *, num_queues); - VirtIOBlockReq *rq; + vq_rq = g_new0(VirtIOBlockReq *, num_queues); WITH_QEMU_LOCK_GUARD(&s->rq_lock) { rq = s->rq; @@ -1209,6 +1213,8 @@ static void virtio_blk_dma_restart_cb(void *opaque, bool running, VirtIOBlockReq *next = rq->next; uint16_t idx = virtio_get_queue_index(rq->vq); + /* Only num_queues vqs were created so vq_rq[idx] is within bounds */ + assert(idx < num_queues); rq->next = vq_rq[idx]; vq_rq[idx] = rq; rq = next; @@ -1485,68 +1491,6 @@ static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f, return 0; } -static bool -validate_iothread_vq_mapping_list(IOThreadVirtQueueMappingList *list, - uint16_t num_queues, Error **errp) -{ - g_autofree unsigned long *vqs = bitmap_new(num_queues); - g_autoptr(GHashTable) iothreads = - g_hash_table_new(g_str_hash, g_str_equal); - - for (IOThreadVirtQueueMappingList *node = list; node; node = node->next) { - const char *name = node->value->iothread; - uint16List *vq; - - if (!iothread_by_id(name)) { - error_setg(errp, "IOThread \"%s\" object does not exist", name); - return false; - } - - if (!g_hash_table_add(iothreads, (gpointer)name)) { - error_setg(errp, - "duplicate IOThread name \"%s\" in iothread-vq-mapping", - name); - return false; - } - - if (node != list) { - if (!!node->value->vqs != !!list->value->vqs) { - error_setg(errp, "either all items in iothread-vq-mapping " - "must have vqs or none of them must have it"); - return false; - } - } - - for (vq = node->value->vqs; vq; vq = vq->next) { - if (vq->value >= num_queues) { - error_setg(errp, "vq index %u for IOThread \"%s\" must be " - "less than num_queues %u in iothread-vq-mapping", - vq->value, name, num_queues); - return false; - } - - if (test_and_set_bit(vq->value, vqs)) { - error_setg(errp, "cannot assign vq %u to IOThread \"%s\" " - "because it is already assigned", vq->value, name); - return false; - } - } - } - - if (list->value->vqs) { - for (uint16_t i = 0; i < num_queues; i++) { - if (!test_bit(i, vqs)) { - error_setg(errp, - "missing vq %u IOThread assignment in iothread-vq-mapping", - i); - return false; - } - } - } - - return true; -} - static void virtio_resize_cb(void *opaque) { VirtIODevice *vdev = opaque; @@ -1613,15 +1557,95 @@ static const BlockDevOps virtio_block_ops = { .drained_end = virtio_blk_drained_end, }; -/* Generate vq:AioContext mappings from a validated iothread-vq-mapping list */ -static void -apply_vq_mapping(IOThreadVirtQueueMappingList *iothread_vq_mapping_list, - AioContext **vq_aio_context, uint16_t num_queues) +static bool +validate_iothread_vq_mapping_list(IOThreadVirtQueueMappingList *list, + uint16_t num_queues, Error **errp) +{ + g_autofree unsigned long *vqs = bitmap_new(num_queues); + g_autoptr(GHashTable) iothreads = + g_hash_table_new(g_str_hash, g_str_equal); + + for (IOThreadVirtQueueMappingList *node = list; node; node = node->next) { + const char *name = node->value->iothread; + uint16List *vq; + + if (!iothread_by_id(name)) { + error_setg(errp, "IOThread \"%s\" object does not exist", name); + return false; + } + + if (!g_hash_table_add(iothreads, (gpointer)name)) { + error_setg(errp, + "duplicate IOThread name \"%s\" in iothread-vq-mapping", + name); + return false; + } + + if (node != list) { + if (!!node->value->vqs != !!list->value->vqs) { + error_setg(errp, "either all items in iothread-vq-mapping " + "must have vqs or none of them must have it"); + return false; + } + } + + for (vq = node->value->vqs; vq; vq = vq->next) { + if (vq->value >= num_queues) { + error_setg(errp, "vq index %u for IOThread \"%s\" must be " + "less than num_queues %u in iothread-vq-mapping", + vq->value, name, num_queues); + return false; + } + + if (test_and_set_bit(vq->value, vqs)) { + error_setg(errp, "cannot assign vq %u to IOThread \"%s\" " + "because it is already assigned", vq->value, name); + return false; + } + } + } + + if (list->value->vqs) { + for (uint16_t i = 0; i < num_queues; i++) { + if (!test_bit(i, vqs)) { + error_setg(errp, + "missing vq %u IOThread assignment in iothread-vq-mapping", + i); + return false; + } + } + } + + return true; +} + +/** + * apply_iothread_vq_mapping: + * @iothread_vq_mapping_list: The mapping of virtqueues to IOThreads. + * @vq_aio_context: The array of AioContext pointers to fill in. + * @num_queues: The length of @vq_aio_context. + * @errp: If an error occurs, a pointer to the area to store the error. + * + * Fill in the AioContext for each virtqueue in the @vq_aio_context array given + * the iothread-vq-mapping parameter in @iothread_vq_mapping_list. + * + * Returns: %true on success, %false on failure. + **/ +static bool apply_iothread_vq_mapping( + IOThreadVirtQueueMappingList *iothread_vq_mapping_list, + AioContext **vq_aio_context, + uint16_t num_queues, + Error **errp) { IOThreadVirtQueueMappingList *node; size_t num_iothreads = 0; size_t cur_iothread = 0; + if (!validate_iothread_vq_mapping_list(iothread_vq_mapping_list, + num_queues, errp)) { + return false; + } + for (node = iothread_vq_mapping_list; node; node = node->next) { num_iothreads++; } @@ -1638,6 +1662,7 @@ apply_vq_mapping(IOThreadVirtQueueMappingList *iothread_vq_mapping_list, /* Explicit vq:IOThread assignment */ for (vq = node->value->vqs; vq; vq = vq->next) { + assert(vq->value < num_queues); vq_aio_context[vq->value] = ctx; } } else { @@ -1650,6 +1675,8 @@ apply_vq_mapping(IOThreadVirtQueueMappingList *iothread_vq_mapping_list, cur_iothread++; } + + return true; } /* Context: BQL held */ @@ -1660,6 +1687,13 @@ static bool virtio_blk_vq_aio_context_init(VirtIOBlock *s, Error **errp) BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); + if (conf->iothread && conf->iothread_vq_mapping_list) { + error_setg(errp, + "iothread and iothread-vq-mapping properties cannot be set " + "at the same time"); + return false; + } + if (conf->iothread || conf->iothread_vq_mapping_list) { if (!k->set_guest_notifiers || !k->ioeventfd_assign) { error_setg(errp, @@ -1685,8 +1719,14 @@ static bool virtio_blk_vq_aio_context_init(VirtIOBlock *s, Error **errp) s->vq_aio_context = g_new(AioContext *, conf->num_queues); if (conf->iothread_vq_mapping_list) { - apply_vq_mapping(conf->iothread_vq_mapping_list, s->vq_aio_context, - conf->num_queues); + if (!apply_iothread_vq_mapping(conf->iothread_vq_mapping_list, + s->vq_aio_context, + conf->num_queues, + errp)) { + g_free(s->vq_aio_context); + s->vq_aio_context = NULL; + return false; + } } else if (conf->iothread) { AioContext *ctx = iothread_get_aio_context(conf->iothread); for (unsigned i = 0; i < conf->num_queues; i++) { @@ -1790,6 +1830,7 @@ static int virtio_blk_start_ioeventfd(VirtIODevice *vdev) * Try to change the AioContext so that block jobs and other operations can * co-locate their activity in the same AioContext. If it fails, nevermind. */ + assert(nvqs > 0); /* enforced during ->realize() */ r = blk_set_aio_context(s->conf.conf.blk, s->vq_aio_context[0], &local_err); if (r < 0) { @@ -1808,17 +1849,14 @@ static int virtio_blk_start_ioeventfd(VirtIODevice *vdev) s->ioeventfd_started = true; smp_wmb(); /* paired with aio_notify_accept() on the read side */ - /* Get this show started by hooking up our callbacks */ - for (i = 0; i < nvqs; i++) { - VirtQueue *vq = virtio_get_queue(vdev, i); - AioContext *ctx = s->vq_aio_context[i]; - - /* Kick right away to begin processing requests already in vring */ - event_notifier_set(virtio_queue_get_host_notifier(vq)); - - if (!blk_in_drain(s->conf.conf.blk)) { - virtio_queue_aio_attach_host_notifier(vq, ctx); - } + /* + * Get this show started by hooking up our callbacks. If drained now, + * virtio_blk_drained_end() will do this later. + * Attaching the notifier also kicks the virtqueues, processing any requests + * they may already have. + */ + if (!blk_in_drain(s->conf.conf.blk)) { + virtio_blk_ioeventfd_attach(s); } return 0; @@ -1924,6 +1962,7 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) VirtIODevice *vdev = VIRTIO_DEVICE(dev); VirtIOBlock *s = VIRTIO_BLK(dev); VirtIOBlkConf *conf = &s->conf; + BlockDriverState *bs; Error *err = NULL; unsigned i; @@ -1969,7 +2008,7 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) return; } - BlockDriverState *bs = blk_bs(conf->conf.blk); + bs = blk_bs(conf->conf.blk); if (bs->bl.zoned != BLK_Z_NONE) { virtio_add_feature(&s->host_features, VIRTIO_BLK_F_ZONED); if (bs->bl.zoned == BLK_Z_HM) { @@ -1996,19 +2035,6 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) return; } - if (conf->iothread_vq_mapping_list) { - if (conf->iothread) { - error_setg(errp, "iothread and iothread-vq-mapping properties " - "cannot be set at the same time"); - return; - } - - if (!validate_iothread_vq_mapping_list(conf->iothread_vq_mapping_list, - conf->num_queues, errp)) { - return; - } - } - s->config_size = virtio_get_config_size(&virtio_blk_cfg_size_params, s->host_features); virtio_init(vdev, VIRTIO_ID_BLOCK, s->config_size); diff --git a/hw/riscv/Kconfig b/hw/riscv/Kconfig index a50717be87..5d644eb7b1 100644 --- a/hw/riscv/Kconfig +++ b/hw/riscv/Kconfig @@ -41,6 +41,7 @@ config RISCV_VIRT select RISCV_IMSIC select SIFIVE_PLIC select SIFIVE_TEST + select SMBIOS select VIRTIO_MMIO select FW_CFG_DMA select PLATFORM_BUS diff --git a/hw/riscv/boot.c b/hw/riscv/boot.c index 0ffca05189..12f9792245 100644 --- a/hw/riscv/boot.c +++ b/hw/riscv/boot.c @@ -36,7 +36,8 @@ bool riscv_is_32bit(RISCVHartArrayState *harts) { - return harts->harts[0].env.misa_mxl_max == MXL_RV32; + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(&harts->harts[0]); + return mcc->misa_mxl_max == MXL_RV32; } /* diff --git a/hw/riscv/numa.c b/hw/riscv/numa.c index d319aefb45..cf686f4ff1 100644 --- a/hw/riscv/numa.c +++ b/hw/riscv/numa.c @@ -167,7 +167,8 @@ void riscv_socket_fdt_write_id(const MachineState *ms, const char *node_name, void riscv_socket_fdt_write_distance_matrix(const MachineState *ms) { int i, j, idx; - uint32_t *dist_matrix, dist_matrix_size; + g_autofree uint32_t *dist_matrix = NULL; + uint32_t dist_matrix_size; if (numa_enabled(ms) && ms->numa_state->have_numa_distance) { dist_matrix_size = riscv_socket_count(ms) * riscv_socket_count(ms); @@ -189,7 +190,6 @@ void riscv_socket_fdt_write_distance_matrix(const MachineState *ms) "numa-distance-map-v1"); qemu_fdt_setprop(ms->fdt, "/distance-map", "distance-matrix", dist_matrix, dist_matrix_size); - g_free(dist_matrix); } } diff --git a/hw/riscv/sifive_u.c b/hw/riscv/sifive_u.c index 5207ec1fa5..af5f923f54 100644 --- a/hw/riscv/sifive_u.c +++ b/hw/riscv/sifive_u.c @@ -171,7 +171,6 @@ static void create_fdt(SiFiveUState *s, const MemMapEntry *memmap, int cpu_phandle = phandle++; nodename = g_strdup_printf("/cpus/cpu@%d", cpu); char *intc = g_strdup_printf("/cpus/cpu@%d/interrupt-controller", cpu); - char *isa; qemu_fdt_add_subnode(fdt, nodename); /* cpu 0 is the management hart that does not have mmu */ if (cpu != 0) { @@ -180,11 +179,10 @@ static void create_fdt(SiFiveUState *s, const MemMapEntry *memmap, } else { qemu_fdt_setprop_string(fdt, nodename, "mmu-type", "riscv,sv48"); } - isa = riscv_isa_string(&s->soc.u_cpus.harts[cpu - 1]); + riscv_isa_write_fdt(&s->soc.u_cpus.harts[cpu - 1], fdt, nodename); } else { - isa = riscv_isa_string(&s->soc.e_cpus.harts[0]); + riscv_isa_write_fdt(&s->soc.e_cpus.harts[0], fdt, nodename); } - qemu_fdt_setprop_string(fdt, nodename, "riscv,isa", isa); qemu_fdt_setprop_string(fdt, nodename, "compatible", "riscv"); qemu_fdt_setprop_string(fdt, nodename, "status", "okay"); qemu_fdt_setprop_cell(fdt, nodename, "reg", cpu); @@ -194,7 +192,6 @@ static void create_fdt(SiFiveUState *s, const MemMapEntry *memmap, qemu_fdt_setprop_string(fdt, intc, "compatible", "riscv,cpu-intc"); qemu_fdt_setprop(fdt, intc, "interrupt-controller", NULL, 0); qemu_fdt_setprop_cell(fdt, intc, "#interrupt-cells", 1); - g_free(isa); g_free(intc); g_free(nodename); } diff --git a/hw/riscv/spike.c b/hw/riscv/spike.c index 81f7e53aed..64074395bc 100644 --- a/hw/riscv/spike.c +++ b/hw/riscv/spike.c @@ -59,7 +59,7 @@ static void create_fdt(SpikeState *s, const MemMapEntry *memmap, MachineState *ms = MACHINE(s); uint32_t *clint_cells; uint32_t cpu_phandle, intc_phandle, phandle = 1; - char *name, *mem_name, *clint_name, *clust_name; + char *mem_name, *clint_name, *clust_name; char *core_name, *cpu_name, *intc_name; static const char * const clint_compat[2] = { "sifive,clint0", "riscv,clint0" @@ -113,9 +113,7 @@ static void create_fdt(SpikeState *s, const MemMapEntry *memmap, } else { qemu_fdt_setprop_string(fdt, cpu_name, "mmu-type", "riscv,sv48"); } - name = riscv_isa_string(&s->soc[socket].harts[cpu]); - qemu_fdt_setprop_string(fdt, cpu_name, "riscv,isa", name); - g_free(name); + riscv_isa_write_fdt(&s->soc[socket].harts[cpu], fdt, cpu_name); qemu_fdt_setprop_string(fdt, cpu_name, "compatible", "riscv"); qemu_fdt_setprop_string(fdt, cpu_name, "status", "okay"); qemu_fdt_setprop_cell(fdt, cpu_name, "reg", diff --git a/hw/riscv/virt-acpi-build.c b/hw/riscv/virt-acpi-build.c index 26c7e4482d..fb8baf64f6 100644 --- a/hw/riscv/virt-acpi-build.c +++ b/hw/riscv/virt-acpi-build.c @@ -196,7 +196,7 @@ static void build_rhct(GArray *table_data, RISCVCPU *cpu = &s->soc[0].harts[0]; uint32_t mmu_offset = 0; uint8_t satp_mode_max; - char *isa; + g_autofree char *isa = NULL; AcpiTable table = { .sig = "RHCT", .rev = 1, .oem_id = s->oem_id, .oem_table_id = s->oem_table_id }; diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c index f9fd1341fc..fd35c74781 100644 --- a/hw/riscv/virt.c +++ b/hw/riscv/virt.c @@ -36,6 +36,7 @@ #include "hw/riscv/boot.h" #include "hw/riscv/numa.h" #include "kvm/kvm_riscv.h" +#include "hw/firmware/smbios.h" #include "hw/intc/riscv_aclint.h" #include "hw/intc/riscv_aplic.h" #include "hw/intc/sifive_plic.h" @@ -215,12 +216,15 @@ static void create_fdt_socket_cpus(RISCVVirtState *s, int socket, int cpu; uint32_t cpu_phandle; MachineState *ms = MACHINE(s); - char *name, *cpu_name, *core_name, *intc_name, *sv_name; bool is_32_bit = riscv_is_32bit(&s->soc[0]); uint8_t satp_mode_max; for (cpu = s->soc[socket].num_harts - 1; cpu >= 0; cpu--) { RISCVCPU *cpu_ptr = &s->soc[socket].harts[cpu]; + g_autofree char *cpu_name = NULL; + g_autofree char *core_name = NULL; + g_autofree char *intc_name = NULL; + g_autofree char *sv_name = NULL; cpu_phandle = (*phandle)++; @@ -233,12 +237,9 @@ static void create_fdt_socket_cpus(RISCVVirtState *s, int socket, sv_name = g_strdup_printf("riscv,%s", satp_mode_str(satp_mode_max, is_32_bit)); qemu_fdt_setprop_string(ms->fdt, cpu_name, "mmu-type", sv_name); - g_free(sv_name); } - name = riscv_isa_string(cpu_ptr); - qemu_fdt_setprop_string(ms->fdt, cpu_name, "riscv,isa", name); - g_free(name); + riscv_isa_write_fdt(cpu_ptr, ms->fdt, cpu_name); if (cpu_ptr->cfg.ext_zicbom) { qemu_fdt_setprop_cell(ms->fdt, cpu_name, "riscv,cbom-block-size", @@ -277,17 +278,13 @@ static void create_fdt_socket_cpus(RISCVVirtState *s, int socket, core_name = g_strdup_printf("%s/core%d", clust_name, cpu); qemu_fdt_add_subnode(ms->fdt, core_name); qemu_fdt_setprop_cell(ms->fdt, core_name, "cpu", cpu_phandle); - - g_free(core_name); - g_free(intc_name); - g_free(cpu_name); } } static void create_fdt_socket_memory(RISCVVirtState *s, const MemMapEntry *memmap, int socket) { - char *mem_name; + g_autofree char *mem_name = NULL; uint64_t addr, size; MachineState *ms = MACHINE(s); @@ -299,7 +296,6 @@ static void create_fdt_socket_memory(RISCVVirtState *s, addr >> 32, addr, size >> 32, size); qemu_fdt_setprop_string(ms->fdt, mem_name, "device_type", "memory"); riscv_socket_fdt_write_id(ms, mem_name, socket); - g_free(mem_name); } static void create_fdt_socket_clint(RISCVVirtState *s, @@ -307,8 +303,8 @@ static void create_fdt_socket_clint(RISCVVirtState *s, uint32_t *intc_phandles) { int cpu; - char *clint_name; - uint32_t *clint_cells; + g_autofree char *clint_name = NULL; + g_autofree uint32_t *clint_cells = NULL; unsigned long clint_addr; MachineState *ms = MACHINE(s); static const char * const clint_compat[2] = { @@ -335,9 +331,6 @@ static void create_fdt_socket_clint(RISCVVirtState *s, qemu_fdt_setprop(ms->fdt, clint_name, "interrupts-extended", clint_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 4); riscv_socket_fdt_write_id(ms, clint_name, socket); - g_free(clint_name); - - g_free(clint_cells); } static void create_fdt_socket_aclint(RISCVVirtState *s, @@ -348,9 +341,9 @@ static void create_fdt_socket_aclint(RISCVVirtState *s, char *name; unsigned long addr, size; uint32_t aclint_cells_size; - uint32_t *aclint_mswi_cells; - uint32_t *aclint_sswi_cells; - uint32_t *aclint_mtimer_cells; + g_autofree uint32_t *aclint_mswi_cells = NULL; + g_autofree uint32_t *aclint_sswi_cells = NULL; + g_autofree uint32_t *aclint_mtimer_cells = NULL; MachineState *ms = MACHINE(s); aclint_mswi_cells = g_new0(uint32_t, s->soc[socket].num_harts * 2); @@ -422,10 +415,6 @@ static void create_fdt_socket_aclint(RISCVVirtState *s, riscv_socket_fdt_write_id(ms, name, socket); g_free(name); } - - g_free(aclint_mswi_cells); - g_free(aclint_mtimer_cells); - g_free(aclint_sswi_cells); } static void create_fdt_socket_plic(RISCVVirtState *s, @@ -434,8 +423,8 @@ static void create_fdt_socket_plic(RISCVVirtState *s, uint32_t *plic_phandles) { int cpu; - char *plic_name; - uint32_t *plic_cells; + g_autofree char *plic_name = NULL; + g_autofree uint32_t *plic_cells; unsigned long plic_addr; MachineState *ms = MACHINE(s); static const char * const plic_compat[2] = { @@ -495,10 +484,6 @@ static void create_fdt_socket_plic(RISCVVirtState *s, memmap[VIRT_PLATFORM_BUS].size, VIRT_PLATFORM_BUS_IRQ); } - - g_free(plic_name); - - g_free(plic_cells); } uint32_t imsic_num_bits(uint32_t count) @@ -517,11 +502,12 @@ static void create_fdt_one_imsic(RISCVVirtState *s, hwaddr base_addr, bool m_mode, uint32_t imsic_guest_bits) { int cpu, socket; - char *imsic_name; + g_autofree char *imsic_name = NULL; MachineState *ms = MACHINE(s); int socket_count = riscv_socket_count(ms); - uint32_t imsic_max_hart_per_socket; - uint32_t *imsic_cells, *imsic_regs, imsic_addr, imsic_size; + uint32_t imsic_max_hart_per_socket, imsic_addr, imsic_size; + g_autofree uint32_t *imsic_cells = NULL; + g_autofree uint32_t *imsic_regs = NULL; imsic_cells = g_new0(uint32_t, ms->smp.cpus * 2); imsic_regs = g_new0(uint32_t, socket_count * 4); @@ -573,10 +559,6 @@ static void create_fdt_one_imsic(RISCVVirtState *s, hwaddr base_addr, IMSIC_MMIO_GROUP_MIN_SHIFT); } qemu_fdt_setprop_cell(ms->fdt, imsic_name, "phandle", msi_phandle); - - g_free(imsic_name); - g_free(imsic_regs); - g_free(imsic_cells); } static void create_fdt_imsic(RISCVVirtState *s, const MemMapEntry *memmap, @@ -608,12 +590,10 @@ static void create_fdt_one_aplic(RISCVVirtState *s, int socket, bool m_mode, int num_harts) { int cpu; - char *aplic_name; - uint32_t *aplic_cells; + g_autofree char *aplic_name = NULL; + g_autofree uint32_t *aplic_cells = g_new0(uint32_t, num_harts * 2); MachineState *ms = MACHINE(s); - aplic_cells = g_new0(uint32_t, num_harts * 2); - for (cpu = 0; cpu < num_harts; cpu++) { aplic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]); aplic_cells[cpu * 2 + 1] = cpu_to_be32(m_mode ? IRQ_M_EXT : IRQ_S_EXT); @@ -648,9 +628,6 @@ static void create_fdt_one_aplic(RISCVVirtState *s, int socket, riscv_socket_fdt_write_id(ms, aplic_name, socket); qemu_fdt_setprop_cell(ms->fdt, aplic_name, "phandle", aplic_phandle); - - g_free(aplic_name); - g_free(aplic_cells); } static void create_fdt_socket_aplic(RISCVVirtState *s, @@ -662,7 +639,7 @@ static void create_fdt_socket_aplic(RISCVVirtState *s, uint32_t *aplic_phandles, int num_harts) { - char *aplic_name; + g_autofree char *aplic_name = NULL; unsigned long aplic_addr; MachineState *ms = MACHINE(s); uint32_t aplic_m_phandle, aplic_s_phandle; @@ -697,23 +674,18 @@ static void create_fdt_socket_aplic(RISCVVirtState *s, VIRT_PLATFORM_BUS_IRQ); } - g_free(aplic_name); - aplic_phandles[socket] = aplic_s_phandle; } static void create_fdt_pmu(RISCVVirtState *s) { - char *pmu_name; + g_autofree char *pmu_name = g_strdup_printf("/pmu"); MachineState *ms = MACHINE(s); RISCVCPU hart = s->soc[0].harts[0]; - pmu_name = g_strdup_printf("/pmu"); qemu_fdt_add_subnode(ms->fdt, pmu_name); qemu_fdt_setprop_string(ms->fdt, pmu_name, "compatible", "riscv,pmu"); riscv_pmu_generate_fdt_node(ms->fdt, hart.pmu_avail_ctrs, pmu_name); - - g_free(pmu_name); } static void create_fdt_sockets(RISCVVirtState *s, const MemMapEntry *memmap, @@ -723,11 +695,11 @@ static void create_fdt_sockets(RISCVVirtState *s, const MemMapEntry *memmap, uint32_t *irq_virtio_phandle, uint32_t *msi_pcie_phandle) { - char *clust_name; int socket, phandle_pos; MachineState *ms = MACHINE(s); uint32_t msi_m_phandle = 0, msi_s_phandle = 0; - uint32_t *intc_phandles, xplic_phandles[MAX_NODES]; + uint32_t xplic_phandles[MAX_NODES]; + g_autofree uint32_t *intc_phandles = NULL; int socket_count = riscv_socket_count(ms); qemu_fdt_add_subnode(ms->fdt, "/cpus"); @@ -741,6 +713,7 @@ static void create_fdt_sockets(RISCVVirtState *s, const MemMapEntry *memmap, phandle_pos = ms->smp.cpus; for (socket = (socket_count - 1); socket >= 0; socket--) { + g_autofree char *clust_name = NULL; phandle_pos -= s->soc[socket].num_harts; clust_name = g_strdup_printf("/cpus/cpu-map/cluster%d", socket); @@ -751,8 +724,6 @@ static void create_fdt_sockets(RISCVVirtState *s, const MemMapEntry *memmap, create_fdt_socket_memory(s, memmap, socket); - g_free(clust_name); - if (tcg_enabled()) { if (s->have_aclint) { create_fdt_socket_aclint(s, memmap, socket, @@ -795,8 +766,6 @@ static void create_fdt_sockets(RISCVVirtState *s, const MemMapEntry *memmap, } } - g_free(intc_phandles); - if (kvm_enabled() && virt_use_kvm_aia(s)) { *irq_mmio_phandle = xplic_phandles[0]; *irq_virtio_phandle = xplic_phandles[0]; @@ -825,12 +794,12 @@ static void create_fdt_virtio(RISCVVirtState *s, const MemMapEntry *memmap, uint32_t irq_virtio_phandle) { int i; - char *name; MachineState *ms = MACHINE(s); for (i = 0; i < VIRTIO_COUNT; i++) { - name = g_strdup_printf("/soc/virtio_mmio@%lx", + g_autofree char *name = g_strdup_printf("/soc/virtio_mmio@%lx", (long)(memmap[VIRT_VIRTIO].base + i * memmap[VIRT_VIRTIO].size)); + qemu_fdt_add_subnode(ms->fdt, name); qemu_fdt_setprop_string(ms->fdt, name, "compatible", "virtio,mmio"); qemu_fdt_setprop_cells(ms->fdt, name, "reg", @@ -845,7 +814,6 @@ static void create_fdt_virtio(RISCVVirtState *s, const MemMapEntry *memmap, qemu_fdt_setprop_cells(ms->fdt, name, "interrupts", VIRTIO_IRQ + i, 0x4); } - g_free(name); } } @@ -853,7 +821,7 @@ static void create_fdt_pcie(RISCVVirtState *s, const MemMapEntry *memmap, uint32_t irq_pcie_phandle, uint32_t msi_pcie_phandle) { - char *name; + g_autofree char *name = NULL; MachineState *ms = MACHINE(s); name = g_strdup_printf("/soc/pci@%lx", @@ -887,7 +855,6 @@ static void create_fdt_pcie(RISCVVirtState *s, const MemMapEntry *memmap, 2, virt_high_pcie_memmap.base, 2, virt_high_pcie_memmap.size); create_pcie_irq_map(s, ms->fdt, name, irq_pcie_phandle); - g_free(name); } static void create_fdt_reset(RISCVVirtState *s, const MemMapEntry *memmap, @@ -934,7 +901,7 @@ static void create_fdt_reset(RISCVVirtState *s, const MemMapEntry *memmap, static void create_fdt_uart(RISCVVirtState *s, const MemMapEntry *memmap, uint32_t irq_mmio_phandle) { - char *name; + g_autofree char *name = NULL; MachineState *ms = MACHINE(s); name = g_strdup_printf("/soc/serial@%lx", (long)memmap[VIRT_UART0].base); @@ -952,13 +919,12 @@ static void create_fdt_uart(RISCVVirtState *s, const MemMapEntry *memmap, } qemu_fdt_setprop_string(ms->fdt, "/chosen", "stdout-path", name); - g_free(name); } static void create_fdt_rtc(RISCVVirtState *s, const MemMapEntry *memmap, uint32_t irq_mmio_phandle) { - char *name; + g_autofree char *name = NULL; MachineState *ms = MACHINE(s); name = g_strdup_printf("/soc/rtc@%lx", (long)memmap[VIRT_RTC].base); @@ -974,41 +940,36 @@ static void create_fdt_rtc(RISCVVirtState *s, const MemMapEntry *memmap, } else { qemu_fdt_setprop_cells(ms->fdt, name, "interrupts", RTC_IRQ, 0x4); } - g_free(name); } static void create_fdt_flash(RISCVVirtState *s, const MemMapEntry *memmap) { - char *name; MachineState *ms = MACHINE(s); hwaddr flashsize = virt_memmap[VIRT_FLASH].size / 2; hwaddr flashbase = virt_memmap[VIRT_FLASH].base; + g_autofree char *name = g_strdup_printf("/flash@%" PRIx64, flashbase); - name = g_strdup_printf("/flash@%" PRIx64, flashbase); qemu_fdt_add_subnode(ms->fdt, name); qemu_fdt_setprop_string(ms->fdt, name, "compatible", "cfi-flash"); qemu_fdt_setprop_sized_cells(ms->fdt, name, "reg", 2, flashbase, 2, flashsize, 2, flashbase + flashsize, 2, flashsize); qemu_fdt_setprop_cell(ms->fdt, name, "bank-width", 4); - g_free(name); } static void create_fdt_fw_cfg(RISCVVirtState *s, const MemMapEntry *memmap) { - char *nodename; MachineState *ms = MACHINE(s); hwaddr base = memmap[VIRT_FW_CFG].base; hwaddr size = memmap[VIRT_FW_CFG].size; + g_autofree char *nodename = g_strdup_printf("/fw-cfg@%" PRIx64, base); - nodename = g_strdup_printf("/fw-cfg@%" PRIx64, base); qemu_fdt_add_subnode(ms->fdt, nodename); qemu_fdt_setprop_string(ms->fdt, nodename, "compatible", "qemu,fw-cfg-mmio"); qemu_fdt_setprop_sized_cells(ms->fdt, nodename, "reg", 2, base, 2, size); qemu_fdt_setprop(ms->fdt, nodename, "dma-coherent", NULL, 0); - g_free(nodename); } static void finalize_fdt(RISCVVirtState *s) @@ -1155,7 +1116,7 @@ static DeviceState *virt_create_plic(const MemMapEntry *memmap, int socket, int base_hartid, int hart_count) { DeviceState *ret; - char *plic_hart_config; + g_autofree char *plic_hart_config = NULL; /* Per-socket PLIC hart topology configuration string */ plic_hart_config = riscv_plic_hart_config_string(hart_count); @@ -1174,8 +1135,6 @@ static DeviceState *virt_create_plic(const MemMapEntry *memmap, int socket, VIRT_PLIC_CONTEXT_STRIDE, memmap[VIRT_PLIC].size); - g_free(plic_hart_config); - return ret; } @@ -1263,6 +1222,45 @@ static void create_platform_bus(RISCVVirtState *s, DeviceState *irqchip) sysbus_mmio_get_region(sysbus, 0)); } +static void virt_build_smbios(RISCVVirtState *s) +{ + MachineClass *mc = MACHINE_GET_CLASS(s); + MachineState *ms = MACHINE(s); + uint8_t *smbios_tables, *smbios_anchor; + size_t smbios_tables_len, smbios_anchor_len; + struct smbios_phys_mem_area mem_array; + const char *product = "QEMU Virtual Machine"; + + if (kvm_enabled()) { + product = "KVM Virtual Machine"; + } + + smbios_set_defaults("QEMU", product, mc->name, false, + true, SMBIOS_ENTRY_POINT_TYPE_64); + + if (riscv_is_32bit(&s->soc[0])) { + smbios_set_default_processor_family(0x200); + } else { + smbios_set_default_processor_family(0x201); + } + + /* build the array of physical mem area from base_memmap */ + mem_array.address = s->memmap[VIRT_DRAM].base; + mem_array.length = ms->ram_size; + + smbios_get_tables(ms, &mem_array, 1, + &smbios_tables, &smbios_tables_len, + &smbios_anchor, &smbios_anchor_len, + &error_fatal); + + if (smbios_anchor) { + fw_cfg_add_file(s->fw_cfg, "etc/smbios/smbios-tables", + smbios_tables, smbios_tables_len); + fw_cfg_add_file(s->fw_cfg, "etc/smbios/smbios-anchor", + smbios_anchor, smbios_anchor_len); + } +} + static void virt_machine_done(Notifier *notifier, void *data) { RISCVVirtState *s = container_of(notifier, RISCVVirtState, @@ -1351,6 +1349,8 @@ static void virt_machine_done(Notifier *notifier, void *data) riscv_setup_direct_kernel(kernel_entry, fdt_load_addr); } + virt_build_smbios(s); + if (virt_is_acpi_enabled(s)) { virt_acpi_setup(s); } @@ -1362,7 +1362,6 @@ static void virt_machine_init(MachineState *machine) RISCVVirtState *s = RISCV_VIRT_MACHINE(machine); MemoryRegion *system_memory = get_system_memory(); MemoryRegion *mask_rom = g_new(MemoryRegion, 1); - char *soc_name; DeviceState *mmio_irqchip, *virtio_irqchip, *pcie_irqchip; int i, base_hartid, hart_count; int socket_count = riscv_socket_count(machine); @@ -1382,6 +1381,8 @@ static void virt_machine_init(MachineState *machine) /* Initialize sockets */ mmio_irqchip = virtio_irqchip = pcie_irqchip = NULL; for (i = 0; i < socket_count; i++) { + g_autofree char *soc_name = g_strdup_printf("soc%d", i); + if (!riscv_socket_check_hartids(machine, i)) { error_report("discontinuous hartids in socket%d", i); exit(1); @@ -1399,10 +1400,8 @@ static void virt_machine_init(MachineState *machine) exit(1); } - soc_name = g_strdup_printf("soc%d", i); object_initialize_child(OBJECT(machine), soc_name, &s->soc[i], TYPE_RISCV_HART_ARRAY); - g_free(soc_name); object_property_set_str(OBJECT(&s->soc[i]), "cpu-type", machine->cpu_type, &error_abort); object_property_set_int(OBJECT(&s->soc[i]), "hartid-base", diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c index 0a2eb11c56..9e40b0c920 100644 --- a/hw/scsi/scsi-bus.c +++ b/hw/scsi/scsi-bus.c @@ -120,17 +120,13 @@ static void scsi_device_for_each_req_async_bh(void *opaque) SCSIRequest *next; /* - * If the AioContext changed before this BH was called then reschedule into - * the new AioContext before accessing ->requests. This can happen when - * scsi_device_for_each_req_async() is called and then the AioContext is - * changed before BHs are run. + * The BB cannot have changed contexts between this BH being scheduled and + * now: BBs' AioContexts, when they have a node attached, can only be + * changed via bdrv_try_change_aio_context(), in a drained section. While + * we have the in-flight counter incremented, that drain must block. */ ctx = blk_get_aio_context(s->conf.blk); - if (ctx != qemu_get_current_aio_context()) { - aio_bh_schedule_oneshot(ctx, scsi_device_for_each_req_async_bh, - g_steal_pointer(&data)); - return; - } + assert(ctx == qemu_get_current_aio_context()); QTAILQ_FOREACH_SAFE(req, &s->requests, next, next) { data->fn(req, data->fn_opaque); @@ -138,11 +134,16 @@ static void scsi_device_for_each_req_async_bh(void *opaque) /* Drop the reference taken by scsi_device_for_each_req_async() */ object_unref(OBJECT(s)); + + /* Paired with blk_inc_in_flight() in scsi_device_for_each_req_async() */ + blk_dec_in_flight(s->conf.blk); } /* * Schedule @fn() to be invoked for each enqueued request in device @s. @fn() * runs in the AioContext that is executing the request. + * Keeps the BlockBackend's in-flight counter incremented until everything is + * done, so draining it will settle all scheduled @fn() calls. */ static void scsi_device_for_each_req_async(SCSIDevice *s, void (*fn)(SCSIRequest *, void *), @@ -163,6 +164,8 @@ static void scsi_device_for_each_req_async(SCSIDevice *s, */ object_ref(OBJECT(s)); + /* Paired with blk_dec_in_flight() in scsi_device_for_each_req_async_bh() */ + blk_inc_in_flight(s->conf.blk); aio_bh_schedule_oneshot(blk_get_aio_context(s->conf.blk), scsi_device_for_each_req_async_bh, data); @@ -373,15 +376,13 @@ static void scsi_qdev_unrealize(DeviceState *qdev) /* handle legacy '-drive if=scsi,...' cmd line args */ SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockBackend *blk, - int unit, bool removable, int bootindex, - bool share_rw, - BlockdevOnError rerror, - BlockdevOnError werror, + int unit, bool removable, BlockConf *conf, const char *serial, Error **errp) { const char *driver; char *name; DeviceState *dev; + SCSIDevice *s; DriveInfo *dinfo; if (blk_is_sg(blk)) { @@ -399,11 +400,10 @@ SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockBackend *blk, object_property_add_child(OBJECT(bus), name, OBJECT(dev)); g_free(name); + s = SCSI_DEVICE(dev); + s->conf = *conf; + qdev_prop_set_uint32(dev, "scsi-id", unit); - if (bootindex >= 0) { - object_property_set_int(OBJECT(dev), "bootindex", bootindex, - &error_abort); - } if (object_property_find(OBJECT(dev), "removable")) { qdev_prop_set_bit(dev, "removable", removable); } @@ -414,19 +414,12 @@ SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockBackend *blk, object_unparent(OBJECT(dev)); return NULL; } - if (!object_property_set_bool(OBJECT(dev), "share-rw", share_rw, errp)) { - object_unparent(OBJECT(dev)); - return NULL; - } - - qdev_prop_set_enum(dev, "rerror", rerror); - qdev_prop_set_enum(dev, "werror", werror); if (!qdev_realize_and_unref(dev, &bus->qbus, errp)) { object_unparent(OBJECT(dev)); return NULL; } - return SCSI_DEVICE(dev); + return s; } void scsi_bus_legacy_handle_cmdline(SCSIBus *bus) @@ -434,6 +427,12 @@ void scsi_bus_legacy_handle_cmdline(SCSIBus *bus) Location loc; DriveInfo *dinfo; int unit; + BlockConf conf = { + .bootindex = -1, + .share_rw = false, + .rerror = BLOCKDEV_ON_ERROR_AUTO, + .werror = BLOCKDEV_ON_ERROR_AUTO, + }; loc_push_none(&loc); for (unit = 0; unit <= bus->info->max_target; unit++) { @@ -443,10 +442,7 @@ void scsi_bus_legacy_handle_cmdline(SCSIBus *bus) } qemu_opts_loc_restore(dinfo->opts); scsi_bus_legacy_add_drive(bus, blk_by_legacy_dinfo(dinfo), - unit, false, -1, false, - BLOCKDEV_ON_ERROR_AUTO, - BLOCKDEV_ON_ERROR_AUTO, - NULL, &error_fatal); + unit, false, &conf, NULL, &error_fatal); } loc_pop(&loc); } @@ -1728,11 +1724,20 @@ static void scsi_device_purge_one_req(SCSIRequest *req, void *opaque) scsi_req_cancel_async(req, NULL); } +/** + * Cancel all requests, and block until they are deleted. + */ void scsi_device_purge_requests(SCSIDevice *sdev, SCSISense sense) { scsi_device_for_each_req_async(sdev, scsi_device_purge_one_req, NULL); + /* + * Await all the scsi_device_purge_one_req() calls scheduled by + * scsi_device_for_each_req_async(), and all I/O requests that were + * cancelled this way, but may still take a bit of time to settle. + */ blk_drain(sdev->conf.blk); + scsi_device_set_ua(sdev, sense); } diff --git a/hw/scsi/virtio-scsi.c b/hw/scsi/virtio-scsi.c index 690aceec45..9f02ceea09 100644 --- a/hw/scsi/virtio-scsi.c +++ b/hw/scsi/virtio-scsi.c @@ -1149,6 +1149,7 @@ static void virtio_scsi_drained_begin(SCSIBus *bus) static void virtio_scsi_drained_end(SCSIBus *bus) { VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus); + VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s); VirtIODevice *vdev = VIRTIO_DEVICE(s); uint32_t total_queues = VIRTIO_SCSI_VQ_NUM_FIXED + s->parent_obj.conf.num_queues; @@ -1166,7 +1167,11 @@ static void virtio_scsi_drained_end(SCSIBus *bus) for (uint32_t i = 0; i < total_queues; i++) { VirtQueue *vq = virtio_get_queue(vdev, i); - virtio_queue_aio_attach_host_notifier(vq, s->ctx); + if (vq == vs->event_vq) { + virtio_queue_aio_attach_host_notifier_no_poll(vq, s->ctx); + } else { + virtio_queue_aio_attach_host_notifier(vq, s->ctx); + } } } diff --git a/hw/smbios/smbios.c b/hw/smbios/smbios.c index 2a90601ac5..c0c5a81e66 100644 --- a/hw/smbios/smbios.c +++ b/hw/smbios/smbios.c @@ -102,6 +102,7 @@ static struct { #define DEFAULT_CPU_SPEED 2000 static struct { + uint16_t processor_family; const char *sock_pfx, *manufacturer, *version, *serial, *asset, *part; uint64_t max_speed; uint64_t current_speed; @@ -110,6 +111,7 @@ static struct { .max_speed = DEFAULT_CPU_SPEED, .current_speed = DEFAULT_CPU_SPEED, .processor_id = 0, + .processor_family = 0x01, /* Other */ }; struct type8_instance { @@ -338,6 +340,10 @@ static const QemuOptDesc qemu_smbios_type4_opts[] = { .type = QEMU_OPT_STRING, .help = "part number", }, { + .name = "processor-family", + .type = QEMU_OPT_NUMBER, + .help = "processor family", + }, { .name = "processor-id", .type = QEMU_OPT_NUMBER, .help = "processor id", @@ -726,7 +732,7 @@ static void smbios_build_type_4_table(MachineState *ms, unsigned instance) snprintf(sock_str, sizeof(sock_str), "%s%2x", type4.sock_pfx, instance); SMBIOS_TABLE_SET_STR(4, socket_designation_str, sock_str); t->processor_type = 0x03; /* CPU */ - t->processor_family = 0x01; /* Other */ + t->processor_family = 0xfe; /* use Processor Family 2 field */ SMBIOS_TABLE_SET_STR(4, processor_manufacturer_str, type4.manufacturer); if (type4.processor_id == 0) { t->processor_id[0] = cpu_to_le32(smbios_cpuid_version); @@ -758,7 +764,7 @@ static void smbios_build_type_4_table(MachineState *ms, unsigned instance) t->thread_count = (threads_per_socket > 255) ? 0xFF : threads_per_socket; t->processor_characteristics = cpu_to_le16(0x02); /* Unknown */ - t->processor_family2 = cpu_to_le16(0x01); /* Other */ + t->processor_family2 = cpu_to_le16(type4.processor_family); if (tbl_len == SMBIOS_TYPE_4_LEN_V30) { t->core_count2 = t->core_enabled2 = cpu_to_le16(cores_per_socket); @@ -983,6 +989,13 @@ void smbios_set_cpuid(uint32_t version, uint32_t features) field = value; \ } +void smbios_set_default_processor_family(uint16_t processor_family) +{ + if (type4.processor_family <= 0x01) { + type4.processor_family = processor_family; + } +} + void smbios_set_defaults(const char *manufacturer, const char *product, const char *version, bool legacy_mode, bool uuid_encoded, SmbiosEntryPointType ep_type) @@ -1402,6 +1415,9 @@ void smbios_entry_add(QemuOpts *opts, Error **errp) return; } save_opt(&type4.sock_pfx, opts, "sock_pfx"); + type4.processor_family = qemu_opt_get_number(opts, + "processor-family", + 0x01 /* Other */); save_opt(&type4.manufacturer, opts, "manufacturer"); save_opt(&type4.version, opts, "version"); save_opt(&type4.serial, opts, "serial"); diff --git a/hw/usb/dev-storage-classic.c b/hw/usb/dev-storage-classic.c index 84d19752b5..50a3ad6285 100644 --- a/hw/usb/dev-storage-classic.c +++ b/hw/usb/dev-storage-classic.c @@ -67,10 +67,7 @@ static void usb_msd_storage_realize(USBDevice *dev, Error **errp) scsi_bus_init(&s->bus, sizeof(s->bus), DEVICE(dev), &usb_msd_scsi_info_storage); scsi_dev = scsi_bus_legacy_add_drive(&s->bus, blk, 0, !!s->removable, - s->conf.bootindex, s->conf.share_rw, - s->conf.rerror, s->conf.werror, - dev->serial, - errp); + &s->conf, dev->serial, errp); blk_unref(blk); if (!scsi_dev) { return; diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index 7549094154..d229755eae 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -3556,6 +3556,17 @@ static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n) void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx) { + /* + * virtio_queue_aio_detach_host_notifier() can leave notifications disabled. + * Re-enable them. (And if detach has not been used before, notifications + * being enabled is still the default state while a notifier is attached; + * see virtio_queue_host_notifier_aio_poll_end(), which will always leave + * notifications enabled once the polling section is left.) + */ + if (!virtio_queue_get_notification(vq)) { + virtio_queue_set_notification(vq, 1); + } + aio_set_event_notifier(ctx, &vq->host_notifier, virtio_queue_host_notifier_read, virtio_queue_host_notifier_aio_poll, @@ -3563,6 +3574,13 @@ void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx) aio_set_event_notifier_poll(ctx, &vq->host_notifier, virtio_queue_host_notifier_aio_poll_begin, virtio_queue_host_notifier_aio_poll_end); + + /* + * We will have ignored notifications about new requests from the guest + * while no notifiers were attached, so "kick" the virt queue to process + * those requests now. + */ + event_notifier_set(&vq->host_notifier); } /* @@ -3573,14 +3591,38 @@ void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx) */ void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue *vq, AioContext *ctx) { + /* See virtio_queue_aio_attach_host_notifier() */ + if (!virtio_queue_get_notification(vq)) { + virtio_queue_set_notification(vq, 1); + } + aio_set_event_notifier(ctx, &vq->host_notifier, virtio_queue_host_notifier_read, NULL, NULL); + + /* + * See virtio_queue_aio_attach_host_notifier(). + * Note that this may be unnecessary for the type of virtqueues this + * function is used for. Still, it will not hurt to have a quick look into + * whether we can/should process any of the virtqueue elements. + */ + event_notifier_set(&vq->host_notifier); } void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx) { aio_set_event_notifier(ctx, &vq->host_notifier, NULL, NULL, NULL); + + /* + * aio_set_event_notifier_poll() does not guarantee whether io_poll_end() + * will run after io_poll_begin(), so by removing the notifier, we do not + * know whether virtio_queue_host_notifier_aio_poll_end() has run after a + * previous virtio_queue_host_notifier_aio_poll_begin(), i.e. whether + * notifications are enabled or disabled. It does not really matter anyway; + * we just removed the notifier, so we do not care about notifications until + * we potentially re-attach it. The attach_host_notifier functions will + * ensure that notifications are enabled again when they are needed. + */ } void virtio_queue_host_notifier_read(EventNotifier *n) diff --git a/include/block/aio.h b/include/block/aio.h index 5d0a114988..8378553eb9 100644 --- a/include/block/aio.h +++ b/include/block/aio.h @@ -480,9 +480,14 @@ void aio_set_event_notifier(AioContext *ctx, AioPollFn *io_poll, EventNotifierHandler *io_poll_ready); -/* Set polling begin/end callbacks for an event notifier that has already been +/* + * Set polling begin/end callbacks for an event notifier that has already been * registered with aio_set_event_notifier. Do nothing if the event notifier is * not registered. + * + * Note that if the io_poll_end() callback (or the entire notifier) is removed + * during polling, it will not be called, so an io_poll_begin() is not + * necessarily always followed by an io_poll_end(). */ void aio_set_event_notifier_poll(AioContext *ctx, EventNotifier *notifier, diff --git a/include/hw/firmware/smbios.h b/include/hw/firmware/smbios.h index 7f3259a630..6e514982d4 100644 --- a/include/hw/firmware/smbios.h +++ b/include/hw/firmware/smbios.h @@ -295,6 +295,7 @@ void smbios_set_cpuid(uint32_t version, uint32_t features); void smbios_set_defaults(const char *manufacturer, const char *product, const char *version, bool legacy_mode, bool uuid_encoded, SmbiosEntryPointType ep_type); +void smbios_set_default_processor_family(uint16_t processor_family); uint8_t *smbios_get_table_legacy(MachineState *ms, size_t *length); void smbios_get_tables(MachineState *ms, const struct smbios_phys_mem_area *mem_array, diff --git a/include/hw/scsi/scsi.h b/include/hw/scsi/scsi.h index 10c4e8288d..c3d5e17e38 100644 --- a/include/hw/scsi/scsi.h +++ b/include/hw/scsi/scsi.h @@ -199,10 +199,7 @@ static inline SCSIBus *scsi_bus_from_device(SCSIDevice *d) } SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockBackend *blk, - int unit, bool removable, int bootindex, - bool share_rw, - BlockdevOnError rerror, - BlockdevOnError werror, + int unit, bool removable, BlockConf *conf, const char *serial, Error **errp); void scsi_bus_set_ua(SCSIBus *bus, SCSISense sense); void scsi_bus_legacy_handle_cmdline(SCSIBus *bus); diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h index 833a9a344f..5c14110c4b 100644 --- a/include/hw/virtio/virtio-blk.h +++ b/include/hw/virtio/virtio-blk.h @@ -55,7 +55,7 @@ struct VirtIOBlock { VirtIODevice parent_obj; BlockBackend *blk; QemuMutex rq_lock; - void *rq; /* protected by rq_lock */ + struct VirtIOBlockReq *rq; /* protected by rq_lock */ VirtIOBlkConf conf; unsigned short sector_mask; bool original_wce; diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h index d614878164..fad9a7e8ff 100644 --- a/include/sysemu/kvm.h +++ b/include/sysemu/kvm.h @@ -538,4 +538,10 @@ bool kvm_arch_cpu_check_are_resettable(void); bool kvm_dirty_ring_enabled(void); uint32_t kvm_dirty_ring_size(void); + +/** + * kvm_hwpoisoned_mem - indicate if there is any hwpoisoned page + * reported for the VM. + */ +bool kvm_hwpoisoned_mem(void); #endif diff --git a/include/tcg/tcg-cond.h b/include/tcg/tcg-cond.h index 2a38a386d4..5cadbd6ff2 100644 --- a/include/tcg/tcg-cond.h +++ b/include/tcg/tcg-cond.h @@ -29,26 +29,34 @@ * Conditions. Note that these are laid out for easy manipulation by * the functions below: * bit 0 is used for inverting; - * bit 1 is signed, - * bit 2 is unsigned, - * bit 3 is used with bit 0 for swapping signed/unsigned. + * bit 1 is used for conditions that need swapping (signed/unsigned). + * bit 2 is used with bit 1 for swapping. + * bit 3 is used for unsigned conditions. */ typedef enum { /* non-signed */ TCG_COND_NEVER = 0 | 0 | 0 | 0, TCG_COND_ALWAYS = 0 | 0 | 0 | 1, + + /* equality */ TCG_COND_EQ = 8 | 0 | 0 | 0, TCG_COND_NE = 8 | 0 | 0 | 1, + + /* "test" i.e. and then compare vs 0 */ + TCG_COND_TSTEQ = 8 | 4 | 0 | 0, + TCG_COND_TSTNE = 8 | 4 | 0 | 1, + /* signed */ TCG_COND_LT = 0 | 0 | 2 | 0, TCG_COND_GE = 0 | 0 | 2 | 1, - TCG_COND_LE = 8 | 0 | 2 | 0, - TCG_COND_GT = 8 | 0 | 2 | 1, + TCG_COND_GT = 0 | 4 | 2 | 0, + TCG_COND_LE = 0 | 4 | 2 | 1, + /* unsigned */ - TCG_COND_LTU = 0 | 4 | 0 | 0, - TCG_COND_GEU = 0 | 4 | 0 | 1, - TCG_COND_LEU = 8 | 4 | 0 | 0, - TCG_COND_GTU = 8 | 4 | 0 | 1, + TCG_COND_LTU = 8 | 0 | 2 | 0, + TCG_COND_GEU = 8 | 0 | 2 | 1, + TCG_COND_GTU = 8 | 4 | 2 | 0, + TCG_COND_LEU = 8 | 4 | 2 | 1, } TCGCond; /* Invert the sense of the comparison. */ @@ -60,25 +68,49 @@ static inline TCGCond tcg_invert_cond(TCGCond c) /* Swap the operands in a comparison. */ static inline TCGCond tcg_swap_cond(TCGCond c) { - return c & 6 ? (TCGCond)(c ^ 9) : c; + return (TCGCond)(c ^ ((c & 2) << 1)); +} + +/* Must a comparison be considered signed? */ +static inline bool is_signed_cond(TCGCond c) +{ + return (c & (8 | 2)) == 2; +} + +/* Must a comparison be considered unsigned? */ +static inline bool is_unsigned_cond(TCGCond c) +{ + return (c & (8 | 2)) == (8 | 2); +} + +/* Must a comparison be considered a test? */ +static inline bool is_tst_cond(TCGCond c) +{ + return (c | 1) == TCG_COND_TSTNE; } /* Create an "unsigned" version of a "signed" comparison. */ static inline TCGCond tcg_unsigned_cond(TCGCond c) { - return c & 2 ? (TCGCond)(c ^ 6) : c; + return is_signed_cond(c) ? (TCGCond)(c + 8) : c; } /* Create a "signed" version of an "unsigned" comparison. */ static inline TCGCond tcg_signed_cond(TCGCond c) { - return c & 4 ? (TCGCond)(c ^ 6) : c; + return is_unsigned_cond(c) ? (TCGCond)(c - 8) : c; } -/* Must a comparison be considered unsigned? */ -static inline bool is_unsigned_cond(TCGCond c) +/* Create the eq/ne version of a tsteq/tstne comparison. */ +static inline TCGCond tcg_tst_eqne_cond(TCGCond c) +{ + return is_tst_cond(c) ? (TCGCond)(c - 4) : c; +} + +/* Create the lt/ge version of a tstne/tsteq comparison of the sign. */ +static inline TCGCond tcg_tst_ltge_cond(TCGCond c) { - return (c & 4) != 0; + return is_tst_cond(c) ? (TCGCond)(c ^ 0xf) : c; } /* @@ -92,7 +124,7 @@ static inline TCGCond tcg_high_cond(TCGCond c) case TCG_COND_LE: case TCG_COND_GEU: case TCG_COND_LEU: - return (TCGCond)(c ^ 8); + return (TCGCond)(c ^ (4 | 1)); default: return c; } diff --git a/migration/migration.c b/migration/migration.c index d5f705ceef..ab21de2cad 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -67,6 +67,7 @@ #include "options.h" #include "sysemu/dirtylimit.h" #include "qemu/sockets.h" +#include "sysemu/kvm.h" static NotifierList migration_state_notifiers = NOTIFIER_LIST_INITIALIZER(migration_state_notifiers); @@ -128,11 +129,17 @@ static bool migration_needs_multiple_sockets(void) return migrate_multifd() || migrate_postcopy_preempt(); } -static bool transport_supports_multi_channels(SocketAddress *saddr) +static bool transport_supports_multi_channels(MigrationAddress *addr) { - return saddr->type == SOCKET_ADDRESS_TYPE_INET || - saddr->type == SOCKET_ADDRESS_TYPE_UNIX || - saddr->type == SOCKET_ADDRESS_TYPE_VSOCK; + if (addr->transport == MIGRATION_ADDRESS_TYPE_SOCKET) { + SocketAddress *saddr = &addr->u.socket; + + return saddr->type == SOCKET_ADDRESS_TYPE_INET || + saddr->type == SOCKET_ADDRESS_TYPE_UNIX || + saddr->type == SOCKET_ADDRESS_TYPE_VSOCK; + } + + return false; } static bool @@ -140,8 +147,7 @@ migration_channels_and_transport_compatible(MigrationAddress *addr, Error **errp) { if (migration_needs_multiple_sockets() && - (addr->transport == MIGRATION_ADDRESS_TYPE_SOCKET) && - !transport_supports_multi_channels(&addr->u.socket)) { + !transport_supports_multi_channels(addr)) { error_setg(errp, "Migration requires multi-channel URIs (e.g. tcp)"); return false; } @@ -311,7 +317,7 @@ void migration_incoming_state_destroy(void) { struct MigrationIncomingState *mis = migration_incoming_get_current(); - multifd_load_cleanup(); + multifd_recv_cleanup(); compress_threads_load_cleanup(); if (mis->to_src_file) { @@ -662,7 +668,7 @@ static void process_incoming_migration_bh(void *opaque) trace_vmstate_downtime_checkpoint("dst-precopy-bh-announced"); - multifd_load_shutdown(); + multifd_recv_shutdown(); dirty_bitmap_mig_before_vm_start(); @@ -759,7 +765,7 @@ fail: MIGRATION_STATUS_FAILED); qemu_fclose(mis->from_src_file); - multifd_load_cleanup(); + multifd_recv_cleanup(); compress_threads_load_cleanup(); exit(EXIT_FAILURE); @@ -885,7 +891,7 @@ void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp) default_channel = !mis->from_src_file; } - if (multifd_load_setup(errp) != 0) { + if (multifd_recv_setup(errp) != 0) { return; } @@ -1331,7 +1337,7 @@ static void migrate_fd_cleanup(MigrationState *s) } bql_lock(); - multifd_save_cleanup(); + multifd_send_shutdown(); qemu_mutex_lock(&s->qemu_file_lock); tmp = s->to_dst_file; s->to_dst_file = NULL; @@ -1906,6 +1912,12 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc, return false; } + if (kvm_hwpoisoned_mem()) { + error_setg(errp, "Can't migrate this vm with hardware poisoned memory, " + "please reboot the vm and try again"); + return false; + } + if (migration_is_blocked(errp)) { return false; } @@ -3315,6 +3327,10 @@ static void *migration_thread(void *opaque) object_ref(OBJECT(s)); update_iteration_initial_status(s); + if (!multifd_send_setup()) { + goto out; + } + bql_lock(); qemu_savevm_state_header(s->to_dst_file); bql_unlock(); @@ -3386,6 +3402,7 @@ static void *migration_thread(void *opaque) urgent = migration_rate_limit(); } +out: trace_migration_thread_after_loop(); migration_iteration_finish(s); object_unref(OBJECT(s)); @@ -3623,15 +3640,6 @@ void migrate_fd_connect(MigrationState *s, Error *error_in) return; } - if (multifd_save_setup(&local_err) != 0) { - migrate_set_error(s, local_err); - error_report_err(local_err); - migrate_set_state(&s->state, MIGRATION_STATUS_SETUP, - MIGRATION_STATUS_FAILED); - migrate_fd_cleanup(s); - return; - } - if (migrate_background_snapshot()) { qemu_thread_create(&s->thread, "bg_snapshot", bg_migration_thread, s, QEMU_THREAD_JOINABLE); diff --git a/migration/multifd-zlib.c b/migration/multifd-zlib.c index 37ce48621e..012e3bdea1 100644 --- a/migration/multifd-zlib.c +++ b/migration/multifd-zlib.c @@ -116,17 +116,20 @@ static void zlib_send_cleanup(MultiFDSendParams *p, Error **errp) */ static int zlib_send_prepare(MultiFDSendParams *p, Error **errp) { + MultiFDPages_t *pages = p->pages; struct zlib_data *z = p->data; z_stream *zs = &z->zs; uint32_t out_size = 0; int ret; uint32_t i; - for (i = 0; i < p->normal_num; i++) { + multifd_send_prepare_header(p); + + for (i = 0; i < pages->num; i++) { uint32_t available = z->zbuff_len - out_size; int flush = Z_NO_FLUSH; - if (i == p->normal_num - 1) { + if (i == pages->num - 1) { flush = Z_SYNC_FLUSH; } @@ -135,7 +138,7 @@ static int zlib_send_prepare(MultiFDSendParams *p, Error **errp) * with compression. zlib does not guarantee that this is safe, * therefore copy the page before calling deflate(). */ - memcpy(z->buf, p->pages->block->host + p->normal[i], p->page_size); + memcpy(z->buf, p->pages->block->host + pages->offset[i], p->page_size); zs->avail_in = p->page_size; zs->next_in = z->buf; @@ -171,6 +174,8 @@ static int zlib_send_prepare(MultiFDSendParams *p, Error **errp) p->next_packet_size = out_size; p->flags |= MULTIFD_FLAG_ZLIB; + multifd_send_fill_packet(p); + return 0; } diff --git a/migration/multifd-zstd.c b/migration/multifd-zstd.c index b471daadcd..dc8fe43e94 100644 --- a/migration/multifd-zstd.c +++ b/migration/multifd-zstd.c @@ -113,21 +113,24 @@ static void zstd_send_cleanup(MultiFDSendParams *p, Error **errp) */ static int zstd_send_prepare(MultiFDSendParams *p, Error **errp) { + MultiFDPages_t *pages = p->pages; struct zstd_data *z = p->data; int ret; uint32_t i; + multifd_send_prepare_header(p); + z->out.dst = z->zbuff; z->out.size = z->zbuff_len; z->out.pos = 0; - for (i = 0; i < p->normal_num; i++) { + for (i = 0; i < pages->num; i++) { ZSTD_EndDirective flush = ZSTD_e_continue; - if (i == p->normal_num - 1) { + if (i == pages->num - 1) { flush = ZSTD_e_flush; } - z->in.src = p->pages->block->host + p->normal[i]; + z->in.src = p->pages->block->host + pages->offset[i]; z->in.size = p->page_size; z->in.pos = 0; @@ -160,6 +163,8 @@ static int zstd_send_prepare(MultiFDSendParams *p, Error **errp) p->next_packet_size = z->out.pos; p->flags |= MULTIFD_FLAG_ZSTD; + multifd_send_fill_packet(p); + return 0; } diff --git a/migration/multifd.c b/migration/multifd.c index 25cbc6dc6b..adfe8c9a0a 100644 --- a/migration/multifd.c +++ b/migration/multifd.c @@ -45,20 +45,54 @@ typedef struct { uint64_t unused2[4]; /* Reserved for future use */ } __attribute__((packed)) MultiFDInit_t; +struct { + MultiFDSendParams *params; + /* array of pages to sent */ + MultiFDPages_t *pages; + /* + * Global number of generated multifd packets. + * + * Note that we used 'uintptr_t' because it'll naturally support atomic + * operations on both 32bit / 64 bits hosts. It means on 32bit systems + * multifd will overflow the packet_num easier, but that should be + * fine. + * + * Another option is to use QEMU's Stat64 then it'll be 64 bits on all + * hosts, however so far it does not support atomic fetch_add() yet. + * Make it easy for now. + */ + uintptr_t packet_num; + /* + * Synchronization point past which no more channels will be + * created. + */ + QemuSemaphore channels_created; + /* send channels ready */ + QemuSemaphore channels_ready; + /* + * Have we already run terminate threads. There is a race when it + * happens that we got one error while we are exiting. + * We will use atomic operations. Only valid values are 0 and 1. + */ + int exiting; + /* multifd ops */ + MultiFDMethods *ops; +} *multifd_send_state; + /* Multifd without compression */ /** * nocomp_send_setup: setup send side * - * For no compression this function does nothing. - * - * Returns 0 for success or -1 for error - * * @p: Params for the channel that we are using * @errp: pointer to an error */ static int nocomp_send_setup(MultiFDSendParams *p, Error **errp) { + if (migrate_zero_copy_send()) { + p->write_flags |= QIO_CHANNEL_WRITE_FLAG_ZERO_COPY; + } + return 0; } @@ -88,16 +122,38 @@ static void nocomp_send_cleanup(MultiFDSendParams *p, Error **errp) */ static int nocomp_send_prepare(MultiFDSendParams *p, Error **errp) { + bool use_zero_copy_send = migrate_zero_copy_send(); MultiFDPages_t *pages = p->pages; + int ret; - for (int i = 0; i < p->normal_num; i++) { - p->iov[p->iovs_num].iov_base = pages->block->host + p->normal[i]; + if (!use_zero_copy_send) { + /* + * Only !zerocopy needs the header in IOV; zerocopy will + * send it separately. + */ + multifd_send_prepare_header(p); + } + + for (int i = 0; i < pages->num; i++) { + p->iov[p->iovs_num].iov_base = pages->block->host + pages->offset[i]; p->iov[p->iovs_num].iov_len = p->page_size; p->iovs_num++; } - p->next_packet_size = p->normal_num * p->page_size; + p->next_packet_size = pages->num * p->page_size; p->flags |= MULTIFD_FLAG_NOCOMP; + + multifd_send_fill_packet(p); + + if (use_zero_copy_send) { + /* Send header first, without zerocopy */ + ret = qio_channel_write_all(p->c, (void *)p->packet, + p->packet_len, errp); + if (ret != 0) { + return -1; + } + } + return 0; } @@ -172,6 +228,17 @@ void multifd_register_ops(int method, MultiFDMethods *ops) multifd_ops[method] = ops; } +/* Reset a MultiFDPages_t* object for the next use */ +static void multifd_pages_reset(MultiFDPages_t *pages) +{ + /* + * We don't need to touch offset[] array, because it will be + * overwritten later when reused. + */ + pages->num = 0; + pages->block = NULL; +} + static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp) { MultiFDInit_t msg = {}; @@ -248,35 +315,44 @@ static MultiFDPages_t *multifd_pages_init(uint32_t n) static void multifd_pages_clear(MultiFDPages_t *pages) { - pages->num = 0; + multifd_pages_reset(pages); pages->allocated = 0; - pages->block = NULL; g_free(pages->offset); pages->offset = NULL; g_free(pages); } -static void multifd_send_fill_packet(MultiFDSendParams *p) +void multifd_send_fill_packet(MultiFDSendParams *p) { MultiFDPacket_t *packet = p->packet; + MultiFDPages_t *pages = p->pages; + uint64_t packet_num; int i; packet->flags = cpu_to_be32(p->flags); packet->pages_alloc = cpu_to_be32(p->pages->allocated); - packet->normal_pages = cpu_to_be32(p->normal_num); + packet->normal_pages = cpu_to_be32(pages->num); packet->next_packet_size = cpu_to_be32(p->next_packet_size); - packet->packet_num = cpu_to_be64(p->packet_num); - if (p->pages->block) { - strncpy(packet->ramblock, p->pages->block->idstr, 256); + packet_num = qatomic_fetch_inc(&multifd_send_state->packet_num); + packet->packet_num = cpu_to_be64(packet_num); + + if (pages->block) { + strncpy(packet->ramblock, pages->block->idstr, 256); } - for (i = 0; i < p->normal_num; i++) { + for (i = 0; i < pages->num; i++) { /* there are architectures where ram_addr_t is 32 bit */ - uint64_t temp = p->normal[i]; + uint64_t temp = pages->offset[i]; packet->offset[i] = cpu_to_be64(temp); } + + p->packets_sent++; + p->total_normal_pages += pages->num; + + trace_multifd_send(p->id, packet_num, pages->num, p->flags, + p->next_packet_size); } static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp) @@ -324,6 +400,11 @@ static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp) p->next_packet_size = be32_to_cpu(packet->next_packet_size); p->packet_num = be64_to_cpu(packet->packet_num); + p->packets_recved++; + p->total_normal_pages += p->normal_num; + + trace_multifd_recv(p->id, p->packet_num, p->normal_num, p->flags, + p->next_packet_size); if (p->normal_num == 0) { return 0; @@ -354,23 +435,22 @@ static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp) return 0; } -struct { - MultiFDSendParams *params; - /* array of pages to sent */ - MultiFDPages_t *pages; - /* global number of generated multifd packets */ - uint64_t packet_num; - /* send channels ready */ - QemuSemaphore channels_ready; - /* - * Have we already run terminate threads. There is a race when it - * happens that we got one error while we are exiting. - * We will use atomic operations. Only valid values are 0 and 1. - */ - int exiting; - /* multifd ops */ - MultiFDMethods *ops; -} *multifd_send_state; +static bool multifd_send_should_exit(void) +{ + return qatomic_read(&multifd_send_state->exiting); +} + +/* + * The migration thread can wait on either of the two semaphores. This + * function can be used to kick the main thread out of waiting on either of + * them. Should mostly only be called when something wrong happened with + * the current multifd send thread. + */ +static void multifd_send_kick_main(MultiFDSendParams *p) +{ + qemu_sem_post(&p->sem_sync); + qemu_sem_post(&multifd_send_state->channels_ready); +} /* * How we use multifd_send_state->pages and channel->pages? @@ -388,20 +468,23 @@ struct { * thread is using the channel mutex when changing it, and the channel * have to had finish with its own, otherwise pending_job can't be * false. + * + * Returns true if succeed, false otherwise. */ - -static int multifd_send_pages(void) +static bool multifd_send_pages(void) { int i; static int next_channel; MultiFDSendParams *p = NULL; /* make happy gcc */ MultiFDPages_t *pages = multifd_send_state->pages; - if (qatomic_read(&multifd_send_state->exiting)) { - return -1; + if (multifd_send_should_exit()) { + return false; } + /* We wait here, until at least one channel is ready */ qemu_sem_wait(&multifd_send_state->channels_ready); + /* * next_channel can remain from a previous migration that was * using more channels, so ensure it doesn't overflow if the @@ -409,69 +492,100 @@ static int multifd_send_pages(void) */ next_channel %= migrate_multifd_channels(); for (i = next_channel;; i = (i + 1) % migrate_multifd_channels()) { - p = &multifd_send_state->params[i]; - - qemu_mutex_lock(&p->mutex); - if (p->quit) { - error_report("%s: channel %d has already quit!", __func__, i); - qemu_mutex_unlock(&p->mutex); - return -1; + if (multifd_send_should_exit()) { + return false; } - if (!p->pending_job) { - p->pending_job++; + p = &multifd_send_state->params[i]; + /* + * Lockless read to p->pending_job is safe, because only multifd + * sender thread can clear it. + */ + if (qatomic_read(&p->pending_job) == false) { next_channel = (i + 1) % migrate_multifd_channels(); break; } - qemu_mutex_unlock(&p->mutex); } - assert(!p->pages->num); - assert(!p->pages->block); - p->packet_num = multifd_send_state->packet_num++; + /* + * Make sure we read p->pending_job before all the rest. Pairs with + * qatomic_store_release() in multifd_send_thread(). + */ + smp_mb_acquire(); + assert(!p->pages->num); multifd_send_state->pages = p->pages; p->pages = pages; - qemu_mutex_unlock(&p->mutex); + /* + * Making sure p->pages is setup before marking pending_job=true. Pairs + * with the qatomic_load_acquire() in multifd_send_thread(). + */ + qatomic_store_release(&p->pending_job, true); qemu_sem_post(&p->sem); - return 1; + return true; } -int multifd_queue_page(RAMBlock *block, ram_addr_t offset) +static inline bool multifd_queue_empty(MultiFDPages_t *pages) { - MultiFDPages_t *pages = multifd_send_state->pages; - bool changed = false; + return pages->num == 0; +} - if (!pages->block) { - pages->block = block; - } +static inline bool multifd_queue_full(MultiFDPages_t *pages) +{ + return pages->num == pages->allocated; +} - if (pages->block == block) { - pages->offset[pages->num] = offset; - pages->num++; +static inline void multifd_enqueue(MultiFDPages_t *pages, ram_addr_t offset) +{ + pages->offset[pages->num++] = offset; +} - if (pages->num < pages->allocated) { - return 1; - } - } else { - changed = true; - } +/* Returns true if enqueue successful, false otherwise */ +bool multifd_queue_page(RAMBlock *block, ram_addr_t offset) +{ + MultiFDPages_t *pages; - if (multifd_send_pages() < 0) { - return -1; +retry: + pages = multifd_send_state->pages; + + /* If the queue is empty, we can already enqueue now */ + if (multifd_queue_empty(pages)) { + pages->block = block; + multifd_enqueue(pages, offset); + return true; } - if (changed) { - return multifd_queue_page(block, offset); + /* + * Not empty, meanwhile we need a flush. It can because of either: + * + * (1) The page is not on the same ramblock of previous ones, or, + * (2) The queue is full. + * + * After flush, always retry. + */ + if (pages->block != block || multifd_queue_full(pages)) { + if (!multifd_send_pages()) { + return false; + } + goto retry; } - return 1; + /* Not empty, and we still have space, do it! */ + multifd_enqueue(pages, offset); + return true; } -static void multifd_send_terminate_threads(Error *err) +/* Multifd send side hit an error; remember it and prepare to quit */ +static void multifd_send_set_error(Error *err) { - int i; - - trace_multifd_send_terminate_threads(err != NULL); + /* + * We don't want to exit each threads twice. Depending on where + * we get the error, or if there are two independent errors in two + * threads at the same time, we can end calling this function + * twice. + */ + if (qatomic_xchg(&multifd_send_state->exiting, 1)) { + return; + } if (err) { MigrationState *s = migrate_get_current(); @@ -484,27 +598,46 @@ static void multifd_send_terminate_threads(Error *err) MIGRATION_STATUS_FAILED); } } +} + +static void multifd_send_terminate_threads(void) +{ + int i; + + trace_multifd_send_terminate_threads(); /* - * We don't want to exit each threads twice. Depending on where - * we get the error, or if there are two independent errors in two - * threads at the same time, we can end calling this function - * twice. + * Tell everyone we're quitting. No xchg() needed here; we simply + * always set it. */ - if (qatomic_xchg(&multifd_send_state->exiting, 1)) { - return; - } + qatomic_set(&multifd_send_state->exiting, 1); + /* + * Firstly, kick all threads out; no matter whether they are just idle, + * or blocked in an IO system call. + */ for (i = 0; i < migrate_multifd_channels(); i++) { MultiFDSendParams *p = &multifd_send_state->params[i]; - qemu_mutex_lock(&p->mutex); - p->quit = true; qemu_sem_post(&p->sem); if (p->c) { qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); } - qemu_mutex_unlock(&p->mutex); + } + + /* + * Finally recycle all the threads. + */ + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDSendParams *p = &multifd_send_state->params[i]; + + if (p->tls_thread_created) { + qemu_thread_join(&p->tls_thread); + } + + if (p->thread_created) { + qemu_thread_join(&p->thread); + } } } @@ -513,57 +646,62 @@ static int multifd_send_channel_destroy(QIOChannel *send) return socket_send_channel_destroy(send); } -void multifd_save_cleanup(void) +static bool multifd_send_cleanup_channel(MultiFDSendParams *p, Error **errp) +{ + if (p->registered_yank) { + migration_ioc_unregister_yank(p->c); + } + multifd_send_channel_destroy(p->c); + p->c = NULL; + qemu_sem_destroy(&p->sem); + qemu_sem_destroy(&p->sem_sync); + g_free(p->name); + p->name = NULL; + multifd_pages_clear(p->pages); + p->pages = NULL; + p->packet_len = 0; + g_free(p->packet); + p->packet = NULL; + g_free(p->iov); + p->iov = NULL; + multifd_send_state->ops->send_cleanup(p, errp); + + return *errp == NULL; +} + +static void multifd_send_cleanup_state(void) +{ + qemu_sem_destroy(&multifd_send_state->channels_created); + qemu_sem_destroy(&multifd_send_state->channels_ready); + g_free(multifd_send_state->params); + multifd_send_state->params = NULL; + multifd_pages_clear(multifd_send_state->pages); + multifd_send_state->pages = NULL; + g_free(multifd_send_state); + multifd_send_state = NULL; +} + +void multifd_send_shutdown(void) { int i; if (!migrate_multifd()) { return; } - multifd_send_terminate_threads(NULL); - for (i = 0; i < migrate_multifd_channels(); i++) { - MultiFDSendParams *p = &multifd_send_state->params[i]; - if (p->running) { - qemu_thread_join(&p->thread); - } - } + multifd_send_terminate_threads(); + for (i = 0; i < migrate_multifd_channels(); i++) { MultiFDSendParams *p = &multifd_send_state->params[i]; Error *local_err = NULL; - if (p->registered_yank) { - migration_ioc_unregister_yank(p->c); - } - multifd_send_channel_destroy(p->c); - p->c = NULL; - qemu_mutex_destroy(&p->mutex); - qemu_sem_destroy(&p->sem); - qemu_sem_destroy(&p->sem_sync); - g_free(p->name); - p->name = NULL; - multifd_pages_clear(p->pages); - p->pages = NULL; - p->packet_len = 0; - g_free(p->packet); - p->packet = NULL; - g_free(p->iov); - p->iov = NULL; - g_free(p->normal); - p->normal = NULL; - multifd_send_state->ops->send_cleanup(p, &local_err); - if (local_err) { + if (!multifd_send_cleanup_channel(p, &local_err)) { migrate_set_error(migrate_get_current(), local_err); error_free(local_err); } } - qemu_sem_destroy(&multifd_send_state->channels_ready); - g_free(multifd_send_state->params); - multifd_send_state->params = NULL; - multifd_pages_clear(multifd_send_state->pages); - multifd_send_state->pages = NULL; - g_free(multifd_send_state); - multifd_send_state = NULL; + + multifd_send_cleanup_state(); } static int multifd_zero_copy_flush(QIOChannel *c) @@ -592,47 +730,38 @@ int multifd_send_sync_main(void) return 0; } if (multifd_send_state->pages->num) { - if (multifd_send_pages() < 0) { + if (!multifd_send_pages()) { error_report("%s: multifd_send_pages fail", __func__); return -1; } } - /* - * When using zero-copy, it's necessary to flush the pages before any of - * the pages can be sent again, so we'll make sure the new version of the - * pages will always arrive _later_ than the old pages. - * - * Currently we achieve this by flushing the zero-page requested writes - * per ram iteration, but in the future we could potentially optimize it - * to be less frequent, e.g. only after we finished one whole scanning of - * all the dirty bitmaps. - */ - flush_zero_copy = migrate_zero_copy_send(); for (i = 0; i < migrate_multifd_channels(); i++) { MultiFDSendParams *p = &multifd_send_state->params[i]; - trace_multifd_send_sync_main_signal(p->id); - - qemu_mutex_lock(&p->mutex); - - if (p->quit) { - error_report("%s: channel %d has already quit", __func__, i); - qemu_mutex_unlock(&p->mutex); + if (multifd_send_should_exit()) { return -1; } - p->packet_num = multifd_send_state->packet_num++; - p->flags |= MULTIFD_FLAG_SYNC; - p->pending_job++; - qemu_mutex_unlock(&p->mutex); + trace_multifd_send_sync_main_signal(p->id); + + /* + * We should be the only user so far, so not possible to be set by + * others concurrently. + */ + assert(qatomic_read(&p->pending_sync) == false); + qatomic_set(&p->pending_sync, true); qemu_sem_post(&p->sem); } for (i = 0; i < migrate_multifd_channels(); i++) { MultiFDSendParams *p = &multifd_send_state->params[i]; + if (multifd_send_should_exit()) { + return -1; + } + qemu_sem_wait(&multifd_send_state->channels_ready); trace_multifd_send_sync_main_wait(p->id); qemu_sem_wait(&p->sem_sync); @@ -652,7 +781,6 @@ static void *multifd_send_thread(void *opaque) MigrationThread *thread = NULL; Error *local_err = NULL; int ret = 0; - bool use_zero_copy_send = migrate_zero_copy_send(); thread = migration_threads_add(p->name, qemu_get_thread_id()); @@ -663,64 +791,28 @@ static void *multifd_send_thread(void *opaque) ret = -1; goto out; } - /* initial packet */ - p->num_packets = 1; while (true) { qemu_sem_post(&multifd_send_state->channels_ready); qemu_sem_wait(&p->sem); - if (qatomic_read(&multifd_send_state->exiting)) { + if (multifd_send_should_exit()) { break; } - qemu_mutex_lock(&p->mutex); - - if (p->pending_job) { - uint64_t packet_num = p->packet_num; - uint32_t flags; - p->normal_num = 0; - - if (use_zero_copy_send) { - p->iovs_num = 0; - } else { - p->iovs_num = 1; - } - for (int i = 0; i < p->pages->num; i++) { - p->normal[p->normal_num] = p->pages->offset[i]; - p->normal_num++; - } + /* + * Read pending_job flag before p->pages. Pairs with the + * qatomic_store_release() in multifd_send_pages(). + */ + if (qatomic_load_acquire(&p->pending_job)) { + MultiFDPages_t *pages = p->pages; - if (p->normal_num) { - ret = multifd_send_state->ops->send_prepare(p, &local_err); - if (ret != 0) { - qemu_mutex_unlock(&p->mutex); - break; - } - } - multifd_send_fill_packet(p); - flags = p->flags; - p->flags = 0; - p->num_packets++; - p->total_normal_pages += p->normal_num; - p->pages->num = 0; - p->pages->block = NULL; - qemu_mutex_unlock(&p->mutex); + p->iovs_num = 0; + assert(pages->num); - trace_multifd_send(p->id, packet_num, p->normal_num, flags, - p->next_packet_size); - - if (use_zero_copy_send) { - /* Send header first, without zerocopy */ - ret = qio_channel_write_all(p->c, (void *)p->packet, - p->packet_len, &local_err); - if (ret != 0) { - break; - } - } else { - /* Send header using the same writev call */ - p->iov[0].iov_len = p->packet_len; - p->iov[0].iov_base = p->packet; + ret = multifd_send_state->ops->send_prepare(p, &local_err); + if (ret != 0) { + break; } ret = qio_channel_writev_full_all(p->c, p->iov, p->iovs_num, NULL, @@ -731,17 +823,35 @@ static void *multifd_send_thread(void *opaque) stat64_add(&mig_stats.multifd_bytes, p->next_packet_size + p->packet_len); + + multifd_pages_reset(p->pages); p->next_packet_size = 0; - qemu_mutex_lock(&p->mutex); - p->pending_job--; - qemu_mutex_unlock(&p->mutex); - if (flags & MULTIFD_FLAG_SYNC) { - qemu_sem_post(&p->sem_sync); - } + /* + * Making sure p->pages is published before saying "we're + * free". Pairs with the smp_mb_acquire() in + * multifd_send_pages(). + */ + qatomic_store_release(&p->pending_job, false); } else { - qemu_mutex_unlock(&p->mutex); - /* sometimes there are spurious wakeups */ + /* + * If not a normal job, must be a sync request. Note that + * pending_sync is a standalone flag (unlike pending_job), so + * it doesn't require explicit memory barriers. + */ + assert(qatomic_read(&p->pending_sync)); + p->flags = MULTIFD_FLAG_SYNC; + multifd_send_fill_packet(p); + ret = qio_channel_write_all(p->c, (void *)p->packet, + p->packet_len, &local_err); + if (ret != 0) { + break; + } + /* p->next_packet_size will always be zero for a SYNC packet */ + stat64_add(&mig_stats.multifd_bytes, p->packet_len); + p->flags = 0; + qatomic_set(&p->pending_sync, false); + qemu_sem_post(&p->sem_sync); } } @@ -749,53 +859,19 @@ out: if (ret) { assert(local_err); trace_multifd_send_error(p->id); - multifd_send_terminate_threads(local_err); - qemu_sem_post(&p->sem_sync); - qemu_sem_post(&multifd_send_state->channels_ready); + multifd_send_set_error(local_err); + multifd_send_kick_main(p); error_free(local_err); } - qemu_mutex_lock(&p->mutex); - p->running = false; - qemu_mutex_unlock(&p->mutex); - rcu_unregister_thread(); migration_threads_remove(thread); - trace_multifd_send_thread_end(p->id, p->num_packets, p->total_normal_pages); + trace_multifd_send_thread_end(p->id, p->packets_sent, p->total_normal_pages); return NULL; } -static bool multifd_channel_connect(MultiFDSendParams *p, - QIOChannel *ioc, - Error **errp); - -static void multifd_tls_outgoing_handshake(QIOTask *task, - gpointer opaque) -{ - MultiFDSendParams *p = opaque; - QIOChannel *ioc = QIO_CHANNEL(qio_task_get_source(task)); - Error *err = NULL; - - if (!qio_task_propagate_error(task, &err)) { - trace_multifd_tls_outgoing_handshake_complete(ioc); - if (multifd_channel_connect(p, ioc, &err)) { - return; - } - } - - trace_multifd_tls_outgoing_handshake_error(ioc, error_get_pretty(err)); - - migrate_set_error(migrate_get_current(), err); - /* - * Error happen, mark multifd_send_thread status as 'quit' although it - * is not created, and then tell who pay attention to me. - */ - p->quit = true; - qemu_sem_post(&multifd_send_state->channels_ready); - qemu_sem_post(&p->sem_sync); - error_free(err); -} +static void multifd_new_send_channel_async(QIOTask *task, gpointer opaque); static void *multifd_tls_handshake_thread(void *opaque) { @@ -803,7 +879,7 @@ static void *multifd_tls_handshake_thread(void *opaque) QIOChannelTLS *tioc = QIO_CHANNEL_TLS(p->c); qio_channel_tls_handshake(tioc, - multifd_tls_outgoing_handshake, + multifd_new_send_channel_async, p, NULL, NULL); @@ -823,11 +899,17 @@ static bool multifd_tls_channel_connect(MultiFDSendParams *p, return false; } + /* + * Ownership of the socket channel now transfers to the newly + * created TLS channel, which has already taken a reference. + */ object_unref(OBJECT(ioc)); trace_multifd_tls_outgoing_handshake_start(ioc, tioc, hostname); qio_channel_set_name(QIO_CHANNEL(tioc), "multifd-tls-outgoing"); p->c = QIO_CHANNEL(tioc); - qemu_thread_create(&p->thread, "multifd-tls-handshake-worker", + + p->tls_thread_created = true; + qemu_thread_create(&p->tls_thread, "multifd-tls-handshake-worker", multifd_tls_handshake_thread, p, QEMU_THREAD_JOINABLE); return true; @@ -837,61 +919,72 @@ static bool multifd_channel_connect(MultiFDSendParams *p, QIOChannel *ioc, Error **errp) { - trace_multifd_set_outgoing_channel( - ioc, object_get_typename(OBJECT(ioc)), - migrate_get_current()->hostname); - - if (migrate_channel_requires_tls_upgrade(ioc)) { - /* - * tls_channel_connect will call back to this - * function after the TLS handshake, - * so we mustn't call multifd_send_thread until then - */ - return multifd_tls_channel_connect(p, ioc, errp); - } + qio_channel_set_delay(ioc, false); migration_ioc_register_yank(ioc); p->registered_yank = true; p->c = ioc; + + p->thread_created = true; qemu_thread_create(&p->thread, p->name, multifd_send_thread, p, QEMU_THREAD_JOINABLE); return true; } -static void multifd_new_send_channel_cleanup(MultiFDSendParams *p, - QIOChannel *ioc, Error *err) -{ - migrate_set_error(migrate_get_current(), err); - /* Error happen, we need to tell who pay attention to me */ - qemu_sem_post(&multifd_send_state->channels_ready); - qemu_sem_post(&p->sem_sync); - /* - * Although multifd_send_thread is not created, but main migration - * thread need to judge whether it is running, so we need to mark - * its status. - */ - p->quit = true; - object_unref(OBJECT(ioc)); - error_free(err); -} - +/* + * When TLS is enabled this function is called once to establish the + * TLS connection and a second time after the TLS handshake to create + * the multifd channel. Without TLS it goes straight into the channel + * creation. + */ static void multifd_new_send_channel_async(QIOTask *task, gpointer opaque) { MultiFDSendParams *p = opaque; QIOChannel *ioc = QIO_CHANNEL(qio_task_get_source(task)); Error *local_err = NULL; + bool ret; trace_multifd_new_send_channel_async(p->id); - if (!qio_task_propagate_error(task, &local_err)) { - qio_channel_set_delay(ioc, false); - p->running = true; - if (multifd_channel_connect(p, ioc, &local_err)) { + + if (qio_task_propagate_error(task, &local_err)) { + ret = false; + goto out; + } + + trace_multifd_set_outgoing_channel(ioc, object_get_typename(OBJECT(ioc)), + migrate_get_current()->hostname); + + if (migrate_channel_requires_tls_upgrade(ioc)) { + ret = multifd_tls_channel_connect(p, ioc, &local_err); + if (ret) { return; } + } else { + ret = multifd_channel_connect(p, ioc, &local_err); + } + +out: + /* + * Here we're not interested whether creation succeeded, only that + * it happened at all. + */ + qemu_sem_post(&multifd_send_state->channels_created); + + if (ret) { + return; } trace_multifd_new_send_channel_async_error(p->id, local_err); - multifd_new_send_channel_cleanup(p, ioc, local_err); + multifd_send_set_error(local_err); + if (!p->c) { + /* + * If no channel has been created, drop the initial + * reference. Otherwise cleanup happens at + * multifd_send_channel_destroy() + */ + object_unref(OBJECT(ioc)); + } + error_free(local_err); } static void multifd_new_send_channel_create(gpointer opaque) @@ -899,20 +992,23 @@ static void multifd_new_send_channel_create(gpointer opaque) socket_send_channel_create(multifd_new_send_channel_async, opaque); } -int multifd_save_setup(Error **errp) +bool multifd_send_setup(void) { - int thread_count; + MigrationState *s = migrate_get_current(); + Error *local_err = NULL; + int thread_count, ret = 0; uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); uint8_t i; if (!migrate_multifd()) { - return 0; + return true; } thread_count = migrate_multifd_channels(); multifd_send_state = g_malloc0(sizeof(*multifd_send_state)); multifd_send_state->params = g_new0(MultiFDSendParams, thread_count); multifd_send_state->pages = multifd_pages_init(page_count); + qemu_sem_init(&multifd_send_state->channels_created, 0); qemu_sem_init(&multifd_send_state->channels_ready, 0); qatomic_set(&multifd_send_state->exiting, 0); multifd_send_state->ops = multifd_ops[migrate_multifd_compression()]; @@ -920,11 +1016,8 @@ int multifd_save_setup(Error **errp) for (i = 0; i < thread_count; i++) { MultiFDSendParams *p = &multifd_send_state->params[i]; - qemu_mutex_init(&p->mutex); qemu_sem_init(&p->sem, 0); qemu_sem_init(&p->sem_sync, 0); - p->quit = false; - p->pending_job = 0; p->id = i; p->pages = multifd_pages_init(page_count); p->packet_len = sizeof(MultiFDPacket_t) @@ -935,29 +1028,39 @@ int multifd_save_setup(Error **errp) p->name = g_strdup_printf("multifdsend_%d", i); /* We need one extra place for the packet header */ p->iov = g_new0(struct iovec, page_count + 1); - p->normal = g_new0(ram_addr_t, page_count); p->page_size = qemu_target_page_size(); p->page_count = page_count; - - if (migrate_zero_copy_send()) { - p->write_flags = QIO_CHANNEL_WRITE_FLAG_ZERO_COPY; - } else { - p->write_flags = 0; - } - + p->write_flags = 0; multifd_new_send_channel_create(p); } + /* + * Wait until channel creation has started for all channels. The + * creation can still fail, but no more channels will be created + * past this point. + */ + for (i = 0; i < thread_count; i++) { + qemu_sem_wait(&multifd_send_state->channels_created); + } + for (i = 0; i < thread_count; i++) { MultiFDSendParams *p = &multifd_send_state->params[i]; - int ret; - ret = multifd_send_state->ops->send_setup(p, errp); + ret = multifd_send_state->ops->send_setup(p, &local_err); if (ret) { - return ret; + break; } } - return 0; + + if (ret) { + migrate_set_error(s, local_err); + error_report_err(local_err); + migrate_set_state(&s->state, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_FAILED); + return false; + } + + return true; } struct { @@ -1006,14 +1109,42 @@ static void multifd_recv_terminate_threads(Error *err) } } -void multifd_load_shutdown(void) +void multifd_recv_shutdown(void) { if (migrate_multifd()) { multifd_recv_terminate_threads(NULL); } } -void multifd_load_cleanup(void) +static void multifd_recv_cleanup_channel(MultiFDRecvParams *p) +{ + migration_ioc_unregister_yank(p->c); + object_unref(OBJECT(p->c)); + p->c = NULL; + qemu_mutex_destroy(&p->mutex); + qemu_sem_destroy(&p->sem_sync); + g_free(p->name); + p->name = NULL; + p->packet_len = 0; + g_free(p->packet); + p->packet = NULL; + g_free(p->iov); + p->iov = NULL; + g_free(p->normal); + p->normal = NULL; + multifd_recv_state->ops->recv_cleanup(p); +} + +static void multifd_recv_cleanup_state(void) +{ + qemu_sem_destroy(&multifd_recv_state->sem_sync); + g_free(multifd_recv_state->params); + multifd_recv_state->params = NULL; + g_free(multifd_recv_state); + multifd_recv_state = NULL; +} + +void multifd_recv_cleanup(void) { int i; @@ -1024,40 +1155,20 @@ void multifd_load_cleanup(void) for (i = 0; i < migrate_multifd_channels(); i++) { MultiFDRecvParams *p = &multifd_recv_state->params[i]; - if (p->running) { - /* - * multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code, - * however try to wakeup it without harm in cleanup phase. - */ - qemu_sem_post(&p->sem_sync); - } + /* + * multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code, + * however try to wakeup it without harm in cleanup phase. + */ + qemu_sem_post(&p->sem_sync); - qemu_thread_join(&p->thread); + if (p->thread_created) { + qemu_thread_join(&p->thread); + } } for (i = 0; i < migrate_multifd_channels(); i++) { - MultiFDRecvParams *p = &multifd_recv_state->params[i]; - - migration_ioc_unregister_yank(p->c); - object_unref(OBJECT(p->c)); - p->c = NULL; - qemu_mutex_destroy(&p->mutex); - qemu_sem_destroy(&p->sem_sync); - g_free(p->name); - p->name = NULL; - p->packet_len = 0; - g_free(p->packet); - p->packet = NULL; - g_free(p->iov); - p->iov = NULL; - g_free(p->normal); - p->normal = NULL; - multifd_recv_state->ops->recv_cleanup(p); + multifd_recv_cleanup_channel(&multifd_recv_state->params[i]); } - qemu_sem_destroy(&multifd_recv_state->sem_sync); - g_free(multifd_recv_state->params); - multifd_recv_state->params = NULL; - g_free(multifd_recv_state); - multifd_recv_state = NULL; + multifd_recv_cleanup_state(); } void multifd_recv_sync_main(void) @@ -1119,10 +1230,6 @@ static void *multifd_recv_thread(void *opaque) flags = p->flags; /* recv methods don't know how to handle the SYNC flag */ p->flags &= ~MULTIFD_FLAG_SYNC; - trace_multifd_recv(p->id, p->packet_num, p->normal_num, flags, - p->next_packet_size); - p->num_packets++; - p->total_normal_pages += p->normal_num; qemu_mutex_unlock(&p->mutex); if (p->normal_num) { @@ -1142,17 +1249,14 @@ static void *multifd_recv_thread(void *opaque) multifd_recv_terminate_threads(local_err); error_free(local_err); } - qemu_mutex_lock(&p->mutex); - p->running = false; - qemu_mutex_unlock(&p->mutex); rcu_unregister_thread(); - trace_multifd_recv_thread_end(p->id, p->num_packets, p->total_normal_pages); + trace_multifd_recv_thread_end(p->id, p->packets_recved, p->total_normal_pages); return NULL; } -int multifd_load_setup(Error **errp) +int multifd_recv_setup(Error **errp) { int thread_count; uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); @@ -1249,10 +1353,8 @@ void multifd_recv_new_channel(QIOChannel *ioc, Error **errp) } p->c = ioc; object_ref(OBJECT(ioc)); - /* initial packet */ - p->num_packets = 1; - p->running = true; + p->thread_created = true; qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p, QEMU_THREAD_JOINABLE); qatomic_inc(&multifd_recv_state->count); diff --git a/migration/multifd.h b/migration/multifd.h index 35d11f103c..8a1cad0996 100644 --- a/migration/multifd.h +++ b/migration/multifd.h @@ -13,16 +13,16 @@ #ifndef QEMU_MIGRATION_MULTIFD_H #define QEMU_MIGRATION_MULTIFD_H -int multifd_save_setup(Error **errp); -void multifd_save_cleanup(void); -int multifd_load_setup(Error **errp); -void multifd_load_cleanup(void); -void multifd_load_shutdown(void); +bool multifd_send_setup(void); +void multifd_send_shutdown(void); +int multifd_recv_setup(Error **errp); +void multifd_recv_cleanup(void); +void multifd_recv_shutdown(void); bool multifd_recv_all_channels_created(void); void multifd_recv_new_channel(QIOChannel *ioc, Error **errp); void multifd_recv_sync_main(void); int multifd_send_sync_main(void); -int multifd_queue_page(RAMBlock *block, ram_addr_t offset); +bool multifd_queue_page(RAMBlock *block, ram_addr_t offset); /* Multifd Compression flags */ #define MULTIFD_FLAG_SYNC (1 << 0) @@ -73,6 +73,9 @@ typedef struct { char *name; /* channel thread id */ QemuThread thread; + bool thread_created; + QemuThread tls_thread; + bool tls_thread_created; /* communication channel */ QIOChannel *c; /* is the yank function registered */ @@ -91,18 +94,19 @@ typedef struct { /* syncs main thread and channels */ QemuSemaphore sem_sync; - /* this mutex protects the following parameters */ - QemuMutex mutex; - /* is this channel thread running */ - bool running; - /* should this thread finish */ - bool quit; /* multifd flags for each packet */ uint32_t flags; - /* global number of generated multifd packets */ - uint64_t packet_num; - /* thread has work to do */ - int pending_job; + /* + * The sender thread has work to do if either of below boolean is set. + * + * @pending_job: a job is pending + * @pending_sync: a sync request is pending + * + * For both of these fields, they're only set by the requesters, and + * cleared by the multifd sender threads. + */ + bool pending_job; + bool pending_sync; /* array of pages to sent. * The owner of 'pages' depends of 'pending_job' value: * pending_job == 0 -> migration_thread can use it. @@ -117,17 +121,13 @@ typedef struct { /* size of the next packet that contains pages */ uint32_t next_packet_size; /* packets sent through this channel */ - uint64_t num_packets; + uint64_t packets_sent; /* non zero pages sent through this channel */ uint64_t total_normal_pages; /* buffers to send */ struct iovec *iov; /* number of iovs used */ uint32_t iovs_num; - /* Pages that are not zero */ - ram_addr_t *normal; - /* num of non zero pages */ - uint32_t normal_num; /* used for compression methods */ void *data; } MultiFDSendParams; @@ -142,6 +142,7 @@ typedef struct { char *name; /* channel thread id */ QemuThread thread; + bool thread_created; /* communication channel */ QIOChannel *c; /* packet allocated len */ @@ -156,8 +157,6 @@ typedef struct { /* this mutex protects the following parameters */ QemuMutex mutex; - /* is this channel thread running */ - bool running; /* should this thread finish */ bool quit; /* multifd flags for each packet */ @@ -171,8 +170,8 @@ typedef struct { MultiFDPacket_t *packet; /* size of the next packet that contains pages */ uint32_t next_packet_size; - /* packets sent through this channel */ - uint64_t num_packets; + /* packets received through this channel */ + uint64_t packets_recved; /* ramblock */ RAMBlock *block; /* ramblock host address */ @@ -205,6 +204,14 @@ typedef struct { } MultiFDMethods; void multifd_register_ops(int method, MultiFDMethods *ops); +void multifd_send_fill_packet(MultiFDSendParams *p); + +static inline void multifd_send_prepare_header(MultiFDSendParams *p) +{ + p->iov[0].iov_len = p->packet_len; + p->iov[0].iov_base = p->packet; + p->iovs_num++; +} -#endif +#endif diff --git a/migration/ram.c b/migration/ram.c index d5b7cd5ac2..4649a81204 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -1252,7 +1252,7 @@ static int ram_save_page(RAMState *rs, PageSearchStatus *pss) static int ram_save_multifd_page(RAMBlock *block, ram_addr_t offset) { - if (multifd_queue_page(block, offset) < 0) { + if (!multifd_queue_page(block, offset)) { return -1; } stat64_add(&mig_stats.normal_pages, 1); diff --git a/migration/trace-events b/migration/trace-events index de4a743c8a..298ad2b0dd 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -141,7 +141,7 @@ multifd_send_error(uint8_t id) "channel %u" multifd_send_sync_main(long packet_num) "packet num %ld" multifd_send_sync_main_signal(uint8_t id) "channel %u" multifd_send_sync_main_wait(uint8_t id) "channel %u" -multifd_send_terminate_threads(bool error) "error %d" +multifd_send_terminate_threads(void) "" multifd_send_thread_end(uint8_t id, uint64_t packets, uint64_t normal_pages) "channel %u packets %" PRIu64 " normal pages %" PRIu64 multifd_send_thread_start(uint8_t id) "%u" multifd_tls_outgoing_handshake_start(void *ioc, void *tioc, const char *hostname) "ioc=%p tioc=%p hostname=%s" diff --git a/qapi/qmp-dispatch.c b/qapi/qmp-dispatch.c index 176b549473..f3488afeef 100644 --- a/qapi/qmp-dispatch.c +++ b/qapi/qmp-dispatch.c @@ -212,8 +212,7 @@ QDict *coroutine_mixed_fn qmp_dispatch(const QmpCommandList *cmds, QObject *requ * executing the command handler so that it can make progress if it * involves an AIO_WAIT_WHILE(). */ - aio_co_schedule(qemu_get_aio_context(), qemu_coroutine_self()); - qemu_coroutine_yield(); + aio_co_reschedule_self(qemu_get_aio_context()); } monitor_set_cur(qemu_coroutine_self(), cur_mon); @@ -227,9 +226,7 @@ QDict *coroutine_mixed_fn qmp_dispatch(const QmpCommandList *cmds, QObject *requ * Move back to iohandler_ctx so that nested event loops for * qemu_aio_context don't start new monitor commands. */ - aio_co_schedule(iohandler_get_aio_context(), - qemu_coroutine_self()); - qemu_coroutine_yield(); + aio_co_reschedule_self(iohandler_get_aio_context()); } } else { /* diff --git a/qemu-options.hx b/qemu-options.hx index 5adbed1101..f7c83362d1 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -2686,7 +2686,7 @@ DEF("smbios", HAS_ARG, QEMU_OPTION_smbios, " specify SMBIOS type 3 fields\n" "-smbios type=4[,sock_pfx=str][,manufacturer=str][,version=str][,serial=str]\n" " [,asset=str][,part=str][,max-speed=%d][,current-speed=%d]\n" - " [,processor-id=%d]\n" + " [,processor-family=%d,processor-id=%d]\n" " specify SMBIOS type 4 fields\n" "-smbios type=8[,external_reference=str][,internal_reference=str][,connector_type=%d][,port_type=%d]\n" " specify SMBIOS type 8 fields\n" @@ -2697,7 +2697,7 @@ DEF("smbios", HAS_ARG, QEMU_OPTION_smbios, " specify SMBIOS type 17 fields\n" "-smbios type=41[,designation=str][,kind=str][,instance=%d][,pcidev=str]\n" " specify SMBIOS type 41 fields\n", - QEMU_ARCH_I386 | QEMU_ARCH_ARM | QEMU_ARCH_LOONGARCH) + QEMU_ARCH_I386 | QEMU_ARCH_ARM | QEMU_ARCH_LOONGARCH | QEMU_ARCH_RISCV) SRST ``-smbios file=binary`` Load SMBIOS entry from binary file. @@ -2714,7 +2714,7 @@ SRST ``-smbios type=3[,manufacturer=str][,version=str][,serial=str][,asset=str][,sku=str]`` Specify SMBIOS type 3 fields -``-smbios type=4[,sock_pfx=str][,manufacturer=str][,version=str][,serial=str][,asset=str][,part=str][,processor-id=%d]`` +``-smbios type=4[,sock_pfx=str][,manufacturer=str][,version=str][,serial=str][,asset=str][,part=str][,processor-family=%d][,processor-id=%d]`` Specify SMBIOS type 4 fields ``-smbios type=11[,value=str][,path=filename]`` diff --git a/target/alpha/translate.c b/target/alpha/translate.c index 4b464f8651..882cf6cea0 100644 --- a/target/alpha/translate.c +++ b/target/alpha/translate.c @@ -453,13 +453,13 @@ static DisasJumpType gen_bdirect(DisasContext *ctx, int ra, int32_t disp) } static DisasJumpType gen_bcond_internal(DisasContext *ctx, TCGCond cond, - TCGv cmp, int32_t disp) + TCGv cmp, uint64_t imm, int32_t disp) { uint64_t dest = ctx->base.pc_next + (disp << 2); TCGLabel *lab_true = gen_new_label(); if (use_goto_tb(ctx, dest)) { - tcg_gen_brcondi_i64(cond, cmp, 0, lab_true); + tcg_gen_brcondi_i64(cond, cmp, imm, lab_true); tcg_gen_goto_tb(0); tcg_gen_movi_i64(cpu_pc, ctx->base.pc_next); @@ -472,81 +472,71 @@ static DisasJumpType gen_bcond_internal(DisasContext *ctx, TCGCond cond, return DISAS_NORETURN; } else { - TCGv_i64 z = load_zero(ctx); + TCGv_i64 i = tcg_constant_i64(imm); TCGv_i64 d = tcg_constant_i64(dest); TCGv_i64 p = tcg_constant_i64(ctx->base.pc_next); - tcg_gen_movcond_i64(cond, cpu_pc, cmp, z, d, p); + tcg_gen_movcond_i64(cond, cpu_pc, cmp, i, d, p); return DISAS_PC_UPDATED; } } static DisasJumpType gen_bcond(DisasContext *ctx, TCGCond cond, int ra, - int32_t disp, int mask) + int32_t disp) { - if (mask) { - TCGv tmp = tcg_temp_new(); - DisasJumpType ret; - - tcg_gen_andi_i64(tmp, load_gpr(ctx, ra), 1); - ret = gen_bcond_internal(ctx, cond, tmp, disp); - return ret; - } - return gen_bcond_internal(ctx, cond, load_gpr(ctx, ra), disp); + return gen_bcond_internal(ctx, cond, load_gpr(ctx, ra), + is_tst_cond(cond), disp); } /* Fold -0.0 for comparison with COND. */ -static void gen_fold_mzero(TCGCond cond, TCGv dest, TCGv src) +static TCGv_i64 gen_fold_mzero(TCGCond *pcond, uint64_t *pimm, TCGv_i64 src) { - uint64_t mzero = 1ull << 63; + TCGv_i64 tmp; - switch (cond) { + *pimm = 0; + switch (*pcond) { case TCG_COND_LE: case TCG_COND_GT: /* For <= or >, the -0.0 value directly compares the way we want. */ - tcg_gen_mov_i64(dest, src); - break; + return src; case TCG_COND_EQ: case TCG_COND_NE: - /* For == or !=, we can simply mask off the sign bit and compare. */ - tcg_gen_andi_i64(dest, src, mzero - 1); - break; + /* For == or !=, we can compare without the sign bit. */ + *pcond = *pcond == TCG_COND_EQ ? TCG_COND_TSTEQ : TCG_COND_TSTNE; + *pimm = INT64_MAX; + return src; case TCG_COND_GE: case TCG_COND_LT: /* For >= or <, map -0.0 to +0.0. */ - tcg_gen_movcond_i64(TCG_COND_NE, dest, src, tcg_constant_i64(mzero), - src, tcg_constant_i64(0)); - break; + tmp = tcg_temp_new_i64(); + tcg_gen_movcond_i64(TCG_COND_EQ, tmp, + src, tcg_constant_i64(INT64_MIN), + tcg_constant_i64(0), src); + return tmp; default: - abort(); + g_assert_not_reached(); } } static DisasJumpType gen_fbcond(DisasContext *ctx, TCGCond cond, int ra, int32_t disp) { - TCGv cmp_tmp = tcg_temp_new(); - DisasJumpType ret; - - gen_fold_mzero(cond, cmp_tmp, load_fpr(ctx, ra)); - ret = gen_bcond_internal(ctx, cond, cmp_tmp, disp); - return ret; + uint64_t imm; + TCGv_i64 tmp = gen_fold_mzero(&cond, &imm, load_fpr(ctx, ra)); + return gen_bcond_internal(ctx, cond, tmp, imm, disp); } static void gen_fcmov(DisasContext *ctx, TCGCond cond, int ra, int rb, int rc) { - TCGv_i64 va, vb, z; - - z = load_zero(ctx); - vb = load_fpr(ctx, rb); - va = tcg_temp_new(); - gen_fold_mzero(cond, va, load_fpr(ctx, ra)); - - tcg_gen_movcond_i64(cond, dest_fpr(ctx, rc), va, z, vb, load_fpr(ctx, rc)); + uint64_t imm; + TCGv_i64 tmp = gen_fold_mzero(&cond, &imm, load_fpr(ctx, ra)); + tcg_gen_movcond_i64(cond, dest_fpr(ctx, rc), + tmp, tcg_constant_i64(imm), + load_fpr(ctx, rb), load_fpr(ctx, rc)); } #define QUAL_RM_N 0x080 /* Round mode nearest even */ @@ -1683,16 +1673,12 @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) break; case 0x14: /* CMOVLBS */ - tmp = tcg_temp_new(); - tcg_gen_andi_i64(tmp, va, 1); - tcg_gen_movcond_i64(TCG_COND_NE, vc, tmp, load_zero(ctx), + tcg_gen_movcond_i64(TCG_COND_TSTNE, vc, va, tcg_constant_i64(1), vb, load_gpr(ctx, rc)); break; case 0x16: /* CMOVLBC */ - tmp = tcg_temp_new(); - tcg_gen_andi_i64(tmp, va, 1); - tcg_gen_movcond_i64(TCG_COND_EQ, vc, tmp, load_zero(ctx), + tcg_gen_movcond_i64(TCG_COND_TSTEQ, vc, va, tcg_constant_i64(1), vb, load_gpr(ctx, rc)); break; case 0x20: @@ -2827,35 +2813,35 @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) break; case 0x38: /* BLBC */ - ret = gen_bcond(ctx, TCG_COND_EQ, ra, disp21, 1); + ret = gen_bcond(ctx, TCG_COND_TSTEQ, ra, disp21); break; case 0x39: /* BEQ */ - ret = gen_bcond(ctx, TCG_COND_EQ, ra, disp21, 0); + ret = gen_bcond(ctx, TCG_COND_EQ, ra, disp21); break; case 0x3A: /* BLT */ - ret = gen_bcond(ctx, TCG_COND_LT, ra, disp21, 0); + ret = gen_bcond(ctx, TCG_COND_LT, ra, disp21); break; case 0x3B: /* BLE */ - ret = gen_bcond(ctx, TCG_COND_LE, ra, disp21, 0); + ret = gen_bcond(ctx, TCG_COND_LE, ra, disp21); break; case 0x3C: /* BLBS */ - ret = gen_bcond(ctx, TCG_COND_NE, ra, disp21, 1); + ret = gen_bcond(ctx, TCG_COND_TSTNE, ra, disp21); break; case 0x3D: /* BNE */ - ret = gen_bcond(ctx, TCG_COND_NE, ra, disp21, 0); + ret = gen_bcond(ctx, TCG_COND_NE, ra, disp21); break; case 0x3E: /* BGE */ - ret = gen_bcond(ctx, TCG_COND_GE, ra, disp21, 0); + ret = gen_bcond(ctx, TCG_COND_GE, ra, disp21); break; case 0x3F: /* BGT */ - ret = gen_bcond(ctx, TCG_COND_GT, ra, disp21, 0); + ret = gen_bcond(ctx, TCG_COND_GT, ra, disp21); break; invalid_opc: ret = gen_invalid(ctx); diff --git a/target/m68k/translate.c b/target/m68k/translate.c index f886190f88..d7d5ff4300 100644 --- a/target/m68k/translate.c +++ b/target/m68k/translate.c @@ -5129,46 +5129,44 @@ undef: static void gen_fcc_cond(DisasCompare *c, DisasContext *s, int cond) { TCGv fpsr; + int imm = 0; - c->v2 = tcg_constant_i32(0); /* TODO: Raise BSUN exception. */ fpsr = tcg_temp_new(); gen_load_fcr(s, fpsr, M68K_FPSR); + c->v1 = fpsr; + switch (cond) { case 0: /* False */ case 16: /* Signaling False */ - c->v1 = c->v2; c->tcond = TCG_COND_NEVER; break; case 1: /* EQual Z */ case 17: /* Signaling EQual Z */ - c->v1 = tcg_temp_new(); - tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_Z); - c->tcond = TCG_COND_NE; + imm = FPSR_CC_Z; + c->tcond = TCG_COND_TSTNE; break; case 2: /* Ordered Greater Than !(A || Z || N) */ case 18: /* Greater Than !(A || Z || N) */ - c->v1 = tcg_temp_new(); - tcg_gen_andi_i32(c->v1, fpsr, - FPSR_CC_A | FPSR_CC_Z | FPSR_CC_N); - c->tcond = TCG_COND_EQ; + imm = FPSR_CC_A | FPSR_CC_Z | FPSR_CC_N; + c->tcond = TCG_COND_TSTEQ; break; case 3: /* Ordered Greater than or Equal Z || !(A || N) */ case 19: /* Greater than or Equal Z || !(A || N) */ c->v1 = tcg_temp_new(); tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_A); tcg_gen_shli_i32(c->v1, c->v1, ctz32(FPSR_CC_N) - ctz32(FPSR_CC_A)); - tcg_gen_andi_i32(fpsr, fpsr, FPSR_CC_Z | FPSR_CC_N); tcg_gen_or_i32(c->v1, c->v1, fpsr); tcg_gen_xori_i32(c->v1, c->v1, FPSR_CC_N); - c->tcond = TCG_COND_NE; + imm = FPSR_CC_Z | FPSR_CC_N; + c->tcond = TCG_COND_TSTNE; break; case 4: /* Ordered Less Than !(!N || A || Z); */ case 20: /* Less Than !(!N || A || Z); */ c->v1 = tcg_temp_new(); tcg_gen_xori_i32(c->v1, fpsr, FPSR_CC_N); - tcg_gen_andi_i32(c->v1, c->v1, FPSR_CC_N | FPSR_CC_A | FPSR_CC_Z); - c->tcond = TCG_COND_EQ; + imm = FPSR_CC_N | FPSR_CC_A | FPSR_CC_Z; + c->tcond = TCG_COND_TSTEQ; break; case 5: /* Ordered Less than or Equal Z || (N && !A) */ case 21: /* Less than or Equal Z || (N && !A) */ @@ -5176,49 +5174,45 @@ static void gen_fcc_cond(DisasCompare *c, DisasContext *s, int cond) tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_A); tcg_gen_shli_i32(c->v1, c->v1, ctz32(FPSR_CC_N) - ctz32(FPSR_CC_A)); tcg_gen_andc_i32(c->v1, fpsr, c->v1); - tcg_gen_andi_i32(c->v1, c->v1, FPSR_CC_Z | FPSR_CC_N); - c->tcond = TCG_COND_NE; + imm = FPSR_CC_Z | FPSR_CC_N; + c->tcond = TCG_COND_TSTNE; break; case 6: /* Ordered Greater or Less than !(A || Z) */ case 22: /* Greater or Less than !(A || Z) */ - c->v1 = tcg_temp_new(); - tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_A | FPSR_CC_Z); - c->tcond = TCG_COND_EQ; + imm = FPSR_CC_A | FPSR_CC_Z; + c->tcond = TCG_COND_TSTEQ; break; case 7: /* Ordered !A */ case 23: /* Greater, Less or Equal !A */ - c->v1 = tcg_temp_new(); - tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_A); - c->tcond = TCG_COND_EQ; + imm = FPSR_CC_A; + c->tcond = TCG_COND_TSTEQ; break; case 8: /* Unordered A */ case 24: /* Not Greater, Less or Equal A */ - c->v1 = tcg_temp_new(); - tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_A); - c->tcond = TCG_COND_NE; + imm = FPSR_CC_A; + c->tcond = TCG_COND_TSTNE; break; case 9: /* Unordered or Equal A || Z */ case 25: /* Not Greater or Less then A || Z */ - c->v1 = tcg_temp_new(); - tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_A | FPSR_CC_Z); - c->tcond = TCG_COND_NE; + imm = FPSR_CC_A | FPSR_CC_Z; + c->tcond = TCG_COND_TSTNE; break; case 10: /* Unordered or Greater Than A || !(N || Z)) */ case 26: /* Not Less or Equal A || !(N || Z)) */ c->v1 = tcg_temp_new(); tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_Z); tcg_gen_shli_i32(c->v1, c->v1, ctz32(FPSR_CC_N) - ctz32(FPSR_CC_Z)); - tcg_gen_andi_i32(fpsr, fpsr, FPSR_CC_A | FPSR_CC_N); tcg_gen_or_i32(c->v1, c->v1, fpsr); tcg_gen_xori_i32(c->v1, c->v1, FPSR_CC_N); - c->tcond = TCG_COND_NE; + imm = FPSR_CC_A | FPSR_CC_N; + c->tcond = TCG_COND_TSTNE; break; case 11: /* Unordered or Greater or Equal A || Z || !N */ case 27: /* Not Less Than A || Z || !N */ c->v1 = tcg_temp_new(); - tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_A | FPSR_CC_Z | FPSR_CC_N); - tcg_gen_xori_i32(c->v1, c->v1, FPSR_CC_N); - c->tcond = TCG_COND_NE; + tcg_gen_xori_i32(c->v1, fpsr, FPSR_CC_N); + imm = FPSR_CC_A | FPSR_CC_Z | FPSR_CC_N; + c->tcond = TCG_COND_TSTNE; break; case 12: /* Unordered or Less Than A || (N && !Z) */ case 28: /* Not Greater than or Equal A || (N && !Z) */ @@ -5226,27 +5220,25 @@ static void gen_fcc_cond(DisasCompare *c, DisasContext *s, int cond) tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_Z); tcg_gen_shli_i32(c->v1, c->v1, ctz32(FPSR_CC_N) - ctz32(FPSR_CC_Z)); tcg_gen_andc_i32(c->v1, fpsr, c->v1); - tcg_gen_andi_i32(c->v1, c->v1, FPSR_CC_A | FPSR_CC_N); - c->tcond = TCG_COND_NE; + imm = FPSR_CC_A | FPSR_CC_N; + c->tcond = TCG_COND_TSTNE; break; case 13: /* Unordered or Less or Equal A || Z || N */ case 29: /* Not Greater Than A || Z || N */ - c->v1 = tcg_temp_new(); - tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_A | FPSR_CC_Z | FPSR_CC_N); - c->tcond = TCG_COND_NE; + imm = FPSR_CC_A | FPSR_CC_Z | FPSR_CC_N; + c->tcond = TCG_COND_TSTNE; break; case 14: /* Not Equal !Z */ case 30: /* Signaling Not Equal !Z */ - c->v1 = tcg_temp_new(); - tcg_gen_andi_i32(c->v1, fpsr, FPSR_CC_Z); - c->tcond = TCG_COND_EQ; + imm = FPSR_CC_Z; + c->tcond = TCG_COND_TSTEQ; break; case 15: /* True */ case 31: /* Signaling True */ - c->v1 = c->v2; c->tcond = TCG_COND_ALWAYS; break; } + c->v2 = tcg_constant_i32(imm); } static void gen_fjmpcc(DisasContext *s, int cond, TCGLabel *l1) diff --git a/target/riscv/cpu-qom.h b/target/riscv/cpu-qom.h index 9219c2fcc3..3670cfe6d9 100644 --- a/target/riscv/cpu-qom.h +++ b/target/riscv/cpu-qom.h @@ -34,7 +34,10 @@ #define TYPE_RISCV_CPU_BASE32 RISCV_CPU_TYPE_NAME("rv32") #define TYPE_RISCV_CPU_BASE64 RISCV_CPU_TYPE_NAME("rv64") #define TYPE_RISCV_CPU_BASE128 RISCV_CPU_TYPE_NAME("x-rv128") +#define TYPE_RISCV_CPU_RV32I RISCV_CPU_TYPE_NAME("rv32i") +#define TYPE_RISCV_CPU_RV32E RISCV_CPU_TYPE_NAME("rv32e") #define TYPE_RISCV_CPU_RV64I RISCV_CPU_TYPE_NAME("rv64i") +#define TYPE_RISCV_CPU_RV64E RISCV_CPU_TYPE_NAME("rv64e") #define TYPE_RISCV_CPU_RVA22U64 RISCV_CPU_TYPE_NAME("rva22u64") #define TYPE_RISCV_CPU_RVA22S64 RISCV_CPU_TYPE_NAME("rva22s64") #define TYPE_RISCV_CPU_IBEX RISCV_CPU_TYPE_NAME("lowrisc-ibex") diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index be21fa09c6..1b8d001d23 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -29,8 +29,10 @@ #include "qapi/visitor.h" #include "qemu/error-report.h" #include "hw/qdev-properties.h" +#include "hw/core/qdev-prop-internal.h" #include "migration/vmstate.h" #include "fpu/softfloat-helpers.h" +#include "sysemu/device_tree.h" #include "sysemu/kvm.h" #include "sysemu/tcg.h" #include "kvm/kvm_riscv.h" @@ -38,9 +40,9 @@ #include "tcg/tcg.h" /* RISC-V CPU definitions */ -static const char riscv_single_letter_exts[] = "IEMAFDQCPVH"; +static const char riscv_single_letter_exts[] = "IEMAFDQCBPVH"; const uint32_t misa_bits[] = {RVI, RVE, RVM, RVA, RVF, RVD, RVV, - RVC, RVS, RVU, RVH, RVJ, RVG, 0}; + RVC, RVS, RVU, RVH, RVJ, RVG, RVB, 0}; /* * From vector_helper.c @@ -58,6 +60,20 @@ bool riscv_cpu_is_32bit(RISCVCPU *cpu) return riscv_cpu_mxl(&cpu->env) == MXL_RV32; } +/* Hash that stores general user set numeric options */ +static GHashTable *general_user_opts; + +static void cpu_option_add_user_setting(const char *optname, uint32_t value) +{ + g_hash_table_insert(general_user_opts, (gpointer)optname, + GUINT_TO_POINTER(value)); +} + +bool riscv_cpu_option_set(const char *optname) +{ + return g_hash_table_contains(general_user_opts, optname); +} + #define ISA_EXT_DATA_ENTRY(_name, _min_ver, _prop) \ {#_name, _min_ver, CPU_CFG_OFFSET(_prop)} @@ -93,7 +109,9 @@ const RISCVIsaExtData isa_edata_arr[] = { ISA_EXT_DATA_ENTRY(zihintpause, PRIV_VERSION_1_10_0, ext_zihintpause), ISA_EXT_DATA_ENTRY(zihpm, PRIV_VERSION_1_12_0, ext_zihpm), ISA_EXT_DATA_ENTRY(zmmul, PRIV_VERSION_1_12_0, ext_zmmul), + ISA_EXT_DATA_ENTRY(zaamo, PRIV_VERSION_1_12_0, ext_zaamo), ISA_EXT_DATA_ENTRY(zacas, PRIV_VERSION_1_12_0, ext_zacas), + ISA_EXT_DATA_ENTRY(zalrsc, PRIV_VERSION_1_12_0, ext_zalrsc), ISA_EXT_DATA_ENTRY(zawrs, PRIV_VERSION_1_12_0, ext_zawrs), ISA_EXT_DATA_ENTRY(zfa, PRIV_VERSION_1_12_0, ext_zfa), ISA_EXT_DATA_ENTRY(zfbfmin, PRIV_VERSION_1_12_0, ext_zfbfmin), @@ -190,6 +208,11 @@ void isa_ext_update_enabled(RISCVCPU *cpu, uint32_t ext_offset, bool en) *ext_enabled = en; } +bool riscv_cpu_is_vendor(Object *cpu_obj) +{ + return object_dynamic_cast(cpu_obj, TYPE_RISCV_VENDOR_CPU) != NULL; +} + const char * const riscv_int_regnames[] = { "x0/zero", "x1/ra", "x2/sp", "x3/gp", "x4/tp", "x5/t0", "x6/t1", "x7/t2", "x8/s0", "x9/s1", "x10/a0", "x11/a1", "x12/a2", "x13/a3", @@ -281,12 +304,16 @@ const char *riscv_cpu_get_trap_name(target_ulong cause, bool async) } } -void riscv_cpu_set_misa(CPURISCVState *env, RISCVMXL mxl, uint32_t ext) +void riscv_cpu_set_misa_ext(CPURISCVState *env, uint32_t ext) { - env->misa_mxl_max = env->misa_mxl = mxl; env->misa_ext_mask = env->misa_ext = ext; } +int riscv_cpu_max_xlen(RISCVCPUClass *mcc) +{ + return 16 << mcc->misa_mxl_max; +} + #ifndef CONFIG_USER_ONLY static uint8_t satp_mode_from_str(const char *satp_mode_str) { @@ -396,11 +423,7 @@ static void riscv_any_cpu_init(Object *obj) { RISCVCPU *cpu = RISCV_CPU(obj); CPURISCVState *env = &cpu->env; -#if defined(TARGET_RISCV32) - riscv_cpu_set_misa(env, MXL_RV32, RVI | RVM | RVA | RVF | RVD | RVC | RVU); -#elif defined(TARGET_RISCV64) - riscv_cpu_set_misa(env, MXL_RV64, RVI | RVM | RVA | RVF | RVD | RVC | RVU); -#endif + riscv_cpu_set_misa_ext(env, RVI | RVM | RVA | RVF | RVD | RVC | RVU); #ifndef CONFIG_USER_ONLY set_satp_mode_max_supported(RISCV_CPU(obj), @@ -421,25 +444,29 @@ static void riscv_max_cpu_init(Object *obj) { RISCVCPU *cpu = RISCV_CPU(obj); CPURISCVState *env = &cpu->env; - RISCVMXL mlx = MXL_RV64; -#ifdef TARGET_RISCV32 - mlx = MXL_RV32; -#endif - riscv_cpu_set_misa(env, mlx, 0); + cpu->cfg.mmu = true; + cpu->cfg.pmp = true; + env->priv_ver = PRIV_VERSION_LATEST; #ifndef CONFIG_USER_ONLY - set_satp_mode_max_supported(RISCV_CPU(obj), mlx == MXL_RV32 ? - VM_1_10_SV32 : VM_1_10_SV57); +#ifdef TARGET_RISCV32 + set_satp_mode_max_supported(cpu, VM_1_10_SV32); +#else + set_satp_mode_max_supported(cpu, VM_1_10_SV57); +#endif #endif } #if defined(TARGET_RISCV64) static void rv64_base_cpu_init(Object *obj) { - CPURISCVState *env = &RISCV_CPU(obj)->env; - /* We set this in the realise function */ - riscv_cpu_set_misa(env, MXL_RV64, 0); + RISCVCPU *cpu = RISCV_CPU(obj); + CPURISCVState *env = &cpu->env; + + cpu->cfg.mmu = true; + cpu->cfg.pmp = true; + /* Set latest version of privileged specification */ env->priv_ver = PRIV_VERSION_LATEST; #ifndef CONFIG_USER_ONLY @@ -451,8 +478,7 @@ static void rv64_sifive_u_cpu_init(Object *obj) { RISCVCPU *cpu = RISCV_CPU(obj); CPURISCVState *env = &cpu->env; - riscv_cpu_set_misa(env, MXL_RV64, - RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU); + riscv_cpu_set_misa_ext(env, RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU); env->priv_ver = PRIV_VERSION_1_10_0; #ifndef CONFIG_USER_ONLY set_satp_mode_max_supported(RISCV_CPU(obj), VM_1_10_SV39); @@ -470,7 +496,7 @@ static void rv64_sifive_e_cpu_init(Object *obj) CPURISCVState *env = &RISCV_CPU(obj)->env; RISCVCPU *cpu = RISCV_CPU(obj); - riscv_cpu_set_misa(env, MXL_RV64, RVI | RVM | RVA | RVC | RVU); + riscv_cpu_set_misa_ext(env, RVI | RVM | RVA | RVC | RVU); env->priv_ver = PRIV_VERSION_1_10_0; #ifndef CONFIG_USER_ONLY set_satp_mode_max_supported(cpu, VM_1_10_MBARE); @@ -487,7 +513,7 @@ static void rv64_thead_c906_cpu_init(Object *obj) CPURISCVState *env = &RISCV_CPU(obj)->env; RISCVCPU *cpu = RISCV_CPU(obj); - riscv_cpu_set_misa(env, MXL_RV64, RVG | RVC | RVS | RVU); + riscv_cpu_set_misa_ext(env, RVG | RVC | RVS | RVU); env->priv_ver = PRIV_VERSION_1_11_0; cpu->cfg.ext_zfa = true; @@ -518,7 +544,7 @@ static void rv64_veyron_v1_cpu_init(Object *obj) CPURISCVState *env = &RISCV_CPU(obj)->env; RISCVCPU *cpu = RISCV_CPU(obj); - riscv_cpu_set_misa(env, MXL_RV64, RVG | RVC | RVS | RVU | RVH); + riscv_cpu_set_misa_ext(env, RVG | RVC | RVS | RVU | RVH); env->priv_ver = PRIV_VERSION_1_12_0; /* Enable ISA extensions */ @@ -555,15 +581,19 @@ static void rv64_veyron_v1_cpu_init(Object *obj) static void rv128_base_cpu_init(Object *obj) { + RISCVCPU *cpu = RISCV_CPU(obj); + CPURISCVState *env = &cpu->env; + if (qemu_tcg_mttcg_enabled()) { /* Missing 128-bit aligned atomics */ error_report("128-bit RISC-V currently does not work with Multi " "Threaded TCG. Please use: -accel tcg,thread=single"); exit(EXIT_FAILURE); } - CPURISCVState *env = &RISCV_CPU(obj)->env; - /* We set this in the realise function */ - riscv_cpu_set_misa(env, MXL_RV128, 0); + + cpu->cfg.mmu = true; + cpu->cfg.pmp = true; + /* Set latest version of privileged specification */ env->priv_ver = PRIV_VERSION_LATEST; #ifndef CONFIG_USER_ONLY @@ -574,30 +604,23 @@ static void rv128_base_cpu_init(Object *obj) static void rv64i_bare_cpu_init(Object *obj) { CPURISCVState *env = &RISCV_CPU(obj)->env; - riscv_cpu_set_misa(env, MXL_RV64, RVI); - - /* Remove the defaults from the parent class */ - RISCV_CPU(obj)->cfg.ext_zicntr = false; - RISCV_CPU(obj)->cfg.ext_zihpm = false; - - /* Set to QEMU's first supported priv version */ - env->priv_ver = PRIV_VERSION_1_10_0; + riscv_cpu_set_misa_ext(env, RVI); +} - /* - * Support all available satp_mode settings. The default - * value will be set to MBARE if the user doesn't set - * satp_mode manually (see set_satp_mode_default()). - */ -#ifndef CONFIG_USER_ONLY - set_satp_mode_max_supported(RISCV_CPU(obj), VM_1_10_SV64); -#endif +static void rv64e_bare_cpu_init(Object *obj) +{ + CPURISCVState *env = &RISCV_CPU(obj)->env; + riscv_cpu_set_misa_ext(env, RVE); } #else static void rv32_base_cpu_init(Object *obj) { - CPURISCVState *env = &RISCV_CPU(obj)->env; - /* We set this in the realise function */ - riscv_cpu_set_misa(env, MXL_RV32, 0); + RISCVCPU *cpu = RISCV_CPU(obj); + CPURISCVState *env = &cpu->env; + + cpu->cfg.mmu = true; + cpu->cfg.pmp = true; + /* Set latest version of privileged specification */ env->priv_ver = PRIV_VERSION_LATEST; #ifndef CONFIG_USER_ONLY @@ -609,8 +632,7 @@ static void rv32_sifive_u_cpu_init(Object *obj) { RISCVCPU *cpu = RISCV_CPU(obj); CPURISCVState *env = &cpu->env; - riscv_cpu_set_misa(env, MXL_RV32, - RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU); + riscv_cpu_set_misa_ext(env, RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU); env->priv_ver = PRIV_VERSION_1_10_0; #ifndef CONFIG_USER_ONLY set_satp_mode_max_supported(RISCV_CPU(obj), VM_1_10_SV32); @@ -628,7 +650,7 @@ static void rv32_sifive_e_cpu_init(Object *obj) CPURISCVState *env = &RISCV_CPU(obj)->env; RISCVCPU *cpu = RISCV_CPU(obj); - riscv_cpu_set_misa(env, MXL_RV32, RVI | RVM | RVA | RVC | RVU); + riscv_cpu_set_misa_ext(env, RVI | RVM | RVA | RVC | RVU); env->priv_ver = PRIV_VERSION_1_10_0; #ifndef CONFIG_USER_ONLY set_satp_mode_max_supported(cpu, VM_1_10_MBARE); @@ -645,7 +667,7 @@ static void rv32_ibex_cpu_init(Object *obj) CPURISCVState *env = &RISCV_CPU(obj)->env; RISCVCPU *cpu = RISCV_CPU(obj); - riscv_cpu_set_misa(env, MXL_RV32, RVI | RVM | RVC | RVU); + riscv_cpu_set_misa_ext(env, RVI | RVM | RVC | RVU); env->priv_ver = PRIV_VERSION_1_12_0; #ifndef CONFIG_USER_ONLY set_satp_mode_max_supported(cpu, VM_1_10_MBARE); @@ -662,7 +684,7 @@ static void rv32_imafcu_nommu_cpu_init(Object *obj) CPURISCVState *env = &RISCV_CPU(obj)->env; RISCVCPU *cpu = RISCV_CPU(obj); - riscv_cpu_set_misa(env, MXL_RV32, RVI | RVM | RVA | RVF | RVC | RVU); + riscv_cpu_set_misa_ext(env, RVI | RVM | RVA | RVF | RVC | RVU); env->priv_ver = PRIV_VERSION_1_10_0; #ifndef CONFIG_USER_ONLY set_satp_mode_max_supported(cpu, VM_1_10_MBARE); @@ -673,6 +695,18 @@ static void rv32_imafcu_nommu_cpu_init(Object *obj) cpu->cfg.ext_zicsr = true; cpu->cfg.pmp = true; } + +static void rv32i_bare_cpu_init(Object *obj) +{ + CPURISCVState *env = &RISCV_CPU(obj)->env; + riscv_cpu_set_misa_ext(env, RVI); +} + +static void rv32e_bare_cpu_init(Object *obj) +{ + CPURISCVState *env = &RISCV_CPU(obj)->env; + riscv_cpu_set_misa_ext(env, RVE); +} #endif static ObjectClass *riscv_cpu_class_by_name(const char *cpu_model) @@ -813,7 +847,7 @@ static void riscv_cpu_dump_state(CPUState *cs, FILE *f, int flags) csr_ops[csrno].name, val); } } - uint16_t vlenb = cpu->cfg.vlen >> 3; + uint16_t vlenb = cpu->cfg.vlenb; for (i = 0; i < 32; i++) { qemu_fprintf(f, " %-8s ", riscv_rvv_regnames[i]); @@ -887,7 +921,7 @@ static void riscv_cpu_reset_hold(Object *obj) mcc->parent_phases.hold(obj); } #ifndef CONFIG_USER_ONLY - env->misa_mxl = env->misa_mxl_max; + env->misa_mxl = mcc->misa_mxl_max; env->priv = PRV_M; env->mstatus &= ~(MSTATUS_MIE | MSTATUS_MPRV); if (env->misa_mxl > MXL_RV32) { @@ -1080,17 +1114,18 @@ void riscv_cpu_finalize_features(RISCVCPU *cpu, Error **errp) } #endif - /* - * KVM accel does not have a specialized finalize() - * callback because its extensions are validated - * in the get()/set() callbacks of each property. - */ if (tcg_enabled()) { riscv_tcg_cpu_finalize_features(cpu, &local_err); if (local_err != NULL) { error_propagate(errp, local_err); return; } + } else if (kvm_enabled()) { + riscv_kvm_cpu_finalize_features(cpu, &local_err); + if (local_err != NULL) { + error_propagate(errp, local_err); + return; + } } } @@ -1263,11 +1298,19 @@ static void riscv_cpu_post_init(Object *obj) static void riscv_cpu_init(Object *obj) { + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(obj); + RISCVCPU *cpu = RISCV_CPU(obj); + CPURISCVState *env = &cpu->env; + + env->misa_mxl = mcc->misa_mxl_max; + #ifndef CONFIG_USER_ONLY qdev_init_gpio_in(DEVICE(obj), riscv_cpu_set_irq, IRQ_LOCAL_MAX + IRQ_LOCAL_GUEST_MAX); #endif /* CONFIG_USER_ONLY */ + general_user_opts = g_hash_table_new(g_str_hash, g_str_equal); + /* * The timer and performance counters extensions were supported * in QEMU before they were added as discrete extensions in the @@ -1277,6 +1320,43 @@ static void riscv_cpu_init(Object *obj) */ RISCV_CPU(obj)->cfg.ext_zicntr = true; RISCV_CPU(obj)->cfg.ext_zihpm = true; + + /* Default values for non-bool cpu properties */ + cpu->cfg.pmu_mask = MAKE_64BIT_MASK(3, 16); + cpu->cfg.vlenb = 128 >> 3; + cpu->cfg.elen = 64; + cpu->cfg.cbom_blocksize = 64; + cpu->cfg.cbop_blocksize = 64; + cpu->cfg.cboz_blocksize = 64; + cpu->env.vext_ver = VEXT_VERSION_1_00_0; +} + +static void riscv_bare_cpu_init(Object *obj) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + + /* + * Bare CPUs do not inherit the timer and performance + * counters from the parent class (see riscv_cpu_init() + * for info on why the parent enables them). + * + * Users have to explicitly enable these counters for + * bare CPUs. + */ + cpu->cfg.ext_zicntr = false; + cpu->cfg.ext_zihpm = false; + + /* Set to QEMU's first supported priv version */ + cpu->env.priv_ver = PRIV_VERSION_1_10_0; + + /* + * Support all available satp_mode settings. The default + * value will be set to MBARE if the user doesn't set + * satp_mode manually (see set_satp_mode_default()). + */ +#ifndef CONFIG_USER_ONLY + set_satp_mode_max_supported(cpu, VM_1_10_SV64); +#endif } typedef struct misa_ext_info { @@ -1304,8 +1384,29 @@ static const MISAExtInfo misa_ext_info_arr[] = { MISA_EXT_INFO(RVJ, "x-j", "Dynamic translated languages"), MISA_EXT_INFO(RVV, "v", "Vector operations"), MISA_EXT_INFO(RVG, "g", "General purpose (IMAFD_Zicsr_Zifencei)"), + MISA_EXT_INFO(RVB, "x-b", "Bit manipulation (Zba_Zbb_Zbs)") }; +static void riscv_cpu_validate_misa_mxl(RISCVCPUClass *mcc) +{ + CPUClass *cc = CPU_CLASS(mcc); + + /* Validate that MISA_MXL is set properly. */ + switch (mcc->misa_mxl_max) { +#ifdef TARGET_RISCV64 + case MXL_RV64: + case MXL_RV128: + cc->gdb_core_xml_file = "riscv-64bit-cpu.xml"; + break; +#endif + case MXL_RV32: + cc->gdb_core_xml_file = "riscv-32bit-cpu.xml"; + break; + default: + g_assert_not_reached(); + } +} + static int riscv_validate_misa_info_idx(uint32_t bit) { int idx; @@ -1451,6 +1552,9 @@ const RISCVCPUMultiExtConfig riscv_cpu_experimental_exts[] = { MULTI_EXT_CFG_BOOL("x-smaia", ext_smaia, false), MULTI_EXT_CFG_BOOL("x-ssaia", ext_ssaia, false), + MULTI_EXT_CFG_BOOL("x-zaamo", ext_zaamo, false), + MULTI_EXT_CFG_BOOL("x-zalrsc", ext_zalrsc, false), + MULTI_EXT_CFG_BOOL("x-zvfh", ext_zvfh, false), MULTI_EXT_CFG_BOOL("x-zvfhmin", ext_zvfhmin, false), @@ -1485,26 +1589,46 @@ const RISCVCPUMultiExtConfig riscv_cpu_deprecated_exts[] = { DEFINE_PROP_END_OF_LIST(), }; +static void cpu_set_prop_err(RISCVCPU *cpu, const char *propname, + Error **errp) +{ + g_autofree char *cpuname = riscv_cpu_get_name(cpu); + error_setg(errp, "CPU '%s' does not allow changing the value of '%s'", + cpuname, propname); +} + static void prop_pmu_num_set(Object *obj, Visitor *v, const char *name, void *opaque, Error **errp) { RISCVCPU *cpu = RISCV_CPU(obj); - uint8_t pmu_num; + uint8_t pmu_num, curr_pmu_num; + uint32_t pmu_mask; visit_type_uint8(v, name, &pmu_num, errp); + curr_pmu_num = ctpop32(cpu->cfg.pmu_mask); + + if (pmu_num != curr_pmu_num && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, name, errp); + error_append_hint(errp, "Current '%s' val: %u\n", + name, curr_pmu_num); + return; + } + if (pmu_num > (RV_MAX_MHPMCOUNTERS - 3)) { error_setg(errp, "Number of counters exceeds maximum available"); return; } if (pmu_num == 0) { - cpu->cfg.pmu_mask = 0; + pmu_mask = 0; } else { - cpu->cfg.pmu_mask = MAKE_64BIT_MASK(3, pmu_num); + pmu_mask = MAKE_64BIT_MASK(3, pmu_num); } warn_report("\"pmu-num\" property is deprecated; use \"pmu-mask\""); + cpu->cfg.pmu_mask = pmu_mask; + cpu_option_add_user_setting("pmu-mask", pmu_mask); } static void prop_pmu_num_get(Object *obj, Visitor *v, const char *name, @@ -1516,30 +1640,525 @@ static void prop_pmu_num_get(Object *obj, Visitor *v, const char *name, visit_type_uint8(v, name, &pmu_num, errp); } -const PropertyInfo prop_pmu_num = { +static const PropertyInfo prop_pmu_num = { .name = "pmu-num", .get = prop_pmu_num_get, .set = prop_pmu_num_set, }; -Property riscv_cpu_options[] = { - DEFINE_PROP_UINT32("pmu-mask", RISCVCPU, cfg.pmu_mask, MAKE_64BIT_MASK(3, 16)), - {.name = "pmu-num", .info = &prop_pmu_num}, /* Deprecated */ +static void prop_pmu_mask_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + uint32_t value; + uint8_t pmu_num; - DEFINE_PROP_BOOL("mmu", RISCVCPU, cfg.mmu, true), - DEFINE_PROP_BOOL("pmp", RISCVCPU, cfg.pmp, true), + visit_type_uint32(v, name, &value, errp); - DEFINE_PROP_STRING("priv_spec", RISCVCPU, cfg.priv_spec), - DEFINE_PROP_STRING("vext_spec", RISCVCPU, cfg.vext_spec), + if (value != cpu->cfg.pmu_mask && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, name, errp); + error_append_hint(errp, "Current '%s' val: %x\n", + name, cpu->cfg.pmu_mask); + return; + } - DEFINE_PROP_UINT16("vlen", RISCVCPU, cfg.vlen, 128), - DEFINE_PROP_UINT16("elen", RISCVCPU, cfg.elen, 64), + pmu_num = ctpop32(value); - DEFINE_PROP_UINT16("cbom_blocksize", RISCVCPU, cfg.cbom_blocksize, 64), - DEFINE_PROP_UINT16("cbop_blocksize", RISCVCPU, cfg.cbop_blocksize, 64), - DEFINE_PROP_UINT16("cboz_blocksize", RISCVCPU, cfg.cboz_blocksize, 64), + if (pmu_num > (RV_MAX_MHPMCOUNTERS - 3)) { + error_setg(errp, "Number of counters exceeds maximum available"); + return; + } - DEFINE_PROP_END_OF_LIST(), + cpu_option_add_user_setting(name, value); + cpu->cfg.pmu_mask = value; +} + +static void prop_pmu_mask_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint8_t pmu_mask = RISCV_CPU(obj)->cfg.pmu_mask; + + visit_type_uint8(v, name, &pmu_mask, errp); +} + +static const PropertyInfo prop_pmu_mask = { + .name = "pmu-mask", + .get = prop_pmu_mask_get, + .set = prop_pmu_mask_set, +}; + +static void prop_mmu_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + bool value; + + visit_type_bool(v, name, &value, errp); + + if (cpu->cfg.mmu != value && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, "mmu", errp); + return; + } + + cpu_option_add_user_setting(name, value); + cpu->cfg.mmu = value; +} + +static void prop_mmu_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + bool value = RISCV_CPU(obj)->cfg.mmu; + + visit_type_bool(v, name, &value, errp); +} + +static const PropertyInfo prop_mmu = { + .name = "mmu", + .get = prop_mmu_get, + .set = prop_mmu_set, +}; + +static void prop_pmp_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + bool value; + + visit_type_bool(v, name, &value, errp); + + if (cpu->cfg.pmp != value && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, name, errp); + return; + } + + cpu_option_add_user_setting(name, value); + cpu->cfg.pmp = value; +} + +static void prop_pmp_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + bool value = RISCV_CPU(obj)->cfg.pmp; + + visit_type_bool(v, name, &value, errp); +} + +static const PropertyInfo prop_pmp = { + .name = "pmp", + .get = prop_pmp_get, + .set = prop_pmp_set, +}; + +static int priv_spec_from_str(const char *priv_spec_str) +{ + int priv_version = -1; + + if (!g_strcmp0(priv_spec_str, PRIV_VER_1_12_0_STR)) { + priv_version = PRIV_VERSION_1_12_0; + } else if (!g_strcmp0(priv_spec_str, PRIV_VER_1_11_0_STR)) { + priv_version = PRIV_VERSION_1_11_0; + } else if (!g_strcmp0(priv_spec_str, PRIV_VER_1_10_0_STR)) { + priv_version = PRIV_VERSION_1_10_0; + } + + return priv_version; +} + +static const char *priv_spec_to_str(int priv_version) +{ + switch (priv_version) { + case PRIV_VERSION_1_10_0: + return PRIV_VER_1_10_0_STR; + case PRIV_VERSION_1_11_0: + return PRIV_VER_1_11_0_STR; + case PRIV_VERSION_1_12_0: + return PRIV_VER_1_12_0_STR; + default: + return NULL; + } +} + +static void prop_priv_spec_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + g_autofree char *value = NULL; + int priv_version = -1; + + visit_type_str(v, name, &value, errp); + + priv_version = priv_spec_from_str(value); + if (priv_version < 0) { + error_setg(errp, "Unsupported privilege spec version '%s'", value); + return; + } + + if (priv_version != cpu->env.priv_ver && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, name, errp); + error_append_hint(errp, "Current '%s' val: %s\n", name, + object_property_get_str(obj, name, NULL)); + return; + } + + cpu_option_add_user_setting(name, priv_version); + cpu->env.priv_ver = priv_version; +} + +static void prop_priv_spec_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + const char *value = priv_spec_to_str(cpu->env.priv_ver); + + visit_type_str(v, name, (char **)&value, errp); +} + +static const PropertyInfo prop_priv_spec = { + .name = "priv_spec", + .get = prop_priv_spec_get, + .set = prop_priv_spec_set, +}; + +static void prop_vext_spec_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + g_autofree char *value = NULL; + + visit_type_str(v, name, &value, errp); + + if (g_strcmp0(value, VEXT_VER_1_00_0_STR) != 0) { + error_setg(errp, "Unsupported vector spec version '%s'", value); + return; + } + + cpu_option_add_user_setting(name, VEXT_VERSION_1_00_0); + cpu->env.vext_ver = VEXT_VERSION_1_00_0; +} + +static void prop_vext_spec_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + const char *value = VEXT_VER_1_00_0_STR; + + visit_type_str(v, name, (char **)&value, errp); +} + +static const PropertyInfo prop_vext_spec = { + .name = "vext_spec", + .get = prop_vext_spec_get, + .set = prop_vext_spec_set, +}; + +static void prop_vlen_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + uint16_t value; + + if (!visit_type_uint16(v, name, &value, errp)) { + return; + } + + if (!is_power_of_2(value)) { + error_setg(errp, "Vector extension VLEN must be power of 2"); + return; + } + + if (value != cpu->cfg.vlenb && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, name, errp); + error_append_hint(errp, "Current '%s' val: %u\n", + name, cpu->cfg.vlenb << 3); + return; + } + + cpu_option_add_user_setting(name, value); + cpu->cfg.vlenb = value >> 3; +} + +static void prop_vlen_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint16_t value = RISCV_CPU(obj)->cfg.vlenb << 3; + + visit_type_uint16(v, name, &value, errp); +} + +static const PropertyInfo prop_vlen = { + .name = "vlen", + .get = prop_vlen_get, + .set = prop_vlen_set, +}; + +static void prop_elen_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + uint16_t value; + + if (!visit_type_uint16(v, name, &value, errp)) { + return; + } + + if (!is_power_of_2(value)) { + error_setg(errp, "Vector extension ELEN must be power of 2"); + return; + } + + if (value != cpu->cfg.elen && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, name, errp); + error_append_hint(errp, "Current '%s' val: %u\n", + name, cpu->cfg.elen); + return; + } + + cpu_option_add_user_setting(name, value); + cpu->cfg.elen = value; +} + +static void prop_elen_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint16_t value = RISCV_CPU(obj)->cfg.elen; + + visit_type_uint16(v, name, &value, errp); +} + +static const PropertyInfo prop_elen = { + .name = "elen", + .get = prop_elen_get, + .set = prop_elen_set, +}; + +static void prop_cbom_blksize_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + uint16_t value; + + if (!visit_type_uint16(v, name, &value, errp)) { + return; + } + + if (value != cpu->cfg.cbom_blocksize && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, name, errp); + error_append_hint(errp, "Current '%s' val: %u\n", + name, cpu->cfg.cbom_blocksize); + return; + } + + cpu_option_add_user_setting(name, value); + cpu->cfg.cbom_blocksize = value; +} + +static void prop_cbom_blksize_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint16_t value = RISCV_CPU(obj)->cfg.cbom_blocksize; + + visit_type_uint16(v, name, &value, errp); +} + +static const PropertyInfo prop_cbom_blksize = { + .name = "cbom_blocksize", + .get = prop_cbom_blksize_get, + .set = prop_cbom_blksize_set, +}; + +static void prop_cbop_blksize_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + uint16_t value; + + if (!visit_type_uint16(v, name, &value, errp)) { + return; + } + + if (value != cpu->cfg.cbop_blocksize && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, name, errp); + error_append_hint(errp, "Current '%s' val: %u\n", + name, cpu->cfg.cbop_blocksize); + return; + } + + cpu_option_add_user_setting(name, value); + cpu->cfg.cbop_blocksize = value; +} + +static void prop_cbop_blksize_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint16_t value = RISCV_CPU(obj)->cfg.cbop_blocksize; + + visit_type_uint16(v, name, &value, errp); +} + +static const PropertyInfo prop_cbop_blksize = { + .name = "cbop_blocksize", + .get = prop_cbop_blksize_get, + .set = prop_cbop_blksize_set, +}; + +static void prop_cboz_blksize_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + RISCVCPU *cpu = RISCV_CPU(obj); + uint16_t value; + + if (!visit_type_uint16(v, name, &value, errp)) { + return; + } + + if (value != cpu->cfg.cboz_blocksize && riscv_cpu_is_vendor(obj)) { + cpu_set_prop_err(cpu, name, errp); + error_append_hint(errp, "Current '%s' val: %u\n", + name, cpu->cfg.cboz_blocksize); + return; + } + + cpu_option_add_user_setting(name, value); + cpu->cfg.cboz_blocksize = value; +} + +static void prop_cboz_blksize_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint16_t value = RISCV_CPU(obj)->cfg.cboz_blocksize; + + visit_type_uint16(v, name, &value, errp); +} + +static const PropertyInfo prop_cboz_blksize = { + .name = "cboz_blocksize", + .get = prop_cboz_blksize_get, + .set = prop_cboz_blksize_set, +}; + +static void prop_mvendorid_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + bool dynamic_cpu = riscv_cpu_is_dynamic(obj); + RISCVCPU *cpu = RISCV_CPU(obj); + uint32_t prev_val = cpu->cfg.mvendorid; + uint32_t value; + + if (!visit_type_uint32(v, name, &value, errp)) { + return; + } + + if (!dynamic_cpu && prev_val != value) { + error_setg(errp, "Unable to change %s mvendorid (0x%x)", + object_get_typename(obj), prev_val); + return; + } + + cpu->cfg.mvendorid = value; +} + +static void prop_mvendorid_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint32_t value = RISCV_CPU(obj)->cfg.mvendorid; + + visit_type_uint32(v, name, &value, errp); +} + +static const PropertyInfo prop_mvendorid = { + .name = "mvendorid", + .get = prop_mvendorid_get, + .set = prop_mvendorid_set, +}; + +static void prop_mimpid_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + bool dynamic_cpu = riscv_cpu_is_dynamic(obj); + RISCVCPU *cpu = RISCV_CPU(obj); + uint64_t prev_val = cpu->cfg.mimpid; + uint64_t value; + + if (!visit_type_uint64(v, name, &value, errp)) { + return; + } + + if (!dynamic_cpu && prev_val != value) { + error_setg(errp, "Unable to change %s mimpid (0x%" PRIu64 ")", + object_get_typename(obj), prev_val); + return; + } + + cpu->cfg.mimpid = value; +} + +static void prop_mimpid_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint64_t value = RISCV_CPU(obj)->cfg.mimpid; + + visit_type_uint64(v, name, &value, errp); +} + +static const PropertyInfo prop_mimpid = { + .name = "mimpid", + .get = prop_mimpid_get, + .set = prop_mimpid_set, +}; + +static void prop_marchid_set(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + bool dynamic_cpu = riscv_cpu_is_dynamic(obj); + RISCVCPU *cpu = RISCV_CPU(obj); + uint64_t prev_val = cpu->cfg.marchid; + uint64_t value, invalid_val; + uint32_t mxlen = 0; + + if (!visit_type_uint64(v, name, &value, errp)) { + return; + } + + if (!dynamic_cpu && prev_val != value) { + error_setg(errp, "Unable to change %s marchid (0x%" PRIu64 ")", + object_get_typename(obj), prev_val); + return; + } + + switch (riscv_cpu_mxl(&cpu->env)) { + case MXL_RV32: + mxlen = 32; + break; + case MXL_RV64: + case MXL_RV128: + mxlen = 64; + break; + default: + g_assert_not_reached(); + } + + invalid_val = 1LL << (mxlen - 1); + + if (value == invalid_val) { + error_setg(errp, "Unable to set marchid with MSB (%u) bit set " + "and the remaining bits zero", mxlen); + return; + } + + cpu->cfg.marchid = value; +} + +static void prop_marchid_get(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + uint64_t value = RISCV_CPU(obj)->cfg.marchid; + + visit_type_uint64(v, name, &value, errp); +} + +static const PropertyInfo prop_marchid = { + .name = "marchid", + .get = prop_marchid_get, + .set = prop_marchid_set, }; /* @@ -1612,6 +2231,26 @@ RISCVCPUProfile *riscv_profiles[] = { static Property riscv_cpu_properties[] = { DEFINE_PROP_BOOL("debug", RISCVCPU, cfg.debug, true), + {.name = "pmu-mask", .info = &prop_pmu_mask}, + {.name = "pmu-num", .info = &prop_pmu_num}, /* Deprecated */ + + {.name = "mmu", .info = &prop_mmu}, + {.name = "pmp", .info = &prop_pmp}, + + {.name = "priv_spec", .info = &prop_priv_spec}, + {.name = "vext_spec", .info = &prop_vext_spec}, + + {.name = "vlen", .info = &prop_vlen}, + {.name = "elen", .info = &prop_elen}, + + {.name = "cbom_blocksize", .info = &prop_cbom_blksize}, + {.name = "cbop_blocksize", .info = &prop_cbop_blksize}, + {.name = "cboz_blocksize", .info = &prop_cboz_blksize}, + + {.name = "mvendorid", .info = &prop_mvendorid}, + {.name = "mimpid", .info = &prop_mimpid}, + {.name = "marchid", .info = &prop_marchid}, + #ifndef CONFIG_USER_ONLY DEFINE_PROP_UINT64("resetvec", RISCVCPU, env.resetvec, DEFAULT_RSTVEC), #endif @@ -1692,115 +2331,7 @@ static const struct SysemuCPUOps riscv_sysemu_ops = { }; #endif -static void cpu_set_mvendorid(Object *obj, Visitor *v, const char *name, - void *opaque, Error **errp) -{ - bool dynamic_cpu = riscv_cpu_is_dynamic(obj); - RISCVCPU *cpu = RISCV_CPU(obj); - uint32_t prev_val = cpu->cfg.mvendorid; - uint32_t value; - - if (!visit_type_uint32(v, name, &value, errp)) { - return; - } - - if (!dynamic_cpu && prev_val != value) { - error_setg(errp, "Unable to change %s mvendorid (0x%x)", - object_get_typename(obj), prev_val); - return; - } - - cpu->cfg.mvendorid = value; -} - -static void cpu_get_mvendorid(Object *obj, Visitor *v, const char *name, - void *opaque, Error **errp) -{ - uint32_t value = RISCV_CPU(obj)->cfg.mvendorid; - - visit_type_uint32(v, name, &value, errp); -} - -static void cpu_set_mimpid(Object *obj, Visitor *v, const char *name, - void *opaque, Error **errp) -{ - bool dynamic_cpu = riscv_cpu_is_dynamic(obj); - RISCVCPU *cpu = RISCV_CPU(obj); - uint64_t prev_val = cpu->cfg.mimpid; - uint64_t value; - - if (!visit_type_uint64(v, name, &value, errp)) { - return; - } - - if (!dynamic_cpu && prev_val != value) { - error_setg(errp, "Unable to change %s mimpid (0x%" PRIu64 ")", - object_get_typename(obj), prev_val); - return; - } - - cpu->cfg.mimpid = value; -} - -static void cpu_get_mimpid(Object *obj, Visitor *v, const char *name, - void *opaque, Error **errp) -{ - uint64_t value = RISCV_CPU(obj)->cfg.mimpid; - - visit_type_uint64(v, name, &value, errp); -} - -static void cpu_set_marchid(Object *obj, Visitor *v, const char *name, - void *opaque, Error **errp) -{ - bool dynamic_cpu = riscv_cpu_is_dynamic(obj); - RISCVCPU *cpu = RISCV_CPU(obj); - uint64_t prev_val = cpu->cfg.marchid; - uint64_t value, invalid_val; - uint32_t mxlen = 0; - - if (!visit_type_uint64(v, name, &value, errp)) { - return; - } - - if (!dynamic_cpu && prev_val != value) { - error_setg(errp, "Unable to change %s marchid (0x%" PRIu64 ")", - object_get_typename(obj), prev_val); - return; - } - - switch (riscv_cpu_mxl(&cpu->env)) { - case MXL_RV32: - mxlen = 32; - break; - case MXL_RV64: - case MXL_RV128: - mxlen = 64; - break; - default: - g_assert_not_reached(); - } - - invalid_val = 1LL << (mxlen - 1); - - if (value == invalid_val) { - error_setg(errp, "Unable to set marchid with MSB (%u) bit set " - "and the remaining bits zero", mxlen); - return; - } - - cpu->cfg.marchid = value; -} - -static void cpu_get_marchid(Object *obj, Visitor *v, const char *name, - void *opaque, Error **errp) -{ - uint64_t value = RISCV_CPU(obj)->cfg.marchid; - - visit_type_uint64(v, name, &value, errp); -} - -static void riscv_cpu_class_init(ObjectClass *c, void *data) +static void riscv_cpu_common_class_init(ObjectClass *c, void *data) { RISCVCPUClass *mcc = RISCV_CPU_CLASS(c); CPUClass *cc = CPU_CLASS(c); @@ -1831,16 +2362,15 @@ static void riscv_cpu_class_init(ObjectClass *c, void *data) cc->gdb_arch_name = riscv_gdb_arch_name; cc->gdb_get_dynamic_xml = riscv_gdb_get_dynamic_xml; - object_class_property_add(c, "mvendorid", "uint32", cpu_get_mvendorid, - cpu_set_mvendorid, NULL, NULL); - - object_class_property_add(c, "mimpid", "uint64", cpu_get_mimpid, - cpu_set_mimpid, NULL, NULL); + device_class_set_props(dc, riscv_cpu_properties); +} - object_class_property_add(c, "marchid", "uint64", cpu_get_marchid, - cpu_set_marchid, NULL, NULL); +static void riscv_cpu_class_init(ObjectClass *c, void *data) +{ + RISCVCPUClass *mcc = RISCV_CPU_CLASS(c); - device_class_set_props(dc, riscv_cpu_properties); + mcc->misa_mxl_max = (uint32_t)(uintptr_t)data; + riscv_cpu_validate_misa_mxl(mcc); } static void riscv_isa_string_ext(RISCVCPU *cpu, char **isa_str, @@ -1863,10 +2393,13 @@ static void riscv_isa_string_ext(RISCVCPU *cpu, char **isa_str, char *riscv_isa_string(RISCVCPU *cpu) { + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cpu); int i; const size_t maxlen = sizeof("rv128") + sizeof(riscv_single_letter_exts); char *isa_str = g_new(char, maxlen); - char *p = isa_str + snprintf(isa_str, maxlen, "rv%d", TARGET_LONG_BITS); + int xlen = riscv_cpu_max_xlen(mcc); + char *p = isa_str + snprintf(isa_str, maxlen, "rv%d", xlen); + for (i = 0; i < sizeof(riscv_single_letter_exts) - 1; i++) { if (cpu->env.misa_ext & RV(riscv_single_letter_exts[i])) { *p++ = qemu_tolower(riscv_single_letter_exts[i]); @@ -1879,39 +2412,102 @@ char *riscv_isa_string(RISCVCPU *cpu) return isa_str; } -#define DEFINE_CPU(type_name, initfn) \ - { \ - .name = type_name, \ - .parent = TYPE_RISCV_CPU, \ - .instance_init = initfn \ +#ifndef CONFIG_USER_ONLY +static char **riscv_isa_extensions_list(RISCVCPU *cpu, int *count) +{ + int maxlen = ARRAY_SIZE(riscv_single_letter_exts) + ARRAY_SIZE(isa_edata_arr); + char **extensions = g_new(char *, maxlen); + + for (int i = 0; i < sizeof(riscv_single_letter_exts) - 1; i++) { + if (cpu->env.misa_ext & RV(riscv_single_letter_exts[i])) { + extensions[*count] = g_new(char, 2); + snprintf(extensions[*count], 2, "%c", + qemu_tolower(riscv_single_letter_exts[i])); + (*count)++; + } + } + + for (const RISCVIsaExtData *edata = isa_edata_arr; edata->name; edata++) { + if (isa_ext_is_enabled(cpu, edata->ext_enable_offset)) { + extensions[*count] = g_strdup(edata->name); + (*count)++; + } + } + + return extensions; +} + +void riscv_isa_write_fdt(RISCVCPU *cpu, void *fdt, char *nodename) +{ + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cpu); + const size_t maxlen = sizeof("rv128i"); + g_autofree char *isa_base = g_new(char, maxlen); + g_autofree char *riscv_isa; + char **isa_extensions; + int count = 0; + int xlen = riscv_cpu_max_xlen(mcc); + + riscv_isa = riscv_isa_string(cpu); + qemu_fdt_setprop_string(fdt, nodename, "riscv,isa", riscv_isa); + + snprintf(isa_base, maxlen, "rv%di", xlen); + qemu_fdt_setprop_string(fdt, nodename, "riscv,isa-base", isa_base); + + isa_extensions = riscv_isa_extensions_list(cpu, &count); + qemu_fdt_setprop_string_array(fdt, nodename, "riscv,isa-extensions", + isa_extensions, count); + + for (int i = 0; i < count; i++) { + g_free(isa_extensions[i]); + } + + g_free(isa_extensions); +} +#endif + +#define DEFINE_CPU(type_name, misa_mxl_max, initfn) \ + { \ + .name = (type_name), \ + .parent = TYPE_RISCV_CPU, \ + .instance_init = (initfn), \ + .class_init = riscv_cpu_class_init, \ + .class_data = (void *)(misa_mxl_max) \ } -#define DEFINE_DYNAMIC_CPU(type_name, initfn) \ - { \ - .name = type_name, \ - .parent = TYPE_RISCV_DYNAMIC_CPU, \ - .instance_init = initfn \ +#define DEFINE_DYNAMIC_CPU(type_name, misa_mxl_max, initfn) \ + { \ + .name = (type_name), \ + .parent = TYPE_RISCV_DYNAMIC_CPU, \ + .instance_init = (initfn), \ + .class_init = riscv_cpu_class_init, \ + .class_data = (void *)(misa_mxl_max) \ } -#define DEFINE_VENDOR_CPU(type_name, initfn) \ - { \ - .name = type_name, \ - .parent = TYPE_RISCV_VENDOR_CPU, \ - .instance_init = initfn \ +#define DEFINE_VENDOR_CPU(type_name, misa_mxl_max, initfn) \ + { \ + .name = (type_name), \ + .parent = TYPE_RISCV_VENDOR_CPU, \ + .instance_init = (initfn), \ + .class_init = riscv_cpu_class_init, \ + .class_data = (void *)(misa_mxl_max) \ } -#define DEFINE_BARE_CPU(type_name, initfn) \ - { \ - .name = type_name, \ - .parent = TYPE_RISCV_BARE_CPU, \ - .instance_init = initfn \ +#define DEFINE_BARE_CPU(type_name, misa_mxl_max, initfn) \ + { \ + .name = (type_name), \ + .parent = TYPE_RISCV_BARE_CPU, \ + .instance_init = (initfn), \ + .class_init = riscv_cpu_class_init, \ + .class_data = (void *)(misa_mxl_max) \ } -#define DEFINE_PROFILE_CPU(type_name, initfn) \ - { \ - .name = type_name, \ - .parent = TYPE_RISCV_BARE_CPU, \ - .instance_init = initfn \ +#define DEFINE_PROFILE_CPU(type_name, misa_mxl_max, initfn) \ + { \ + .name = (type_name), \ + .parent = TYPE_RISCV_BARE_CPU, \ + .instance_init = (initfn), \ + .class_init = riscv_cpu_class_init, \ + .class_data = (void *)(misa_mxl_max) \ } static const TypeInfo riscv_cpu_type_infos[] = { @@ -1924,7 +2520,7 @@ static const TypeInfo riscv_cpu_type_infos[] = { .instance_post_init = riscv_cpu_post_init, .abstract = true, .class_size = sizeof(RISCVCPUClass), - .class_init = riscv_cpu_class_init, + .class_init = riscv_cpu_common_class_init, }, { .name = TYPE_RISCV_DYNAMIC_CPU, @@ -1939,27 +2535,33 @@ static const TypeInfo riscv_cpu_type_infos[] = { { .name = TYPE_RISCV_BARE_CPU, .parent = TYPE_RISCV_CPU, + .instance_init = riscv_bare_cpu_init, .abstract = true, }, - DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_ANY, riscv_any_cpu_init), - DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_MAX, riscv_max_cpu_init), #if defined(TARGET_RISCV32) - DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_BASE32, rv32_base_cpu_init), - DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_IBEX, rv32_ibex_cpu_init), - DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_E31, rv32_sifive_e_cpu_init), - DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_E34, rv32_imafcu_nommu_cpu_init), - DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_U34, rv32_sifive_u_cpu_init), + DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_ANY, MXL_RV32, riscv_any_cpu_init), + DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_MAX, MXL_RV32, riscv_max_cpu_init), + DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_BASE32, MXL_RV32, rv32_base_cpu_init), + DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_IBEX, MXL_RV32, rv32_ibex_cpu_init), + DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_E31, MXL_RV32, rv32_sifive_e_cpu_init), + DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_E34, MXL_RV32, rv32_imafcu_nommu_cpu_init), + DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_U34, MXL_RV32, rv32_sifive_u_cpu_init), + DEFINE_BARE_CPU(TYPE_RISCV_CPU_RV32I, MXL_RV32, rv32i_bare_cpu_init), + DEFINE_BARE_CPU(TYPE_RISCV_CPU_RV32E, MXL_RV32, rv32e_bare_cpu_init), #elif defined(TARGET_RISCV64) - DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_BASE64, rv64_base_cpu_init), - DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_E51, rv64_sifive_e_cpu_init), - DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_U54, rv64_sifive_u_cpu_init), - DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SHAKTI_C, rv64_sifive_u_cpu_init), - DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_THEAD_C906, rv64_thead_c906_cpu_init), - DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_VEYRON_V1, rv64_veyron_v1_cpu_init), - DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_BASE128, rv128_base_cpu_init), - DEFINE_BARE_CPU(TYPE_RISCV_CPU_RV64I, rv64i_bare_cpu_init), - DEFINE_PROFILE_CPU(TYPE_RISCV_CPU_RVA22U64, rva22u64_profile_cpu_init), - DEFINE_PROFILE_CPU(TYPE_RISCV_CPU_RVA22S64, rva22s64_profile_cpu_init), + DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_ANY, MXL_RV64, riscv_any_cpu_init), + DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_MAX, MXL_RV64, riscv_max_cpu_init), + DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_BASE64, MXL_RV64, rv64_base_cpu_init), + DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_E51, MXL_RV64, rv64_sifive_e_cpu_init), + DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SIFIVE_U54, MXL_RV64, rv64_sifive_u_cpu_init), + DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_SHAKTI_C, MXL_RV64, rv64_sifive_u_cpu_init), + DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_THEAD_C906, MXL_RV64, rv64_thead_c906_cpu_init), + DEFINE_VENDOR_CPU(TYPE_RISCV_CPU_VEYRON_V1, MXL_RV64, rv64_veyron_v1_cpu_init), + DEFINE_DYNAMIC_CPU(TYPE_RISCV_CPU_BASE128, MXL_RV128, rv128_base_cpu_init), + DEFINE_BARE_CPU(TYPE_RISCV_CPU_RV64I, MXL_RV64, rv64i_bare_cpu_init), + DEFINE_BARE_CPU(TYPE_RISCV_CPU_RV64E, MXL_RV64, rv64e_bare_cpu_init), + DEFINE_PROFILE_CPU(TYPE_RISCV_CPU_RVA22U64, MXL_RV64, rva22u64_profile_cpu_init), + DEFINE_PROFILE_CPU(TYPE_RISCV_CPU_RVA22S64, MXL_RV64, rva22s64_profile_cpu_init), #endif }; diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index f63ee9cc58..f52dce78ba 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -69,6 +69,7 @@ typedef struct CPUArchState CPURISCVState; #define RVH RV('H') #define RVJ RV('J') #define RVG RV('G') +#define RVB RV('B') extern const uint32_t misa_bits[]; const char *riscv_get_misa_ext_name(uint32_t bit); @@ -93,6 +94,9 @@ typedef struct riscv_cpu_profile { extern RISCVCPUProfile *riscv_profiles[]; /* Privileged specification version */ +#define PRIV_VER_1_10_0_STR "v1.10.0" +#define PRIV_VER_1_11_0_STR "v1.11.0" +#define PRIV_VER_1_12_0_STR "v1.12.0" enum { PRIV_VERSION_1_10_0 = 0, PRIV_VERSION_1_11_0, @@ -102,6 +106,7 @@ enum { }; #define VEXT_VERSION_1_00_0 0x00010000 +#define VEXT_VER_1_00_0_STR "v1.0" enum { TRANSLATE_SUCCESS, @@ -180,12 +185,10 @@ struct CPUArchState { target_ulong guest_phys_fault_addr; target_ulong priv_ver; - target_ulong bext_ver; target_ulong vext_ver; /* RISCVMXL, but uint32_t for vmstate migration */ uint32_t misa_mxl; /* current mxl */ - uint32_t misa_mxl_max; /* max mxl for this cpu */ uint32_t misa_ext; /* current extensions */ uint32_t misa_ext_mask; /* max ext for this cpu */ uint32_t xl; /* current xlen */ @@ -361,6 +364,7 @@ struct CPUArchState { target_ulong tdata1[RV_MAX_TRIGGERS]; target_ulong tdata2[RV_MAX_TRIGGERS]; target_ulong tdata3[RV_MAX_TRIGGERS]; + target_ulong mcontext; struct CPUBreakpoint *cpu_breakpoint[RV_MAX_TRIGGERS]; struct CPUWatchpoint *cpu_watchpoint[RV_MAX_TRIGGERS]; QEMUTimer *itrigger_timer[RV_MAX_TRIGGERS]; @@ -466,6 +470,7 @@ struct RISCVCPUClass { DeviceRealize parent_realize; ResettablePhases parent_phases; + uint32_t misa_mxl_max; /* max mxl for this cpu */ }; static inline int riscv_has_ext(CPURISCVState *env, target_ulong ext) @@ -506,8 +511,11 @@ bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size, MMUAccessType access_type, int mmu_idx, bool probe, uintptr_t retaddr); char *riscv_isa_string(RISCVCPU *cpu); +int riscv_cpu_max_xlen(RISCVCPUClass *mcc); +bool riscv_cpu_option_set(const char *optname); #ifndef CONFIG_USER_ONLY +void riscv_isa_write_fdt(RISCVCPU *cpu, void *fdt, char *nodename); void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr, vaddr addr, unsigned size, MMUAccessType access_type, @@ -682,11 +690,17 @@ static inline RISCVMXL riscv_cpu_sxl(CPURISCVState *env) * = 256 >> 7 * = 2 */ -static inline uint32_t vext_get_vlmax(RISCVCPU *cpu, target_ulong vtype) +static inline uint32_t vext_get_vlmax(uint32_t vlenb, uint32_t vsew, + int8_t lmul) { - uint8_t sew = FIELD_EX64(vtype, VTYPE, VSEW); - int8_t lmul = sextract32(FIELD_EX64(vtype, VTYPE, VLMUL), 0, 3); - return cpu->cfg.vlen >> (sew + 3 - lmul); + uint32_t vlen = vlenb << 3; + + /* + * We need to use 'vlen' instead of 'vlenb' to + * preserve the '+ 3' in the formula. Otherwise + * we risk a negative shift if vsew < lmul. + */ + return vlen >> (vsew + 3 - lmul); } void cpu_get_tb_cpu_state(CPURISCVState *env, vaddr *pc, @@ -769,7 +783,8 @@ enum riscv_pmu_event_idx { /* used by tcg/tcg-cpu.c*/ void isa_ext_update_enabled(RISCVCPU *cpu, uint32_t ext_offset, bool en); bool isa_ext_is_enabled(RISCVCPU *cpu, uint32_t ext_offset); -void riscv_cpu_set_misa(CPURISCVState *env, RISCVMXL mxl, uint32_t ext); +void riscv_cpu_set_misa_ext(CPURISCVState *env, uint32_t ext); +bool riscv_cpu_is_vendor(Object *cpu_obj); typedef struct RISCVCPUMultiExtConfig { const char *name; @@ -782,7 +797,6 @@ extern const RISCVCPUMultiExtConfig riscv_cpu_vendor_exts[]; extern const RISCVCPUMultiExtConfig riscv_cpu_experimental_exts[]; extern const RISCVCPUMultiExtConfig riscv_cpu_named_features[]; extern const RISCVCPUMultiExtConfig riscv_cpu_deprecated_exts[]; -extern Property riscv_cpu_options[]; typedef struct isa_ext_data { const char *name; diff --git a/target/riscv/cpu_bits.h b/target/riscv/cpu_bits.h index ebd7917d49..fc2068ee4d 100644 --- a/target/riscv/cpu_bits.h +++ b/target/riscv/cpu_bits.h @@ -32,14 +32,6 @@ #define FSR_NXA (FPEXC_NX << FSR_AEXC_SHIFT) #define FSR_AEXC (FSR_NVA | FSR_OFA | FSR_UFA | FSR_DZA | FSR_NXA) -/* Vector Fixed-Point round model */ -#define FSR_VXRM_SHIFT 9 -#define FSR_VXRM (0x3 << FSR_VXRM_SHIFT) - -/* Vector Fixed-Point saturation flag */ -#define FSR_VXSAT_SHIFT 8 -#define FSR_VXSAT (0x1 << FSR_VXSAT_SHIFT) - /* Control and Status Registers */ /* User Trap Setup */ @@ -361,6 +353,7 @@ #define CSR_TDATA2 0x7a2 #define CSR_TDATA3 0x7a3 #define CSR_TINFO 0x7a4 +#define CSR_MCONTEXT 0x7a8 /* Debug Mode Registers */ #define CSR_DCSR 0x7b0 @@ -905,4 +898,10 @@ typedef enum RISCVException { /* JVT CSR bits */ #define JVT_MODE 0x3F #define JVT_BASE (~0x3F) + +/* Debug Sdtrig CSR masks */ +#define MCONTEXT32 0x0000003F +#define MCONTEXT64 0x0000000000001FFFULL +#define MCONTEXT32_HCONTEXT 0x0000007F +#define MCONTEXT64_HCONTEXT 0x0000000000003FFFULL #endif diff --git a/target/riscv/cpu_cfg.h b/target/riscv/cpu_cfg.h index 780ae6ef17..833bf58217 100644 --- a/target/riscv/cpu_cfg.h +++ b/target/riscv/cpu_cfg.h @@ -78,7 +78,9 @@ struct RISCVCPUConfig { bool ext_svnapot; bool ext_svpbmt; bool ext_zdinx; + bool ext_zaamo; bool ext_zacas; + bool ext_zalrsc; bool ext_zawrs; bool ext_zfa; bool ext_zfbfmin; @@ -139,11 +141,7 @@ struct RISCVCPUConfig { bool ext_XVentanaCondOps; uint32_t pmu_mask; - char *priv_spec; - char *user_spec; - char *bext_spec; - char *vext_spec; - uint16_t vlen; + uint16_t vlenb; uint16_t elen; uint16_t cbom_blocksize; uint16_t cbop_blocksize; diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c index b6b23b7d03..d462d95ee1 100644 --- a/target/riscv/cpu_helper.c +++ b/target/riscv/cpu_helper.c @@ -81,13 +81,16 @@ void cpu_get_tb_cpu_state(CPURISCVState *env, vaddr *pc, * which is not supported by GVEC. So we set vl_eq_vlmax flag to true * only when maxsz >= 8 bytes. */ - uint32_t vlmax = vext_get_vlmax(cpu, env->vtype); - uint32_t sew = FIELD_EX64(env->vtype, VTYPE, VSEW); - uint32_t maxsz = vlmax << sew; + + /* lmul encoded as in DisasContext::lmul */ + int8_t lmul = sextract32(FIELD_EX64(env->vtype, VTYPE, VLMUL), 0, 3); + uint32_t vsew = FIELD_EX64(env->vtype, VTYPE, VSEW); + uint32_t vlmax = vext_get_vlmax(cpu->cfg.vlenb, vsew, lmul); + uint32_t maxsz = vlmax << vsew; bool vl_eq_vlmax = (env->vstart == 0) && (vlmax == env->vl) && (maxsz >= 8); flags = FIELD_DP32(flags, TB_FLAGS, VILL, env->vill); - flags = FIELD_DP32(flags, TB_FLAGS, SEW, sew); + flags = FIELD_DP32(flags, TB_FLAGS, SEW, vsew); flags = FIELD_DP32(flags, TB_FLAGS, LMUL, FIELD_EX64(env->vtype, VTYPE, VLMUL)); flags = FIELD_DP32(flags, TB_FLAGS, VL_EQ_VLMAX, vl_eq_vlmax); diff --git a/target/riscv/csr.c b/target/riscv/csr.c index 674ea075a4..d4e8ac13b9 100644 --- a/target/riscv/csr.c +++ b/target/riscv/csr.c @@ -242,7 +242,7 @@ static RISCVException any32(CPURISCVState *env, int csrno) } -static int aia_any(CPURISCVState *env, int csrno) +static RISCVException aia_any(CPURISCVState *env, int csrno) { if (!riscv_cpu_cfg(env)->ext_smaia) { return RISCV_EXCP_ILLEGAL_INST; @@ -251,7 +251,7 @@ static int aia_any(CPURISCVState *env, int csrno) return any(env, csrno); } -static int aia_any32(CPURISCVState *env, int csrno) +static RISCVException aia_any32(CPURISCVState *env, int csrno) { if (!riscv_cpu_cfg(env)->ext_smaia) { return RISCV_EXCP_ILLEGAL_INST; @@ -269,7 +269,7 @@ static RISCVException smode(CPURISCVState *env, int csrno) return RISCV_EXCP_ILLEGAL_INST; } -static int smode32(CPURISCVState *env, int csrno) +static RISCVException smode32(CPURISCVState *env, int csrno) { if (riscv_cpu_mxl(env) != MXL_RV32) { return RISCV_EXCP_ILLEGAL_INST; @@ -278,7 +278,7 @@ static int smode32(CPURISCVState *env, int csrno) return smode(env, csrno); } -static int aia_smode(CPURISCVState *env, int csrno) +static RISCVException aia_smode(CPURISCVState *env, int csrno) { if (!riscv_cpu_cfg(env)->ext_ssaia) { return RISCV_EXCP_ILLEGAL_INST; @@ -287,7 +287,7 @@ static int aia_smode(CPURISCVState *env, int csrno) return smode(env, csrno); } -static int aia_smode32(CPURISCVState *env, int csrno) +static RISCVException aia_smode32(CPURISCVState *env, int csrno) { if (!riscv_cpu_cfg(env)->ext_ssaia) { return RISCV_EXCP_ILLEGAL_INST; @@ -496,7 +496,7 @@ static RISCVException pointer_masking(CPURISCVState *env, int csrno) return RISCV_EXCP_ILLEGAL_INST; } -static int aia_hmode(CPURISCVState *env, int csrno) +static RISCVException aia_hmode(CPURISCVState *env, int csrno) { if (!riscv_cpu_cfg(env)->ext_ssaia) { return RISCV_EXCP_ILLEGAL_INST; @@ -505,7 +505,7 @@ static int aia_hmode(CPURISCVState *env, int csrno) return hmode(env, csrno); } -static int aia_hmode32(CPURISCVState *env, int csrno) +static RISCVException aia_hmode32(CPURISCVState *env, int csrno) { if (!riscv_cpu_cfg(env)->ext_ssaia) { return RISCV_EXCP_ILLEGAL_INST; @@ -681,9 +681,10 @@ static RISCVException read_vl(CPURISCVState *env, int csrno, return RISCV_EXCP_NONE; } -static int read_vlenb(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_vlenb(CPURISCVState *env, int csrno, + target_ulong *val) { - *val = riscv_cpu_cfg(env)->vlen >> 3; + *val = riscv_cpu_cfg(env)->vlenb; return RISCV_EXCP_NONE; } @@ -738,17 +739,19 @@ static RISCVException write_vstart(CPURISCVState *env, int csrno, * The vstart CSR is defined to have only enough writable bits * to hold the largest element index, i.e. lg2(VLEN) bits. */ - env->vstart = val & ~(~0ULL << ctzl(riscv_cpu_cfg(env)->vlen)); + env->vstart = val & ~(~0ULL << ctzl(riscv_cpu_cfg(env)->vlenb << 3)); return RISCV_EXCP_NONE; } -static int read_vcsr(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_vcsr(CPURISCVState *env, int csrno, + target_ulong *val) { *val = (env->vxrm << VCSR_VXRM_SHIFT) | (env->vxsat << VCSR_VXSAT_SHIFT); return RISCV_EXCP_NONE; } -static int write_vcsr(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_vcsr(CPURISCVState *env, int csrno, + target_ulong val) { #if !defined(CONFIG_USER_ONLY) env->mstatus |= MSTATUS_VS; @@ -798,13 +801,15 @@ static RISCVException read_timeh(CPURISCVState *env, int csrno, return RISCV_EXCP_NONE; } -static int read_hpmcounter(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_hpmcounter(CPURISCVState *env, int csrno, + target_ulong *val) { *val = get_ticks(false); return RISCV_EXCP_NONE; } -static int read_hpmcounterh(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_hpmcounterh(CPURISCVState *env, int csrno, + target_ulong *val) { *val = get_ticks(true); return RISCV_EXCP_NONE; @@ -812,7 +817,8 @@ static int read_hpmcounterh(CPURISCVState *env, int csrno, target_ulong *val) #else /* CONFIG_USER_ONLY */ -static int read_mhpmevent(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_mhpmevent(CPURISCVState *env, int csrno, + target_ulong *val) { int evt_index = csrno - CSR_MCOUNTINHIBIT; @@ -821,7 +827,8 @@ static int read_mhpmevent(CPURISCVState *env, int csrno, target_ulong *val) return RISCV_EXCP_NONE; } -static int write_mhpmevent(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_mhpmevent(CPURISCVState *env, int csrno, + target_ulong val) { int evt_index = csrno - CSR_MCOUNTINHIBIT; uint64_t mhpmevt_val = val; @@ -837,7 +844,8 @@ static int write_mhpmevent(CPURISCVState *env, int csrno, target_ulong val) return RISCV_EXCP_NONE; } -static int read_mhpmeventh(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_mhpmeventh(CPURISCVState *env, int csrno, + target_ulong *val) { int evt_index = csrno - CSR_MHPMEVENT3H + 3; @@ -846,7 +854,8 @@ static int read_mhpmeventh(CPURISCVState *env, int csrno, target_ulong *val) return RISCV_EXCP_NONE; } -static int write_mhpmeventh(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_mhpmeventh(CPURISCVState *env, int csrno, + target_ulong val) { int evt_index = csrno - CSR_MHPMEVENT3H + 3; uint64_t mhpmevth_val = val; @@ -860,7 +869,8 @@ static int write_mhpmeventh(CPURISCVState *env, int csrno, target_ulong val) return RISCV_EXCP_NONE; } -static int write_mhpmcounter(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_mhpmcounter(CPURISCVState *env, int csrno, + target_ulong val) { int ctr_idx = csrno - CSR_MCYCLE; PMUCTRState *counter = &env->pmu_ctrs[ctr_idx]; @@ -885,7 +895,8 @@ static int write_mhpmcounter(CPURISCVState *env, int csrno, target_ulong val) return RISCV_EXCP_NONE; } -static int write_mhpmcounterh(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_mhpmcounterh(CPURISCVState *env, int csrno, + target_ulong val) { int ctr_idx = csrno - CSR_MCYCLEH; PMUCTRState *counter = &env->pmu_ctrs[ctr_idx]; @@ -945,7 +956,8 @@ static RISCVException riscv_pmu_read_ctr(CPURISCVState *env, target_ulong *val, return RISCV_EXCP_NONE; } -static int read_hpmcounter(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_hpmcounter(CPURISCVState *env, int csrno, + target_ulong *val) { uint16_t ctr_index; @@ -960,7 +972,8 @@ static int read_hpmcounter(CPURISCVState *env, int csrno, target_ulong *val) return riscv_pmu_read_ctr(env, val, false, ctr_index); } -static int read_hpmcounterh(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_hpmcounterh(CPURISCVState *env, int csrno, + target_ulong *val) { uint16_t ctr_index; @@ -975,7 +988,8 @@ static int read_hpmcounterh(CPURISCVState *env, int csrno, target_ulong *val) return riscv_pmu_read_ctr(env, val, true, ctr_index); } -static int read_scountovf(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_scountovf(CPURISCVState *env, int csrno, + target_ulong *val) { int mhpmevt_start = CSR_MHPMEVENT3 - CSR_MCOUNTINHIBIT; int i; @@ -1638,7 +1652,8 @@ static RISCVException rmw_mvienh(CPURISCVState *env, int csrno, return ret; } -static int read_mtopi(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_mtopi(CPURISCVState *env, int csrno, + target_ulong *val) { int irq; uint8_t iprio; @@ -1678,8 +1693,9 @@ static int aia_xlate_vs_csrno(CPURISCVState *env, int csrno) }; } -static int rmw_xiselect(CPURISCVState *env, int csrno, target_ulong *val, - target_ulong new_val, target_ulong wr_mask) +static RISCVException rmw_xiselect(CPURISCVState *env, int csrno, + target_ulong *val, target_ulong new_val, + target_ulong wr_mask) { target_ulong *iselect; @@ -1758,8 +1774,9 @@ static int rmw_iprio(target_ulong xlen, return 0; } -static int rmw_xireg(CPURISCVState *env, int csrno, target_ulong *val, - target_ulong new_val, target_ulong wr_mask) +static RISCVException rmw_xireg(CPURISCVState *env, int csrno, + target_ulong *val, target_ulong new_val, + target_ulong wr_mask) { bool virt, isel_reserved; uint8_t *iprio; @@ -1833,8 +1850,9 @@ done: return RISCV_EXCP_NONE; } -static int rmw_xtopei(CPURISCVState *env, int csrno, target_ulong *val, - target_ulong new_val, target_ulong wr_mask) +static RISCVException rmw_xtopei(CPURISCVState *env, int csrno, + target_ulong *val, target_ulong new_val, + target_ulong wr_mask) { bool virt; int ret = -EINVAL; @@ -3031,7 +3049,8 @@ static RISCVException write_satp(CPURISCVState *env, int csrno, return RISCV_EXCP_NONE; } -static int read_vstopi(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_vstopi(CPURISCVState *env, int csrno, + target_ulong *val) { int irq, ret; target_ulong topei; @@ -3120,7 +3139,8 @@ static int read_vstopi(CPURISCVState *env, int csrno, target_ulong *val) return RISCV_EXCP_NONE; } -static int read_stopi(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_stopi(CPURISCVState *env, int csrno, + target_ulong *val) { int irq; uint8_t iprio; @@ -3576,19 +3596,21 @@ static RISCVException write_htimedeltah(CPURISCVState *env, int csrno, return RISCV_EXCP_NONE; } -static int read_hvictl(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_hvictl(CPURISCVState *env, int csrno, + target_ulong *val) { *val = env->hvictl; return RISCV_EXCP_NONE; } -static int write_hvictl(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_hvictl(CPURISCVState *env, int csrno, + target_ulong val) { env->hvictl = val & HVICTL_VALID_MASK; return RISCV_EXCP_NONE; } -static int read_hvipriox(CPURISCVState *env, int first_index, +static RISCVException read_hvipriox(CPURISCVState *env, int first_index, uint8_t *iprio, target_ulong *val) { int i, irq, rdzero, num_irqs = 4 * (riscv_cpu_mxl_bits(env) / 32); @@ -3614,7 +3636,7 @@ static int read_hvipriox(CPURISCVState *env, int first_index, return RISCV_EXCP_NONE; } -static int write_hvipriox(CPURISCVState *env, int first_index, +static RISCVException write_hvipriox(CPURISCVState *env, int first_index, uint8_t *iprio, target_ulong val) { int i, irq, rdzero, num_irqs = 4 * (riscv_cpu_mxl_bits(env) / 32); @@ -3640,42 +3662,50 @@ static int write_hvipriox(CPURISCVState *env, int first_index, return RISCV_EXCP_NONE; } -static int read_hviprio1(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_hviprio1(CPURISCVState *env, int csrno, + target_ulong *val) { return read_hvipriox(env, 0, env->hviprio, val); } -static int write_hviprio1(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_hviprio1(CPURISCVState *env, int csrno, + target_ulong val) { return write_hvipriox(env, 0, env->hviprio, val); } -static int read_hviprio1h(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_hviprio1h(CPURISCVState *env, int csrno, + target_ulong *val) { return read_hvipriox(env, 4, env->hviprio, val); } -static int write_hviprio1h(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_hviprio1h(CPURISCVState *env, int csrno, + target_ulong val) { return write_hvipriox(env, 4, env->hviprio, val); } -static int read_hviprio2(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_hviprio2(CPURISCVState *env, int csrno, + target_ulong *val) { return read_hvipriox(env, 8, env->hviprio, val); } -static int write_hviprio2(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_hviprio2(CPURISCVState *env, int csrno, + target_ulong val) { return write_hvipriox(env, 8, env->hviprio, val); } -static int read_hviprio2h(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_hviprio2h(CPURISCVState *env, int csrno, + target_ulong *val) { return read_hvipriox(env, 12, env->hviprio, val); } -static int write_hviprio2h(CPURISCVState *env, int csrno, target_ulong val) +static RISCVException write_hviprio2h(CPURISCVState *env, int csrno, + target_ulong val) { return write_hvipriox(env, 12, env->hviprio, val); } @@ -3699,7 +3729,8 @@ static RISCVException write_vsstatus(CPURISCVState *env, int csrno, return RISCV_EXCP_NONE; } -static int read_vstvec(CPURISCVState *env, int csrno, target_ulong *val) +static RISCVException read_vstvec(CPURISCVState *env, int csrno, + target_ulong *val) { *val = env->vstvec; return RISCV_EXCP_NONE; @@ -3906,6 +3937,31 @@ static RISCVException read_tinfo(CPURISCVState *env, int csrno, return RISCV_EXCP_NONE; } +static RISCVException read_mcontext(CPURISCVState *env, int csrno, + target_ulong *val) +{ + *val = env->mcontext; + return RISCV_EXCP_NONE; +} + +static RISCVException write_mcontext(CPURISCVState *env, int csrno, + target_ulong val) +{ + bool rv32 = riscv_cpu_mxl(env) == MXL_RV32 ? true : false; + int32_t mask; + + if (riscv_has_ext(env, RVH)) { + /* Spec suggest 7-bit for RV32 and 14-bit for RV64 w/ H extension */ + mask = rv32 ? MCONTEXT32_HCONTEXT : MCONTEXT64_HCONTEXT; + } else { + /* Spec suggest 6-bit for RV32 and 13-bit for RV64 w/o H extension */ + mask = rv32 ? MCONTEXT32 : MCONTEXT64; + } + + env->mcontext = val & mask; + return RISCV_EXCP_NONE; +} + /* * Functions to access Pointer Masking feature registers * We have to check if current priv lvl could modify @@ -4800,11 +4856,12 @@ riscv_csr_operations csr_ops[CSR_TABLE_SIZE] = { [CSR_PMPADDR15] = { "pmpaddr15", pmp, read_pmpaddr, write_pmpaddr }, /* Debug CSRs */ - [CSR_TSELECT] = { "tselect", debug, read_tselect, write_tselect }, - [CSR_TDATA1] = { "tdata1", debug, read_tdata, write_tdata }, - [CSR_TDATA2] = { "tdata2", debug, read_tdata, write_tdata }, - [CSR_TDATA3] = { "tdata3", debug, read_tdata, write_tdata }, - [CSR_TINFO] = { "tinfo", debug, read_tinfo, write_ignore }, + [CSR_TSELECT] = { "tselect", debug, read_tselect, write_tselect }, + [CSR_TDATA1] = { "tdata1", debug, read_tdata, write_tdata }, + [CSR_TDATA2] = { "tdata2", debug, read_tdata, write_tdata }, + [CSR_TDATA3] = { "tdata3", debug, read_tdata, write_tdata }, + [CSR_TINFO] = { "tinfo", debug, read_tinfo, write_ignore }, + [CSR_MCONTEXT] = { "mcontext", debug, read_mcontext, write_mcontext }, /* User Pointer Masking */ [CSR_UMTE] = { "umte", pointer_masking, read_umte, write_umte }, diff --git a/target/riscv/debug.c b/target/riscv/debug.c index 4945d1a1f2..e30d99cc2f 100644 --- a/target/riscv/debug.c +++ b/target/riscv/debug.c @@ -940,4 +940,6 @@ void riscv_trigger_reset_hold(CPURISCVState *env) env->cpu_watchpoint[i] = NULL; timer_del(env->itrigger_timer[i]); } + + env->mcontext = 0; } diff --git a/target/riscv/gdbstub.c b/target/riscv/gdbstub.c index 58b3ace0fe..ca9b71f7bb 100644 --- a/target/riscv/gdbstub.c +++ b/target/riscv/gdbstub.c @@ -49,6 +49,7 @@ static const struct TypeSize vec_lanes[] = { int riscv_cpu_gdb_read_register(CPUState *cs, GByteArray *mem_buf, int n) { + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cs); RISCVCPU *cpu = RISCV_CPU(cs); CPURISCVState *env = &cpu->env; target_ulong tmp; @@ -61,7 +62,7 @@ int riscv_cpu_gdb_read_register(CPUState *cs, GByteArray *mem_buf, int n) return 0; } - switch (env->misa_mxl_max) { + switch (mcc->misa_mxl_max) { case MXL_RV32: return gdb_get_reg32(mem_buf, tmp); case MXL_RV64: @@ -75,12 +76,13 @@ int riscv_cpu_gdb_read_register(CPUState *cs, GByteArray *mem_buf, int n) int riscv_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n) { + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cs); RISCVCPU *cpu = RISCV_CPU(cs); CPURISCVState *env = &cpu->env; int length = 0; target_ulong tmp; - switch (env->misa_mxl_max) { + switch (mcc->misa_mxl_max) { case MXL_RV32: tmp = (int32_t)ldl_p(mem_buf); length = 4; @@ -130,7 +132,7 @@ static int riscv_gdb_set_fpu(CPURISCVState *env, uint8_t *mem_buf, int n) static int riscv_gdb_get_vector(CPURISCVState *env, GByteArray *buf, int n) { - uint16_t vlenb = riscv_cpu_cfg(env)->vlen >> 3; + uint16_t vlenb = riscv_cpu_cfg(env)->vlenb; if (n < 32) { int i; int cnt = 0; @@ -146,7 +148,7 @@ static int riscv_gdb_get_vector(CPURISCVState *env, GByteArray *buf, int n) static int riscv_gdb_set_vector(CPURISCVState *env, uint8_t *mem_buf, int n) { - uint16_t vlenb = riscv_cpu_cfg(env)->vlen >> 3; + uint16_t vlenb = riscv_cpu_cfg(env)->vlenb; if (n < 32) { int i; for (i = 0; i < vlenb; i += 8) { @@ -214,11 +216,12 @@ static int riscv_gdb_set_virtual(CPURISCVState *cs, uint8_t *mem_buf, int n) static int riscv_gen_dynamic_csr_xml(CPUState *cs, int base_reg) { + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cs); RISCVCPU *cpu = RISCV_CPU(cs); CPURISCVState *env = &cpu->env; GString *s = g_string_new(NULL); riscv_csr_predicate_fn predicate; - int bitsize = 16 << env->misa_mxl_max; + int bitsize = riscv_cpu_max_xlen(mcc); int i; #if !defined(CONFIG_USER_ONLY) @@ -266,7 +269,7 @@ static int ricsv_gen_dynamic_vector_xml(CPUState *cs, int base_reg) RISCVCPU *cpu = RISCV_CPU(cs); GString *s = g_string_new(NULL); g_autoptr(GString) ts = g_string_new(""); - int reg_width = cpu->cfg.vlen; + int reg_width = cpu->cfg.vlenb << 3; int num_regs = 0; int i; @@ -310,6 +313,7 @@ static int ricsv_gen_dynamic_vector_xml(CPUState *cs, int base_reg) void riscv_cpu_register_gdb_regs_for_features(CPUState *cs) { + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cs); RISCVCPU *cpu = RISCV_CPU(cs); CPURISCVState *env = &cpu->env; if (env->misa_ext & RVD) { @@ -326,7 +330,7 @@ void riscv_cpu_register_gdb_regs_for_features(CPUState *cs) ricsv_gen_dynamic_vector_xml(cs, base_reg), "riscv-vector.xml", 0); } - switch (env->misa_mxl_max) { + switch (mcc->misa_mxl_max) { case MXL_RV32: gdb_register_coprocessor(cs, riscv_gdb_get_virtual, riscv_gdb_set_virtual, diff --git a/target/riscv/insn_trans/trans_rva.c.inc b/target/riscv/insn_trans/trans_rva.c.inc index 5f194a447b..267930e5bc 100644 --- a/target/riscv/insn_trans/trans_rva.c.inc +++ b/target/riscv/insn_trans/trans_rva.c.inc @@ -18,6 +18,18 @@ * this program. If not, see <http://www.gnu.org/licenses/>. */ +#define REQUIRE_A_OR_ZAAMO(ctx) do { \ + if (!ctx->cfg_ptr->ext_zaamo && !has_ext(ctx, RVA)) { \ + return false; \ + } \ +} while (0) + +#define REQUIRE_A_OR_ZALRSC(ctx) do { \ + if (!ctx->cfg_ptr->ext_zalrsc && !has_ext(ctx, RVA)) { \ + return false; \ + } \ +} while (0) + static bool gen_lr(DisasContext *ctx, arg_atomic *a, MemOp mop) { TCGv src1; @@ -96,132 +108,143 @@ static bool gen_amo(DisasContext *ctx, arg_atomic *a, static bool trans_lr_w(DisasContext *ctx, arg_lr_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZALRSC(ctx); return gen_lr(ctx, a, (MO_ALIGN | MO_TESL)); } static bool trans_sc_w(DisasContext *ctx, arg_sc_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZALRSC(ctx); return gen_sc(ctx, a, (MO_ALIGN | MO_TESL)); } static bool trans_amoswap_w(DisasContext *ctx, arg_amoswap_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_xchg_tl, (MO_ALIGN | MO_TESL)); } static bool trans_amoadd_w(DisasContext *ctx, arg_amoadd_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_add_tl, (MO_ALIGN | MO_TESL)); } static bool trans_amoxor_w(DisasContext *ctx, arg_amoxor_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_xor_tl, (MO_ALIGN | MO_TESL)); } static bool trans_amoand_w(DisasContext *ctx, arg_amoand_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_and_tl, (MO_ALIGN | MO_TESL)); } static bool trans_amoor_w(DisasContext *ctx, arg_amoor_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_or_tl, (MO_ALIGN | MO_TESL)); } static bool trans_amomin_w(DisasContext *ctx, arg_amomin_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_smin_tl, (MO_ALIGN | MO_TESL)); } static bool trans_amomax_w(DisasContext *ctx, arg_amomax_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_smax_tl, (MO_ALIGN | MO_TESL)); } static bool trans_amominu_w(DisasContext *ctx, arg_amominu_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_umin_tl, (MO_ALIGN | MO_TESL)); } static bool trans_amomaxu_w(DisasContext *ctx, arg_amomaxu_w *a) { - REQUIRE_EXT(ctx, RVA); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_umax_tl, (MO_ALIGN | MO_TESL)); } static bool trans_lr_d(DisasContext *ctx, arg_lr_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZALRSC(ctx); return gen_lr(ctx, a, MO_ALIGN | MO_TEUQ); } static bool trans_sc_d(DisasContext *ctx, arg_sc_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZALRSC(ctx); return gen_sc(ctx, a, (MO_ALIGN | MO_TEUQ)); } static bool trans_amoswap_d(DisasContext *ctx, arg_amoswap_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_xchg_tl, (MO_ALIGN | MO_TEUQ)); } static bool trans_amoadd_d(DisasContext *ctx, arg_amoadd_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_add_tl, (MO_ALIGN | MO_TEUQ)); } static bool trans_amoxor_d(DisasContext *ctx, arg_amoxor_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_xor_tl, (MO_ALIGN | MO_TEUQ)); } static bool trans_amoand_d(DisasContext *ctx, arg_amoand_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_and_tl, (MO_ALIGN | MO_TEUQ)); } static bool trans_amoor_d(DisasContext *ctx, arg_amoor_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_or_tl, (MO_ALIGN | MO_TEUQ)); } static bool trans_amomin_d(DisasContext *ctx, arg_amomin_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_smin_tl, (MO_ALIGN | MO_TEUQ)); } static bool trans_amomax_d(DisasContext *ctx, arg_amomax_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_smax_tl, (MO_ALIGN | MO_TEUQ)); } static bool trans_amominu_d(DisasContext *ctx, arg_amominu_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_umin_tl, (MO_ALIGN | MO_TEUQ)); } static bool trans_amomaxu_d(DisasContext *ctx, arg_amomaxu_d *a) { REQUIRE_64BIT(ctx); + REQUIRE_A_OR_ZAAMO(ctx); return gen_amo(ctx, a, &tcg_gen_atomic_fetch_umax_tl, (MO_ALIGN | MO_TEUQ)); } diff --git a/target/riscv/insn_trans/trans_rvbf16.c.inc b/target/riscv/insn_trans/trans_rvbf16.c.inc index 4e39c00884..8ee99df3f3 100644 --- a/target/riscv/insn_trans/trans_rvbf16.c.inc +++ b/target/riscv/insn_trans/trans_rvbf16.c.inc @@ -83,8 +83,8 @@ static bool trans_vfncvtbf16_f_f_w(DisasContext *ctx, arg_vfncvtbf16_f_f_w *a) data = FIELD_DP32(data, VDATA, VMA, ctx->vma); tcg_gen_gvec_3_ptr(vreg_ofs(ctx, a->rd), vreg_ofs(ctx, 0), vreg_ofs(ctx, a->rs2), tcg_env, - ctx->cfg_ptr->vlen / 8, - ctx->cfg_ptr->vlen / 8, data, + ctx->cfg_ptr->vlenb, + ctx->cfg_ptr->vlenb, data, gen_helper_vfncvtbf16_f_f_w); mark_vs_dirty(ctx); gen_set_label(over); @@ -112,8 +112,8 @@ static bool trans_vfwcvtbf16_f_f_v(DisasContext *ctx, arg_vfwcvtbf16_f_f_v *a) data = FIELD_DP32(data, VDATA, VMA, ctx->vma); tcg_gen_gvec_3_ptr(vreg_ofs(ctx, a->rd), vreg_ofs(ctx, 0), vreg_ofs(ctx, a->rs2), tcg_env, - ctx->cfg_ptr->vlen / 8, - ctx->cfg_ptr->vlen / 8, data, + ctx->cfg_ptr->vlenb, + ctx->cfg_ptr->vlenb, data, gen_helper_vfwcvtbf16_f_f_v); mark_vs_dirty(ctx); gen_set_label(over); @@ -143,8 +143,8 @@ static bool trans_vfwmaccbf16_vv(DisasContext *ctx, arg_vfwmaccbf16_vv *a) tcg_gen_gvec_4_ptr(vreg_ofs(ctx, a->rd), vreg_ofs(ctx, 0), vreg_ofs(ctx, a->rs1), vreg_ofs(ctx, a->rs2), tcg_env, - ctx->cfg_ptr->vlen / 8, - ctx->cfg_ptr->vlen / 8, data, + ctx->cfg_ptr->vlenb, + ctx->cfg_ptr->vlenb, data, gen_helper_vfwmaccbf16_vv); mark_vs_dirty(ctx); gen_set_label(over); diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index 3871f0ea73..9e101ab434 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -217,7 +217,7 @@ static bool trans_vsetivli(DisasContext *s, arg_vsetivli *a) /* vector register offset from env */ static uint32_t vreg_ofs(DisasContext *s, int reg) { - return offsetof(CPURISCVState, vreg) + reg * s->cfg_ptr->vlen / 8; + return offsetof(CPURISCVState, vreg) + reg * s->cfg_ptr->vlenb; } /* check functions */ @@ -627,11 +627,11 @@ static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data, * As simd_desc supports at most 2048 bytes, and in this implementation, * the max vector group length is 4096 bytes. So split it into two parts. * - * The first part is vlen in bytes, encoded in maxsz of simd_desc. + * The first part is vlen in bytes (vlenb), encoded in maxsz of simd_desc. * The second part is lmul, encoded in data of simd_desc. */ - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd)); tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0)); @@ -791,8 +791,8 @@ static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2, mask = tcg_temp_new_ptr(); base = get_gpr(s, rs1, EXT_NONE); stride = get_gpr(s, rs2, EXT_NONE); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd)); tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0)); @@ -897,8 +897,8 @@ static bool ldst_index_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, mask = tcg_temp_new_ptr(); index = tcg_temp_new_ptr(); base = get_gpr(s, rs1, EXT_NONE); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd)); tcg_gen_addi_ptr(index, tcg_env, vreg_ofs(s, vs2)); @@ -1036,8 +1036,8 @@ static bool ldff_trans(uint32_t vd, uint32_t rs1, uint32_t data, dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); base = get_gpr(s, rs1, EXT_NONE); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd)); tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0)); @@ -1086,7 +1086,7 @@ static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf, uint32_t width, gen_helper_ldst_whole *fn, DisasContext *s, bool is_store) { - uint32_t evl = (s->cfg_ptr->vlen / 8) * nf / width; + uint32_t evl = s->cfg_ptr->vlenb * nf / width; TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_GEU, cpu_vstart, evl, over); @@ -1096,8 +1096,8 @@ static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf, uint32_t data = FIELD_DP32(0, VDATA, NF, nf); dest = tcg_temp_new_ptr(); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); base = get_gpr(s, rs1, EXT_NONE); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd)); @@ -1160,12 +1160,12 @@ GEN_LDST_WHOLE_TRANS(vs8r_v, 8, 1, true) /* * MAXSZ returns the maximum vector size can be operated in bytes, * which is used in GVEC IR when vl_eq_vlmax flag is set to true - * to accerlate vector operation. + * to accelerate vector operation. */ static inline uint32_t MAXSZ(DisasContext *s) { - int scale = s->lmul - 3; - return s->cfg_ptr->vlen >> -scale; + int max_sz = s->cfg_ptr->vlenb * 8; + return max_sz >> (3 - s->lmul); } static bool opivv_check(DisasContext *s, arg_rmrr *a) @@ -1199,8 +1199,8 @@ do_opivv_gvec(DisasContext *s, arg_rmrr *a, GVecGen3Fn *gvec_fn, data = FIELD_DP32(data, VDATA, VMA, s->vma); tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2), - tcg_env, s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, fn); + tcg_env, s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fn); } mark_vs_dirty(s); gen_set_label(over); @@ -1248,8 +1248,8 @@ static bool opivx_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, uint32_t vm, data = FIELD_DP32(data, VDATA, VTA, s->vta); data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s); data = FIELD_DP32(data, VDATA, VMA, s->vma); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd)); tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, vs2)); @@ -1410,8 +1410,8 @@ static bool opivi_trans(uint32_t vd, uint32_t imm, uint32_t vs2, uint32_t vm, data = FIELD_DP32(data, VDATA, VTA, s->vta); data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s); data = FIELD_DP32(data, VDATA, VMA, s->vma); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd)); tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, vs2)); @@ -1492,8 +1492,8 @@ static bool do_opivv_widen(DisasContext *s, arg_rmrr *a, tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2), - tcg_env, s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, + tcg_env, s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fn); mark_vs_dirty(s); gen_set_label(over); @@ -1568,8 +1568,8 @@ static bool do_opiwv_widen(DisasContext *s, arg_rmrr *a, tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2), - tcg_env, s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, fn); + tcg_env, s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fn); mark_vs_dirty(s); gen_set_label(over); return true; @@ -1639,8 +1639,8 @@ static bool opivv_trans(uint32_t vd, uint32_t vs1, uint32_t vs2, uint32_t vm, data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s); data = FIELD_DP32(data, VDATA, VMA, s->vma); tcg_gen_gvec_4_ptr(vreg_ofs(s, vd), vreg_ofs(s, 0), vreg_ofs(s, vs1), - vreg_ofs(s, vs2), tcg_env, s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, fn); + vreg_ofs(s, vs2), tcg_env, s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fn); mark_vs_dirty(s); gen_set_label(over); return true; @@ -1831,8 +1831,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, data, \ + s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data, \ fns[s->sew]); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -2036,8 +2036,8 @@ static bool trans_vmv_v_v(DisasContext *s, arg_vmv_v_v *a) tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs1), - tcg_env, s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, + tcg_env, s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fns[s->sew]); gen_set_label(over); } @@ -2082,8 +2082,8 @@ static bool trans_vmv_v_x(DisasContext *s, arg_vmv_v_x *a) }; tcg_gen_ext_tl_i64(s1_i64, s1); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd)); fns[s->sew](dest, s1_i64, tcg_env, desc); } @@ -2121,8 +2121,8 @@ static bool trans_vmv_v_i(DisasContext *s, arg_vmv_v_i *a) s1 = tcg_constant_i64(simm); dest = tcg_temp_new_ptr(); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd)); fns[s->sew](dest, s1, tcg_env, desc); @@ -2275,8 +2275,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, data, \ + s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -2303,8 +2303,8 @@ static bool opfvf_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); src2 = tcg_temp_new_ptr(); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd)); tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, vs2)); @@ -2391,8 +2391,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, data, \ + s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -2465,8 +2465,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, data, \ + s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -2581,8 +2581,8 @@ static bool do_opfv(DisasContext *s, arg_rmr *a, data = FIELD_DP32(data, VDATA, VMA, s->vma); tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs2), tcg_env, - s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, fn); + s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fn); mark_vs_dirty(s); gen_set_label(over); return true; @@ -2691,8 +2691,8 @@ static bool trans_vfmv_v_f(DisasContext *s, arg_vfmv_v_f *a) do_nanbox(s, t1, cpu_fpr[a->rs1]); dest = tcg_temp_new_ptr(); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd)); fns[s->sew - 1](dest, t1, tcg_env, desc); @@ -2770,8 +2770,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ data = FIELD_DP32(data, VDATA, VMA, s->vma); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, data, \ + s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -2821,8 +2821,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ data = FIELD_DP32(data, VDATA, VMA, s->vma); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, data, \ + s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data, \ fns[s->sew]); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -2888,8 +2888,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ data = FIELD_DP32(data, VDATA, VMA, s->vma); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, data, \ + s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -2937,8 +2937,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ data = FIELD_DP32(data, VDATA, VMA, s->vma); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, data, \ + s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data, \ fns[s->sew]); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -3027,8 +3027,8 @@ static bool trans_##NAME(DisasContext *s, arg_r *a) \ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, data, fn); \ + s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data, fn); \ mark_vs_dirty(s); \ gen_set_label(over); \ return true; \ @@ -3061,8 +3061,8 @@ static bool trans_vcpop_m(DisasContext *s, arg_rmr *a) mask = tcg_temp_new_ptr(); src2 = tcg_temp_new_ptr(); dst = dest_gpr(s, a->rd); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, a->rs2)); tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0)); @@ -3090,8 +3090,8 @@ static bool trans_vfirst_m(DisasContext *s, arg_rmr *a) mask = tcg_temp_new_ptr(); src2 = tcg_temp_new_ptr(); dst = dest_gpr(s, a->rd); - desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data)); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, a->rs2)); tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0)); @@ -3128,8 +3128,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ data = FIELD_DP32(data, VDATA, VMA, s->vma); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), \ vreg_ofs(s, 0), vreg_ofs(s, a->rs2), \ - tcg_env, s->cfg_ptr->vlen / 8, \ - s->cfg_ptr->vlen / 8, \ + tcg_env, s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, \ data, fn); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -3171,8 +3171,8 @@ static bool trans_viota_m(DisasContext *s, arg_viota_m *a) }; tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs2), tcg_env, - s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, fns[s->sew]); + s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fns[s->sew]); mark_vs_dirty(s); gen_set_label(over); return true; @@ -3200,8 +3200,8 @@ static bool trans_vid_v(DisasContext *s, arg_vid_v *a) gen_helper_vid_v_w, gen_helper_vid_v_d, }; tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), - tcg_env, s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, + tcg_env, s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fns[s->sew]); mark_vs_dirty(s); gen_set_label(over); @@ -3535,8 +3535,7 @@ static bool trans_vrgather_vx(DisasContext *s, arg_rmrr *a) } if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { - int scale = s->lmul - (s->sew + 3); - int vlmax = s->cfg_ptr->vlen >> -scale; + int vlmax = vext_get_vlmax(s->cfg_ptr->vlenb, s->sew, s->lmul); TCGv_i64 dest = tcg_temp_new_i64(); if (a->rs1 == 0) { @@ -3566,8 +3565,7 @@ static bool trans_vrgather_vi(DisasContext *s, arg_rmrr *a) } if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { - int scale = s->lmul - (s->sew + 3); - int vlmax = s->cfg_ptr->vlen >> -scale; + int vlmax = vext_get_vlmax(s->cfg_ptr->vlenb, s->sew, s->lmul); if (a->rs1 >= vlmax) { tcg_gen_gvec_dup_imm(MO_64, vreg_ofs(s, a->rd), MAXSZ(s), MAXSZ(s), 0); @@ -3620,8 +3618,8 @@ static bool trans_vcompress_vm(DisasContext *s, arg_r *a) data = FIELD_DP32(data, VDATA, VTA, s->vta); tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2), - tcg_env, s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, + tcg_env, s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fns[s->sew]); mark_vs_dirty(s); gen_set_label(over); @@ -3641,7 +3639,7 @@ static bool trans_##NAME(DisasContext *s, arg_##NAME * a) \ vext_check_isa_ill(s) && \ QEMU_IS_ALIGNED(a->rd, LEN) && \ QEMU_IS_ALIGNED(a->rs2, LEN)) { \ - uint32_t maxsz = (s->cfg_ptr->vlen >> 3) * LEN; \ + uint32_t maxsz = s->cfg_ptr->vlenb * LEN; \ if (s->vstart_eq_zero) { \ tcg_gen_gvec_mov(s->sew, vreg_ofs(s, a->rd), \ vreg_ofs(s, a->rs2), maxsz, maxsz); \ @@ -3723,8 +3721,8 @@ static bool int_ext_op(DisasContext *s, arg_rmr *a, uint8_t seq) tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs2), tcg_env, - s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, fn); + s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, fn); mark_vs_dirty(s); gen_set_label(over); diff --git a/target/riscv/insn_trans/trans_rvvk.c.inc b/target/riscv/insn_trans/trans_rvvk.c.inc index 3801c16829..a5cdd1b67f 100644 --- a/target/riscv/insn_trans/trans_rvvk.c.inc +++ b/target/riscv/insn_trans/trans_rvvk.c.inc @@ -174,7 +174,7 @@ GEN_OPIVX_GVEC_TRANS_CHECK(vandn_vx, andcs, zvkb_vx_check) data = FIELD_DP32(data, VDATA, VMA, s->vma); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, s->cfg_ptr->vlen / 8, \ + s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, \ data, fns[s->sew]); \ mark_vs_dirty(s); \ gen_set_label(over); \ @@ -267,7 +267,7 @@ GEN_OPIVI_WIDEN_TRANS(vwsll_vi, IMM_ZX, vwsll_vx, vwsll_vx_check) rd_v = tcg_temp_new_ptr(); \ rs2_v = tcg_temp_new_ptr(); \ desc = tcg_constant_i32( \ - simd_desc(s->cfg_ptr->vlen / 8, s->cfg_ptr->vlen / 8, data)); \ + simd_desc(s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data)); \ tcg_gen_addi_ptr(rd_v, tcg_env, vreg_ofs(s, a->rd)); \ tcg_gen_addi_ptr(rs2_v, tcg_env, vreg_ofs(s, a->rs2)); \ gen_helper_##NAME(rd_v, rs2_v, tcg_env, desc); \ @@ -345,7 +345,7 @@ GEN_V_UNMASKED_TRANS(vaesem_vs, vaes_check_vs, ZVKNED_EGS) rs2_v = tcg_temp_new_ptr(); \ uimm_v = tcg_constant_i32(a->rs1); \ desc = tcg_constant_i32( \ - simd_desc(s->cfg_ptr->vlen / 8, s->cfg_ptr->vlen / 8, data)); \ + simd_desc(s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data)); \ tcg_gen_addi_ptr(rd_v, tcg_env, vreg_ofs(s, a->rd)); \ tcg_gen_addi_ptr(rs2_v, tcg_env, vreg_ofs(s, a->rs2)); \ gen_helper_##NAME(rd_v, rs2_v, uimm_v, tcg_env, desc); \ @@ -413,7 +413,7 @@ GEN_VI_UNMASKED_TRANS(vaeskf2_vi, vaeskf2_check, ZVKNED_EGS) \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), tcg_env, \ - s->cfg_ptr->vlen / 8, s->cfg_ptr->vlen / 8, \ + s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, \ data, gen_helper_##NAME); \ \ mark_vs_dirty(s); \ @@ -466,8 +466,8 @@ static bool trans_vsha2cl_vv(DisasContext *s, arg_rmrr *a) data = FIELD_DP32(data, VDATA, VMA, s->vma); tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs1), - vreg_ofs(s, a->rs2), tcg_env, s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, + vreg_ofs(s, a->rs2), tcg_env, s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, s->sew == MO_32 ? gen_helper_vsha2cl32_vv : gen_helper_vsha2cl64_vv); @@ -500,8 +500,8 @@ static bool trans_vsha2ch_vv(DisasContext *s, arg_rmrr *a) data = FIELD_DP32(data, VDATA, VMA, s->vma); tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs1), - vreg_ofs(s, a->rs2), tcg_env, s->cfg_ptr->vlen / 8, - s->cfg_ptr->vlen / 8, data, + vreg_ofs(s, a->rs2), tcg_env, s->cfg_ptr->vlenb, + s->cfg_ptr->vlenb, data, s->sew == MO_32 ? gen_helper_vsha2ch32_vv : gen_helper_vsha2ch64_vv); diff --git a/target/riscv/insn_trans/trans_xthead.c.inc b/target/riscv/insn_trans/trans_xthead.c.inc index dbb6411239..22488412d4 100644 --- a/target/riscv/insn_trans/trans_xthead.c.inc +++ b/target/riscv/insn_trans/trans_xthead.c.inc @@ -992,7 +992,6 @@ static bool trans_th_sfence_vmas(DisasContext *ctx, arg_th_sfence_vmas *a) #endif } -#ifndef CONFIG_USER_ONLY static void gen_th_sync_local(DisasContext *ctx) { /* @@ -1003,14 +1002,12 @@ static void gen_th_sync_local(DisasContext *ctx) tcg_gen_exit_tb(NULL, 0); ctx->base.is_jmp = DISAS_NORETURN; } -#endif static bool trans_th_sync(DisasContext *ctx, arg_th_sync *a) { (void) a; REQUIRE_XTHEADSYNC(ctx); -#ifndef CONFIG_USER_ONLY REQUIRE_PRIV_MSU(ctx); /* @@ -1019,9 +1016,6 @@ static bool trans_th_sync(DisasContext *ctx, arg_th_sync *a) gen_th_sync_local(ctx); return true; -#else - return false; -#endif } static bool trans_th_sync_i(DisasContext *ctx, arg_th_sync_i *a) @@ -1029,7 +1023,6 @@ static bool trans_th_sync_i(DisasContext *ctx, arg_th_sync_i *a) (void) a; REQUIRE_XTHEADSYNC(ctx); -#ifndef CONFIG_USER_ONLY REQUIRE_PRIV_MSU(ctx); /* @@ -1038,9 +1031,6 @@ static bool trans_th_sync_i(DisasContext *ctx, arg_th_sync_i *a) gen_th_sync_local(ctx); return true; -#else - return false; -#endif } static bool trans_th_sync_is(DisasContext *ctx, arg_th_sync_is *a) diff --git a/target/riscv/kvm/kvm-cpu.c b/target/riscv/kvm/kvm-cpu.c index 680a729cd8..422e4f121c 100644 --- a/target/riscv/kvm/kvm-cpu.c +++ b/target/riscv/kvm/kvm-cpu.c @@ -86,6 +86,27 @@ static uint64_t kvm_riscv_reg_id_u64(uint64_t type, uint64_t idx) return KVM_REG_RISCV | KVM_REG_SIZE_U64 | type | idx; } +static uint64_t kvm_encode_reg_size_id(uint64_t id, size_t size_b) +{ + uint64_t size_ctz = __builtin_ctz(size_b); + + return id | (size_ctz << KVM_REG_SIZE_SHIFT); +} + +static uint64_t kvm_riscv_vector_reg_id(RISCVCPU *cpu, + uint64_t idx) +{ + uint64_t id; + size_t size_b; + + g_assert(idx < 32); + + id = KVM_REG_RISCV | KVM_REG_RISCV_VECTOR | KVM_REG_RISCV_VECTOR_REG(idx); + size_b = cpu->cfg.vlenb; + + return kvm_encode_reg_size_id(id, size_b); +} + #define RISCV_CORE_REG(env, name) \ kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, \ KVM_REG_RISCV_CORE_REG(name)) @@ -145,7 +166,7 @@ typedef struct KVMCPUConfig { const char *name; const char *description; target_ulong offset; - int kvm_reg_id; + uint64_t kvm_reg_id; bool user_set; bool supported; } KVMCPUConfig; @@ -352,29 +373,12 @@ static KVMCPUConfig kvm_cboz_blocksize = { .kvm_reg_id = KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) }; -static void kvm_cpu_set_cbomz_blksize(Object *obj, Visitor *v, - const char *name, - void *opaque, Error **errp) -{ - KVMCPUConfig *cbomz_cfg = opaque; - RISCVCPU *cpu = RISCV_CPU(obj); - uint16_t value, *host_val; - - if (!visit_type_uint16(v, name, &value, errp)) { - return; - } - - host_val = kvmconfig_get_cfg_addr(cpu, cbomz_cfg); - - if (value != *host_val) { - error_report("Unable to set %s to a different value than " - "the host (%u)", - cbomz_cfg->name, *host_val); - exit(EXIT_FAILURE); - } - - cbomz_cfg->user_set = true; -} +static KVMCPUConfig kvm_v_vlenb = { + .name = "vlenb", + .offset = CPU_CFG_OFFSET(vlenb), + .kvm_reg_id = KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_VECTOR | + KVM_REG_RISCV_VECTOR_CSR_REG(vlenb) +}; static void kvm_riscv_update_cpu_cfg_isa_ext(RISCVCPU *cpu, CPUState *cs) { @@ -493,14 +497,6 @@ static void kvm_riscv_add_cpu_user_properties(Object *cpu_obj) NULL, multi_cfg); } - object_property_add(cpu_obj, "cbom_blocksize", "uint16", - NULL, kvm_cpu_set_cbomz_blksize, - NULL, &kvm_cbom_blocksize); - - object_property_add(cpu_obj, "cboz_blocksize", "uint16", - NULL, kvm_cpu_set_cbomz_blksize, - NULL, &kvm_cboz_blocksize); - riscv_cpu_add_kvm_unavail_prop_array(cpu_obj, riscv_cpu_extensions); riscv_cpu_add_kvm_unavail_prop_array(cpu_obj, riscv_cpu_vendor_exts); riscv_cpu_add_kvm_unavail_prop_array(cpu_obj, riscv_cpu_experimental_exts); @@ -716,9 +712,11 @@ static void kvm_riscv_put_regs_timer(CPUState *cs) static int kvm_riscv_get_regs_vector(CPUState *cs) { - CPURISCVState *env = &RISCV_CPU(cs)->env; + RISCVCPU *cpu = RISCV_CPU(cs); + CPURISCVState *env = &cpu->env; target_ulong reg; - int ret = 0; + uint64_t vreg_id; + int vreg_idx, ret = 0; if (!riscv_has_ext(env, RVV)) { return 0; @@ -742,14 +740,39 @@ static int kvm_riscv_get_regs_vector(CPUState *cs) } env->vtype = reg; + if (kvm_v_vlenb.supported) { + ret = kvm_get_one_reg(cs, RISCV_VECTOR_CSR_REG(env, vlenb), ®); + if (ret) { + return ret; + } + cpu->cfg.vlenb = reg; + + for (int i = 0; i < 32; i++) { + /* + * vreg[] is statically allocated using RV_VLEN_MAX. + * Use it instead of vlenb to calculate vreg_idx for + * simplicity. + */ + vreg_idx = i * RV_VLEN_MAX / 64; + vreg_id = kvm_riscv_vector_reg_id(cpu, i); + + ret = kvm_get_one_reg(cs, vreg_id, &env->vreg[vreg_idx]); + if (ret) { + return ret; + } + } + } + return 0; } static int kvm_riscv_put_regs_vector(CPUState *cs) { - CPURISCVState *env = &RISCV_CPU(cs)->env; + RISCVCPU *cpu = RISCV_CPU(cs); + CPURISCVState *env = &cpu->env; target_ulong reg; - int ret = 0; + uint64_t vreg_id; + int vreg_idx, ret = 0; if (!riscv_has_ext(env, RVV)) { return 0; @@ -769,6 +792,29 @@ static int kvm_riscv_put_regs_vector(CPUState *cs) reg = env->vtype; ret = kvm_set_one_reg(cs, RISCV_VECTOR_CSR_REG(env, vtype), ®); + if (ret) { + return ret; + } + + if (kvm_v_vlenb.supported) { + reg = cpu->cfg.vlenb; + ret = kvm_set_one_reg(cs, RISCV_VECTOR_CSR_REG(env, vlenb), ®); + + for (int i = 0; i < 32; i++) { + /* + * vreg[] is statically allocated using RV_VLEN_MAX. + * Use it instead of vlenb to calculate vreg_idx for + * simplicity. + */ + vreg_idx = i * RV_VLEN_MAX / 64; + vreg_id = kvm_riscv_vector_reg_id(cpu, i); + + ret = kvm_set_one_reg(cs, vreg_id, &env->vreg[vreg_idx]); + if (ret) { + return ret; + } + } + } return ret; } @@ -953,6 +999,33 @@ static int uint64_cmp(const void *a, const void *b) return 0; } +static void kvm_riscv_read_vlenb(RISCVCPU *cpu, KVMScratchCPU *kvmcpu, + struct kvm_reg_list *reglist) +{ + struct kvm_one_reg reg; + struct kvm_reg_list *reg_search; + uint64_t val; + int ret; + + reg_search = bsearch(&kvm_v_vlenb.kvm_reg_id, reglist->reg, reglist->n, + sizeof(uint64_t), uint64_cmp); + + if (reg_search) { + reg.id = kvm_v_vlenb.kvm_reg_id; + reg.addr = (uint64_t)&val; + + ret = ioctl(kvmcpu->cpufd, KVM_GET_ONE_REG, ®); + if (ret != 0) { + error_report("Unable to read vlenb register, error code: %s", + strerrorname_np(errno)); + exit(EXIT_FAILURE); + } + + kvm_v_vlenb.supported = true; + cpu->cfg.vlenb = val; + } +} + static void kvm_riscv_init_multiext_cfg(RISCVCPU *cpu, KVMScratchCPU *kvmcpu) { KVMCPUConfig *multi_ext_cfg; @@ -1027,6 +1100,10 @@ static void kvm_riscv_init_multiext_cfg(RISCVCPU *cpu, KVMScratchCPU *kvmcpu) if (cpu->cfg.ext_zicboz) { kvm_riscv_read_cbomz_blksize(cpu, kvmcpu, &kvm_cboz_blocksize); } + + if (riscv_has_ext(&cpu->env, RVV)) { + kvm_riscv_read_vlenb(cpu, kvmcpu, reglist); + } } static void riscv_init_kvm_registers(Object *cpu_obj) @@ -1559,19 +1636,10 @@ void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift, static void kvm_cpu_instance_init(CPUState *cs) { Object *obj = OBJECT(RISCV_CPU(cs)); - DeviceState *dev = DEVICE(obj); riscv_init_kvm_registers(obj); kvm_riscv_add_cpu_user_properties(obj); - - for (Property *prop = riscv_cpu_options; prop && prop->name; prop++) { - /* Check if we have a specific KVM handler for the option */ - if (object_property_find(obj, prop->name)) { - continue; - } - qdev_property_add_static(dev, prop); - } } /* @@ -1598,6 +1666,88 @@ static bool kvm_cpu_realize(CPUState *cs, Error **errp) return true; } +void riscv_kvm_cpu_finalize_features(RISCVCPU *cpu, Error **errp) +{ + CPURISCVState *env = &cpu->env; + KVMScratchCPU kvmcpu; + struct kvm_one_reg reg; + uint64_t val; + int ret; + + /* short-circuit without spinning the scratch CPU */ + if (!cpu->cfg.ext_zicbom && !cpu->cfg.ext_zicboz && + !riscv_has_ext(env, RVV)) { + return; + } + + if (!kvm_riscv_create_scratch_vcpu(&kvmcpu)) { + error_setg(errp, "Unable to create scratch KVM cpu"); + return; + } + + if (cpu->cfg.ext_zicbom && + riscv_cpu_option_set(kvm_cbom_blocksize.name)) { + + reg.id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CONFIG, + kvm_cbom_blocksize.kvm_reg_id); + reg.addr = (uint64_t)&val; + ret = ioctl(kvmcpu.cpufd, KVM_GET_ONE_REG, ®); + if (ret != 0) { + error_setg(errp, "Unable to read cbom_blocksize, error %d", errno); + return; + } + + if (cpu->cfg.cbom_blocksize != val) { + error_setg(errp, "Unable to set cbom_blocksize to a different " + "value than the host (%lu)", val); + return; + } + } + + if (cpu->cfg.ext_zicboz && + riscv_cpu_option_set(kvm_cboz_blocksize.name)) { + + reg.id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CONFIG, + kvm_cboz_blocksize.kvm_reg_id); + reg.addr = (uint64_t)&val; + ret = ioctl(kvmcpu.cpufd, KVM_GET_ONE_REG, ®); + if (ret != 0) { + error_setg(errp, "Unable to read cboz_blocksize, error %d", errno); + return; + } + + if (cpu->cfg.cboz_blocksize != val) { + error_setg(errp, "Unable to set cboz_blocksize to a different " + "value than the host (%lu)", val); + return; + } + } + + /* Users are setting vlen, not vlenb */ + if (riscv_has_ext(env, RVV) && riscv_cpu_option_set("vlen")) { + if (!kvm_v_vlenb.supported) { + error_setg(errp, "Unable to set 'vlenb': register not supported"); + return; + } + + reg.id = kvm_v_vlenb.kvm_reg_id; + reg.addr = (uint64_t)&val; + ret = ioctl(kvmcpu.cpufd, KVM_GET_ONE_REG, ®); + if (ret != 0) { + error_setg(errp, "Unable to read vlenb register, error %d", errno); + return; + } + + if (cpu->cfg.vlenb != val) { + error_setg(errp, "Unable to set 'vlen' to a different " + "value than the host (%lu)", val * 8); + return; + } + } + + kvm_riscv_destroy_scratch_vcpu(&kvmcpu); +} + static void kvm_cpu_accel_class_init(ObjectClass *oc, void *data) { AccelCPUClass *acc = ACCEL_CPU_CLASS(oc); @@ -1619,14 +1769,14 @@ static void kvm_cpu_accel_register_types(void) } type_init(kvm_cpu_accel_register_types); -static void riscv_host_cpu_init(Object *obj) +static void riscv_host_cpu_class_init(ObjectClass *c, void *data) { - CPURISCVState *env = &RISCV_CPU(obj)->env; + RISCVCPUClass *mcc = RISCV_CPU_CLASS(c); #if defined(TARGET_RISCV32) - env->misa_mxl_max = env->misa_mxl = MXL_RV32; + mcc->misa_mxl_max = MXL_RV32; #elif defined(TARGET_RISCV64) - env->misa_mxl_max = env->misa_mxl = MXL_RV64; + mcc->misa_mxl_max = MXL_RV64; #endif } @@ -1634,7 +1784,7 @@ static const TypeInfo riscv_kvm_cpu_type_infos[] = { { .name = TYPE_RISCV_CPU_HOST, .parent = TYPE_RISCV_CPU, - .instance_init = riscv_host_cpu_init, + .class_init = riscv_host_cpu_class_init, } }; diff --git a/target/riscv/kvm/kvm_riscv.h b/target/riscv/kvm/kvm_riscv.h index 8329cfab82..4bd98fddc7 100644 --- a/target/riscv/kvm/kvm_riscv.h +++ b/target/riscv/kvm/kvm_riscv.h @@ -27,5 +27,6 @@ void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift, uint64_t guest_num); void riscv_kvm_aplic_request(void *opaque, int irq, int level); int kvm_riscv_sync_mpstate_to_kvm(RISCVCPU *cpu, int state); +void riscv_kvm_cpu_finalize_features(RISCVCPU *cpu, Error **errp); #endif diff --git a/target/riscv/machine.c b/target/riscv/machine.c index 72fe2374dc..81cf22894e 100644 --- a/target/riscv/machine.c +++ b/target/riscv/machine.c @@ -178,10 +178,9 @@ static const VMStateDescription vmstate_pointermasking = { static bool rv128_needed(void *opaque) { - RISCVCPU *cpu = opaque; - CPURISCVState *env = &cpu->env; + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(opaque); - return env->misa_mxl_max == MXL_RV128; + return mcc->misa_mxl_max == MXL_RV128; } static const VMStateDescription vmstate_rv128 = { @@ -372,7 +371,7 @@ const VMStateDescription vmstate_riscv_cpu = { VMSTATE_UINTTL(env.vext_ver, RISCVCPU), VMSTATE_UINT32(env.misa_mxl, RISCVCPU), VMSTATE_UINT32(env.misa_ext, RISCVCPU), - VMSTATE_UINT32(env.misa_mxl_max, RISCVCPU), + VMSTATE_UNUSED(4), VMSTATE_UINT32(env.misa_ext_mask, RISCVCPU), VMSTATE_UINTTL(env.priv, RISCVCPU), VMSTATE_BOOL(env.virt_enabled, RISCVCPU), diff --git a/target/riscv/tcg/tcg-cpu.c b/target/riscv/tcg/tcg-cpu.c index b7da92783b..dd5228c288 100644 --- a/target/riscv/tcg/tcg-cpu.c +++ b/target/riscv/tcg/tcg-cpu.c @@ -268,97 +268,24 @@ static void riscv_cpu_validate_misa_priv(CPURISCVState *env, Error **errp) } } -static void riscv_cpu_validate_misa_mxl(RISCVCPU *cpu, Error **errp) -{ - RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cpu); - CPUClass *cc = CPU_CLASS(mcc); - CPURISCVState *env = &cpu->env; - - /* Validate that MISA_MXL is set properly. */ - switch (env->misa_mxl_max) { -#ifdef TARGET_RISCV64 - case MXL_RV64: - case MXL_RV128: - cc->gdb_core_xml_file = "riscv-64bit-cpu.xml"; - break; -#endif - case MXL_RV32: - cc->gdb_core_xml_file = "riscv-32bit-cpu.xml"; - break; - default: - g_assert_not_reached(); - } - - if (env->misa_mxl_max != env->misa_mxl) { - error_setg(errp, "misa_mxl_max must be equal to misa_mxl"); - return; - } -} - -static void riscv_cpu_validate_priv_spec(RISCVCPU *cpu, Error **errp) -{ - CPURISCVState *env = &cpu->env; - int priv_version = -1; - - if (cpu->cfg.priv_spec) { - if (!g_strcmp0(cpu->cfg.priv_spec, "v1.12.0")) { - priv_version = PRIV_VERSION_1_12_0; - } else if (!g_strcmp0(cpu->cfg.priv_spec, "v1.11.0")) { - priv_version = PRIV_VERSION_1_11_0; - } else if (!g_strcmp0(cpu->cfg.priv_spec, "v1.10.0")) { - priv_version = PRIV_VERSION_1_10_0; - } else { - error_setg(errp, - "Unsupported privilege spec version '%s'", - cpu->cfg.priv_spec); - return; - } - - env->priv_ver = priv_version; - } -} - static void riscv_cpu_validate_v(CPURISCVState *env, RISCVCPUConfig *cfg, Error **errp) { - if (!is_power_of_2(cfg->vlen)) { - error_setg(errp, "Vector extension VLEN must be power of 2"); - return; - } + uint32_t vlen = cfg->vlenb << 3; - if (cfg->vlen > RV_VLEN_MAX || cfg->vlen < 128) { + if (vlen > RV_VLEN_MAX || vlen < 128) { error_setg(errp, "Vector extension implementation only supports VLEN " "in the range [128, %d]", RV_VLEN_MAX); return; } - if (!is_power_of_2(cfg->elen)) { - error_setg(errp, "Vector extension ELEN must be power of 2"); - return; - } - if (cfg->elen > 64 || cfg->elen < 8) { error_setg(errp, "Vector extension implementation only supports ELEN " "in the range [8, 64]"); return; } - - if (cfg->vext_spec) { - if (!g_strcmp0(cfg->vext_spec, "v1.0")) { - env->vext_ver = VEXT_VERSION_1_00_0; - } else { - error_setg(errp, "Unsupported vector spec version '%s'", - cfg->vext_spec); - return; - } - } else if (env->vext_ver == 0) { - qemu_log("vector version is not specified, " - "use the default value v1.0\n"); - - env->vext_ver = VEXT_VERSION_1_00_0; - } } static void riscv_cpu_disable_priv_spec_isa_exts(RISCVCPU *cpu) @@ -442,12 +369,42 @@ static void riscv_cpu_validate_g(RISCVCPU *cpu) } } +static void riscv_cpu_validate_b(RISCVCPU *cpu) +{ + const char *warn_msg = "RVB mandates disabled extension %s"; + + if (!cpu->cfg.ext_zba) { + if (!cpu_cfg_ext_is_user_set(CPU_CFG_OFFSET(ext_zba))) { + cpu->cfg.ext_zba = true; + } else { + warn_report(warn_msg, "zba"); + } + } + + if (!cpu->cfg.ext_zbb) { + if (!cpu_cfg_ext_is_user_set(CPU_CFG_OFFSET(ext_zbb))) { + cpu->cfg.ext_zbb = true; + } else { + warn_report(warn_msg, "zbb"); + } + } + + if (!cpu->cfg.ext_zbs) { + if (!cpu_cfg_ext_is_user_set(CPU_CFG_OFFSET(ext_zbs))) { + cpu->cfg.ext_zbs = true; + } else { + warn_report(warn_msg, "zbs"); + } + } +} + /* * Check consistency between chosen extensions while setting * cpu->cfg accordingly. */ void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) { + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cpu); CPURISCVState *env = &cpu->env; Error *local_err = NULL; @@ -455,6 +412,10 @@ void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) riscv_cpu_validate_g(cpu); } + if (riscv_has_ext(env, RVB)) { + riscv_cpu_validate_b(cpu); + } + if (riscv_has_ext(env, RVI) && riscv_has_ext(env, RVE)) { error_setg(errp, "I and E extensions are incompatible"); @@ -610,7 +571,7 @@ void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) cpu_cfg_ext_auto_update(cpu, CPU_CFG_OFFSET(ext_zcb), true); cpu_cfg_ext_auto_update(cpu, CPU_CFG_OFFSET(ext_zcmp), true); cpu_cfg_ext_auto_update(cpu, CPU_CFG_OFFSET(ext_zcmt), true); - if (riscv_has_ext(env, RVF) && env->misa_mxl_max == MXL_RV32) { + if (riscv_has_ext(env, RVF) && mcc->misa_mxl_max == MXL_RV32) { cpu_cfg_ext_auto_update(cpu, CPU_CFG_OFFSET(ext_zcf), true); } } @@ -618,7 +579,7 @@ void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) /* zca, zcd and zcf has a PRIV 1.12.0 restriction */ if (riscv_has_ext(env, RVC) && env->priv_ver >= PRIV_VERSION_1_12_0) { cpu_cfg_ext_auto_update(cpu, CPU_CFG_OFFSET(ext_zca), true); - if (riscv_has_ext(env, RVF) && env->misa_mxl_max == MXL_RV32) { + if (riscv_has_ext(env, RVF) && mcc->misa_mxl_max == MXL_RV32) { cpu_cfg_ext_auto_update(cpu, CPU_CFG_OFFSET(ext_zcf), true); } if (riscv_has_ext(env, RVD)) { @@ -626,7 +587,7 @@ void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) } } - if (env->misa_mxl_max != MXL_RV32 && cpu->cfg.ext_zcf) { + if (mcc->misa_mxl_max != MXL_RV32 && cpu->cfg.ext_zcf) { error_setg(errp, "Zcf extension is only relevant to RV32"); return; } @@ -876,12 +837,6 @@ void riscv_tcg_cpu_finalize_features(RISCVCPU *cpu, Error **errp) CPURISCVState *env = &cpu->env; Error *local_err = NULL; - riscv_cpu_validate_priv_spec(cpu, &local_err); - if (local_err != NULL) { - error_propagate(errp, local_err); - return; - } - riscv_cpu_validate_misa_priv(env, &local_err); if (local_err != NULL) { error_propagate(errp, local_err); @@ -917,11 +872,6 @@ static bool riscv_cpu_is_generic(Object *cpu_obj) return object_dynamic_cast(cpu_obj, TYPE_RISCV_DYNAMIC_CPU) != NULL; } -static bool riscv_cpu_is_vendor(Object *cpu_obj) -{ - return object_dynamic_cast(cpu_obj, TYPE_RISCV_VENDOR_CPU) != NULL; -} - /* * We'll get here via the following path: * @@ -932,7 +882,6 @@ static bool riscv_cpu_is_vendor(Object *cpu_obj) static bool riscv_tcg_cpu_realize(CPUState *cs, Error **errp) { RISCVCPU *cpu = RISCV_CPU(cs); - Error *local_err = NULL; if (!riscv_cpu_tcg_compatible(cpu)) { g_autofree char *name = riscv_cpu_get_name(cpu); @@ -941,14 +890,9 @@ static bool riscv_tcg_cpu_realize(CPUState *cs, Error **errp) return false; } - riscv_cpu_validate_misa_mxl(cpu, &local_err); - if (local_err != NULL) { - error_propagate(errp, local_err); - return false; - } - #ifndef CONFIG_USER_ONLY CPURISCVState *env = &cpu->env; + Error *local_err = NULL; CPU(cs)->tcg_cflags |= CF_PCREL; @@ -1056,6 +1000,7 @@ static const RISCVCPUMisaExtConfig misa_ext_cfgs[] = { MISA_CFG(RVJ, false), MISA_CFG(RVV, false), MISA_CFG(RVG, false), + MISA_CFG(RVB, false), }; /* @@ -1326,10 +1271,6 @@ static void riscv_cpu_add_user_properties(Object *obj) riscv_cpu_add_multiext_prop_array(obj, riscv_cpu_deprecated_exts); riscv_cpu_add_profiles(obj); - - for (Property *prop = riscv_cpu_options; prop && prop->name; prop++) { - qdev_property_add_static(DEVICE(obj), prop); - } } /* @@ -1343,7 +1284,7 @@ static void riscv_init_max_cpu_extensions(Object *obj) const RISCVCPUMultiExtConfig *prop; /* Enable RVG, RVJ and RVV that are disabled by default */ - riscv_cpu_set_misa(env, env->misa_mxl, env->misa_ext | RVG | RVJ | RVV); + riscv_cpu_set_misa_ext(env, env->misa_ext | RVG | RVJ | RVV); for (prop = riscv_cpu_extensions; prop && prop->name; prop++) { isa_ext_update_enabled(cpu, prop->offset, true); diff --git a/target/riscv/translate.c b/target/riscv/translate.c index ab18899122..177418b2b9 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -1168,6 +1168,7 @@ static void riscv_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) { DisasContext *ctx = container_of(dcbase, DisasContext, base); CPURISCVState *env = cpu_env(cs); + RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cs); RISCVCPU *cpu = RISCV_CPU(cs); uint32_t tb_flags = ctx->base.tb->flags; @@ -1189,7 +1190,7 @@ static void riscv_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) ctx->cfg_vta_all_1s = cpu->cfg.rvv_ta_all_1s; ctx->vstart_eq_zero = FIELD_EX32(tb_flags, TB_FLAGS, VSTART_EQ_ZERO); ctx->vl_eq_vlmax = FIELD_EX32(tb_flags, TB_FLAGS, VL_EQ_VLMAX); - ctx->misa_mxl_max = env->misa_mxl_max; + ctx->misa_mxl_max = mcc->misa_mxl_max; ctx->xl = FIELD_EX32(tb_flags, TB_FLAGS, XL); ctx->address_xl = FIELD_EX32(tb_flags, TB_FLAGS, AXL); ctx->cs = cs; diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index fe0d5d053c..84cec73eb2 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -35,19 +35,28 @@ target_ulong HELPER(vsetvl)(CPURISCVState *env, target_ulong s1, { int vlmax, vl; RISCVCPU *cpu = env_archcpu(env); - uint64_t lmul = FIELD_EX64(s2, VTYPE, VLMUL); - uint16_t sew = 8 << FIELD_EX64(s2, VTYPE, VSEW); + uint64_t vlmul = FIELD_EX64(s2, VTYPE, VLMUL); + uint8_t vsew = FIELD_EX64(s2, VTYPE, VSEW); + uint16_t sew = 8 << vsew; uint8_t ediv = FIELD_EX64(s2, VTYPE, VEDIV); int xlen = riscv_cpu_xlen(env); bool vill = (s2 >> (xlen - 1)) & 0x1; target_ulong reserved = s2 & MAKE_64BIT_MASK(R_VTYPE_RESERVED_SHIFT, xlen - 1 - R_VTYPE_RESERVED_SHIFT); + int8_t lmul; - if (lmul & 4) { - /* Fractional LMUL - check LMUL * VLEN >= SEW */ - if (lmul == 4 || - cpu->cfg.vlen >> (8 - lmul) < sew) { + if (vlmul & 4) { + /* + * Fractional LMUL, check: + * + * VLEN * LMUL >= SEW + * VLEN >> (8 - lmul) >= sew + * (vlenb << 3) >> (8 - lmul) >= sew + * vlenb >> (8 - 3 - lmul) >= sew + */ + if (vlmul == 4 || + cpu->cfg.vlenb >> (8 - 3 - vlmul) < sew) { vill = true; } } @@ -61,7 +70,9 @@ target_ulong HELPER(vsetvl)(CPURISCVState *env, target_ulong s1, return 0; } - vlmax = vext_get_vlmax(cpu, s2); + /* lmul encoded as in DisasContext::lmul */ + lmul = sextract32(FIELD_EX64(s2, VTYPE, VLMUL), 0, 3); + vlmax = vext_get_vlmax(cpu->cfg.vlenb, vsew, lmul); if (s1 <= vlmax) { vl = s1; } else { @@ -559,7 +570,7 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, { uint32_t i, k, off, pos; uint32_t nf = vext_nf(desc); - uint32_t vlenb = riscv_cpu_cfg(env)->vlen >> 3; + uint32_t vlenb = riscv_cpu_cfg(env)->vlenb; uint32_t max_elems = vlenb >> log2_esz; k = env->vstart / max_elems; @@ -930,7 +941,7 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ { \ uint32_t vl = env->vl; \ uint32_t vm = vext_vm(desc); \ - uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ @@ -968,7 +979,7 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ { \ uint32_t vl = env->vl; \ uint32_t vm = vext_vm(desc); \ - uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ @@ -1172,7 +1183,7 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ - uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ @@ -1237,7 +1248,7 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ - uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ @@ -3972,7 +3983,7 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ - uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ @@ -4012,7 +4023,7 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ - uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ @@ -4529,7 +4540,7 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ uint32_t desc) \ { \ uint32_t vl = env->vl; \ - uint32_t total_elems = riscv_cpu_cfg(env)->vlen; \ + uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3;\ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ int a, b; \ @@ -4616,7 +4627,7 @@ static void vmsetm(void *vd, void *v0, void *vs2, CPURISCVState *env, { uint32_t vm = vext_vm(desc); uint32_t vl = env->vl; - uint32_t total_elems = riscv_cpu_cfg(env)->vlen; + uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; uint32_t vta_all_1s = vext_vta_all_1s(desc); uint32_t vma = vext_vma(desc); int i; diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index 6e4e1f2082..0d0c672c95 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -754,10 +754,10 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask) case CC_OP_TM_64: switch (mask) { case 8: - cond = TCG_COND_EQ; + cond = TCG_COND_TSTEQ; break; case 4 | 2 | 1: - cond = TCG_COND_NE; + cond = TCG_COND_TSTNE; break; default: goto do_dynamic; @@ -768,11 +768,11 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask) case CC_OP_ICM: switch (mask) { case 8: - cond = TCG_COND_EQ; + cond = TCG_COND_TSTEQ; break; case 4 | 2 | 1: case 4 | 2: - cond = TCG_COND_NE; + cond = TCG_COND_TSTNE; break; default: goto do_dynamic; @@ -854,18 +854,14 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask) c->u.s64.a = cc_dst; c->u.s64.b = tcg_constant_i64(0); break; + case CC_OP_LTGT_64: case CC_OP_LTUGTU_64: - c->u.s64.a = cc_src; - c->u.s64.b = cc_dst; - break; - case CC_OP_TM_32: case CC_OP_TM_64: case CC_OP_ICM: - c->u.s64.a = tcg_temp_new_i64(); - c->u.s64.b = tcg_constant_i64(0); - tcg_gen_and_i64(c->u.s64.a, cc_src, cc_dst); + c->u.s64.a = cc_src; + c->u.s64.b = cc_dst; break; case CC_OP_ADDU: @@ -889,67 +885,45 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask) case CC_OP_STATIC: c->is_64 = false; c->u.s32.a = cc_op; - switch (mask) { - case 0x8 | 0x4 | 0x2: /* cc != 3 */ - cond = TCG_COND_NE; + + /* Fold half of the cases using bit 3 to invert. */ + switch (mask & 8 ? mask ^ 0xf : mask) { + case 0x1: /* cc == 3 */ + cond = TCG_COND_EQ; c->u.s32.b = tcg_constant_i32(3); break; - case 0x8 | 0x4 | 0x1: /* cc != 2 */ - cond = TCG_COND_NE; - c->u.s32.b = tcg_constant_i32(2); - break; - case 0x8 | 0x2 | 0x1: /* cc != 1 */ - cond = TCG_COND_NE; - c->u.s32.b = tcg_constant_i32(1); - break; - case 0x8 | 0x2: /* cc == 0 || cc == 2 => (cc & 1) == 0 */ + case 0x2: /* cc == 2 */ cond = TCG_COND_EQ; - c->u.s32.a = tcg_temp_new_i32(); - c->u.s32.b = tcg_constant_i32(0); - tcg_gen_andi_i32(c->u.s32.a, cc_op, 1); - break; - case 0x8 | 0x4: /* cc < 2 */ - cond = TCG_COND_LTU; c->u.s32.b = tcg_constant_i32(2); break; - case 0x8: /* cc == 0 */ - cond = TCG_COND_EQ; - c->u.s32.b = tcg_constant_i32(0); - break; - case 0x4 | 0x2 | 0x1: /* cc != 0 */ - cond = TCG_COND_NE; - c->u.s32.b = tcg_constant_i32(0); - break; - case 0x4 | 0x1: /* cc == 1 || cc == 3 => (cc & 1) != 0 */ - cond = TCG_COND_NE; - c->u.s32.a = tcg_temp_new_i32(); - c->u.s32.b = tcg_constant_i32(0); - tcg_gen_andi_i32(c->u.s32.a, cc_op, 1); - break; case 0x4: /* cc == 1 */ cond = TCG_COND_EQ; c->u.s32.b = tcg_constant_i32(1); break; - case 0x2 | 0x1: /* cc > 1 */ + case 0x2 | 0x1: /* cc == 2 || cc == 3 => cc > 1 */ cond = TCG_COND_GTU; c->u.s32.b = tcg_constant_i32(1); break; - case 0x2: /* cc == 2 */ - cond = TCG_COND_EQ; - c->u.s32.b = tcg_constant_i32(2); + case 0x4 | 0x1: /* cc == 1 || cc == 3 => (cc & 1) != 0 */ + cond = TCG_COND_TSTNE; + c->u.s32.b = tcg_constant_i32(1); break; - case 0x1: /* cc == 3 */ - cond = TCG_COND_EQ; - c->u.s32.b = tcg_constant_i32(3); + case 0x4 | 0x2: /* cc == 1 || cc == 2 => (cc - 1) <= 1 */ + cond = TCG_COND_LEU; + c->u.s32.a = tcg_temp_new_i32(); + c->u.s32.b = tcg_constant_i32(1); + tcg_gen_addi_i32(c->u.s32.a, cc_op, -1); break; - default: - /* CC is masked by something else: (8 >> cc) & mask. */ + case 0x4 | 0x2 | 0x1: /* cc != 0 */ cond = TCG_COND_NE; - c->u.s32.a = tcg_temp_new_i32(); c->u.s32.b = tcg_constant_i32(0); - tcg_gen_shr_i32(c->u.s32.a, tcg_constant_i32(8), cc_op); - tcg_gen_andi_i32(c->u.s32.a, c->u.s32.a, mask); break; + default: + /* case 0: never, handled above. */ + g_assert_not_reached(); + } + if (mask & 8) { + cond = tcg_invert_cond(cond); } break; diff --git a/target/sparc/translate.c b/target/sparc/translate.c index 7df6f83b13..d9304a5bc4 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -488,6 +488,7 @@ static void gen_op_subccc(TCGv dst, TCGv src1, TCGv src2) static void gen_op_mulscc(TCGv dst, TCGv src1, TCGv src2) { TCGv zero = tcg_constant_tl(0); + TCGv one = tcg_constant_tl(1); TCGv t_src1 = tcg_temp_new(); TCGv t_src2 = tcg_temp_new(); TCGv t0 = tcg_temp_new(); @@ -499,8 +500,7 @@ static void gen_op_mulscc(TCGv dst, TCGv src1, TCGv src2) * if (!(env->y & 1)) * src2 = 0; */ - tcg_gen_andi_tl(t0, cpu_y, 0x1); - tcg_gen_movcond_tl(TCG_COND_EQ, t_src2, t0, zero, zero, t_src2); + tcg_gen_movcond_tl(TCG_COND_TSTEQ, t_src2, cpu_y, one, zero, t_src2); /* * b2 = src1 & 1; diff --git a/tcg/aarch64/tcg-target-con-set.h b/tcg/aarch64/tcg-target-con-set.h index 3fdee26a3d..44fcc1206e 100644 --- a/tcg/aarch64/tcg-target-con-set.h +++ b/tcg/aarch64/tcg-target-con-set.h @@ -10,7 +10,7 @@ * tcg-target-con-str.h; the constraint combination is inclusive or. */ C_O0_I1(r) -C_O0_I2(r, rA) +C_O0_I2(r, rC) C_O0_I2(rZ, r) C_O0_I2(w, r) C_O0_I3(rZ, rZ, r) @@ -22,6 +22,7 @@ C_O1_I2(r, 0, rZ) C_O1_I2(r, r, r) C_O1_I2(r, r, rA) C_O1_I2(r, r, rAL) +C_O1_I2(r, r, rC) C_O1_I2(r, r, ri) C_O1_I2(r, r, rL) C_O1_I2(r, rZ, rZ) @@ -31,6 +32,6 @@ C_O1_I2(w, w, wN) C_O1_I2(w, w, wO) C_O1_I2(w, w, wZ) C_O1_I3(w, w, w, w) -C_O1_I4(r, r, rA, rZ, rZ) +C_O1_I4(r, r, rC, rZ, rZ) C_O2_I1(r, r, r) C_O2_I4(r, r, rZ, rZ, rA, rMZ) diff --git a/tcg/aarch64/tcg-target-con-str.h b/tcg/aarch64/tcg-target-con-str.h index fb1a845b4f..48e1722c68 100644 --- a/tcg/aarch64/tcg-target-con-str.h +++ b/tcg/aarch64/tcg-target-con-str.h @@ -16,6 +16,7 @@ REGS('w', ALL_VECTOR_REGS) * CONST(letter, TCG_CT_CONST_* bit set) */ CONST('A', TCG_CT_CONST_AIMM) +CONST('C', TCG_CT_CONST_CMP) CONST('L', TCG_CT_CONST_LIMM) CONST('M', TCG_CT_CONST_MONE) CONST('O', TCG_CT_CONST_ORRI) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index a3efa1e67a..dec8ecc1b6 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -105,6 +105,18 @@ static bool reloc_pc19(tcg_insn_unit *src_rw, const tcg_insn_unit *target) return false; } +static bool reloc_pc14(tcg_insn_unit *src_rw, const tcg_insn_unit *target) +{ + const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); + ptrdiff_t offset = target - src_rx; + + if (offset == sextract64(offset, 0, 14)) { + *src_rw = deposit32(*src_rw, 5, 14, offset); + return true; + } + return false; +} + static bool patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend) { @@ -115,6 +127,8 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, return reloc_pc26(code_ptr, (const tcg_insn_unit *)value); case R_AARCH64_CONDBR19: return reloc_pc19(code_ptr, (const tcg_insn_unit *)value); + case R_AARCH64_TSTBR14: + return reloc_pc14(code_ptr, (const tcg_insn_unit *)value); default: g_assert_not_reached(); } @@ -126,6 +140,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, #define TCG_CT_CONST_MONE 0x800 #define TCG_CT_CONST_ORRI 0x1000 #define TCG_CT_CONST_ANDI 0x2000 +#define TCG_CT_CONST_CMP 0x4000 #define ALL_GENERAL_REGS 0xffffffffu #define ALL_VECTOR_REGS 0xffffffff00000000ull @@ -270,7 +285,8 @@ static bool is_shimm1632(uint32_t v32, int *cmode, int *imm8) } } -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -278,6 +294,15 @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) if (type == TCG_TYPE_I32) { val = (int32_t)val; } + + if (ct & TCG_CT_CONST_CMP) { + if (is_tst_cond(cond)) { + ct |= TCG_CT_CONST_LIMM; + } else { + ct |= TCG_CT_CONST_AIMM; + } + } + if ((ct & TCG_CT_CONST_AIMM) && (is_aimm(val) || is_aimm(-val))) { return 1; } @@ -344,6 +369,9 @@ static const enum aarch64_cond_code tcg_cond_to_aarch64[] = { [TCG_COND_GTU] = COND_HI, [TCG_COND_GEU] = COND_HS, [TCG_COND_LEU] = COND_LS, + /* bit test */ + [TCG_COND_TSTEQ] = COND_EQ, + [TCG_COND_TSTNE] = COND_NE, }; typedef enum { @@ -366,6 +394,10 @@ typedef enum { /* Conditional branch (immediate). */ I3202_B_C = 0x54000000, + /* Test and branch (immediate). */ + I3205_TBZ = 0x36000000, + I3205_TBNZ = 0x37000000, + /* Unconditional branch (immediate). */ I3206_B = 0x14000000, I3206_BL = 0x94000000, @@ -646,6 +678,14 @@ static void tcg_out_insn_3202(TCGContext *s, AArch64Insn insn, tcg_out32(s, insn | tcg_cond_to_aarch64[c] | (imm19 & 0x7ffff) << 5); } +static void tcg_out_insn_3205(TCGContext *s, AArch64Insn insn, + TCGReg rt, int imm6, int imm14) +{ + insn |= (imm6 & 0x20) << (31 - 5); + insn |= (imm6 & 0x1f) << 19; + tcg_out32(s, insn | (imm14 & 0x3fff) << 5 | rt); +} + static void tcg_out_insn_3206(TCGContext *s, AArch64Insn insn, int imm26) { tcg_out32(s, insn | (imm26 & 0x03ffffff)); @@ -1341,19 +1381,26 @@ static inline void tcg_out_dep(TCGContext *s, TCGType ext, TCGReg rd, tcg_out_bfm(s, ext, rd, rn, a, b); } -static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a, +static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGCond cond, TCGReg a, tcg_target_long b, bool const_b) { - if (const_b) { - /* Using CMP or CMN aliases. */ - if (b >= 0) { + if (is_tst_cond(cond)) { + if (!const_b) { + tcg_out_insn(s, 3510, ANDS, ext, TCG_REG_XZR, a, b); + } else { + tcg_debug_assert(is_limm(b)); + tcg_out_logicali(s, I3404_ANDSI, 0, TCG_REG_XZR, a, b); + } + } else { + if (!const_b) { + tcg_out_insn(s, 3502, SUBS, ext, TCG_REG_XZR, a, b); + } else if (b >= 0) { + tcg_debug_assert(is_aimm(b)); tcg_out_insn(s, 3401, SUBSI, ext, TCG_REG_XZR, a, b); } else { + tcg_debug_assert(is_aimm(-b)); tcg_out_insn(s, 3401, ADDSI, ext, TCG_REG_XZR, a, -b); } - } else { - /* Using CMP alias SUBS wzr, Wn, Wm */ - tcg_out_insn(s, 3502, SUBS, ext, TCG_REG_XZR, a, b); } } @@ -1394,30 +1441,75 @@ static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l) static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a, TCGArg b, bool b_const, TCGLabel *l) { - intptr_t offset; - bool need_cmp; + int tbit = -1; + bool need_cmp = true; - if (b_const && b == 0 && (c == TCG_COND_EQ || c == TCG_COND_NE)) { - need_cmp = false; - } else { - need_cmp = true; - tcg_out_cmp(s, ext, a, b, b_const); + switch (c) { + case TCG_COND_EQ: + case TCG_COND_NE: + /* cmp xN,0; b.ne L -> cbnz xN,L */ + if (b_const && b == 0) { + need_cmp = false; + } + break; + case TCG_COND_LT: + case TCG_COND_GE: + /* cmp xN,0; b.mi L -> tbnz xN,63,L */ + if (b_const && b == 0) { + c = (c == TCG_COND_LT ? TCG_COND_TSTNE : TCG_COND_TSTEQ); + tbit = ext ? 63 : 31; + need_cmp = false; + } + break; + case TCG_COND_TSTEQ: + case TCG_COND_TSTNE: + /* tst xN,0xffffffff; b.ne L -> cbnz wN,L */ + if (b_const && b == UINT32_MAX) { + ext = TCG_TYPE_I32; + need_cmp = false; + break; + } + /* tst xN,1<<B; b.ne L -> tbnz xN,B,L */ + if (b_const && is_power_of_2(b)) { + tbit = ctz64(b); + need_cmp = false; + } + break; + default: + break; } - if (!l->has_value) { + if (need_cmp) { + tcg_out_cmp(s, ext, c, a, b, b_const); tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0); - offset = tcg_in32(s) >> 5; - } else { - offset = tcg_pcrel_diff(s, l->u.value_ptr) >> 2; - tcg_debug_assert(offset == sextract64(offset, 0, 19)); + tcg_out_insn(s, 3202, B_C, c, 0); + return; } - if (need_cmp) { - tcg_out_insn(s, 3202, B_C, c, offset); - } else if (c == TCG_COND_EQ) { - tcg_out_insn(s, 3201, CBZ, ext, a, offset); + if (tbit >= 0) { + tcg_out_reloc(s, s->code_ptr, R_AARCH64_TSTBR14, l, 0); + switch (c) { + case TCG_COND_TSTEQ: + tcg_out_insn(s, 3205, TBZ, a, tbit, 0); + break; + case TCG_COND_TSTNE: + tcg_out_insn(s, 3205, TBNZ, a, tbit, 0); + break; + default: + g_assert_not_reached(); + } } else { - tcg_out_insn(s, 3201, CBNZ, ext, a, offset); + tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0); + switch (c) { + case TCG_COND_EQ: + tcg_out_insn(s, 3201, CBZ, ext, a, 0); + break; + case TCG_COND_NE: + tcg_out_insn(s, 3201, CBNZ, ext, a, 0); + break; + default: + g_assert_not_reached(); + } } } @@ -1574,7 +1666,7 @@ static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d, } else { AArch64Insn sel = I3506_CSEL; - tcg_out_cmp(s, ext, a0, 0, 1); + tcg_out_cmp(s, ext, TCG_COND_NE, a0, 0, 1); tcg_out_insn(s, 3507, CLZ, ext, TCG_REG_TMP0, a1); if (const_b) { @@ -1719,7 +1811,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, addr_adj, compare_mask); /* Perform the address comparison. */ - tcg_out_cmp(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP2, 0); + tcg_out_cmp(s, addr_type, TCG_COND_NE, TCG_REG_TMP0, TCG_REG_TMP2, 0); /* If not equal, we jump to the slow path. */ ldst->label_ptr[0] = s->code_ptr; @@ -2275,7 +2367,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, a2 = (int32_t)a2; /* FALLTHRU */ case INDEX_op_setcond_i64: - tcg_out_cmp(s, ext, a1, a2, c2); + tcg_out_cmp(s, ext, args[3], a1, a2, c2); /* Use CSET alias of CSINC Wd, WZR, WZR, invert(cond). */ tcg_out_insn(s, 3506, CSINC, TCG_TYPE_I32, a0, TCG_REG_XZR, TCG_REG_XZR, tcg_invert_cond(args[3])); @@ -2285,7 +2377,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, a2 = (int32_t)a2; /* FALLTHRU */ case INDEX_op_negsetcond_i64: - tcg_out_cmp(s, ext, a1, a2, c2); + tcg_out_cmp(s, ext, args[3], a1, a2, c2); /* Use CSETM alias of CSINV Wd, WZR, WZR, invert(cond). */ tcg_out_insn(s, 3506, CSINV, ext, a0, TCG_REG_XZR, TCG_REG_XZR, tcg_invert_cond(args[3])); @@ -2295,7 +2387,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, a2 = (int32_t)a2; /* FALLTHRU */ case INDEX_op_movcond_i64: - tcg_out_cmp(s, ext, a1, a2, c2); + tcg_out_cmp(s, ext, args[5], a1, a2, c2); tcg_out_insn(s, 3506, CSEL, ext, a0, REG0(3), REG0(4), args[5]); break; @@ -2895,11 +2987,13 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_add_i64: case INDEX_op_sub_i32: case INDEX_op_sub_i64: + return C_O1_I2(r, r, rA); + case INDEX_op_setcond_i32: case INDEX_op_setcond_i64: case INDEX_op_negsetcond_i32: case INDEX_op_negsetcond_i64: - return C_O1_I2(r, r, rA); + return C_O1_I2(r, r, rC); case INDEX_op_mul_i32: case INDEX_op_mul_i64: @@ -2949,11 +3043,11 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - return C_O0_I2(r, rA); + return C_O0_I2(r, rC); case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: - return C_O1_I4(r, r, rA, rZ, rZ); + return C_O1_I4(r, r, rC, rZ, rZ); case INDEX_op_qemu_ld_a32_i32: case INDEX_op_qemu_ld_a64_i32: diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index 33f15a564a..ef5ebe91bd 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -138,6 +138,8 @@ typedef enum { #define TCG_TARGET_HAS_qemu_ldst_i128 1 #endif +#define TCG_TARGET_HAS_tst 1 + #define TCG_TARGET_HAS_v64 1 #define TCG_TARGET_HAS_v128 1 #define TCG_TARGET_HAS_v256 0 diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index a9aa8aa91c..ffd23ef789 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -501,7 +501,8 @@ static bool is_shimm1632(uint32_t v32, int *cmode, int *imm8) * mov operand2: values represented with x << (2 * y), x < 0x100 * add, sub, eor...: ditto */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -1190,6 +1191,33 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) } } +static TCGCond tcg_out_cmp(TCGContext *s, TCGCond cond, TCGReg a, + TCGArg b, int b_const) +{ + if (!is_tst_cond(cond)) { + tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0, a, b, b_const); + return cond; + } + + cond = tcg_tst_eqne_cond(cond); + if (b_const) { + int imm12 = encode_imm(b); + + /* + * The compare constraints allow rIN, but TST does not support N. + * Be prepared to load the constant into a scratch register. + */ + if (imm12 >= 0) { + tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, a, imm12); + return cond; + } + tcg_out_movi32(s, COND_AL, TCG_REG_TMP, b); + b = TCG_REG_TMP; + } + tcg_out_dat_reg(s, COND_AL, ARITH_TST, 0, a, b, SHIFT_IMM_LSL(0)); + return cond; +} + static TCGCond tcg_out_cmp2(TCGContext *s, const TCGArg *args, const int *const_args) { @@ -1217,6 +1245,13 @@ static TCGCond tcg_out_cmp2(TCGContext *s, const TCGArg *args, tcg_out_dat_rI(s, COND_EQ, ARITH_CMP, 0, al, bl, const_bl); return cond; + case TCG_COND_TSTEQ: + case TCG_COND_TSTNE: + /* Similar, but with TST instead of CMP. */ + tcg_out_dat_rI(s, COND_AL, ARITH_TST, 0, ah, bh, const_bh); + tcg_out_dat_rI(s, COND_EQ, ARITH_TST, 0, al, bl, const_bl); + return tcg_tst_eqne_cond(cond); + case TCG_COND_LT: case TCG_COND_GE: /* We perform a double-word subtraction and examine the result. @@ -1808,9 +1843,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, /* Constraints mean that v2 is always in the same register as dest, * so we only need to do "if condition passed, move v1 to dest". */ - tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0, - args[1], args[2], const_args[2]); - tcg_out_dat_rIK(s, tcg_cond_to_arm_cond[args[5]], ARITH_MOV, + c = tcg_out_cmp(s, args[5], args[1], args[2], const_args[2]); + tcg_out_dat_rIK(s, tcg_cond_to_arm_cond[c], ARITH_MOV, ARITH_MVN, args[0], 0, args[3], const_args[3]); break; case INDEX_op_add_i32: @@ -1960,25 +1994,21 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_brcond_i32: - tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0, - args[0], args[1], const_args[1]); - tcg_out_goto_label(s, tcg_cond_to_arm_cond[args[2]], - arg_label(args[3])); + c = tcg_out_cmp(s, args[2], args[0], args[1], const_args[1]); + tcg_out_goto_label(s, tcg_cond_to_arm_cond[c], arg_label(args[3])); break; case INDEX_op_setcond_i32: - tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0, - args[1], args[2], const_args[2]); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[args[3]], + c = tcg_out_cmp(s, args[3], args[1], args[2], const_args[2]); + tcg_out_dat_imm(s, tcg_cond_to_arm_cond[c], ARITH_MOV, args[0], 0, 1); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(args[3])], + tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(c)], ARITH_MOV, args[0], 0, 0); break; case INDEX_op_negsetcond_i32: - tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0, - args[1], args[2], const_args[2]); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[args[3]], + c = tcg_out_cmp(s, args[3], args[1], args[2], const_args[2]); + tcg_out_dat_imm(s, tcg_cond_to_arm_cond[c], ARITH_MVN, args[0], 0, 0); - tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(args[3])], + tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(c)], ARITH_MOV, args[0], 0, 0); break; diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index a712cc80ad..a43875cb09 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -125,6 +125,8 @@ extern bool use_neon_instructions; #define TCG_TARGET_HAS_qemu_ldst_i128 0 +#define TCG_TARGET_HAS_tst 1 + #define TCG_TARGET_HAS_v64 use_neon_instructions #define TCG_TARGET_HAS_v128 use_neon_instructions #define TCG_TARGET_HAS_v256 0 diff --git a/tcg/i386/tcg-target-con-set.h b/tcg/i386/tcg-target-con-set.h index 7d00a7dde8..e24241cfa2 100644 --- a/tcg/i386/tcg-target-con-set.h +++ b/tcg/i386/tcg-target-con-set.h @@ -20,7 +20,7 @@ C_O0_I2(L, L) C_O0_I2(qi, r) C_O0_I2(re, r) C_O0_I2(ri, r) -C_O0_I2(r, re) +C_O0_I2(r, reT) C_O0_I2(s, L) C_O0_I2(x, r) C_O0_I3(L, L, L) @@ -34,7 +34,7 @@ C_O1_I1(r, r) C_O1_I1(x, r) C_O1_I1(x, x) C_O1_I2(q, 0, qi) -C_O1_I2(q, r, re) +C_O1_I2(q, r, reT) C_O1_I2(r, 0, ci) C_O1_I2(r, 0, r) C_O1_I2(r, 0, re) @@ -50,7 +50,7 @@ C_N1_I2(r, r, r) C_N1_I2(r, r, rW) C_O1_I3(x, 0, x, x) C_O1_I3(x, x, x, x) -C_O1_I4(r, r, re, r, 0) +C_O1_I4(r, r, reT, r, 0) C_O1_I4(r, r, r, ri, ri) C_O2_I1(r, r, L) C_O2_I2(a, d, a, r) diff --git a/tcg/i386/tcg-target-con-str.h b/tcg/i386/tcg-target-con-str.h index 95a30e58cd..cc22db227b 100644 --- a/tcg/i386/tcg-target-con-str.h +++ b/tcg/i386/tcg-target-con-str.h @@ -28,5 +28,6 @@ REGS('s', ALL_BYTEL_REGS & ~SOFTMMU_RESERVE_REGS) /* qemu_st8_i32 data */ */ CONST('e', TCG_CT_CONST_S32) CONST('I', TCG_CT_CONST_I32) +CONST('T', TCG_CT_CONST_TST) CONST('W', TCG_CT_CONST_WSZ) CONST('Z', TCG_CT_CONST_U32) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index d268199fc1..c6ba498623 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -132,6 +132,7 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define TCG_CT_CONST_U32 0x200 #define TCG_CT_CONST_I32 0x400 #define TCG_CT_CONST_WSZ 0x800 +#define TCG_CT_CONST_TST 0x1000 /* Registers used with L constraint, which are the first argument registers on x86_64, and two random call clobbered registers on @@ -195,13 +196,15 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, } /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { if (ct & TCG_CT_CONST) { return 1; } if (type == TCG_TYPE_I32) { - if (ct & (TCG_CT_CONST_S32 | TCG_CT_CONST_U32 | TCG_CT_CONST_I32)) { + if (ct & (TCG_CT_CONST_S32 | TCG_CT_CONST_U32 | + TCG_CT_CONST_I32 | TCG_CT_CONST_TST)) { return 1; } } else { @@ -214,6 +217,17 @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) { return 1; } + /* + * This will be used in combination with TCG_CT_CONST_S32, + * so "normal" TESTQ is already matched. Also accept: + * TESTQ -> TESTL (uint32_t) + * TESTQ -> BT (is_power_of_2) + */ + if ((ct & TCG_CT_CONST_TST) + && is_tst_cond(cond) + && (val == (uint32_t)val || is_power_of_2(val))) { + return 1; + } } if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) { return 1; @@ -395,6 +409,7 @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) #define OPC_SHLX (0xf7 | P_EXT38 | P_DATA16) #define OPC_SHRX (0xf7 | P_EXT38 | P_SIMDF2) #define OPC_SHRD_Ib (0xac | P_EXT) +#define OPC_TESTB (0x84) #define OPC_TESTL (0x85) #define OPC_TZCNT (0xbc | P_EXT | P_SIMDF3) #define OPC_UD2 (0x0b | P_EXT) @@ -441,6 +456,12 @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) #define OPC_GRP3_Ev (0xf7) #define OPC_GRP5 (0xff) #define OPC_GRP14 (0x73 | P_EXT | P_DATA16) +#define OPC_GRPBT (0xba | P_EXT) + +#define OPC_GRPBT_BT 4 +#define OPC_GRPBT_BTS 5 +#define OPC_GRPBT_BTR 6 +#define OPC_GRPBT_BTC 7 /* Group 1 opcode extensions for 0x80-0x83. These are also used as modifiers for OPC_ARITH. */ @@ -505,6 +526,8 @@ static const uint8_t tcg_cond_to_jcc[] = { [TCG_COND_GEU] = JCC_JAE, [TCG_COND_LEU] = JCC_JBE, [TCG_COND_GTU] = JCC_JA, + [TCG_COND_TSTEQ] = JCC_JE, + [TCG_COND_TSTNE] = JCC_JNE, }; #if TCG_TARGET_REG_BITS == 64 @@ -1448,27 +1471,101 @@ static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, bool small) } } -static void tcg_out_cmp(TCGContext *s, TCGArg arg1, TCGArg arg2, - int const_arg2, int rexw) +static int tcg_out_cmp(TCGContext *s, TCGCond cond, TCGArg arg1, + TCGArg arg2, int const_arg2, int rexw) { - if (const_arg2) { - if (arg2 == 0) { - /* test r, r */ + int jz, js; + + if (!is_tst_cond(cond)) { + if (!const_arg2) { + tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2); + } else if (arg2 == 0) { tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1); } else { + tcg_debug_assert(!rexw || arg2 == (int32_t)arg2); tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0); } - } else { - tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2); + return tcg_cond_to_jcc[cond]; + } + + jz = tcg_cond_to_jcc[cond]; + js = (cond == TCG_COND_TSTNE ? JCC_JS : JCC_JNS); + + if (!const_arg2) { + tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg2); + return jz; + } + + if (arg2 <= 0xff && (TCG_TARGET_REG_BITS == 64 || arg1 < 4)) { + if (arg2 == 0x80) { + tcg_out_modrm(s, OPC_TESTB | P_REXB_R, arg1, arg1); + return js; + } + if (arg2 == 0xff) { + tcg_out_modrm(s, OPC_TESTB | P_REXB_R, arg1, arg1); + return jz; + } + tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, arg1); + tcg_out8(s, arg2); + return jz; + } + + if ((arg2 & ~0xff00) == 0 && arg1 < 4) { + if (arg2 == 0x8000) { + tcg_out_modrm(s, OPC_TESTB, arg1 + 4, arg1 + 4); + return js; + } + if (arg2 == 0xff00) { + tcg_out_modrm(s, OPC_TESTB, arg1 + 4, arg1 + 4); + return jz; + } + tcg_out_modrm(s, OPC_GRP3_Eb, EXT3_TESTi, arg1 + 4); + tcg_out8(s, arg2 >> 8); + return jz; + } + + if (arg2 == 0xffff) { + tcg_out_modrm(s, OPC_TESTL | P_DATA16, arg1, arg1); + return jz; } + if (arg2 == 0xffffffffu) { + tcg_out_modrm(s, OPC_TESTL, arg1, arg1); + return jz; + } + + if (is_power_of_2(rexw ? arg2 : (uint32_t)arg2)) { + int jc = (cond == TCG_COND_TSTNE ? JCC_JB : JCC_JAE); + int sh = ctz64(arg2); + + rexw = (sh & 32 ? P_REXW : 0); + if ((sh & 31) == 31) { + tcg_out_modrm(s, OPC_TESTL | rexw, arg1, arg1); + return js; + } else { + tcg_out_modrm(s, OPC_GRPBT | rexw, OPC_GRPBT_BT, arg1); + tcg_out8(s, sh); + return jc; + } + } + + if (rexw) { + if (arg2 == (uint32_t)arg2) { + rexw = 0; + } else { + tcg_debug_assert(arg2 == (int32_t)arg2); + } + } + tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_TESTi, arg1); + tcg_out32(s, arg2); + return jz; } static void tcg_out_brcond(TCGContext *s, int rexw, TCGCond cond, TCGArg arg1, TCGArg arg2, int const_arg2, TCGLabel *label, bool small) { - tcg_out_cmp(s, arg1, arg2, const_arg2, rexw); - tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small); + int jcc = tcg_out_cmp(s, cond, arg1, arg2, const_arg2, rexw); + tcg_out_jxx(s, jcc, label, small); } #if TCG_TARGET_REG_BITS == 32 @@ -1477,18 +1574,21 @@ static void tcg_out_brcond2(TCGContext *s, const TCGArg *args, { TCGLabel *label_next = gen_new_label(); TCGLabel *label_this = arg_label(args[5]); + TCGCond cond = args[4]; - switch(args[4]) { + switch (cond) { case TCG_COND_EQ: - tcg_out_brcond(s, 0, TCG_COND_NE, args[0], args[2], const_args[2], - label_next, 1); - tcg_out_brcond(s, 0, TCG_COND_EQ, args[1], args[3], const_args[3], + case TCG_COND_TSTEQ: + tcg_out_brcond(s, 0, tcg_invert_cond(cond), + args[0], args[2], const_args[2], label_next, 1); + tcg_out_brcond(s, 0, cond, args[1], args[3], const_args[3], label_this, small); break; case TCG_COND_NE: - tcg_out_brcond(s, 0, TCG_COND_NE, args[0], args[2], const_args[2], + case TCG_COND_TSTNE: + tcg_out_brcond(s, 0, cond, args[0], args[2], const_args[2], label_this, small); - tcg_out_brcond(s, 0, TCG_COND_NE, args[1], args[3], const_args[3], + tcg_out_brcond(s, 0, cond, args[1], args[3], const_args[3], label_this, small); break; case TCG_COND_LT: @@ -1560,6 +1660,7 @@ static void tcg_out_setcond(TCGContext *s, int rexw, TCGCond cond, { bool inv = false; bool cleared; + int jcc; switch (cond) { case TCG_COND_NE: @@ -1596,7 +1697,7 @@ static void tcg_out_setcond(TCGContext *s, int rexw, TCGCond cond, * We can then use NEG or INC to produce the desired result. * This is always smaller than the SETCC expansion. */ - tcg_out_cmp(s, arg1, arg2, const_arg2, rexw); + tcg_out_cmp(s, TCG_COND_LTU, arg1, arg2, const_arg2, rexw); /* X - X - C = -C = (C ? -1 : 0) */ tgen_arithr(s, ARITH_SBB + (neg ? rexw : 0), dest, dest); @@ -1643,8 +1744,8 @@ static void tcg_out_setcond(TCGContext *s, int rexw, TCGCond cond, cleared = true; } - tcg_out_cmp(s, arg1, arg2, const_arg2, rexw); - tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest); + jcc = tcg_out_cmp(s, cond, arg1, arg2, const_arg2, rexw); + tcg_out_modrm(s, OPC_SETCC | jcc, 0, dest); if (!cleared) { tcg_out_ext8u(s, dest, dest); @@ -1698,14 +1799,14 @@ static void tcg_out_setcond2(TCGContext *s, const TCGArg *args, } #endif -static void tcg_out_cmov(TCGContext *s, TCGCond cond, int rexw, +static void tcg_out_cmov(TCGContext *s, int jcc, int rexw, TCGReg dest, TCGReg v1) { if (have_cmov) { - tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond] | rexw, dest, v1); + tcg_out_modrm(s, OPC_CMOVCC | jcc | rexw, dest, v1); } else { TCGLabel *over = gen_new_label(); - tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1); + tcg_out_jxx(s, jcc ^ 1, over, 1); tcg_out_mov(s, TCG_TYPE_I32, dest, v1); tcg_out_label(s, over); } @@ -1715,8 +1816,8 @@ static void tcg_out_movcond(TCGContext *s, int rexw, TCGCond cond, TCGReg dest, TCGReg c1, TCGArg c2, int const_c2, TCGReg v1) { - tcg_out_cmp(s, c1, c2, const_c2, rexw); - tcg_out_cmov(s, cond, rexw, dest, v1); + int jcc = tcg_out_cmp(s, cond, c1, c2, const_c2, rexw); + tcg_out_cmov(s, jcc, rexw, dest, v1); } static void tcg_out_ctz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, @@ -1728,12 +1829,12 @@ static void tcg_out_ctz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, tcg_debug_assert(arg2 == (rexw ? 64 : 32)); } else { tcg_debug_assert(dest != arg2); - tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2); + tcg_out_cmov(s, JCC_JB, rexw, dest, arg2); } } else { tcg_debug_assert(dest != arg2); tcg_out_modrm(s, OPC_BSF + rexw, dest, arg1); - tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2); + tcg_out_cmov(s, JCC_JE, rexw, dest, arg2); } } @@ -1746,7 +1847,7 @@ static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, tcg_debug_assert(arg2 == (rexw ? 64 : 32)); } else { tcg_debug_assert(dest != arg2); - tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2); + tcg_out_cmov(s, JCC_JB, rexw, dest, arg2); } } else { tcg_debug_assert(!const_a2); @@ -1758,8 +1859,8 @@ static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, tgen_arithi(s, ARITH_XOR + rexw, dest, rexw ? 63 : 31, 0); /* Since we have destroyed the flags from BSR, we have to re-test. */ - tcg_out_cmp(s, arg1, 0, 1, rexw); - tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2); + int jcc = tcg_out_cmp(s, TCG_COND_EQ, arg1, 0, 1, rexw); + tcg_out_cmov(s, jcc, rexw, dest, arg2); } } @@ -1824,23 +1925,6 @@ static void tcg_out_nopn(TCGContext *s, int n) tcg_out8(s, 0x90); } -/* Test register R vs immediate bits I, setting Z flag for EQ/NE. */ -static void __attribute__((unused)) -tcg_out_testi(TCGContext *s, TCGReg r, uint32_t i) -{ - /* - * This is used for testing alignment, so we can usually use testb. - * For i686, we have to use testl for %esi/%edi. - */ - if (i <= 0xff && (TCG_TARGET_REG_BITS == 64 || r < 4)) { - tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, r); - tcg_out8(s, i); - } else { - tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, r); - tcg_out32(s, i); - } -} - typedef struct { TCGReg base; int index; @@ -2101,16 +2185,17 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_L0, TCG_REG_L0, offsetof(CPUTLBEntry, addend)); } else if (a_mask) { - ldst = new_ldst_label(s); + int jcc; + ldst = new_ldst_label(s); ldst->is_ld = is_ld; ldst->oi = oi; ldst->addrlo_reg = addrlo; ldst->addrhi_reg = addrhi; - tcg_out_testi(s, addrlo, a_mask); /* jne slow_path */ - tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); + jcc = tcg_out_cmp(s, TCG_COND_TSTNE, addrlo, a_mask, true, false); + tcg_out_opc(s, OPC_JCC_long + jcc, 0, 0, 0); ldst->label_ptr[0] = s->code_ptr; s->code_ptr += 4; } @@ -2256,9 +2341,10 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, } else { TCGLabel *l1 = gen_new_label(); TCGLabel *l2 = gen_new_label(); + int jcc; - tcg_out_testi(s, h.base, 15); - tcg_out_jxx(s, JCC_JNE, l1, true); + jcc = tcg_out_cmp(s, TCG_COND_TSTNE, h.base, 15, true, false); + tcg_out_jxx(s, jcc, l1, true); tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_VxWx + h.seg, TCG_TMP_VEC, 0, @@ -2384,9 +2470,10 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, } else { TCGLabel *l1 = gen_new_label(); TCGLabel *l2 = gen_new_label(); + int jcc; - tcg_out_testi(s, h.base, 15); - tcg_out_jxx(s, JCC_JNE, l1, true); + jcc = tcg_out_cmp(s, TCG_COND_TSTNE, h.base, 15, true, false); + tcg_out_jxx(s, jcc, l1, true); tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_WxVx + h.seg, TCG_TMP_VEC, 0, @@ -3373,7 +3460,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - return C_O0_I2(r, re); + return C_O0_I2(r, reT); case INDEX_op_bswap16_i32: case INDEX_op_bswap16_i64: @@ -3421,11 +3508,11 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_setcond_i64: case INDEX_op_negsetcond_i32: case INDEX_op_negsetcond_i64: - return C_O1_I2(q, r, re); + return C_O1_I2(q, r, reT); case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: - return C_O1_I4(r, r, re, r, 0); + return C_O1_I4(r, r, reT, r, 0); case INDEX_op_div2_i32: case INDEX_op_div2_i64: diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index fa34deec47..a10d4e1fce 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -198,6 +198,8 @@ typedef enum { #define TCG_TARGET_HAS_qemu_ldst_i128 \ (TCG_TARGET_REG_BITS == 64 && (cpuinfo & CPUINFO_ATOMIC_VMOVDQA)) +#define TCG_TARGET_HAS_tst 1 + /* We do not support older SSE systems, only beginning with AVX1. */ #define TCG_TARGET_HAS_v64 have_avx1 #define TCG_TARGET_HAS_v128 have_avx1 diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index dcf0205458..69c5b8ac4f 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -183,7 +183,8 @@ static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) } /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { if (ct & TCG_CT_CONST) { return true; diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index 9c70ebfefc..fede627bf7 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -169,6 +169,8 @@ typedef enum { #define TCG_TARGET_HAS_qemu_ldst_i128 (cpuinfo & CPUINFO_LSX) +#define TCG_TARGET_HAS_tst 0 + #define TCG_TARGET_HAS_v64 0 #define TCG_TARGET_HAS_v128 (cpuinfo & CPUINFO_LSX) #define TCG_TARGET_HAS_v256 0 diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index 8328dbdecc..3b5b5c6d5b 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -188,7 +188,8 @@ static bool is_p2m1(tcg_target_long val) } /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { if (ct & TCG_CT_CONST) { return 1; diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index b98ffae1d0..a996aa171d 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -194,6 +194,8 @@ extern bool use_mips32r2_instructions; #define TCG_TARGET_HAS_qemu_ldst_i128 0 +#define TCG_TARGET_HAS_tst 0 + #define TCG_TARGET_DEFAULT_MO 0 #define TCG_TARGET_NEED_LDST_LABELS #define TCG_TARGET_NEED_POOL_LABELS diff --git a/tcg/optimize.c b/tcg/optimize.c index f2d01654c5..79e701652b 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -124,11 +124,22 @@ static inline bool ts_is_const(TCGTemp *ts) return ts_info(ts)->is_const; } +static inline bool ts_is_const_val(TCGTemp *ts, uint64_t val) +{ + TempOptInfo *ti = ts_info(ts); + return ti->is_const && ti->val == val; +} + static inline bool arg_is_const(TCGArg arg) { return ts_is_const(arg_temp(arg)); } +static inline bool arg_is_const_val(TCGArg arg, uint64_t val) +{ + return ts_is_const_val(arg_temp(arg), val); +} + static inline bool ts_is_copy(TCGTemp *ts) { return ts_info(ts)->next_copy != ts; @@ -353,6 +364,13 @@ static TCGArg arg_new_constant(OptContext *ctx, uint64_t val) return temp_arg(ts); } +static TCGArg arg_new_temp(OptContext *ctx) +{ + TCGTemp *ts = tcg_temp_new_internal(ctx->type, TEMP_EBB); + init_ts_info(ctx, ts); + return temp_arg(ts); +} + static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); @@ -614,9 +632,15 @@ static bool do_constant_folding_cond_32(uint32_t x, uint32_t y, TCGCond c) return x <= y; case TCG_COND_GTU: return x > y; - default: - g_assert_not_reached(); + case TCG_COND_TSTEQ: + return (x & y) == 0; + case TCG_COND_TSTNE: + return (x & y) != 0; + case TCG_COND_ALWAYS: + case TCG_COND_NEVER: + break; } + g_assert_not_reached(); } static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c) @@ -642,12 +666,18 @@ static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c) return x <= y; case TCG_COND_GTU: return x > y; - default: - g_assert_not_reached(); + case TCG_COND_TSTEQ: + return (x & y) == 0; + case TCG_COND_TSTNE: + return (x & y) != 0; + case TCG_COND_ALWAYS: + case TCG_COND_NEVER: + break; } + g_assert_not_reached(); } -static bool do_constant_folding_cond_eq(TCGCond c) +static int do_constant_folding_cond_eq(TCGCond c) { switch (c) { case TCG_COND_GT: @@ -662,9 +692,14 @@ static bool do_constant_folding_cond_eq(TCGCond c) case TCG_COND_LEU: case TCG_COND_EQ: return 1; - default: - g_assert_not_reached(); + case TCG_COND_TSTEQ: + case TCG_COND_TSTNE: + return -1; + case TCG_COND_ALWAYS: + case TCG_COND_NEVER: + break; } + g_assert_not_reached(); } /* @@ -689,11 +724,13 @@ static int do_constant_folding_cond(TCGType type, TCGArg x, } } else if (args_are_copies(x, y)) { return do_constant_folding_cond_eq(c); - } else if (arg_is_const(y) && arg_info(y)->val == 0) { + } else if (arg_is_const_val(y, 0)) { switch (c) { case TCG_COND_LTU: + case TCG_COND_TSTNE: return 0; case TCG_COND_GEU: + case TCG_COND_TSTEQ: return 1; default: return -1; @@ -702,43 +739,6 @@ static int do_constant_folding_cond(TCGType type, TCGArg x, return -1; } -/* - * Return -1 if the condition can't be simplified, - * and the result of the condition (0 or 1) if it can. - */ -static int do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) -{ - TCGArg al = p1[0], ah = p1[1]; - TCGArg bl = p2[0], bh = p2[1]; - - if (arg_is_const(bl) && arg_is_const(bh)) { - tcg_target_ulong blv = arg_info(bl)->val; - tcg_target_ulong bhv = arg_info(bh)->val; - uint64_t b = deposit64(blv, 32, 32, bhv); - - if (arg_is_const(al) && arg_is_const(ah)) { - tcg_target_ulong alv = arg_info(al)->val; - tcg_target_ulong ahv = arg_info(ah)->val; - uint64_t a = deposit64(alv, 32, 32, ahv); - return do_constant_folding_cond_64(a, b, c); - } - if (b == 0) { - switch (c) { - case TCG_COND_LTU: - return 0; - case TCG_COND_GEU: - return 1; - default: - break; - } - } - } - if (args_are_copies(al, bl) && args_are_copies(ah, bh)) { - return do_constant_folding_cond_eq(c); - } - return -1; -} - /** * swap_commutative: * @dest: TCGArg of the destination argument, or NO_DEST. @@ -785,6 +785,166 @@ static bool swap_commutative2(TCGArg *p1, TCGArg *p2) return false; } +/* + * Return -1 if the condition can't be simplified, + * and the result of the condition (0 or 1) if it can. + */ +static int do_constant_folding_cond1(OptContext *ctx, TCGOp *op, TCGArg dest, + TCGArg *p1, TCGArg *p2, TCGArg *pcond) +{ + TCGCond cond; + bool swap; + int r; + + swap = swap_commutative(dest, p1, p2); + cond = *pcond; + if (swap) { + *pcond = cond = tcg_swap_cond(cond); + } + + r = do_constant_folding_cond(ctx->type, *p1, *p2, cond); + if (r >= 0) { + return r; + } + if (!is_tst_cond(cond)) { + return -1; + } + + /* + * TSTNE x,x -> NE x,0 + * TSTNE x,-1 -> NE x,0 + */ + if (args_are_copies(*p1, *p2) || arg_is_const_val(*p2, -1)) { + *p2 = arg_new_constant(ctx, 0); + *pcond = tcg_tst_eqne_cond(cond); + return -1; + } + + /* TSTNE x,sign -> LT x,0 */ + if (arg_is_const_val(*p2, (ctx->type == TCG_TYPE_I32 + ? INT32_MIN : INT64_MIN))) { + *p2 = arg_new_constant(ctx, 0); + *pcond = tcg_tst_ltge_cond(cond); + return -1; + } + + /* Expand to AND with a temporary if no backend support. */ + if (!TCG_TARGET_HAS_tst) { + TCGOpcode and_opc = (ctx->type == TCG_TYPE_I32 + ? INDEX_op_and_i32 : INDEX_op_and_i64); + TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, and_opc, 3); + TCGArg tmp = arg_new_temp(ctx); + + op2->args[0] = tmp; + op2->args[1] = *p1; + op2->args[2] = *p2; + + *p1 = tmp; + *p2 = arg_new_constant(ctx, 0); + *pcond = tcg_tst_eqne_cond(cond); + } + return -1; +} + +static int do_constant_folding_cond2(OptContext *ctx, TCGOp *op, TCGArg *args) +{ + TCGArg al, ah, bl, bh; + TCGCond c; + bool swap; + int r; + + swap = swap_commutative2(args, args + 2); + c = args[4]; + if (swap) { + args[4] = c = tcg_swap_cond(c); + } + + al = args[0]; + ah = args[1]; + bl = args[2]; + bh = args[3]; + + if (arg_is_const(bl) && arg_is_const(bh)) { + tcg_target_ulong blv = arg_info(bl)->val; + tcg_target_ulong bhv = arg_info(bh)->val; + uint64_t b = deposit64(blv, 32, 32, bhv); + + if (arg_is_const(al) && arg_is_const(ah)) { + tcg_target_ulong alv = arg_info(al)->val; + tcg_target_ulong ahv = arg_info(ah)->val; + uint64_t a = deposit64(alv, 32, 32, ahv); + + r = do_constant_folding_cond_64(a, b, c); + if (r >= 0) { + return r; + } + } + + if (b == 0) { + switch (c) { + case TCG_COND_LTU: + case TCG_COND_TSTNE: + return 0; + case TCG_COND_GEU: + case TCG_COND_TSTEQ: + return 1; + default: + break; + } + } + + /* TSTNE x,-1 -> NE x,0 */ + if (b == -1 && is_tst_cond(c)) { + args[3] = args[2] = arg_new_constant(ctx, 0); + args[4] = tcg_tst_eqne_cond(c); + return -1; + } + + /* TSTNE x,sign -> LT x,0 */ + if (b == INT64_MIN && is_tst_cond(c)) { + /* bl must be 0, so copy that to bh */ + args[3] = bl; + args[4] = tcg_tst_ltge_cond(c); + return -1; + } + } + + if (args_are_copies(al, bl) && args_are_copies(ah, bh)) { + r = do_constant_folding_cond_eq(c); + if (r >= 0) { + return r; + } + + /* TSTNE x,x -> NE x,0 */ + if (is_tst_cond(c)) { + args[3] = args[2] = arg_new_constant(ctx, 0); + args[4] = tcg_tst_eqne_cond(c); + return -1; + } + } + + /* Expand to AND with a temporary if no backend support. */ + if (!TCG_TARGET_HAS_tst && is_tst_cond(c)) { + TCGOp *op1 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3); + TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3); + TCGArg t1 = arg_new_temp(ctx); + TCGArg t2 = arg_new_temp(ctx); + + op1->args[0] = t1; + op1->args[1] = al; + op1->args[2] = bl; + op2->args[0] = t2; + op2->args[1] = ah; + op2->args[2] = bh; + + args[0] = t1; + args[1] = t2; + args[3] = args[2] = arg_new_constant(ctx, 0); + args[4] = tcg_tst_eqne_cond(c); + } + return -1; +} + static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args) { for (int i = 0; i < nb_args; i++) { @@ -954,7 +1114,7 @@ static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx) /* If the binary operation has first argument @i, fold to @i. */ static bool fold_ix_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { - if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == i) { + if (arg_is_const_val(op->args[1], i)) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); } return false; @@ -963,7 +1123,7 @@ static bool fold_ix_to_i(OptContext *ctx, TCGOp *op, uint64_t i) /* If the binary operation has first argument @i, fold to NOT. */ static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i) { - if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == i) { + if (arg_is_const_val(op->args[1], i)) { return fold_to_not(ctx, op, 2); } return false; @@ -972,7 +1132,7 @@ static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i) /* If the binary operation has second argument @i, fold to @i. */ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { - if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + if (arg_is_const_val(op->args[2], i)) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); } return false; @@ -981,7 +1141,7 @@ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) /* If the binary operation has second argument @i, fold to identity. */ static bool fold_xi_to_x(OptContext *ctx, TCGOp *op, uint64_t i) { - if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + if (arg_is_const_val(op->args[2], i)) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); } return false; @@ -990,7 +1150,7 @@ static bool fold_xi_to_x(OptContext *ctx, TCGOp *op, uint64_t i) /* If the binary operation has second argument @i, fold to NOT. */ static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i) { - if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + if (arg_is_const_val(op->args[2], i)) { return fold_to_not(ctx, op, 1); } return false; @@ -1182,14 +1342,8 @@ static bool fold_andc(OptContext *ctx, TCGOp *op) static bool fold_brcond(OptContext *ctx, TCGOp *op) { - TCGCond cond = op->args[2]; - int i; - - if (swap_commutative(NO_DEST, &op->args[0], &op->args[1])) { - op->args[2] = cond = tcg_swap_cond(cond); - } - - i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); + int i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[0], + &op->args[1], &op->args[2]); if (i == 0) { tcg_op_remove(ctx->tcg, op); return true; @@ -1203,15 +1357,13 @@ static bool fold_brcond(OptContext *ctx, TCGOp *op) static bool fold_brcond2(OptContext *ctx, TCGOp *op) { - TCGCond cond = op->args[4]; - TCGArg label = op->args[5]; + TCGCond cond; + TCGArg label; int i, inv = 0; - if (swap_commutative2(&op->args[0], &op->args[2])) { - op->args[4] = cond = tcg_swap_cond(cond); - } - - i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); + i = do_constant_folding_cond2(ctx, op, &op->args[0]); + cond = op->args[4]; + label = op->args[5]; if (i >= 0) { goto do_brcond_const; } @@ -1223,8 +1375,8 @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) * Simplify LT/GE comparisons vs zero to a single compare * vs the high word of the input. */ - if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0 && - arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0) { + if (arg_is_const_val(op->args[2], 0) && + arg_is_const_val(op->args[3], 0)) { goto do_brcond_high; } break; @@ -1252,24 +1404,37 @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) case 0: goto do_brcond_const; case 1: - op->opc = INDEX_op_brcond_i32; - op->args[1] = op->args[2]; - op->args[2] = cond; - op->args[3] = label; - break; + goto do_brcond_low; + } + break; + + case TCG_COND_TSTEQ: + case TCG_COND_TSTNE: + if (arg_is_const_val(op->args[2], 0)) { + goto do_brcond_high; + } + if (arg_is_const_val(op->args[3], 0)) { + goto do_brcond_low; } break; default: break; + do_brcond_low: + op->opc = INDEX_op_brcond_i32; + op->args[1] = op->args[2]; + op->args[2] = cond; + op->args[3] = label; + return fold_brcond(ctx, op); + do_brcond_high: op->opc = INDEX_op_brcond_i32; op->args[0] = op->args[1]; op->args[1] = op->args[3]; op->args[2] = cond; op->args[3] = label; - break; + return fold_brcond(ctx, op); do_brcond_const: if (i == 0) { @@ -1448,9 +1613,7 @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) } /* Inserting a value into zero at offset 0. */ - if (arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0 - && op->args[3] == 0) { + if (arg_is_const_val(op->args[1], 0) && op->args[3] == 0) { uint64_t mask = MAKE_64BIT_MASK(0, op->args[4]); op->opc = and_opc; @@ -1461,8 +1624,7 @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) } /* Inserting zero into a value. */ - if (arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { + if (arg_is_const_val(op->args[2], 0)) { uint64_t mask = deposit64(-1, op->args[3], op->args[4], 0); op->opc = and_opc; @@ -1687,21 +1849,18 @@ static bool fold_mov(OptContext *ctx, TCGOp *op) static bool fold_movcond(OptContext *ctx, TCGOp *op) { - TCGCond cond = op->args[5]; int i; - if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { - op->args[5] = cond = tcg_swap_cond(cond); - } /* * Canonicalize the "false" input reg to match the destination reg so * that the tcg backend can implement a "move if true" operation. */ if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { - op->args[5] = cond = tcg_invert_cond(cond); + op->args[5] = tcg_invert_cond(op->args[5]); } - i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); + i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[1], + &op->args[2], &op->args[5]); if (i >= 0) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); } @@ -1715,6 +1874,7 @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) uint64_t tv = arg_info(op->args[3])->val; uint64_t fv = arg_info(op->args[4])->val; TCGOpcode opc, negopc = 0; + TCGCond cond = op->args[5]; switch (ctx->type) { case TCG_TYPE_I32: @@ -1940,19 +2100,107 @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) return false; } -static bool fold_setcond(OptContext *ctx, TCGOp *op) +static void fold_setcond_tst_pow2(OptContext *ctx, TCGOp *op, bool neg) { + TCGOpcode and_opc, sub_opc, xor_opc, neg_opc, shr_opc, uext_opc, sext_opc; TCGCond cond = op->args[3]; - int i; + TCGArg ret, src1, src2; + TCGOp *op2; + uint64_t val; + int sh; + bool inv; + + if (!is_tst_cond(cond) || !arg_is_const(op->args[2])) { + return; + } + + src2 = op->args[2]; + val = arg_info(src2)->val; + if (!is_power_of_2(val)) { + return; + } + sh = ctz64(val); - if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) { - op->args[3] = cond = tcg_swap_cond(cond); + switch (ctx->type) { + case TCG_TYPE_I32: + and_opc = INDEX_op_and_i32; + sub_opc = INDEX_op_sub_i32; + xor_opc = INDEX_op_xor_i32; + shr_opc = INDEX_op_shr_i32; + neg_opc = INDEX_op_neg_i32; + if (TCG_TARGET_extract_i32_valid(sh, 1)) { + uext_opc = TCG_TARGET_HAS_extract_i32 ? INDEX_op_extract_i32 : 0; + sext_opc = TCG_TARGET_HAS_sextract_i32 ? INDEX_op_sextract_i32 : 0; + } + break; + case TCG_TYPE_I64: + and_opc = INDEX_op_and_i64; + sub_opc = INDEX_op_sub_i64; + xor_opc = INDEX_op_xor_i64; + shr_opc = INDEX_op_shr_i64; + neg_opc = INDEX_op_neg_i64; + if (TCG_TARGET_extract_i64_valid(sh, 1)) { + uext_opc = TCG_TARGET_HAS_extract_i64 ? INDEX_op_extract_i64 : 0; + sext_opc = TCG_TARGET_HAS_sextract_i64 ? INDEX_op_sextract_i64 : 0; + } + break; + default: + g_assert_not_reached(); + } + + ret = op->args[0]; + src1 = op->args[1]; + inv = cond == TCG_COND_TSTEQ; + + if (sh && sext_opc && neg && !inv) { + op->opc = sext_opc; + op->args[1] = src1; + op->args[2] = sh; + op->args[3] = 1; + return; + } else if (sh && uext_opc) { + op->opc = uext_opc; + op->args[1] = src1; + op->args[2] = sh; + op->args[3] = 1; + } else { + if (sh) { + op2 = tcg_op_insert_before(ctx->tcg, op, shr_opc, 3); + op2->args[0] = ret; + op2->args[1] = src1; + op2->args[2] = arg_new_constant(ctx, sh); + src1 = ret; + } + op->opc = and_opc; + op->args[1] = src1; + op->args[2] = arg_new_constant(ctx, 1); } - i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); + if (neg && inv) { + op2 = tcg_op_insert_after(ctx->tcg, op, sub_opc, 3); + op2->args[0] = ret; + op2->args[1] = ret; + op2->args[2] = arg_new_constant(ctx, 1); + } else if (inv) { + op2 = tcg_op_insert_after(ctx->tcg, op, xor_opc, 3); + op2->args[0] = ret; + op2->args[1] = ret; + op2->args[2] = arg_new_constant(ctx, 1); + } else if (neg) { + op2 = tcg_op_insert_after(ctx->tcg, op, neg_opc, 2); + op2->args[0] = ret; + op2->args[1] = ret; + } +} + +static bool fold_setcond(OptContext *ctx, TCGOp *op) +{ + int i = do_constant_folding_cond1(ctx, op, op->args[0], &op->args[1], + &op->args[2], &op->args[3]); if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); } + fold_setcond_tst_pow2(ctx, op, false); ctx->z_mask = 1; ctx->s_mask = smask_from_zmask(1); @@ -1961,34 +2209,25 @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) static bool fold_negsetcond(OptContext *ctx, TCGOp *op) { - TCGCond cond = op->args[3]; - int i; - - if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) { - op->args[3] = cond = tcg_swap_cond(cond); - } - - i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); + int i = do_constant_folding_cond1(ctx, op, op->args[0], &op->args[1], + &op->args[2], &op->args[3]); if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], -i); } + fold_setcond_tst_pow2(ctx, op, true); /* Value is {0,-1} so all bits are repetitions of the sign. */ ctx->s_mask = -1; return false; } - static bool fold_setcond2(OptContext *ctx, TCGOp *op) { - TCGCond cond = op->args[5]; + TCGCond cond; int i, inv = 0; - if (swap_commutative2(&op->args[1], &op->args[3])) { - op->args[5] = cond = tcg_swap_cond(cond); - } - - i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); + i = do_constant_folding_cond2(ctx, op, &op->args[1]); + cond = op->args[5]; if (i >= 0) { goto do_setcond_const; } @@ -2000,8 +2239,8 @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) * Simplify LT/GE comparisons vs zero to a single compare * vs the high word of the input. */ - if (arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0 && - arg_is_const(op->args[4]) && arg_info(op->args[4])->val == 0) { + if (arg_is_const_val(op->args[3], 0) && + arg_is_const_val(op->args[4], 0)) { goto do_setcond_high; } break; @@ -2029,22 +2268,35 @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) case 0: goto do_setcond_const; case 1: - op->args[2] = op->args[3]; - op->args[3] = cond; - op->opc = INDEX_op_setcond_i32; - break; + goto do_setcond_low; + } + break; + + case TCG_COND_TSTEQ: + case TCG_COND_TSTNE: + if (arg_is_const_val(op->args[2], 0)) { + goto do_setcond_high; + } + if (arg_is_const_val(op->args[4], 0)) { + goto do_setcond_low; } break; default: break; + do_setcond_low: + op->args[2] = op->args[3]; + op->args[3] = cond; + op->opc = INDEX_op_setcond_i32; + return fold_setcond(ctx, op); + do_setcond_high: op->args[1] = op->args[2]; op->args[2] = op->args[4]; op->args[3] = cond; op->opc = INDEX_op_setcond_i32; - break; + return fold_setcond(ctx, op); } ctx->z_mask = 1; diff --git a/tcg/ppc/tcg-target-con-set.h b/tcg/ppc/tcg-target-con-set.h index cb47b29452..9f99bde505 100644 --- a/tcg/ppc/tcg-target-con-set.h +++ b/tcg/ppc/tcg-target-con-set.h @@ -11,7 +11,7 @@ */ C_O0_I1(r) C_O0_I2(r, r) -C_O0_I2(r, ri) +C_O0_I2(r, rC) C_O0_I2(v, r) C_O0_I3(r, r, r) C_O0_I3(o, m, r) @@ -26,13 +26,14 @@ C_O1_I2(r, rI, ri) C_O1_I2(r, rI, rT) C_O1_I2(r, r, r) C_O1_I2(r, r, ri) +C_O1_I2(r, r, rC) C_O1_I2(r, r, rI) C_O1_I2(r, r, rT) C_O1_I2(r, r, rU) C_O1_I2(r, r, rZW) C_O1_I2(v, v, v) C_O1_I3(v, v, v, v) -C_O1_I4(r, r, ri, rZ, rZ) +C_O1_I4(r, r, rC, rZ, rZ) C_O1_I4(r, r, r, ri, ri) C_O2_I1(r, r, r) C_N1O1_I1(o, m, r) diff --git a/tcg/ppc/tcg-target-con-str.h b/tcg/ppc/tcg-target-con-str.h index 20846901de..16b687216e 100644 --- a/tcg/ppc/tcg-target-con-str.h +++ b/tcg/ppc/tcg-target-con-str.h @@ -16,6 +16,7 @@ REGS('v', ALL_VECTOR_REGS) * Define constraint letters for constants: * CONST(letter, TCG_CT_CONST_* bit set) */ +CONST('C', TCG_CT_CONST_CMP) CONST('I', TCG_CT_CONST_S16) CONST('M', TCG_CT_CONST_MONE) CONST('T', TCG_CT_CONST_S32) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 54816967bc..7f3829beeb 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -92,11 +92,13 @@ #define SZR (TCG_TARGET_REG_BITS / 8) #define TCG_CT_CONST_S16 0x100 +#define TCG_CT_CONST_U16 0x200 #define TCG_CT_CONST_S32 0x400 #define TCG_CT_CONST_U32 0x800 #define TCG_CT_CONST_ZERO 0x1000 #define TCG_CT_CONST_MONE 0x2000 #define TCG_CT_CONST_WSZ 0x4000 +#define TCG_CT_CONST_CMP 0x8000 #define ALL_GENERAL_REGS 0xffffffffu #define ALL_VECTOR_REGS 0xffffffff00000000ull @@ -281,31 +283,78 @@ static bool reloc_pc34(tcg_insn_unit *src_rw, const tcg_insn_unit *target) return false; } +static bool mask_operand(uint32_t c, int *mb, int *me); +static bool mask64_operand(uint64_t c, int *mb, int *me); + /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t sval, int ct, + TCGType type, TCGCond cond, int vece) { + uint64_t uval = sval; + int mb, me; + if (ct & TCG_CT_CONST) { return 1; } - /* The only 32-bit constraint we use aside from - TCG_CT_CONST is TCG_CT_CONST_S16. */ if (type == TCG_TYPE_I32) { - val = (int32_t)val; + uval = (uint32_t)sval; + sval = (int32_t)sval; + } + + if (ct & TCG_CT_CONST_CMP) { + switch (cond) { + case TCG_COND_EQ: + case TCG_COND_NE: + ct |= TCG_CT_CONST_S16 | TCG_CT_CONST_U16; + break; + case TCG_COND_LT: + case TCG_COND_GE: + case TCG_COND_LE: + case TCG_COND_GT: + ct |= TCG_CT_CONST_S16; + break; + case TCG_COND_LTU: + case TCG_COND_GEU: + case TCG_COND_LEU: + case TCG_COND_GTU: + ct |= TCG_CT_CONST_U16; + break; + case TCG_COND_TSTEQ: + case TCG_COND_TSTNE: + if ((uval & ~0xffff) == 0 || (uval & ~0xffff0000ull) == 0) { + return 1; + } + if (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32 + ? mask_operand(uval, &mb, &me) + : mask64_operand(uval << clz64(uval), &mb, &me)) { + return 1; + } + return 0; + default: + g_assert_not_reached(); + } } - if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) { + if ((ct & TCG_CT_CONST_S16) && sval == (int16_t)sval) { + return 1; + } + if ((ct & TCG_CT_CONST_U16) && uval == (uint16_t)uval) { return 1; - } else if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) { + } + if ((ct & TCG_CT_CONST_S32) && sval == (int32_t)sval) { return 1; - } else if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) { + } + if ((ct & TCG_CT_CONST_U32) && uval == (uint32_t)uval) { return 1; - } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) { + } + if ((ct & TCG_CT_CONST_ZERO) && sval == 0) { return 1; - } else if ((ct & TCG_CT_CONST_MONE) && val == -1) { + } + if ((ct & TCG_CT_CONST_MONE) && sval == -1) { return 1; - } else if ((ct & TCG_CT_CONST_WSZ) - && val == (type == TCG_TYPE_I32 ? 32 : 64)) { + } + if ((ct & TCG_CT_CONST_WSZ) && sval == (type == TCG_TYPE_I32 ? 32 : 64)) { return 1; } return 0; @@ -669,31 +718,35 @@ enum { CR_SO }; -static const uint32_t tcg_to_bc[] = { - [TCG_COND_EQ] = BC | BI(7, CR_EQ) | BO_COND_TRUE, - [TCG_COND_NE] = BC | BI(7, CR_EQ) | BO_COND_FALSE, - [TCG_COND_LT] = BC | BI(7, CR_LT) | BO_COND_TRUE, - [TCG_COND_GE] = BC | BI(7, CR_LT) | BO_COND_FALSE, - [TCG_COND_LE] = BC | BI(7, CR_GT) | BO_COND_FALSE, - [TCG_COND_GT] = BC | BI(7, CR_GT) | BO_COND_TRUE, - [TCG_COND_LTU] = BC | BI(7, CR_LT) | BO_COND_TRUE, - [TCG_COND_GEU] = BC | BI(7, CR_LT) | BO_COND_FALSE, - [TCG_COND_LEU] = BC | BI(7, CR_GT) | BO_COND_FALSE, - [TCG_COND_GTU] = BC | BI(7, CR_GT) | BO_COND_TRUE, +static const uint32_t tcg_to_bc[16] = { + [TCG_COND_EQ] = BC | BI(0, CR_EQ) | BO_COND_TRUE, + [TCG_COND_NE] = BC | BI(0, CR_EQ) | BO_COND_FALSE, + [TCG_COND_TSTEQ] = BC | BI(0, CR_EQ) | BO_COND_TRUE, + [TCG_COND_TSTNE] = BC | BI(0, CR_EQ) | BO_COND_FALSE, + [TCG_COND_LT] = BC | BI(0, CR_LT) | BO_COND_TRUE, + [TCG_COND_GE] = BC | BI(0, CR_LT) | BO_COND_FALSE, + [TCG_COND_LE] = BC | BI(0, CR_GT) | BO_COND_FALSE, + [TCG_COND_GT] = BC | BI(0, CR_GT) | BO_COND_TRUE, + [TCG_COND_LTU] = BC | BI(0, CR_LT) | BO_COND_TRUE, + [TCG_COND_GEU] = BC | BI(0, CR_LT) | BO_COND_FALSE, + [TCG_COND_LEU] = BC | BI(0, CR_GT) | BO_COND_FALSE, + [TCG_COND_GTU] = BC | BI(0, CR_GT) | BO_COND_TRUE, }; /* The low bit here is set if the RA and RB fields must be inverted. */ -static const uint32_t tcg_to_isel[] = { - [TCG_COND_EQ] = ISEL | BC_(7, CR_EQ), - [TCG_COND_NE] = ISEL | BC_(7, CR_EQ) | 1, - [TCG_COND_LT] = ISEL | BC_(7, CR_LT), - [TCG_COND_GE] = ISEL | BC_(7, CR_LT) | 1, - [TCG_COND_LE] = ISEL | BC_(7, CR_GT) | 1, - [TCG_COND_GT] = ISEL | BC_(7, CR_GT), - [TCG_COND_LTU] = ISEL | BC_(7, CR_LT), - [TCG_COND_GEU] = ISEL | BC_(7, CR_LT) | 1, - [TCG_COND_LEU] = ISEL | BC_(7, CR_GT) | 1, - [TCG_COND_GTU] = ISEL | BC_(7, CR_GT), +static const uint32_t tcg_to_isel[16] = { + [TCG_COND_EQ] = ISEL | BC_(0, CR_EQ), + [TCG_COND_NE] = ISEL | BC_(0, CR_EQ) | 1, + [TCG_COND_TSTEQ] = ISEL | BC_(0, CR_EQ), + [TCG_COND_TSTNE] = ISEL | BC_(0, CR_EQ) | 1, + [TCG_COND_LT] = ISEL | BC_(0, CR_LT), + [TCG_COND_GE] = ISEL | BC_(0, CR_LT) | 1, + [TCG_COND_LE] = ISEL | BC_(0, CR_GT) | 1, + [TCG_COND_GT] = ISEL | BC_(0, CR_GT), + [TCG_COND_LTU] = ISEL | BC_(0, CR_LT), + [TCG_COND_GEU] = ISEL | BC_(0, CR_LT) | 1, + [TCG_COND_LEU] = ISEL | BC_(0, CR_GT) | 1, + [TCG_COND_GTU] = ISEL | BC_(0, CR_GT), }; static bool patch_reloc(tcg_insn_unit *code_ptr, int type, @@ -838,19 +891,31 @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) return true; } -static inline void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs, - int sh, int mb) +static void tcg_out_rld_rc(TCGContext *s, int op, TCGReg ra, TCGReg rs, + int sh, int mb, bool rc) { tcg_debug_assert(TCG_TARGET_REG_BITS == 64); sh = SH(sh & 0x1f) | (((sh >> 5) & 1) << 1); mb = MB64((mb >> 5) | ((mb << 1) & 0x3f)); - tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb); + tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb | rc); +} + +static void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs, + int sh, int mb) +{ + tcg_out_rld_rc(s, op, ra, rs, sh, mb, false); +} + +static void tcg_out_rlw_rc(TCGContext *s, int op, TCGReg ra, TCGReg rs, + int sh, int mb, int me, bool rc) +{ + tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me) | rc); } -static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs, - int sh, int mb, int me) +static void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs, + int sh, int mb, int me) { - tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me)); + tcg_out_rlw_rc(s, op, ra, rs, sh, mb, me, false); } static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src) @@ -1668,6 +1733,50 @@ static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, return false; } +/* + * Set dest non-zero if and only if (arg1 & arg2) is non-zero. + * If RC, then also set RC0. + */ +static void tcg_out_test(TCGContext *s, TCGReg dest, TCGReg arg1, TCGArg arg2, + bool const_arg2, TCGType type, bool rc) +{ + int mb, me; + + if (!const_arg2) { + tcg_out32(s, AND | SAB(arg1, dest, arg2) | rc); + return; + } + + if (type == TCG_TYPE_I32) { + arg2 = (uint32_t)arg2; + } else if (arg2 == (uint32_t)arg2) { + type = TCG_TYPE_I32; + } + + if ((arg2 & ~0xffff) == 0) { + tcg_out32(s, ANDI | SAI(arg1, dest, arg2)); + return; + } + if ((arg2 & ~0xffff0000ull) == 0) { + tcg_out32(s, ANDIS | SAI(arg1, dest, arg2 >> 16)); + return; + } + if (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32) { + if (mask_operand(arg2, &mb, &me)) { + tcg_out_rlw_rc(s, RLWINM, dest, arg1, 0, mb, me, rc); + return; + } + } else { + int sh = clz64(arg2); + if (mask64_operand(arg2 << sh, &mb, &me)) { + tcg_out_rld_rc(s, RLDICR, dest, arg1, sh, me, rc); + return; + } + } + /* Constraints should satisfy this. */ + g_assert_not_reached(); +} + static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2, int const_arg2, int cr, TCGType type) { @@ -1676,7 +1785,10 @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2, tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32); - /* Simplify the comparisons below wrt CMPI. */ + /* + * Simplify the comparisons below wrt CMPI. + * All of the tests are 16-bit, so a 32-bit sign extend always works. + */ if (type == TCG_TYPE_I32) { arg2 = (int32_t)arg2; } @@ -1699,6 +1811,12 @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2, imm = 0; break; + case TCG_COND_TSTEQ: + case TCG_COND_TSTNE: + tcg_debug_assert(cr == 0); + tcg_out_test(s, TCG_REG_R0, arg1, arg2, const_arg2, type, true); + return; + case TCG_COND_LT: case TCG_COND_GE: case TCG_COND_LE: @@ -1826,7 +1944,7 @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, if (have_isa_3_10) { tcg_insn_unit bi, opc; - tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type); + tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 0, type); /* Re-use tcg_to_bc for BI and BO_COND_{TRUE,FALSE}. */ bi = tcg_to_bc[cond] & (0x1f << 16); @@ -1879,7 +1997,7 @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, if (have_isel) { int isel, tab; - tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type); + tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 0, type); isel = tcg_to_isel[cond]; @@ -1909,6 +2027,16 @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, tcg_out_setcond_ne0(s, type, arg0, arg1, neg); break; + case TCG_COND_TSTEQ: + tcg_out_test(s, TCG_REG_R0, arg1, arg2, const_arg2, type, false); + tcg_out_setcond_eq0(s, type, arg0, TCG_REG_R0, neg); + break; + + case TCG_COND_TSTNE: + tcg_out_test(s, TCG_REG_R0, arg1, arg2, const_arg2, type, false); + tcg_out_setcond_ne0(s, type, arg0, TCG_REG_R0, neg); + break; + case TCG_COND_LE: case TCG_COND_LEU: inv = true; @@ -1945,22 +2073,28 @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, } } -static void tcg_out_bc(TCGContext *s, int bc, TCGLabel *l) +static void tcg_out_bc(TCGContext *s, TCGCond cond, int bd) +{ + tcg_out32(s, tcg_to_bc[cond] | bd); +} + +static void tcg_out_bc_lab(TCGContext *s, TCGCond cond, TCGLabel *l) { + int bd = 0; if (l->has_value) { - bc |= reloc_pc14_val(tcg_splitwx_to_rx(s->code_ptr), l->u.value_ptr); + bd = reloc_pc14_val(tcg_splitwx_to_rx(s->code_ptr), l->u.value_ptr); } else { tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, l, 0); } - tcg_out32(s, bc); + tcg_out_bc(s, cond, bd); } static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGArg arg1, TCGArg arg2, int const_arg2, TCGLabel *l, TCGType type) { - tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type); - tcg_out_bc(s, tcg_to_bc[cond], l); + tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 0, type); + tcg_out_bc_lab(s, cond, l); } static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond, @@ -1973,7 +2107,7 @@ static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond, return; } - tcg_out_cmp(s, cond, c1, c2, const_c2, 7, type); + tcg_out_cmp(s, cond, c1, c2, const_c2, 0, type); if (have_isel) { int isel = tcg_to_isel[cond]; @@ -2002,7 +2136,7 @@ static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond, } } /* Branch forward over one insn */ - tcg_out32(s, tcg_to_bc[cond] | 8); + tcg_out_bc(s, cond, 8); if (v2 == 0) { tcg_out_movi(s, type, dest, 0); } else { @@ -2017,17 +2151,17 @@ static void tcg_out_cntxz(TCGContext *s, TCGType type, uint32_t opc, if (const_a2 && a2 == (type == TCG_TYPE_I32 ? 32 : 64)) { tcg_out32(s, opc | RA(a0) | RS(a1)); } else { - tcg_out_cmp(s, TCG_COND_EQ, a1, 0, 1, 7, type); + tcg_out_cmp(s, TCG_COND_EQ, a1, 0, 1, 0, type); /* Note that the only other valid constant for a2 is 0. */ if (have_isel) { tcg_out32(s, opc | RA(TCG_REG_R0) | RS(a1)); tcg_out32(s, tcg_to_isel[TCG_COND_EQ] | TAB(a0, a2, TCG_REG_R0)); } else if (!const_a2 && a0 == a2) { - tcg_out32(s, tcg_to_bc[TCG_COND_EQ] | 8); + tcg_out_bc(s, TCG_COND_EQ, 8); tcg_out32(s, opc | RA(a0) | RS(a1)); } else { tcg_out32(s, opc | RA(a0) | RS(a1)); - tcg_out32(s, tcg_to_bc[TCG_COND_NE] | 8); + tcg_out_bc(s, TCG_COND_NE, 8); if (const_a2) { tcg_out_movi(s, type, a0, 0); } else { @@ -2072,7 +2206,22 @@ static void tcg_out_cmp2(TCGContext *s, const TCGArg *args, do_equality: tcg_out_cmp(s, cond, al, bl, blconst, 6, TCG_TYPE_I32); tcg_out_cmp(s, cond, ah, bh, bhconst, 7, TCG_TYPE_I32); - tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ)); + tcg_out32(s, op | BT(0, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ)); + break; + + case TCG_COND_TSTEQ: + case TCG_COND_TSTNE: + if (blconst) { + tcg_out_andi32(s, TCG_REG_R0, al, bl); + } else { + tcg_out32(s, AND | SAB(al, TCG_REG_R0, bl)); + } + if (bhconst) { + tcg_out_andi32(s, TCG_REG_TMP1, ah, bh); + } else { + tcg_out32(s, AND | SAB(ah, TCG_REG_TMP1, bh)); + } + tcg_out32(s, OR | SAB(TCG_REG_R0, TCG_REG_R0, TCG_REG_TMP1) | 1); break; case TCG_COND_LT: @@ -2090,8 +2239,8 @@ static void tcg_out_cmp2(TCGContext *s, const TCGArg *args, tcg_out_cmp(s, cond, ah, bh, bhconst, 6, TCG_TYPE_I32); tcg_out_cmp(s, cond2, al, bl, blconst, 7, TCG_TYPE_I32); - tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, bit2)); - tcg_out32(s, CROR | BT(7, CR_EQ) | BA(6, bit1) | BB(7, CR_EQ)); + tcg_out32(s, op | BT(0, CR_EQ) | BA(6, CR_EQ) | BB(7, bit2)); + tcg_out32(s, CROR | BT(0, CR_EQ) | BA(6, bit1) | BB(0, CR_EQ)); break; default: @@ -2103,15 +2252,15 @@ static void tcg_out_setcond2(TCGContext *s, const TCGArg *args, const int *const_args) { tcg_out_cmp2(s, args + 1, const_args + 1); - tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7)); - tcg_out_rlw(s, RLWINM, args[0], TCG_REG_R0, 31, 31, 31); + tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(0)); + tcg_out_rlw(s, RLWINM, args[0], TCG_REG_R0, CR_EQ + 0*4 + 1, 31, 31); } -static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args, - const int *const_args) +static void tcg_out_brcond2(TCGContext *s, const TCGArg *args, + const int *const_args) { tcg_out_cmp2(s, args, const_args); - tcg_out_bc(s, BC | BI(7, CR_EQ) | BO_COND_TRUE, arg_label(args[5])); + tcg_out_bc_lab(s, TCG_COND_EQ, arg_label(args[5])); } static void tcg_out_mb(TCGContext *s, TCGArg a0) @@ -2435,17 +2584,17 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_cmp(s, TCG_COND_EQ, addrhi, TCG_REG_TMP2, 0, 6, TCG_TYPE_I32); - /* Combine comparisons into cr7. */ - tcg_out32(s, CRAND | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ)); + /* Combine comparisons into cr0. */ + tcg_out32(s, CRAND | BT(0, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ)); } else { - /* Full comparison into cr7. */ + /* Full comparison into cr0. */ tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP2, - 0, 7, addr_type); + 0, 0, addr_type); } /* Load a pointer into the current opcode w/conditional branch-link. */ ldst->label_ptr[0] = s->code_ptr; - tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK); + tcg_out_bc(s, TCG_COND_NE, LK); h->base = TCG_REG_TMP1; } else { @@ -3979,8 +4128,6 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_sar_i32: case INDEX_op_rotl_i32: case INDEX_op_rotr_i32: - case INDEX_op_setcond_i32: - case INDEX_op_negsetcond_i32: case INDEX_op_and_i64: case INDEX_op_andc_i64: case INDEX_op_shl_i64: @@ -3988,8 +4135,6 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_sar_i64: case INDEX_op_rotl_i64: case INDEX_op_rotr_i64: - case INDEX_op_setcond_i64: - case INDEX_op_negsetcond_i64: return C_O1_I2(r, r, ri); case INDEX_op_mul_i32: @@ -4033,11 +4178,16 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: - return C_O0_I2(r, ri); - + return C_O0_I2(r, rC); + case INDEX_op_setcond_i32: + case INDEX_op_setcond_i64: + case INDEX_op_negsetcond_i32: + case INDEX_op_negsetcond_i64: + return C_O1_I2(r, r, rC); case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: - return C_O1_I4(r, r, ri, rZ, rZ); + return C_O1_I4(r, r, rC, rZ, rZ); + case INDEX_op_deposit_i32: case INDEX_op_deposit_i64: return C_O1_I2(r, 0, rZ); diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index 5295e4f9ab..04a7aba4d3 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -143,6 +143,8 @@ typedef enum { #define TCG_TARGET_HAS_qemu_ldst_i128 \ (TCG_TARGET_REG_BITS == 64 && have_isa_2_07) +#define TCG_TARGET_HAS_tst 1 + /* * While technically Altivec could support V64, it has no 64-bit store * instruction and substituting two 32-bit stores makes the generated diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index 34e10e77d9..639363039b 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -145,7 +145,8 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define sextreg sextract64 /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { if (ct & TCG_CT_CONST) { return 1; diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index a4edc3dc74..2c1b680b93 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -158,6 +158,8 @@ extern bool have_zbb; #define TCG_TARGET_HAS_qemu_ldst_i128 0 +#define TCG_TARGET_HAS_tst 0 + #define TCG_TARGET_DEFAULT_MO (0) #define TCG_TARGET_NEED_LDST_LABELS diff --git a/tcg/s390x/tcg-target-con-set.h b/tcg/s390x/tcg-target-con-set.h index 9a42037499..f75955eaa8 100644 --- a/tcg/s390x/tcg-target-con-set.h +++ b/tcg/s390x/tcg-target-con-set.h @@ -15,7 +15,7 @@ C_O0_I1(r) C_O0_I2(r, r) C_O0_I2(r, ri) -C_O0_I2(r, rA) +C_O0_I2(r, rC) C_O0_I2(v, r) C_O0_I3(o, m, r) C_O1_I1(r, r) @@ -27,7 +27,7 @@ C_O1_I2(r, 0, rI) C_O1_I2(r, 0, rJ) C_O1_I2(r, r, r) C_O1_I2(r, r, ri) -C_O1_I2(r, r, rA) +C_O1_I2(r, r, rC) C_O1_I2(r, r, rI) C_O1_I2(r, r, rJ) C_O1_I2(r, r, rK) @@ -39,10 +39,10 @@ C_O1_I2(v, v, r) C_O1_I2(v, v, v) C_O1_I3(v, v, v, v) C_O1_I4(r, r, ri, rI, r) -C_O1_I4(r, r, rA, rI, r) +C_O1_I4(r, r, rC, rI, r) C_O2_I1(o, m, r) C_O2_I2(o, m, 0, r) C_O2_I2(o, m, r, r) C_O2_I3(o, m, 0, 1, r) C_N1_O1_I4(r, r, 0, 1, ri, r) -C_N1_O1_I4(r, r, 0, 1, rA, r) +C_N1_O1_I4(r, r, 0, 1, rJU, r) diff --git a/tcg/s390x/tcg-target-con-str.h b/tcg/s390x/tcg-target-con-str.h index 25675b449e..745f6c0df5 100644 --- a/tcg/s390x/tcg-target-con-str.h +++ b/tcg/s390x/tcg-target-con-str.h @@ -16,10 +16,11 @@ REGS('o', 0xaaaa) /* odd numbered general regs */ * Define constraint letters for constants: * CONST(letter, TCG_CT_CONST_* bit set) */ -CONST('A', TCG_CT_CONST_S33) +CONST('C', TCG_CT_CONST_CMP) CONST('I', TCG_CT_CONST_S16) CONST('J', TCG_CT_CONST_S32) CONST('K', TCG_CT_CONST_P32) CONST('N', TCG_CT_CONST_INV) CONST('R', TCG_CT_CONST_INVRISBG) +CONST('U', TCG_CT_CONST_U32) CONST('Z', TCG_CT_CONST_ZERO) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 7f6b84aa2c..ad587325fc 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -30,11 +30,12 @@ #define TCG_CT_CONST_S16 (1 << 8) #define TCG_CT_CONST_S32 (1 << 9) -#define TCG_CT_CONST_S33 (1 << 10) +#define TCG_CT_CONST_U32 (1 << 10) #define TCG_CT_CONST_ZERO (1 << 11) #define TCG_CT_CONST_P32 (1 << 12) #define TCG_CT_CONST_INV (1 << 13) #define TCG_CT_CONST_INVRISBG (1 << 14) +#define TCG_CT_CONST_CMP (1 << 15) #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16) #define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32) @@ -111,6 +112,9 @@ typedef enum S390Opcode { RI_OILH = 0xa50a, RI_OILL = 0xa50b, RI_TMLL = 0xa701, + RI_TMLH = 0xa700, + RI_TMHL = 0xa703, + RI_TMHH = 0xa702, RIEb_CGRJ = 0xec64, RIEb_CLGRJ = 0xec65, @@ -403,10 +407,15 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define S390_CC_NEVER 0 #define S390_CC_ALWAYS 15 +#define S390_TM_EQ 8 /* CC == 0 */ +#define S390_TM_NE 7 /* CC in {1,2,3} */ + /* Condition codes that result from a COMPARE and COMPARE LOGICAL. */ -static const uint8_t tcg_cond_to_s390_cond[] = { +static const uint8_t tcg_cond_to_s390_cond[16] = { [TCG_COND_EQ] = S390_CC_EQ, [TCG_COND_NE] = S390_CC_NE, + [TCG_COND_TSTEQ] = S390_CC_EQ, + [TCG_COND_TSTNE] = S390_CC_NE, [TCG_COND_LT] = S390_CC_LT, [TCG_COND_LE] = S390_CC_LE, [TCG_COND_GT] = S390_CC_GT, @@ -420,9 +429,11 @@ static const uint8_t tcg_cond_to_s390_cond[] = { /* Condition codes that result from a LOAD AND TEST. Here, we have no unsigned instruction variation, however since the test is vs zero we can re-map the outcomes appropriately. */ -static const uint8_t tcg_cond_to_ltr_cond[] = { +static const uint8_t tcg_cond_to_ltr_cond[16] = { [TCG_COND_EQ] = S390_CC_EQ, [TCG_COND_NE] = S390_CC_NE, + [TCG_COND_TSTEQ] = S390_CC_ALWAYS, + [TCG_COND_TSTNE] = S390_CC_NEVER, [TCG_COND_LT] = S390_CC_LT, [TCG_COND_LE] = S390_CC_LE, [TCG_COND_GT] = S390_CC_GT, @@ -538,42 +549,74 @@ static bool risbg_mask(uint64_t c) } /* Test if a constant matches the constraint. */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { + uint64_t uval = val; + if (ct & TCG_CT_CONST) { - return 1; + return true; } - if (type == TCG_TYPE_I32) { + uval = (uint32_t)val; val = (int32_t)val; } - /* The following are mutually exclusive. */ - if (ct & TCG_CT_CONST_S16) { - return val == (int16_t)val; - } else if (ct & TCG_CT_CONST_S32) { - return val == (int32_t)val; - } else if (ct & TCG_CT_CONST_S33) { - return val >= -0xffffffffll && val <= 0xffffffffll; - } else if (ct & TCG_CT_CONST_ZERO) { - return val == 0; + if (ct & TCG_CT_CONST_CMP) { + switch (cond) { + case TCG_COND_EQ: + case TCG_COND_NE: + ct |= TCG_CT_CONST_S32 | TCG_CT_CONST_U32; /* CGFI or CLGFI */ + break; + case TCG_COND_LT: + case TCG_COND_GE: + case TCG_COND_LE: + case TCG_COND_GT: + ct |= TCG_CT_CONST_S32; /* CGFI */ + break; + case TCG_COND_LTU: + case TCG_COND_GEU: + case TCG_COND_LEU: + case TCG_COND_GTU: + ct |= TCG_CT_CONST_U32; /* CLGFI */ + break; + case TCG_COND_TSTNE: + case TCG_COND_TSTEQ: + if (is_const_p16(uval) >= 0) { + return true; /* TMxx */ + } + if (risbg_mask(uval)) { + return true; /* RISBG */ + } + break; + default: + g_assert_not_reached(); + } + } + + if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) { + return true; + } + if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) { + return true; + } + if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) { + return true; + } + if ((ct & TCG_CT_CONST_ZERO) && val == 0) { + return true; } if (ct & TCG_CT_CONST_INV) { val = ~val; } - /* - * Note that is_const_p16 is a subset of is_const_p32, - * so we don't need both constraints. - */ if ((ct & TCG_CT_CONST_P32) && is_const_p32(val) >= 0) { return true; } if ((ct & TCG_CT_CONST_INVRISBG) && risbg_mask(~val)) { return true; } - - return 0; + return false; } /* Emit instructions according to the given instruction format. */ @@ -843,6 +886,9 @@ static const S390Opcode oi_insns[4] = { static const S390Opcode lif_insns[2] = { RIL_LLILF, RIL_LLIHF, }; +static const S390Opcode tm_insns[4] = { + RI_TMLL, RI_TMLH, RI_TMHL, RI_TMHH +}; /* load a register with an immediate value */ static void tcg_out_movi(TCGContext *s, TCGType type, @@ -1203,6 +1249,36 @@ static int tgen_cmp2(TCGContext *s, TCGType type, TCGCond c, TCGReg r1, TCGCond inv_c = tcg_invert_cond(c); S390Opcode op; + if (is_tst_cond(c)) { + tcg_debug_assert(!need_carry); + + if (!c2const) { + if (type == TCG_TYPE_I32) { + tcg_out_insn(s, RRFa, NRK, TCG_REG_R0, r1, c2); + } else { + tcg_out_insn(s, RRFa, NGRK, TCG_REG_R0, r1, c2); + } + goto exit; + } + + if (type == TCG_TYPE_I32) { + c2 = (uint32_t)c2; + } + + int i = is_const_p16(c2); + if (i >= 0) { + tcg_out_insn_RI(s, tm_insns[i], r1, c2 >> (i * 16)); + *inv_cc = c == TCG_COND_TSTEQ ? S390_TM_NE : S390_TM_EQ; + return *inv_cc ^ 15; + } + + if (risbg_mask(c2)) { + tgen_andi_risbg(s, TCG_REG_R0, r1, c2); + goto exit; + } + g_assert_not_reached(); + } + if (c2const) { if (c2 == 0) { if (!(is_unsigned && need_carry)) { @@ -1228,22 +1304,34 @@ static int tgen_cmp2(TCGContext *s, TCGType type, TCGCond c, TCGReg r1, goto exit; } - /* - * Constraints are for a signed 33-bit operand, which is a - * convenient superset of this signed/unsigned test. - */ - if (c2 == (is_unsigned ? (TCGArg)(uint32_t)c2 : (TCGArg)(int32_t)c2)) { - op = (is_unsigned ? RIL_CLGFI : RIL_CGFI); - tcg_out_insn_RIL(s, op, r1, c2); - goto exit; + /* Should match TCG_CT_CONST_CMP. */ + switch (c) { + case TCG_COND_LT: + case TCG_COND_GE: + case TCG_COND_LE: + case TCG_COND_GT: + tcg_debug_assert(c2 == (int32_t)c2); + op = RIL_CGFI; + break; + case TCG_COND_EQ: + case TCG_COND_NE: + if (c2 == (int32_t)c2) { + op = RIL_CGFI; + break; + } + /* fall through */ + case TCG_COND_LTU: + case TCG_COND_GEU: + case TCG_COND_LEU: + case TCG_COND_GTU: + tcg_debug_assert(c2 == (uint32_t)c2); + op = RIL_CLGFI; + break; + default: + g_assert_not_reached(); } - - /* Load everything else into a register. */ - tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, c2); - c2 = TCG_TMP0; - } - - if (type == TCG_TYPE_I32) { + tcg_out_insn_RIL(s, op, r1, c2); + } else if (type == TCG_TYPE_I32) { op = (is_unsigned ? RR_CLR : RR_CR); tcg_out_insn_RR(s, op, r1, c2); } else { @@ -1516,46 +1604,49 @@ static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c, TCGReg r1, TCGArg c2, int c2const, TCGLabel *l) { int cc; - bool is_unsigned = is_unsigned_cond(c); - bool in_range; - S390Opcode opc; - cc = tcg_cond_to_s390_cond[c]; + if (!is_tst_cond(c)) { + bool is_unsigned = is_unsigned_cond(c); + bool in_range; + S390Opcode opc; - if (!c2const) { - opc = (type == TCG_TYPE_I32 - ? (is_unsigned ? RIEb_CLRJ : RIEb_CRJ) - : (is_unsigned ? RIEb_CLGRJ : RIEb_CGRJ)); - tgen_compare_branch(s, opc, cc, r1, c2, l); - return; - } + cc = tcg_cond_to_s390_cond[c]; - /* - * COMPARE IMMEDIATE AND BRANCH RELATIVE has an 8-bit immediate field. - * If the immediate we've been given does not fit that range, we'll - * fall back to separate compare and branch instructions using the - * larger comparison range afforded by COMPARE IMMEDIATE. - */ - if (type == TCG_TYPE_I32) { - if (is_unsigned) { - opc = RIEc_CLIJ; - in_range = (uint32_t)c2 == (uint8_t)c2; - } else { - opc = RIEc_CIJ; - in_range = (int32_t)c2 == (int8_t)c2; + if (!c2const) { + opc = (type == TCG_TYPE_I32 + ? (is_unsigned ? RIEb_CLRJ : RIEb_CRJ) + : (is_unsigned ? RIEb_CLGRJ : RIEb_CGRJ)); + tgen_compare_branch(s, opc, cc, r1, c2, l); + return; } - } else { - if (is_unsigned) { - opc = RIEc_CLGIJ; - in_range = (uint64_t)c2 == (uint8_t)c2; + + /* + * COMPARE IMMEDIATE AND BRANCH RELATIVE has an 8-bit immediate field. + * If the immediate we've been given does not fit that range, we'll + * fall back to separate compare and branch instructions using the + * larger comparison range afforded by COMPARE IMMEDIATE. + */ + if (type == TCG_TYPE_I32) { + if (is_unsigned) { + opc = RIEc_CLIJ; + in_range = (uint32_t)c2 == (uint8_t)c2; + } else { + opc = RIEc_CIJ; + in_range = (int32_t)c2 == (int8_t)c2; + } } else { - opc = RIEc_CGIJ; - in_range = (int64_t)c2 == (int8_t)c2; + if (is_unsigned) { + opc = RIEc_CLGIJ; + in_range = (uint64_t)c2 == (uint8_t)c2; + } else { + opc = RIEc_CGIJ; + in_range = (int64_t)c2 == (int8_t)c2; + } + } + if (in_range) { + tgen_compare_imm_branch(s, opc, cc, r1, c2, l); + return; } - } - if (in_range) { - tgen_compare_imm_branch(s, opc, cc, r1, c2, l); - return; } cc = tgen_cmp(s, type, c, r1, c2, c2const, false); @@ -1834,11 +1925,10 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->oi = oi; ldst->addrlo_reg = addr_reg; - /* We are expecting a_bits to max out at 7, much lower than TMLL. */ tcg_debug_assert(a_mask <= 0xffff); tcg_out_insn(s, RI, TMLL, addr_reg, a_mask); - tcg_out16(s, RI_BRC | (7 << 4)); /* CC in {1,2,3} */ + tcg_out16(s, RI_BRC | (S390_TM_NE << 4)); ldst->label_ptr[0] = s->code_ptr++; } @@ -1919,7 +2009,7 @@ static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi, l2 = gen_new_label(); tcg_out_insn(s, RI, TMLL, addr_reg, 15); - tgen_branch(s, 7, l1); /* CC in {1,2,3} */ + tgen_branch(s, S390_TM_NE, l1); } tcg_debug_assert(!need_bswap); @@ -3136,7 +3226,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) return C_O1_I2(r, r, ri); case INDEX_op_setcond_i64: case INDEX_op_negsetcond_i64: - return C_O1_I2(r, r, rA); + return C_O1_I2(r, r, rC); case INDEX_op_clz_i64: return C_O1_I2(r, r, rI); @@ -3186,7 +3276,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_brcond_i32: return C_O0_I2(r, ri); case INDEX_op_brcond_i64: - return C_O0_I2(r, rA); + return C_O0_I2(r, rC); case INDEX_op_bswap16_i32: case INDEX_op_bswap16_i64: @@ -3239,7 +3329,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_movcond_i32: return C_O1_I4(r, r, ri, rI, r); case INDEX_op_movcond_i64: - return C_O1_I4(r, r, rA, rI, r); + return C_O1_I4(r, r, rC, rI, r); case INDEX_op_div2_i32: case INDEX_op_div2_i64: @@ -3258,7 +3348,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_add2_i64: case INDEX_op_sub2_i64: - return C_N1_O1_I4(r, r, 0, 1, rA, r); + return C_N1_O1_I4(r, r, 0, 1, rJU, r); case INDEX_op_st_vec: return C_O0_I2(v, r); diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index e69b0d2ddd..ae448c3a3a 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -138,6 +138,8 @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_qemu_ldst_i128 1 +#define TCG_TARGET_HAS_tst 1 + #define TCG_TARGET_HAS_v64 HAVE_FACILITY(VECTOR) #define TCG_TARGET_HAS_v128 HAVE_FACILITY(VECTOR) #define TCG_TARGET_HAS_v256 0 diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index a91defd0ac..176c98740b 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -322,7 +322,8 @@ static bool patch_reloc(tcg_insn_unit *src_rw, int type, } /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -606,9 +607,11 @@ static void tcg_out_div32(TCGContext *s, TCGReg rd, TCGReg rs1, uns ? ARITH_UDIV : ARITH_SDIV); } -static const uint8_t tcg_cond_to_bcond[] = { +static const uint8_t tcg_cond_to_bcond[16] = { [TCG_COND_EQ] = COND_E, [TCG_COND_NE] = COND_NE, + [TCG_COND_TSTEQ] = COND_E, + [TCG_COND_TSTNE] = COND_NE, [TCG_COND_LT] = COND_L, [TCG_COND_GE] = COND_GE, [TCG_COND_LE] = COND_LE, @@ -619,7 +622,7 @@ static const uint8_t tcg_cond_to_bcond[] = { [TCG_COND_GTU] = COND_GU, }; -static const uint8_t tcg_cond_to_rcond[] = { +static const uint8_t tcg_cond_to_rcond[16] = { [TCG_COND_EQ] = RCOND_Z, [TCG_COND_NE] = RCOND_NZ, [TCG_COND_LT] = RCOND_LZ, @@ -645,15 +648,17 @@ static void tcg_out_bpcc(TCGContext *s, int scond, int flags, TCGLabel *l) tcg_out_bpcc0(s, scond, flags, off19); } -static void tcg_out_cmp(TCGContext *s, TCGReg c1, int32_t c2, int c2const) +static void tcg_out_cmp(TCGContext *s, TCGCond cond, + TCGReg c1, int32_t c2, int c2const) { - tcg_out_arithc(s, TCG_REG_G0, c1, c2, c2const, ARITH_SUBCC); + tcg_out_arithc(s, TCG_REG_G0, c1, c2, c2const, + is_tst_cond(cond) ? ARITH_ANDCC : ARITH_SUBCC); } static void tcg_out_brcond_i32(TCGContext *s, TCGCond cond, TCGReg arg1, int32_t arg2, int const_arg2, TCGLabel *l) { - tcg_out_cmp(s, arg1, arg2, const_arg2); + tcg_out_cmp(s, cond, arg1, arg2, const_arg2); tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_ICC | BPCC_PT, l); tcg_out_nop(s); } @@ -670,7 +675,7 @@ static void tcg_out_movcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg c1, int32_t c2, int c2const, int32_t v1, int v1const) { - tcg_out_cmp(s, c1, c2, c2const); + tcg_out_cmp(s, cond, c1, c2, c2const); tcg_out_movcc(s, cond, MOVCC_ICC, ret, v1, v1const); } @@ -678,7 +683,8 @@ static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond, TCGReg arg1, int32_t arg2, int const_arg2, TCGLabel *l) { /* For 64-bit signed comparisons vs zero, we can avoid the compare. */ - if (arg2 == 0 && !is_unsigned_cond(cond)) { + int rcond = tcg_cond_to_rcond[cond]; + if (arg2 == 0 && rcond) { int off16 = 0; if (l->has_value) { @@ -687,19 +693,18 @@ static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond, TCGReg arg1, tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP16, l, 0); } tcg_out32(s, INSN_OP(0) | INSN_OP2(3) | BPR_PT | INSN_RS1(arg1) - | INSN_COND(tcg_cond_to_rcond[cond]) | off16); + | INSN_COND(rcond) | off16); } else { - tcg_out_cmp(s, arg1, arg2, const_arg2); + tcg_out_cmp(s, cond, arg1, arg2, const_arg2); tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_XCC | BPCC_PT, l); } tcg_out_nop(s); } -static void tcg_out_movr(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg c1, +static void tcg_out_movr(TCGContext *s, int rcond, TCGReg ret, TCGReg c1, int32_t v1, int v1const) { - tcg_out32(s, ARITH_MOVR | INSN_RD(ret) | INSN_RS1(c1) - | (tcg_cond_to_rcond[cond] << 10) + tcg_out32(s, ARITH_MOVR | INSN_RD(ret) | INSN_RS1(c1) | (rcond << 10) | (v1const ? INSN_IMM10(v1) : INSN_RS2(v1))); } @@ -710,11 +715,11 @@ static void tcg_out_movcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, /* For 64-bit signed comparisons vs zero, we can avoid the compare. Note that the immediate range is one bit smaller, so we must check for that as well. */ - if (c2 == 0 && !is_unsigned_cond(cond) - && (!v1const || check_fit_i32(v1, 10))) { - tcg_out_movr(s, cond, ret, c1, v1, v1const); + int rcond = tcg_cond_to_rcond[cond]; + if (c2 == 0 && rcond && (!v1const || check_fit_i32(v1, 10))) { + tcg_out_movr(s, rcond, ret, c1, v1, v1const); } else { - tcg_out_cmp(s, c1, c2, c2const); + tcg_out_cmp(s, cond, c1, c2, c2const); tcg_out_movcc(s, cond, MOVCC_XCC, ret, v1, v1const); } } @@ -742,6 +747,15 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, cond = (cond == TCG_COND_EQ ? TCG_COND_GEU : TCG_COND_LTU); break; + case TCG_COND_TSTEQ: + case TCG_COND_TSTNE: + /* Transform to inequality vs zero. */ + tcg_out_arithc(s, TCG_REG_T1, c1, c2, c2const, ARITH_AND); + c1 = TCG_REG_G0; + c2 = TCG_REG_T1, c2const = 0; + cond = (cond == TCG_COND_TSTEQ ? TCG_COND_GEU : TCG_COND_LTU); + break; + case TCG_COND_GTU: case TCG_COND_LEU: /* If we don't need to load a constant into a register, we can @@ -758,13 +772,13 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, /* FALLTHRU */ default: - tcg_out_cmp(s, c1, c2, c2const); + tcg_out_cmp(s, cond, c1, c2, c2const); tcg_out_movi_s13(s, ret, 0); tcg_out_movcc(s, cond, MOVCC_ICC, ret, neg ? -1 : 1, 1); return; } - tcg_out_cmp(s, c1, c2, c2const); + tcg_out_cmp(s, cond, c1, c2, c2const); if (cond == TCG_COND_LTU) { if (neg) { /* 0 - 0 - C = -C = (C ? -1 : 0) */ @@ -787,6 +801,8 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret, static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg c1, int32_t c2, int c2const, bool neg) { + int rcond; + if (use_vis3_instructions && !neg) { switch (cond) { case TCG_COND_NE: @@ -796,7 +812,7 @@ static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, c2 = c1, c2const = 0, c1 = TCG_REG_G0; /* FALLTHRU */ case TCG_COND_LTU: - tcg_out_cmp(s, c1, c2, c2const); + tcg_out_cmp(s, cond, c1, c2, c2const); tcg_out_arith(s, ret, TCG_REG_G0, TCG_REG_G0, ARITH_ADDXC); return; default: @@ -806,11 +822,12 @@ static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret, /* For 64-bit signed comparisons vs zero, we can avoid the compare if the input does not overlap the output. */ - if (c2 == 0 && !is_unsigned_cond(cond) && c1 != ret) { + rcond = tcg_cond_to_rcond[cond]; + if (c2 == 0 && rcond && c1 != ret) { tcg_out_movi_s13(s, ret, 0); - tcg_out_movr(s, cond, ret, c1, neg ? -1 : 1, 1); + tcg_out_movr(s, rcond, ret, c1, neg ? -1 : 1, 1); } else { - tcg_out_cmp(s, c1, c2, c2const); + tcg_out_cmp(s, cond, c1, c2, c2const); tcg_out_movi_s13(s, ret, 0); tcg_out_movcc(s, cond, MOVCC_XCC, ret, neg ? -1 : 1, 1); } @@ -1098,7 +1115,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_movi_s32(s, TCG_REG_T3, compare_mask); tcg_out_arith(s, TCG_REG_T3, addr_reg, TCG_REG_T3, ARITH_AND); } - tcg_out_cmp(s, TCG_REG_T2, TCG_REG_T3, 0); + tcg_out_cmp(s, TCG_COND_NE, TCG_REG_T2, TCG_REG_T3, 0); ldst = new_ldst_label(s); ldst->is_ld = is_ld; diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h index f8cf145266..a18906a14e 100644 --- a/tcg/sparc64/tcg-target.h +++ b/tcg/sparc64/tcg-target.h @@ -149,6 +149,8 @@ extern bool use_vis3_instructions; #define TCG_TARGET_HAS_qemu_ldst_i128 0 +#define TCG_TARGET_HAS_tst 1 + #define TCG_AREG0 TCG_REG_I0 #define TCG_TARGET_DEFAULT_MO (0) diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index 6c9d9e48db..9b0d982f65 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -83,6 +83,8 @@ static inline TCGv_i64 TCGV128_HIGH(TCGv_i128 t) bool tcg_target_has_memory_bswap(MemOp memop); +TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind); + /* * Locate or create a read-only temporary that is a constant. * This kind of temporary need not be freed, but for convenience diff --git a/tcg/tcg.c b/tcg/tcg.c index eeff4c1d51..d6670237fb 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -173,7 +173,8 @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, const TCGHelperInfo *info); static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot); -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece); +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece); #ifdef TCG_TARGET_NEED_LDST_LABELS static int tcg_out_ldst_finalize(TCGContext *s); #endif @@ -1655,7 +1656,7 @@ TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name) return temp_tcgv_ptr(ts); } -static TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind) +TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind) { TCGContext *s = tcg_ctx; TCGTemp *ts; @@ -2482,7 +2483,9 @@ static const char * const cond_name[] = [TCG_COND_LTU] = "ltu", [TCG_COND_GEU] = "geu", [TCG_COND_LEU] = "leu", - [TCG_COND_GTU] = "gtu" + [TCG_COND_GTU] = "gtu", + [TCG_COND_TSTEQ] = "tsteq", + [TCG_COND_TSTNE] = "tstne", }; static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] = @@ -4784,6 +4787,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) TCGTemp *ts; TCGArg new_args[TCG_MAX_OP_ARGS]; int const_args[TCG_MAX_OP_ARGS]; + TCGCond op_cond; nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; @@ -4796,6 +4800,33 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) i_allocated_regs = s->reserved_regs; o_allocated_regs = s->reserved_regs; + switch (op->opc) { + case INDEX_op_brcond_i32: + case INDEX_op_brcond_i64: + op_cond = op->args[2]; + break; + case INDEX_op_setcond_i32: + case INDEX_op_setcond_i64: + case INDEX_op_negsetcond_i32: + case INDEX_op_negsetcond_i64: + case INDEX_op_cmp_vec: + op_cond = op->args[3]; + break; + case INDEX_op_brcond2_i32: + op_cond = op->args[4]; + break; + case INDEX_op_movcond_i32: + case INDEX_op_movcond_i64: + case INDEX_op_setcond2_i32: + case INDEX_op_cmpsel_vec: + op_cond = op->args[5]; + break; + default: + /* No condition within opcode. */ + op_cond = TCG_COND_ALWAYS; + break; + } + /* satisfy input constraints */ for (k = 0; k < nb_iargs; k++) { TCGRegSet i_preferred_regs, i_required_regs; @@ -4809,7 +4840,8 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) ts = arg_temp(arg); if (ts->val_type == TEMP_VAL_CONST - && tcg_target_const_match(ts->val, ts->type, arg_ct->ct, TCGOP_VECE(op))) { + && tcg_target_const_match(ts->val, arg_ct->ct, ts->type, + op_cond, TCGOP_VECE(op))) { /* constant is OK for instruction */ const_args[i] = 1; new_args[i] = ts->val; diff --git a/tcg/tci.c b/tcg/tci.c index 3cc851b7bd..39adcb7d82 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -228,6 +228,12 @@ static bool tci_compare32(uint32_t u0, uint32_t u1, TCGCond condition) case TCG_COND_GTU: result = (u0 > u1); break; + case TCG_COND_TSTEQ: + result = (u0 & u1) == 0; + break; + case TCG_COND_TSTNE: + result = (u0 & u1) != 0; + break; default: g_assert_not_reached(); } @@ -270,6 +276,12 @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) case TCG_COND_GTU: result = (u0 > u1); break; + case TCG_COND_TSTEQ: + result = (u0 & u1) == 0; + break; + case TCG_COND_TSTNE: + result = (u0 & u1) != 0; + break; default: g_assert_not_reached(); } @@ -1041,6 +1053,8 @@ static const char *str_c(TCGCond c) [TCG_COND_GEU] = "geu", [TCG_COND_LEU] = "leu", [TCG_COND_GTU] = "gtu", + [TCG_COND_TSTEQ] = "tsteq", + [TCG_COND_TSTNE] = "tstne", }; assert((unsigned)c < ARRAY_SIZE(cond)); diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index 461f4b47ff..c740864b96 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -913,7 +913,8 @@ static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, } /* Test if a constant matches the constraint. */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { return ct & TCG_CT_CONST; } diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index 2a13816c8e..a076f401d2 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -117,6 +117,8 @@ #define TCG_TARGET_HAS_qemu_ldst_i128 0 +#define TCG_TARGET_HAS_tst 1 + /* Number of registers available. */ #define TCG_TARGET_NB_REGS 16 diff --git a/tests/qemu-iotests/check b/tests/qemu-iotests/check index f2e9d27dcf..56d88ca423 100755 --- a/tests/qemu-iotests/check +++ b/tests/qemu-iotests/check @@ -184,7 +184,8 @@ if __name__ == '__main__': sys.exit(str(e)) if args.dry_run: - print('\n'.join([os.path.basename(t) for t in tests])) + with env: + print('\n'.join([os.path.basename(t) for t in tests])) else: with TestRunner(env, tap=args.tap, color=args.color) as tr: diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py index 3ff38f2661..588f30a4f1 100644 --- a/tests/qemu-iotests/testenv.py +++ b/tests/qemu-iotests/testenv.py @@ -126,7 +126,7 @@ class TestEnv(ContextManager['TestEnv']): self.tmp_sock_dir = False Path(self.sock_dir).mkdir(parents=True, exist_ok=True) except KeyError: - self.sock_dir = tempfile.mkdtemp() + self.sock_dir = tempfile.mkdtemp(prefix="qemu-iotests-") self.tmp_sock_dir = True self.sample_img_dir = os.getenv('SAMPLE_IMG_DIR', diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c index 7675519cfa..8a5bb1752e 100644 --- a/tests/qtest/migration-test.c +++ b/tests/qtest/migration-test.c @@ -819,7 +819,7 @@ static int test_migrate_start(QTestState **from, QTestState **to, } else if (strcmp(arch, "aarch64") == 0) { memory_size = "150M"; machine_alias = "virt"; - machine_opts = "gic-version=max"; + machine_opts = "gic-version=3"; arch_opts = g_strdup_printf("-cpu max -kernel %s", bootpath); start_address = ARM_TEST_MEM_START; end_address = ARM_TEST_MEM_END; |