diff options
Diffstat (limited to '')
397 files changed, 21819 insertions, 7724 deletions
diff --git a/.gitlab-ci.d/base.yml b/.gitlab-ci.d/base.yml index 188a770799..ef173a34e6 100644 --- a/.gitlab-ci.d/base.yml +++ b/.gitlab-ci.d/base.yml @@ -68,7 +68,7 @@ variables: ############################################################# # Stage 2: fine tune execution of jobs in specific scenarios - # where the catch all logic is inapprorpaite + # where the catch all logic is inappropriate ############################################################# # Optional jobs should not be run unless manually triggered diff --git a/.gitlab-ci.d/cirrus.yml b/.gitlab-ci.d/cirrus.yml index 41d64d6680..e7f1f83c2c 100644 --- a/.gitlab-ci.d/cirrus.yml +++ b/.gitlab-ci.d/cirrus.yml @@ -15,8 +15,10 @@ stage: build image: registry.gitlab.com/libvirt/libvirt-ci/cirrus-run:master needs: [] + # 20 mins larger than "timeout_in" in cirrus/build.yml + # as there's often a 5-10 minute delay before Cirrus CI + # actually starts the task timeout: 80m - allow_failure: true script: - source .gitlab-ci.d/cirrus/$NAME.vars - sed -e "s|[@]CI_REPOSITORY_URL@|$CI_REPOSITORY_URL|g" diff --git a/.gitlab-ci.d/cirrus/build.yml b/.gitlab-ci.d/cirrus/build.yml index a9444902ec..29d55c4aa3 100644 --- a/.gitlab-ci.d/cirrus/build.yml +++ b/.gitlab-ci.d/cirrus/build.yml @@ -16,6 +16,8 @@ env: TEST_TARGETS: "@TEST_TARGETS@" build_task: + # A little shorter than GitLab timeout in ../cirrus.yml + timeout_in: 60m install_script: - @UPDATE_COMMAND@ - @INSTALL_COMMAND@ @PKGS@ diff --git a/MAINTAINERS b/MAINTAINERS index 00562f924f..355b1960ce 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -298,11 +298,9 @@ F: hw/openrisc/ F: tests/tcg/openrisc/ PowerPC TCG CPUs +M: Nicholas Piggin <npiggin@gmail.com> M: Daniel Henrique Barboza <danielhb413@gmail.com> R: Cédric Le Goater <clg@kaod.org> -R: David Gibson <david@gibson.dropbear.id.au> -R: Greg Kurz <groug@kaod.org> -R: Nicholas Piggin <npiggin@gmail.com> L: qemu-ppc@nongnu.org S: Odd Fixes F: target/ppc/ @@ -438,10 +436,9 @@ F: target/mips/kvm* F: target/mips/sysemu/ PPC KVM CPUs -M: Daniel Henrique Barboza <danielhb413@gmail.com> +M: Nicholas Piggin <npiggin@gmail.com> +R: Daniel Henrique Barboza <danielhb413@gmail.com> R: Cédric Le Goater <clg@kaod.org> -R: David Gibson <david@gibson.dropbear.id.au> -R: Greg Kurz <groug@kaod.org> S: Odd Fixes F: target/ppc/kvm.c @@ -1430,10 +1427,10 @@ F: include/hw/rtc/m48t59.h F: tests/avocado/ppc_prep_40p.py sPAPR (pseries) -M: Daniel Henrique Barboza <danielhb413@gmail.com> +M: Nicholas Piggin <npiggin@gmail.com> +R: Daniel Henrique Barboza <danielhb413@gmail.com> R: Cédric Le Goater <clg@kaod.org> R: David Gibson <david@gibson.dropbear.id.au> -R: Greg Kurz <groug@kaod.org> R: Harsh Prateek Bora <harshpb@linux.ibm.com> L: qemu-ppc@nongnu.org S: Odd Fixes @@ -1452,8 +1449,8 @@ F: tests/avocado/ppc_pseries.py PowerNV (Non-Virtualized) M: Cédric Le Goater <clg@kaod.org> +M: Nicholas Piggin <npiggin@gmail.com> R: Frédéric Barrat <fbarrat@linux.ibm.com> -R: Nicholas Piggin <npiggin@gmail.com> L: qemu-ppc@nongnu.org S: Odd Fixes F: docs/system/ppc/powernv.rst @@ -1497,12 +1494,9 @@ F: include/hw/pci-host/mv64361.h Virtual Open Firmware (VOF) M: Alexey Kardashevskiy <aik@ozlabs.ru> -R: Cédric Le Goater <clg@kaod.org> -R: Daniel Henrique Barboza <danielhb413@gmail.com> R: David Gibson <david@gibson.dropbear.id.au> -R: Greg Kurz <groug@kaod.org> L: qemu-ppc@nongnu.org -S: Maintained +S: Odd Fixes F: hw/ppc/spapr_vof* F: hw/ppc/vof* F: include/hw/ppc/vof* @@ -2957,12 +2951,17 @@ W: http://info.iet.unipi.it/~luigi/netmap/ S: Maintained F: net/netmap.c +AF_XDP network backend +R: Ilya Maximets <i.maximets@ovn.org> +F: net/af-xdp.c + Host Memory Backends M: David Hildenbrand <david@redhat.com> M: Igor Mammedov <imammedo@redhat.com> S: Maintained F: backends/hostmem*.c F: include/sysemu/hostmem.h +F: docs/system/vm-templating.rst T: git https://gitlab.com/ehabkost/qemu.git machine-next Cryptodev Backends diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index c643d66190..3270f65c20 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -1193,6 +1193,7 @@ void tlb_set_page_full(CPUState *cpu, int mmu_idx, write_flags = read_flags; if (is_ram) { iotlb = memory_region_get_ram_addr(section->mr) + xlat; + assert(!(iotlb & ~TARGET_PAGE_MASK)); /* * Computing is_clean is expensive; avoid all that unless * the page is actually writable. @@ -1255,16 +1256,18 @@ void tlb_set_page_full(CPUState *cpu, int mmu_idx, /* refill the tlb */ /* - * At this point iotlb contains a physical section number in the lower - * TARGET_PAGE_BITS, and either - * + the ram_addr_t of the page base of the target RAM (RAM) - * + the offset within section->mr of the page base (I/O, ROMD) + * When memory region is ram, iotlb contains a TARGET_PAGE_BITS + * aligned ram_addr_t of the page base of the target RAM. + * Otherwise, iotlb contains + * - a physical section number in the lower TARGET_PAGE_BITS + * - the offset within section->mr of the page base (I/O, ROMD) with the + * TARGET_PAGE_BITS masked off. * We subtract addr_page (which is page aligned and thus won't * disturb the low bits) to give an offset which can be added to the * (non-page-aligned) vaddr of the eventual memory access to get * the MemoryRegion offset for the access. Note that the vaddr we * subtract here is that of the page base, and not the same as the - * vaddr we add back in io_readx()/io_writex()/get_page_addr_code(). + * vaddr we add back in io_prepare()/get_page_addr_code(). */ desc->fulltlb[index] = *full; full = &desc->fulltlb[index]; @@ -1347,116 +1350,41 @@ static inline void cpu_unaligned_access(CPUState *cpu, vaddr addr, mmu_idx, retaddr); } -static inline void cpu_transaction_failed(CPUState *cpu, hwaddr physaddr, - vaddr addr, unsigned size, - MMUAccessType access_type, - int mmu_idx, MemTxAttrs attrs, - MemTxResult response, - uintptr_t retaddr) -{ - CPUClass *cc = CPU_GET_CLASS(cpu); - - if (!cpu->ignore_memory_transaction_failures && - cc->tcg_ops->do_transaction_failed) { - cc->tcg_ops->do_transaction_failed(cpu, physaddr, addr, size, - access_type, mmu_idx, attrs, - response, retaddr); - } -} - -/* - * Save a potentially trashed CPUTLBEntryFull for later lookup by plugin. - * This is read by tlb_plugin_lookup if the fulltlb entry doesn't match - * because of the side effect of io_writex changing memory layout. - */ -static void save_iotlb_data(CPUState *cs, MemoryRegionSection *section, - hwaddr mr_offset) -{ -#ifdef CONFIG_PLUGIN - SavedIOTLB *saved = &cs->saved_iotlb; - saved->section = section; - saved->mr_offset = mr_offset; -#endif -} - -static uint64_t io_readx(CPUArchState *env, CPUTLBEntryFull *full, - int mmu_idx, vaddr addr, uintptr_t retaddr, - MMUAccessType access_type, MemOp op) +static MemoryRegionSection * +io_prepare(hwaddr *out_offset, CPUArchState *env, hwaddr xlat, + MemTxAttrs attrs, vaddr addr, uintptr_t retaddr) { CPUState *cpu = env_cpu(env); - hwaddr mr_offset; MemoryRegionSection *section; - MemoryRegion *mr; - uint64_t val; - MemTxResult r; + hwaddr mr_offset; - section = iotlb_to_section(cpu, full->xlat_section, full->attrs); - mr = section->mr; - mr_offset = (full->xlat_section & TARGET_PAGE_MASK) + addr; + section = iotlb_to_section(cpu, xlat, attrs); + mr_offset = (xlat & TARGET_PAGE_MASK) + addr; cpu->mem_io_pc = retaddr; if (!cpu->can_do_io) { cpu_io_recompile(cpu, retaddr); } - /* - * The memory_region_dispatch may trigger a flush/resize - * so for plugins we save the iotlb_data just in case. - */ - save_iotlb_data(cpu, section, mr_offset); - - { - QEMU_IOTHREAD_LOCK_GUARD(); - r = memory_region_dispatch_read(mr, mr_offset, &val, op, full->attrs); - } - - if (r != MEMTX_OK) { - hwaddr physaddr = mr_offset + - section->offset_within_address_space - - section->offset_within_region; - - cpu_transaction_failed(cpu, physaddr, addr, memop_size(op), access_type, - mmu_idx, full->attrs, r, retaddr); - } - return val; + *out_offset = mr_offset; + return section; } -static void io_writex(CPUArchState *env, CPUTLBEntryFull *full, - int mmu_idx, uint64_t val, vaddr addr, - uintptr_t retaddr, MemOp op) +static void io_failed(CPUArchState *env, CPUTLBEntryFull *full, vaddr addr, + unsigned size, MMUAccessType access_type, int mmu_idx, + MemTxResult response, uintptr_t retaddr) { CPUState *cpu = env_cpu(env); - hwaddr mr_offset; - MemoryRegionSection *section; - MemoryRegion *mr; - MemTxResult r; - - section = iotlb_to_section(cpu, full->xlat_section, full->attrs); - mr = section->mr; - mr_offset = (full->xlat_section & TARGET_PAGE_MASK) + addr; - if (!cpu->can_do_io) { - cpu_io_recompile(cpu, retaddr); - } - cpu->mem_io_pc = retaddr; - /* - * The memory_region_dispatch may trigger a flush/resize - * so for plugins we save the iotlb_data just in case. - */ - save_iotlb_data(cpu, section, mr_offset); - - { - QEMU_IOTHREAD_LOCK_GUARD(); - r = memory_region_dispatch_write(mr, mr_offset, val, op, full->attrs); - } + if (!cpu->ignore_memory_transaction_failures) { + CPUClass *cc = CPU_GET_CLASS(cpu); - if (r != MEMTX_OK) { - hwaddr physaddr = mr_offset + - section->offset_within_address_space - - section->offset_within_region; + if (cc->tcg_ops->do_transaction_failed) { + hwaddr physaddr = full->phys_addr | (addr & ~TARGET_PAGE_MASK); - cpu_transaction_failed(cpu, physaddr, addr, memop_size(op), - MMU_DATA_STORE, mmu_idx, full->attrs, r, - retaddr); + cc->tcg_ops->do_transaction_failed(cpu, physaddr, addr, size, + access_type, mmu_idx, + full->attrs, response, retaddr); + } } } @@ -1726,45 +1654,41 @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, vaddr addr, * in the softmmu lookup code (or helper). We don't handle re-fills or * checking the victim table. This is purely informational. * - * This almost never fails as the memory access being instrumented - * should have just filled the TLB. The one corner case is io_writex - * which can cause TLB flushes and potential resizing of the TLBs - * losing the information we need. In those cases we need to recover - * data from a copy of the CPUTLBEntryFull. As long as this always occurs - * from the same thread (which a mem callback will be) this is safe. + * The one corner case is i/o write, which can cause changes to the + * address space. Those changes, and the corresponding tlb flush, + * should be delayed until the next TB, so even then this ought not fail. + * But check, Just in Case. */ - bool tlb_plugin_lookup(CPUState *cpu, vaddr addr, int mmu_idx, bool is_store, struct qemu_plugin_hwaddr *data) { CPUArchState *env = cpu->env_ptr; CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr); uintptr_t index = tlb_index(env, mmu_idx, addr); - uint64_t tlb_addr = is_store ? tlb_addr_write(tlbe) : tlbe->addr_read; - - if (likely(tlb_hit(tlb_addr, addr))) { - /* We must have an iotlb entry for MMIO */ - if (tlb_addr & TLB_MMIO) { - CPUTLBEntryFull *full; - full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; - data->is_io = true; - data->v.io.section = - iotlb_to_section(cpu, full->xlat_section, full->attrs); - data->v.io.offset = (full->xlat_section & TARGET_PAGE_MASK) + addr; - } else { - data->is_io = false; - data->v.ram.hostaddr = (void *)((uintptr_t)addr + tlbe->addend); - } - return true; - } else { - SavedIOTLB *saved = &cpu->saved_iotlb; + MMUAccessType access_type = is_store ? MMU_DATA_STORE : MMU_DATA_LOAD; + uint64_t tlb_addr = tlb_read_idx(tlbe, access_type); + CPUTLBEntryFull *full; + + if (unlikely(!tlb_hit(tlb_addr, addr))) { + return false; + } + + full = &env_tlb(env)->d[mmu_idx].fulltlb[index]; + data->phys_addr = full->phys_addr | (addr & ~TARGET_PAGE_MASK); + + /* We must have an iotlb entry for MMIO */ + if (tlb_addr & TLB_MMIO) { + MemoryRegionSection *section = + iotlb_to_section(cpu, full->xlat_section & ~TARGET_PAGE_MASK, + full->attrs); data->is_io = true; - data->v.io.section = saved->section; - data->v.io.offset = saved->mr_offset; - return true; + data->mr = section->mr; + } else { + data->is_io = false; + data->mr = NULL; } + return true; } - #endif /* @@ -2084,47 +2008,90 @@ static void *atomic_mmu_lookup(CPUArchState *env, vaddr addr, MemOpIdx oi, * Load @size bytes from @addr, which is memory-mapped i/o. * The bytes are concatenated in big-endian order with @ret_be. */ -static uint64_t do_ld_mmio_beN(CPUArchState *env, CPUTLBEntryFull *full, - uint64_t ret_be, vaddr addr, int size, - int mmu_idx, MMUAccessType type, uintptr_t ra) +static uint64_t int_ld_mmio_beN(CPUArchState *env, CPUTLBEntryFull *full, + uint64_t ret_be, vaddr addr, int size, + int mmu_idx, MMUAccessType type, uintptr_t ra, + MemoryRegion *mr, hwaddr mr_offset) { - uint64_t t; - - tcg_debug_assert(size > 0 && size <= 8); do { + MemOp this_mop; + unsigned this_size; + uint64_t val; + MemTxResult r; + /* Read aligned pieces up to 8 bytes. */ - switch ((size | (int)addr) & 7) { - case 1: - case 3: - case 5: - case 7: - t = io_readx(env, full, mmu_idx, addr, ra, type, MO_UB); - ret_be = (ret_be << 8) | t; - size -= 1; - addr += 1; - break; - case 2: - case 6: - t = io_readx(env, full, mmu_idx, addr, ra, type, MO_BEUW); - ret_be = (ret_be << 16) | t; - size -= 2; - addr += 2; - break; - case 4: - t = io_readx(env, full, mmu_idx, addr, ra, type, MO_BEUL); - ret_be = (ret_be << 32) | t; - size -= 4; - addr += 4; - break; - case 0: - return io_readx(env, full, mmu_idx, addr, ra, type, MO_BEUQ); - default: - qemu_build_not_reached(); + this_mop = ctz32(size | (int)addr | 8); + this_size = 1 << this_mop; + this_mop |= MO_BE; + + r = memory_region_dispatch_read(mr, mr_offset, &val, + this_mop, full->attrs); + if (unlikely(r != MEMTX_OK)) { + io_failed(env, full, addr, this_size, type, mmu_idx, r, ra); } + if (this_size == 8) { + return val; + } + + ret_be = (ret_be << (this_size * 8)) | val; + addr += this_size; + mr_offset += this_size; + size -= this_size; } while (size); + return ret_be; } +static uint64_t do_ld_mmio_beN(CPUArchState *env, CPUTLBEntryFull *full, + uint64_t ret_be, vaddr addr, int size, + int mmu_idx, MMUAccessType type, uintptr_t ra) +{ + MemoryRegionSection *section; + MemoryRegion *mr; + hwaddr mr_offset; + MemTxAttrs attrs; + uint64_t ret; + + tcg_debug_assert(size > 0 && size <= 8); + + attrs = full->attrs; + section = io_prepare(&mr_offset, env, full->xlat_section, attrs, addr, ra); + mr = section->mr; + + qemu_mutex_lock_iothread(); + ret = int_ld_mmio_beN(env, full, ret_be, addr, size, mmu_idx, + type, ra, mr, mr_offset); + qemu_mutex_unlock_iothread(); + + return ret; +} + +static Int128 do_ld16_mmio_beN(CPUArchState *env, CPUTLBEntryFull *full, + uint64_t ret_be, vaddr addr, int size, + int mmu_idx, uintptr_t ra) +{ + MemoryRegionSection *section; + MemoryRegion *mr; + hwaddr mr_offset; + MemTxAttrs attrs; + uint64_t a, b; + + tcg_debug_assert(size > 8 && size <= 16); + + attrs = full->attrs; + section = io_prepare(&mr_offset, env, full->xlat_section, attrs, addr, ra); + mr = section->mr; + + qemu_mutex_lock_iothread(); + a = int_ld_mmio_beN(env, full, ret_be, addr, size - 8, mmu_idx, + MMU_DATA_LOAD, ra, mr, mr_offset); + b = int_ld_mmio_beN(env, full, ret_be, addr + size - 8, 8, mmu_idx, + MMU_DATA_LOAD, ra, mr, mr_offset + size - 8); + qemu_mutex_unlock_iothread(); + + return int128_make128(b, a); +} + /** * do_ld_bytes_beN * @p: translation parameters @@ -2267,7 +2234,6 @@ static uint64_t do_ld_beN(CPUArchState *env, MMULookupPageData *p, unsigned tmp, half_size; if (unlikely(p->flags & TLB_MMIO)) { - QEMU_IOTHREAD_LOCK_GUARD(); return do_ld_mmio_beN(env, p->full, ret_be, p->addr, p->size, mmu_idx, type, ra); } @@ -2318,12 +2284,7 @@ static Int128 do_ld16_beN(CPUArchState *env, MMULookupPageData *p, MemOp atom; if (unlikely(p->flags & TLB_MMIO)) { - QEMU_IOTHREAD_LOCK_GUARD(); - a = do_ld_mmio_beN(env, p->full, a, p->addr, size - 8, - mmu_idx, MMU_DATA_LOAD, ra); - b = do_ld_mmio_beN(env, p->full, 0, p->addr + 8, 8, - mmu_idx, MMU_DATA_LOAD, ra); - return int128_make128(b, a); + return do_ld16_mmio_beN(env, p->full, a, p->addr, size, mmu_idx, ra); } /* @@ -2368,7 +2329,7 @@ static uint8_t do_ld_1(CPUArchState *env, MMULookupPageData *p, int mmu_idx, MMUAccessType type, uintptr_t ra) { if (unlikely(p->flags & TLB_MMIO)) { - return io_readx(env, p->full, mmu_idx, p->addr, ra, type, MO_UB); + return do_ld_mmio_beN(env, p->full, 0, p->addr, 1, mmu_idx, type, ra); } else { return *(uint8_t *)p->haddr; } @@ -2380,7 +2341,6 @@ static uint16_t do_ld_2(CPUArchState *env, MMULookupPageData *p, int mmu_idx, uint16_t ret; if (unlikely(p->flags & TLB_MMIO)) { - QEMU_IOTHREAD_LOCK_GUARD(); ret = do_ld_mmio_beN(env, p->full, 0, p->addr, 2, mmu_idx, type, ra); if ((memop & MO_BSWAP) == MO_LE) { ret = bswap16(ret); @@ -2401,7 +2361,6 @@ static uint32_t do_ld_4(CPUArchState *env, MMULookupPageData *p, int mmu_idx, uint32_t ret; if (unlikely(p->flags & TLB_MMIO)) { - QEMU_IOTHREAD_LOCK_GUARD(); ret = do_ld_mmio_beN(env, p->full, 0, p->addr, 4, mmu_idx, type, ra); if ((memop & MO_BSWAP) == MO_LE) { ret = bswap32(ret); @@ -2422,7 +2381,6 @@ static uint64_t do_ld_8(CPUArchState *env, MMULookupPageData *p, int mmu_idx, uint64_t ret; if (unlikely(p->flags & TLB_MMIO)) { - QEMU_IOTHREAD_LOCK_GUARD(); ret = do_ld_mmio_beN(env, p->full, 0, p->addr, 8, mmu_idx, type, ra); if ((memop & MO_BSWAP) == MO_LE) { ret = bswap64(ret); @@ -2581,12 +2539,8 @@ static Int128 do_ld16_mmu(CPUArchState *env, vaddr addr, crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD, &l); if (likely(!crosspage)) { if (unlikely(l.page[0].flags & TLB_MMIO)) { - QEMU_IOTHREAD_LOCK_GUARD(); - a = do_ld_mmio_beN(env, l.page[0].full, 0, addr, 8, - l.mmu_idx, MMU_DATA_LOAD, ra); - b = do_ld_mmio_beN(env, l.page[0].full, 0, addr + 8, 8, - l.mmu_idx, MMU_DATA_LOAD, ra); - ret = int128_make128(b, a); + ret = do_ld16_mmio_beN(env, l.page[0].full, 0, addr, 16, + l.mmu_idx, ra); if ((l.memop & MO_BSWAP) == MO_LE) { ret = bswap128(ret); } @@ -2727,48 +2681,90 @@ Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr, * The bytes to store are extracted in little-endian order from @val_le; * return the bytes of @val_le beyond @p->size that have not been stored. */ -static uint64_t do_st_mmio_leN(CPUArchState *env, CPUTLBEntryFull *full, - uint64_t val_le, vaddr addr, int size, - int mmu_idx, uintptr_t ra) +static uint64_t int_st_mmio_leN(CPUArchState *env, CPUTLBEntryFull *full, + uint64_t val_le, vaddr addr, int size, + int mmu_idx, uintptr_t ra, + MemoryRegion *mr, hwaddr mr_offset) { - tcg_debug_assert(size > 0 && size <= 8); - do { + MemOp this_mop; + unsigned this_size; + MemTxResult r; + /* Store aligned pieces up to 8 bytes. */ - switch ((size | (int)addr) & 7) { - case 1: - case 3: - case 5: - case 7: - io_writex(env, full, mmu_idx, val_le, addr, ra, MO_UB); - val_le >>= 8; - size -= 1; - addr += 1; - break; - case 2: - case 6: - io_writex(env, full, mmu_idx, val_le, addr, ra, MO_LEUW); - val_le >>= 16; - size -= 2; - addr += 2; - break; - case 4: - io_writex(env, full, mmu_idx, val_le, addr, ra, MO_LEUL); - val_le >>= 32; - size -= 4; - addr += 4; - break; - case 0: - io_writex(env, full, mmu_idx, val_le, addr, ra, MO_LEUQ); + this_mop = ctz32(size | (int)addr | 8); + this_size = 1 << this_mop; + this_mop |= MO_LE; + + r = memory_region_dispatch_write(mr, mr_offset, val_le, + this_mop, full->attrs); + if (unlikely(r != MEMTX_OK)) { + io_failed(env, full, addr, this_size, MMU_DATA_STORE, + mmu_idx, r, ra); + } + if (this_size == 8) { return 0; - default: - qemu_build_not_reached(); } + + val_le >>= this_size * 8; + addr += this_size; + mr_offset += this_size; + size -= this_size; } while (size); return val_le; } +static uint64_t do_st_mmio_leN(CPUArchState *env, CPUTLBEntryFull *full, + uint64_t val_le, vaddr addr, int size, + int mmu_idx, uintptr_t ra) +{ + MemoryRegionSection *section; + hwaddr mr_offset; + MemoryRegion *mr; + MemTxAttrs attrs; + uint64_t ret; + + tcg_debug_assert(size > 0 && size <= 8); + + attrs = full->attrs; + section = io_prepare(&mr_offset, env, full->xlat_section, attrs, addr, ra); + mr = section->mr; + + qemu_mutex_lock_iothread(); + ret = int_st_mmio_leN(env, full, val_le, addr, size, mmu_idx, + ra, mr, mr_offset); + qemu_mutex_unlock_iothread(); + + return ret; +} + +static uint64_t do_st16_mmio_leN(CPUArchState *env, CPUTLBEntryFull *full, + Int128 val_le, vaddr addr, int size, + int mmu_idx, uintptr_t ra) +{ + MemoryRegionSection *section; + MemoryRegion *mr; + hwaddr mr_offset; + MemTxAttrs attrs; + uint64_t ret; + + tcg_debug_assert(size > 8 && size <= 16); + + attrs = full->attrs; + section = io_prepare(&mr_offset, env, full->xlat_section, attrs, addr, ra); + mr = section->mr; + + qemu_mutex_lock_iothread(); + int_st_mmio_leN(env, full, int128_getlo(val_le), addr, 8, + mmu_idx, ra, mr, mr_offset); + ret = int_st_mmio_leN(env, full, int128_gethi(val_le), addr + 8, + size - 8, mmu_idx, ra, mr, mr_offset + 8); + qemu_mutex_unlock_iothread(); + + return ret; +} + /* * Wrapper for the above. */ @@ -2780,7 +2776,6 @@ static uint64_t do_st_leN(CPUArchState *env, MMULookupPageData *p, unsigned tmp, half_size; if (unlikely(p->flags & TLB_MMIO)) { - QEMU_IOTHREAD_LOCK_GUARD(); return do_st_mmio_leN(env, p->full, val_le, p->addr, p->size, mmu_idx, ra); } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { @@ -2835,11 +2830,8 @@ static uint64_t do_st16_leN(CPUArchState *env, MMULookupPageData *p, MemOp atom; if (unlikely(p->flags & TLB_MMIO)) { - QEMU_IOTHREAD_LOCK_GUARD(); - do_st_mmio_leN(env, p->full, int128_getlo(val_le), - p->addr, 8, mmu_idx, ra); - return do_st_mmio_leN(env, p->full, int128_gethi(val_le), - p->addr + 8, size - 8, mmu_idx, ra); + return do_st16_mmio_leN(env, p->full, val_le, p->addr, + size, mmu_idx, ra); } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { return int128_gethi(val_le) >> ((size - 8) * 8); } @@ -2883,7 +2875,7 @@ static void do_st_1(CPUArchState *env, MMULookupPageData *p, uint8_t val, int mmu_idx, uintptr_t ra) { if (unlikely(p->flags & TLB_MMIO)) { - io_writex(env, p->full, mmu_idx, val, p->addr, ra, MO_UB); + do_st_mmio_leN(env, p->full, val, p->addr, 1, mmu_idx, ra); } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { /* nothing */ } else { @@ -2898,7 +2890,6 @@ static void do_st_2(CPUArchState *env, MMULookupPageData *p, uint16_t val, if ((memop & MO_BSWAP) != MO_LE) { val = bswap16(val); } - QEMU_IOTHREAD_LOCK_GUARD(); do_st_mmio_leN(env, p->full, val, p->addr, 2, mmu_idx, ra); } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { /* nothing */ @@ -2918,7 +2909,6 @@ static void do_st_4(CPUArchState *env, MMULookupPageData *p, uint32_t val, if ((memop & MO_BSWAP) != MO_LE) { val = bswap32(val); } - QEMU_IOTHREAD_LOCK_GUARD(); do_st_mmio_leN(env, p->full, val, p->addr, 4, mmu_idx, ra); } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { /* nothing */ @@ -2938,7 +2928,6 @@ static void do_st_8(CPUArchState *env, MMULookupPageData *p, uint64_t val, if ((memop & MO_BSWAP) != MO_LE) { val = bswap64(val); } - QEMU_IOTHREAD_LOCK_GUARD(); do_st_mmio_leN(env, p->full, val, p->addr, 8, mmu_idx, ra); } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { /* nothing */ @@ -3066,11 +3055,7 @@ static void do_st16_mmu(CPUArchState *env, vaddr addr, Int128 val, if ((l.memop & MO_BSWAP) != MO_LE) { val = bswap128(val); } - a = int128_getlo(val); - b = int128_gethi(val); - QEMU_IOTHREAD_LOCK_GUARD(); - do_st_mmio_leN(env, l.page[0].full, a, addr, 8, l.mmu_idx, ra); - do_st_mmio_leN(env, l.page[0].full, b, addr + 8, 8, l.mmu_idx, ra); + do_st16_mmio_leN(env, l.page[0].full, val, addr, 16, l.mmu_idx, ra); } else if (unlikely(l.page[0].flags & TLB_DISCARD_WRITE)) { /* nothing */ } else { diff --git a/accel/tcg/tcg-accel-ops-mttcg.c b/accel/tcg/tcg-accel-ops-mttcg.c index b276262007..4b0dfb4be7 100644 --- a/accel/tcg/tcg-accel-ops-mttcg.c +++ b/accel/tcg/tcg-accel-ops-mttcg.c @@ -100,14 +100,9 @@ static void *mttcg_cpu_thread_fn(void *arg) break; case EXCP_HALTED: /* - * during start-up the vCPU is reset and the thread is - * kicked several times. If we don't ensure we go back - * to sleep in the halted state we won't cleanly - * start-up when the vCPU is enabled. - * - * cpu->halted should ensure we sleep in wait_io_event + * Usually cpu->halted is set, but may have already been + * reset by another thread by the time we arrive here. */ - g_assert(cpu->halted); break; case EXCP_ATOMIC: qemu_mutex_unlock_iothread(); diff --git a/accel/tcg/tcg-runtime-gvec.c b/accel/tcg/tcg-runtime-gvec.c index 6c99f952ca..afca89baa1 100644 --- a/accel/tcg/tcg-runtime-gvec.c +++ b/accel/tcg/tcg-runtime-gvec.c @@ -1042,6 +1042,32 @@ DO_CMP2(64) #undef DO_CMP1 #undef DO_CMP2 +#define DO_CMP1(NAME, TYPE, OP) \ +void HELPER(NAME)(void *d, void *a, uint64_t b64, uint32_t desc) \ +{ \ + intptr_t oprsz = simd_oprsz(desc); \ + TYPE inv = simd_data(desc), b = b64; \ + for (intptr_t i = 0; i < oprsz; i += sizeof(TYPE)) { \ + *(TYPE *)(d + i) = -((*(TYPE *)(a + i) OP b) ^ inv); \ + } \ + clear_high(d, oprsz, desc); \ +} + +#define DO_CMP2(SZ) \ + DO_CMP1(gvec_eqs##SZ, uint##SZ##_t, ==) \ + DO_CMP1(gvec_lts##SZ, int##SZ##_t, <) \ + DO_CMP1(gvec_les##SZ, int##SZ##_t, <=) \ + DO_CMP1(gvec_ltus##SZ, uint##SZ##_t, <) \ + DO_CMP1(gvec_leus##SZ, uint##SZ##_t, <=) + +DO_CMP2(8) +DO_CMP2(16) +DO_CMP2(32) +DO_CMP2(64) + +#undef DO_CMP1 +#undef DO_CMP2 + void HELPER(gvec_ssadd8)(void *d, void *a, void *b, uint32_t desc) { intptr_t oprsz = simd_oprsz(desc); diff --git a/accel/tcg/tcg-runtime.h b/accel/tcg/tcg-runtime.h index 186899a2c7..c23b5e66c4 100644 --- a/accel/tcg/tcg-runtime.h +++ b/accel/tcg/tcg-runtime.h @@ -297,4 +297,29 @@ DEF_HELPER_FLAGS_4(gvec_leu16, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_leu32, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_leu64, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_eqs8, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_eqs16, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_eqs32, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_eqs64, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(gvec_lts8, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_lts16, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_lts32, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_lts64, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(gvec_les8, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_les16, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_les32, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_les64, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(gvec_ltus8, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_ltus16, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_ltus32, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_ltus64, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(gvec_leus8, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_leus16, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_leus32, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(gvec_leus64, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + DEF_HELPER_FLAGS_5(gvec_bitsel, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) diff --git a/audio/jackaudio.c b/audio/jackaudio.c index 5bdf3d7a78..e1eaa3477d 100644 --- a/audio/jackaudio.c +++ b/audio/jackaudio.c @@ -70,6 +70,9 @@ typedef struct QJackClient { int buffersize; jack_port_t **port; QJackBuffer fifo; + + /* Used as workspace by qjack_process() */ + float **process_buffers; } QJackClient; @@ -267,22 +270,21 @@ static int qjack_process(jack_nframes_t nframes, void *arg) } /* get the buffers for the ports */ - float *buffers[c->nchannels]; for (int i = 0; i < c->nchannels; ++i) { - buffers[i] = jack_port_get_buffer(c->port[i], nframes); + c->process_buffers[i] = jack_port_get_buffer(c->port[i], nframes); } if (c->out) { if (likely(c->enabled)) { - qjack_buffer_read_l(&c->fifo, buffers, nframes); + qjack_buffer_read_l(&c->fifo, c->process_buffers, nframes); } else { for (int i = 0; i < c->nchannels; ++i) { - memset(buffers[i], 0, nframes * sizeof(float)); + memset(c->process_buffers[i], 0, nframes * sizeof(float)); } } } else { if (likely(c->enabled)) { - qjack_buffer_write_l(&c->fifo, buffers, nframes); + qjack_buffer_write_l(&c->fifo, c->process_buffers, nframes); } } @@ -400,7 +402,8 @@ static void qjack_client_connect_ports(QJackClient *c) static int qjack_client_init(QJackClient *c) { jack_status_t status; - char client_name[jack_client_name_size()]; + int client_name_len = jack_client_name_size(); /* includes NUL */ + g_autofree char *client_name = g_new(char, client_name_len); jack_options_t options = JackNullOption; if (c->state == QJACK_STATE_RUNNING) { @@ -409,7 +412,7 @@ static int qjack_client_init(QJackClient *c) c->connect_ports = true; - snprintf(client_name, sizeof(client_name), "%s-%s", + snprintf(client_name, client_name_len, "%s-%s", c->out ? "out" : "in", c->opt->client_name ? c->opt->client_name : audio_application_name()); @@ -447,6 +450,9 @@ static int qjack_client_init(QJackClient *c) jack_get_client_name(c->client)); } + /* Allocate working buffer for process callback */ + c->process_buffers = g_new(float *, c->nchannels); + jack_set_process_callback(c->client, qjack_process , c); jack_set_port_registration_callback(c->client, qjack_port_registration, c); jack_set_xrun_callback(c->client, qjack_xrun, c); @@ -578,6 +584,7 @@ static void qjack_client_fini_locked(QJackClient *c) qjack_buffer_free(&c->fifo); g_free(c->port); + g_free(c->process_buffers); c->state = QJACK_STATE_DISCONNECTED; /* fallthrough */ diff --git a/backends/cryptodev.c b/backends/cryptodev.c index 4d183f7237..e5006bd215 100644 --- a/backends/cryptodev.c +++ b/backends/cryptodev.c @@ -252,10 +252,11 @@ static void cryptodev_backend_throttle_timer_cb(void *opaque) continue; } - throttle_account(&backend->ts, true, ret); + throttle_account(&backend->ts, THROTTLE_WRITE, ret); cryptodev_backend_operation(backend, op_info); if (throttle_enabled(&backend->tc) && - throttle_schedule_timer(&backend->ts, &backend->tt, true)) { + throttle_schedule_timer(&backend->ts, &backend->tt, + THROTTLE_WRITE)) { break; } } @@ -271,7 +272,7 @@ int cryptodev_backend_crypto_operation( goto do_account; } - if (throttle_schedule_timer(&backend->ts, &backend->tt, true) || + if (throttle_schedule_timer(&backend->ts, &backend->tt, THROTTLE_WRITE) || !QTAILQ_EMPTY(&backend->opinfos)) { QTAILQ_INSERT_TAIL(&backend->opinfos, op_info, next); return 0; @@ -283,7 +284,7 @@ do_account: return ret; } - throttle_account(&backend->ts, true, ret); + throttle_account(&backend->ts, THROTTLE_WRITE, ret); return cryptodev_backend_operation(backend, op_info); } @@ -341,8 +342,7 @@ static void cryptodev_backend_set_throttle(CryptoDevBackend *backend, int field, if (!enabled) { throttle_init(&backend->ts); throttle_timers_init(&backend->tt, qemu_get_aio_context(), - QEMU_CLOCK_REALTIME, - cryptodev_backend_throttle_timer_cb, /* FIXME */ + QEMU_CLOCK_REALTIME, NULL, cryptodev_backend_throttle_timer_cb, backend); } diff --git a/backends/hostmem-file.c b/backends/hostmem-file.c index b4335a80e6..361d4a8103 100644 --- a/backends/hostmem-file.c +++ b/backends/hostmem-file.c @@ -18,6 +18,8 @@ #include "sysemu/hostmem.h" #include "qom/object_interfaces.h" #include "qom/object.h" +#include "qapi/visitor.h" +#include "qapi/qapi-visit-common.h" OBJECT_DECLARE_SIMPLE_TYPE(HostMemoryBackendFile, MEMORY_BACKEND_FILE) @@ -31,6 +33,7 @@ struct HostMemoryBackendFile { bool discard_data; bool is_pmem; bool readonly; + OnOffAuto rom; }; static void @@ -53,15 +56,39 @@ file_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) return; } + switch (fb->rom) { + case ON_OFF_AUTO_AUTO: + /* Traditionally, opening the file readonly always resulted in ROM. */ + fb->rom = fb->readonly ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; + break; + case ON_OFF_AUTO_ON: + if (!fb->readonly) { + error_setg(errp, "property 'rom' = 'on' is not supported with" + " 'readonly' = 'off'"); + return; + } + break; + case ON_OFF_AUTO_OFF: + if (fb->readonly && backend->share) { + error_setg(errp, "property 'rom' = 'off' is incompatible with" + " 'readonly' = 'on' and 'share' = 'on'"); + return; + } + break; + default: + assert(false); + } + name = host_memory_backend_get_name(backend); ram_flags = backend->share ? RAM_SHARED : 0; + ram_flags |= fb->readonly ? RAM_READONLY_FD : 0; + ram_flags |= fb->rom == ON_OFF_AUTO_ON ? RAM_READONLY : 0; ram_flags |= backend->reserve ? 0 : RAM_NORESERVE; ram_flags |= fb->is_pmem ? RAM_PMEM : 0; ram_flags |= RAM_NAMED_FILE; memory_region_init_ram_from_file(&backend->mr, OBJECT(backend), name, backend->size, fb->align, ram_flags, - fb->mem_path, fb->offset, fb->readonly, - errp); + fb->mem_path, fb->offset, errp); g_free(name); #endif } @@ -201,6 +228,32 @@ static void file_memory_backend_set_readonly(Object *obj, bool value, fb->readonly = value; } +static void file_memory_backend_get_rom(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(obj); + OnOffAuto rom = fb->rom; + + visit_type_OnOffAuto(v, name, &rom, errp); +} + +static void file_memory_backend_set_rom(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + HostMemoryBackend *backend = MEMORY_BACKEND(obj); + HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(obj); + + if (host_memory_backend_mr_inited(backend)) { + error_setg(errp, "cannot change property '%s' of %s.", name, + object_get_typename(obj)); + return; + } + + visit_type_OnOffAuto(v, name, &fb->rom, errp); +} + static void file_backend_unparent(Object *obj) { HostMemoryBackend *backend = MEMORY_BACKEND(obj); @@ -243,6 +296,10 @@ file_backend_class_init(ObjectClass *oc, void *data) object_class_property_add_bool(oc, "readonly", file_memory_backend_get_readonly, file_memory_backend_set_readonly); + object_class_property_add(oc, "rom", "OnOffAuto", + file_memory_backend_get_rom, file_memory_backend_set_rom, NULL, NULL); + object_class_property_set_description(oc, "rom", + "Whether to create Read Only Memory (ROM)"); } static void file_backend_instance_finalize(Object *o) diff --git a/block.c b/block.c index 8da89aaa62..e7f349b25c 100644 --- a/block.c +++ b/block.c @@ -91,9 +91,11 @@ static BlockDriverState *bdrv_open_inherit(const char *filename, static bool bdrv_recurse_has_child(BlockDriverState *bs, BlockDriverState *child); -static void bdrv_replace_child_noperm(BdrvChild *child, - BlockDriverState *new_bs); -static void bdrv_remove_child(BdrvChild *child, Transaction *tran); +static void GRAPH_WRLOCK +bdrv_replace_child_noperm(BdrvChild *child, BlockDriverState *new_bs); + +static void GRAPH_WRLOCK +bdrv_remove_child(BdrvChild *child, Transaction *tran); static int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue, @@ -1699,7 +1701,9 @@ bdrv_open_driver(BlockDriverState *bs, BlockDriver *drv, const char *node_name, open_failed: bs->drv = NULL; if (bs->file != NULL) { + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, bs->file); + bdrv_graph_wrunlock(); assert(!bs->file); } g_free(bs->opaque); @@ -2115,7 +2119,6 @@ static int bdrv_fill_options(QDict **options, const char *filename, typedef struct BlockReopenQueueEntry { bool prepared; - bool perms_checked; BDRVReopenState state; QTAILQ_ENTRY(BlockReopenQueueEntry) entry; } BlockReopenQueueEntry; @@ -2201,7 +2204,8 @@ static bool bdrv_a_allow_b(BdrvChild *a, BdrvChild *b, Error **errp) return false; } -static bool bdrv_parent_perms_conflict(BlockDriverState *bs, Error **errp) +static bool GRAPH_RDLOCK +bdrv_parent_perms_conflict(BlockDriverState *bs, Error **errp) { BdrvChild *a, *b; GLOBAL_STATE_CODE(); @@ -2226,11 +2230,12 @@ static bool bdrv_parent_perms_conflict(BlockDriverState *bs, Error **errp) return false; } -static void bdrv_child_perm(BlockDriverState *bs, BlockDriverState *child_bs, - BdrvChild *c, BdrvChildRole role, - BlockReopenQueue *reopen_queue, - uint64_t parent_perm, uint64_t parent_shared, - uint64_t *nperm, uint64_t *nshared) +static void GRAPH_RDLOCK +bdrv_child_perm(BlockDriverState *bs, BlockDriverState *child_bs, + BdrvChild *c, BdrvChildRole role, + BlockReopenQueue *reopen_queue, + uint64_t parent_perm, uint64_t parent_shared, + uint64_t *nperm, uint64_t *nshared) { assert(bs->drv && bs->drv->bdrv_child_perm); GLOBAL_STATE_CODE(); @@ -2254,8 +2259,8 @@ static void bdrv_child_perm(BlockDriverState *bs, BlockDriverState *child_bs, * simplest way to satisfy this criteria: use only result of * bdrv_topological_dfs() or NULL as @list parameter. */ -static GSList *bdrv_topological_dfs(GSList *list, GHashTable *found, - BlockDriverState *bs) +static GSList * GRAPH_RDLOCK +bdrv_topological_dfs(GSList *list, GHashTable *found, BlockDriverState *bs) { BdrvChild *child; g_autoptr(GHashTable) local_found = NULL; @@ -2318,7 +2323,7 @@ static void bdrv_child_set_perm(BdrvChild *c, uint64_t perm, tran_add(tran, &bdrv_child_set_pem_drv, s); } -static void bdrv_drv_set_perm_commit(void *opaque) +static void GRAPH_RDLOCK bdrv_drv_set_perm_commit(void *opaque) { BlockDriverState *bs = opaque; uint64_t cumulative_perms, cumulative_shared_perms; @@ -2331,7 +2336,7 @@ static void bdrv_drv_set_perm_commit(void *opaque) } } -static void bdrv_drv_set_perm_abort(void *opaque) +static void GRAPH_RDLOCK bdrv_drv_set_perm_abort(void *opaque) { BlockDriverState *bs = opaque; GLOBAL_STATE_CODE(); @@ -2346,9 +2351,13 @@ TransactionActionDrv bdrv_drv_set_perm_drv = { .commit = bdrv_drv_set_perm_commit, }; -static int bdrv_drv_set_perm(BlockDriverState *bs, uint64_t perm, - uint64_t shared_perm, Transaction *tran, - Error **errp) +/* + * After calling this function, the transaction @tran may only be completed + * while holding a reader lock for the graph. + */ +static int GRAPH_RDLOCK +bdrv_drv_set_perm(BlockDriverState *bs, uint64_t perm, uint64_t shared_perm, + Transaction *tran, Error **errp) { GLOBAL_STATE_CODE(); if (!bs->drv) { @@ -2374,20 +2383,22 @@ typedef struct BdrvReplaceChildState { BlockDriverState *old_bs; } BdrvReplaceChildState; -static void bdrv_replace_child_commit(void *opaque) +static void GRAPH_WRLOCK bdrv_replace_child_commit(void *opaque) { BdrvReplaceChildState *s = opaque; GLOBAL_STATE_CODE(); - bdrv_unref(s->old_bs); + bdrv_schedule_unref(s->old_bs); } -static void bdrv_replace_child_abort(void *opaque) +static void GRAPH_WRLOCK bdrv_replace_child_abort(void *opaque) { BdrvReplaceChildState *s = opaque; BlockDriverState *new_bs = s->child->bs; GLOBAL_STATE_CODE(); + assert_bdrv_graph_writable(); + /* old_bs reference is transparently moved from @s to @s->child */ if (!s->child->bs) { /* @@ -2404,6 +2415,7 @@ static void bdrv_replace_child_abort(void *opaque) } assert(s->child->quiesced_parent); bdrv_replace_child_noperm(s->child, s->old_bs); + bdrv_unref(new_bs); } @@ -2421,10 +2433,14 @@ static TransactionActionDrv bdrv_replace_child_drv = { * Both @child->bs and @new_bs (if non-NULL) must be drained. @new_bs must be * kept drained until the transaction is completed. * + * After calling this function, the transaction @tran may only be completed + * while holding a writer lock for the graph. + * * The function doesn't update permissions, caller is responsible for this. */ -static void bdrv_replace_child_tran(BdrvChild *child, BlockDriverState *new_bs, - Transaction *tran) +static void GRAPH_WRLOCK +bdrv_replace_child_tran(BdrvChild *child, BlockDriverState *new_bs, + Transaction *tran) { BdrvReplaceChildState *s = g_new(BdrvReplaceChildState, 1); @@ -2440,6 +2456,7 @@ static void bdrv_replace_child_tran(BdrvChild *child, BlockDriverState *new_bs, if (new_bs) { bdrv_ref(new_bs); } + bdrv_replace_child_noperm(child, new_bs); /* old_bs reference is transparently moved from @child to @s */ } @@ -2447,9 +2464,13 @@ static void bdrv_replace_child_tran(BdrvChild *child, BlockDriverState *new_bs, /* * Refresh permissions in @bs subtree. The function is intended to be called * after some graph modification that was done without permission update. + * + * After calling this function, the transaction @tran may only be completed + * while holding a reader lock for the graph. */ -static int bdrv_node_refresh_perm(BlockDriverState *bs, BlockReopenQueue *q, - Transaction *tran, Error **errp) +static int GRAPH_RDLOCK +bdrv_node_refresh_perm(BlockDriverState *bs, BlockReopenQueue *q, + Transaction *tran, Error **errp) { BlockDriver *drv = bs->drv; BdrvChild *c; @@ -2522,9 +2543,13 @@ static int bdrv_node_refresh_perm(BlockDriverState *bs, BlockReopenQueue *q, /* * @list is a product of bdrv_topological_dfs() (may be called several times) - * a topologically sorted subgraph. + * + * After calling this function, the transaction @tran may only be completed + * while holding a reader lock for the graph. */ -static int bdrv_do_refresh_perms(GSList *list, BlockReopenQueue *q, - Transaction *tran, Error **errp) +static int GRAPH_RDLOCK +bdrv_do_refresh_perms(GSList *list, BlockReopenQueue *q, Transaction *tran, + Error **errp) { int ret; BlockDriverState *bs; @@ -2550,9 +2575,13 @@ static int bdrv_do_refresh_perms(GSList *list, BlockReopenQueue *q, * @list is any list of nodes. List is completed by all subtrees and * topologically sorted. It's not a problem if some node occurs in the @list * several times. + * + * After calling this function, the transaction @tran may only be completed + * while holding a reader lock for the graph. */ -static int bdrv_list_refresh_perms(GSList *list, BlockReopenQueue *q, - Transaction *tran, Error **errp) +static int GRAPH_RDLOCK +bdrv_list_refresh_perms(GSList *list, BlockReopenQueue *q, Transaction *tran, + Error **errp) { g_autoptr(GHashTable) found = g_hash_table_new(NULL, NULL); g_autoptr(GSList) refresh_list = NULL; @@ -2611,9 +2640,14 @@ char *bdrv_perm_names(uint64_t perm) } -/* @tran is allowed to be NULL. In this case no rollback is possible */ -static int bdrv_refresh_perms(BlockDriverState *bs, Transaction *tran, - Error **errp) +/* + * @tran is allowed to be NULL. In this case no rollback is possible. + * + * After calling this function, the transaction @tran may only be completed + * while holding a reader lock for the graph. + */ +static int GRAPH_RDLOCK +bdrv_refresh_perms(BlockDriverState *bs, Transaction *tran, Error **errp) { int ret; Transaction *local_tran = NULL; @@ -2859,8 +2893,8 @@ uint64_t bdrv_qapi_perm_to_blk_perm(BlockPermission qapi_perm) * If @new_bs is non-NULL, the parent of @child must already be drained through * @child and the caller must hold the AioContext lock for @new_bs. */ -static void bdrv_replace_child_noperm(BdrvChild *child, - BlockDriverState *new_bs) +static void GRAPH_WRLOCK +bdrv_replace_child_noperm(BdrvChild *child, BlockDriverState *new_bs) { BlockDriverState *old_bs = child->bs; int new_bs_quiesce_counter; @@ -2895,8 +2929,6 @@ static void bdrv_replace_child_noperm(BdrvChild *child, assert(bdrv_get_aio_context(old_bs) == bdrv_get_aio_context(new_bs)); } - /* TODO Pull this up into the callers to avoid polling here */ - bdrv_graph_wrlock(new_bs); if (old_bs) { if (child->klass->detach) { child->klass->detach(child); @@ -2912,7 +2944,6 @@ static void bdrv_replace_child_noperm(BdrvChild *child, child->klass->attach(child); } } - bdrv_graph_wrunlock(); /* * If the parent was drained through this BdrvChild previously, but new_bs @@ -2947,12 +2978,14 @@ typedef struct BdrvAttachChildCommonState { AioContext *old_child_ctx; } BdrvAttachChildCommonState; -static void bdrv_attach_child_common_abort(void *opaque) +static void GRAPH_WRLOCK bdrv_attach_child_common_abort(void *opaque) { BdrvAttachChildCommonState *s = opaque; BlockDriverState *bs = s->child->bs; GLOBAL_STATE_CODE(); + assert_bdrv_graph_writable(); + bdrv_replace_child_noperm(s->child, NULL); if (bdrv_get_aio_context(bs) != s->old_child_ctx) { @@ -2977,7 +3010,7 @@ static void bdrv_attach_child_common_abort(void *opaque) tran_commit(tran); } - bdrv_unref(bs); + bdrv_schedule_unref(bs); bdrv_child_free(s->child); } @@ -2991,19 +3024,23 @@ static TransactionActionDrv bdrv_attach_child_common_drv = { * * Function doesn't update permissions, caller is responsible for this. * + * After calling this function, the transaction @tran may only be completed + * while holding a writer lock for the graph. + * * Returns new created child. * * The caller must hold the AioContext lock for @child_bs. Both @parent_bs and * @child_bs can move to a different AioContext in this function. Callers must * make sure that their AioContext locking is still correct after this. */ -static BdrvChild *bdrv_attach_child_common(BlockDriverState *child_bs, - const char *child_name, - const BdrvChildClass *child_class, - BdrvChildRole child_role, - uint64_t perm, uint64_t shared_perm, - void *opaque, - Transaction *tran, Error **errp) +static BdrvChild * GRAPH_WRLOCK +bdrv_attach_child_common(BlockDriverState *child_bs, + const char *child_name, + const BdrvChildClass *child_class, + BdrvChildRole child_role, + uint64_t perm, uint64_t shared_perm, + void *opaque, + Transaction *tran, Error **errp) { BdrvChild *new_child; AioContext *parent_ctx, *new_child_ctx; @@ -3106,14 +3143,18 @@ static BdrvChild *bdrv_attach_child_common(BlockDriverState *child_bs, * The caller must hold the AioContext lock for @child_bs. Both @parent_bs and * @child_bs can move to a different AioContext in this function. Callers must * make sure that their AioContext locking is still correct after this. + * + * After calling this function, the transaction @tran may only be completed + * while holding a writer lock for the graph. */ -static BdrvChild *bdrv_attach_child_noperm(BlockDriverState *parent_bs, - BlockDriverState *child_bs, - const char *child_name, - const BdrvChildClass *child_class, - BdrvChildRole child_role, - Transaction *tran, - Error **errp) +static BdrvChild * GRAPH_WRLOCK +bdrv_attach_child_noperm(BlockDriverState *parent_bs, + BlockDriverState *child_bs, + const char *child_name, + const BdrvChildClass *child_class, + BdrvChildRole child_role, + Transaction *tran, + Error **errp) { uint64_t perm, shared_perm; @@ -3158,6 +3199,8 @@ BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs, GLOBAL_STATE_CODE(); + bdrv_graph_wrlock(child_bs); + child = bdrv_attach_child_common(child_bs, child_name, child_class, child_role, perm, shared_perm, opaque, tran, errp); @@ -3170,6 +3213,7 @@ BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs, out: tran_finalize(tran, ret); + bdrv_graph_wrunlock(); bdrv_unref(child_bs); @@ -3215,7 +3259,7 @@ BdrvChild *bdrv_attach_child(BlockDriverState *parent_bs, out: tran_finalize(tran, ret); - bdrv_unref(child_bs); + bdrv_schedule_unref(child_bs); return ret < 0 ? NULL : child; } @@ -3245,7 +3289,7 @@ void bdrv_root_unref_child(BdrvChild *child) NULL); } - bdrv_unref(child_bs); + bdrv_schedule_unref(child_bs); } typedef struct BdrvSetInheritsFrom { @@ -3289,8 +3333,9 @@ static void bdrv_set_inherits_from(BlockDriverState *bs, * @root that point to @root, where necessary. * @tran is allowed to be NULL. In this case no rollback is possible */ -static void bdrv_unset_inherits_from(BlockDriverState *root, BdrvChild *child, - Transaction *tran) +static void GRAPH_WRLOCK +bdrv_unset_inherits_from(BlockDriverState *root, BdrvChild *child, + Transaction *tran) { BdrvChild *c; @@ -3327,7 +3372,8 @@ void bdrv_unref_child(BlockDriverState *parent, BdrvChild *child) } -static void bdrv_parent_cb_change_media(BlockDriverState *bs, bool load) +static void GRAPH_RDLOCK +bdrv_parent_cb_change_media(BlockDriverState *bs, bool load) { BdrvChild *c; GLOBAL_STATE_CODE(); @@ -3368,16 +3414,23 @@ static BdrvChildRole bdrv_backing_role(BlockDriverState *bs) * Sets the bs->backing or bs->file link of a BDS. A new reference is created; * callers which don't need their own reference any more must call bdrv_unref(). * + * If the respective child is already present (i.e. we're detaching a node), + * that child node must be drained. + * * Function doesn't update permissions, caller is responsible for this. * * The caller must hold the AioContext lock for @child_bs. Both @parent_bs and * @child_bs can move to a different AioContext in this function. Callers must * make sure that their AioContext locking is still correct after this. + * + * After calling this function, the transaction @tran may only be completed + * while holding a writer lock for the graph. */ -static int bdrv_set_file_or_backing_noperm(BlockDriverState *parent_bs, - BlockDriverState *child_bs, - bool is_backing, - Transaction *tran, Error **errp) +static int GRAPH_WRLOCK +bdrv_set_file_or_backing_noperm(BlockDriverState *parent_bs, + BlockDriverState *child_bs, + bool is_backing, + Transaction *tran, Error **errp) { bool update_inherits_from = bdrv_inherits_from_recursive(child_bs, parent_bs); @@ -3428,6 +3481,7 @@ static int bdrv_set_file_or_backing_noperm(BlockDriverState *parent_bs, } if (child) { + assert(child->bs->quiesce_counter); bdrv_unset_inherits_from(parent_bs, child, tran); bdrv_remove_child(child, tran); } @@ -3454,9 +3508,7 @@ static int bdrv_set_file_or_backing_noperm(BlockDriverState *parent_bs, } out: - bdrv_graph_rdlock_main_loop(); bdrv_refresh_limits(parent_bs, tran, NULL); - bdrv_graph_rdunlock_main_loop(); return 0; } @@ -3465,10 +3517,17 @@ out: * The caller must hold the AioContext lock for @backing_hd. Both @bs and * @backing_hd can move to a different AioContext in this function. Callers must * make sure that their AioContext locking is still correct after this. + * + * If a backing child is already present (i.e. we're detaching a node), that + * child node must be drained. + * + * After calling this function, the transaction @tran may only be completed + * while holding a writer lock for the graph. */ -static int bdrv_set_backing_noperm(BlockDriverState *bs, - BlockDriverState *backing_hd, - Transaction *tran, Error **errp) +static int GRAPH_WRLOCK +bdrv_set_backing_noperm(BlockDriverState *bs, + BlockDriverState *backing_hd, + Transaction *tran, Error **errp) { GLOBAL_STATE_CODE(); return bdrv_set_file_or_backing_noperm(bs, backing_hd, true, tran, errp); @@ -3483,6 +3542,10 @@ int bdrv_set_backing_hd_drained(BlockDriverState *bs, GLOBAL_STATE_CODE(); assert(bs->quiesce_counter > 0); + if (bs->backing) { + assert(bs->backing->bs->quiesce_counter > 0); + } + bdrv_graph_wrlock(backing_hd); ret = bdrv_set_backing_noperm(bs, backing_hd, tran, errp); if (ret < 0) { @@ -3492,18 +3555,22 @@ int bdrv_set_backing_hd_drained(BlockDriverState *bs, ret = bdrv_refresh_perms(bs, tran, errp); out: tran_finalize(tran, ret); + bdrv_graph_wrunlock(); return ret; } int bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd, Error **errp) { + BlockDriverState *drain_bs = bs->backing ? bs->backing->bs : bs; int ret; GLOBAL_STATE_CODE(); - bdrv_drained_begin(bs); + bdrv_ref(drain_bs); + bdrv_drained_begin(drain_bs); ret = bdrv_set_backing_hd_drained(bs, backing_hd, errp); - bdrv_drained_end(bs); + bdrv_drained_end(drain_bs); + bdrv_unref(drain_bs); return ret; } @@ -3713,11 +3780,13 @@ BdrvChild *bdrv_open_child(const char *filename, return NULL; } + bdrv_graph_wrlock(NULL); ctx = bdrv_get_aio_context(bs); aio_context_acquire(ctx); child = bdrv_attach_child(parent, bs, bdref_key, child_class, child_role, errp); aio_context_release(ctx); + bdrv_graph_wrunlock(); return child; } @@ -3902,6 +3971,9 @@ bdrv_open_inherit(const char *filename, const char *reference, QDict *options, GLOBAL_STATE_CODE(); assert(!qemu_in_coroutine()); + /* TODO We'll eventually have to take a writer lock in this function */ + GRAPH_RDLOCK_GUARD_MAINLOOP(); + if (reference) { bool options_non_empty = options ? qdict_size(options) : false; qobject_unref(options); @@ -4541,7 +4613,10 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp) * reconfiguring the fd and that's why it does it in raw_check_perm(), not * in raw_reopen_prepare() which is called with "old" permissions. */ + bdrv_graph_rdlock_main_loop(); ret = bdrv_list_refresh_perms(refresh_list, bs_queue, tran, errp); + bdrv_graph_rdunlock_main_loop(); + if (ret < 0) { goto abort; } @@ -4562,7 +4637,9 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp) aio_context_release(ctx); } + bdrv_graph_wrlock(NULL); tran_commit(tran); + bdrv_graph_wrunlock(); QTAILQ_FOREACH_REVERSE(bs_entry, bs_queue, entry) { BlockDriverState *bs = bs_entry->state.bs; @@ -4579,7 +4656,10 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp) goto cleanup; abort: + bdrv_graph_wrlock(NULL); tran_abort(tran); + bdrv_graph_wrunlock(); + QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) { if (bs_entry->prepared) { ctx = bdrv_get_aio_context(bs_entry->state.bs); @@ -4645,6 +4725,9 @@ int bdrv_reopen_set_read_only(BlockDriverState *bs, bool read_only, * true and reopen_state->new_backing_bs contains a pointer to the new * backing BlockDriverState (or NULL). * + * After calling this function, the transaction @tran may only be completed + * while holding a writer lock for the graph. + * * Return 0 on success, otherwise return < 0 and set @errp. * * The caller must hold the AioContext lock of @reopen_state->bs. @@ -4729,6 +4812,11 @@ static int bdrv_reopen_parse_file_or_backing(BDRVReopenState *reopen_state, reopen_state->old_file_bs = old_child_bs; } + if (old_child_bs) { + bdrv_ref(old_child_bs); + bdrv_drained_begin(old_child_bs); + } + old_ctx = bdrv_get_aio_context(bs); ctx = bdrv_get_aio_context(new_child_bs); if (old_ctx != ctx) { @@ -4736,14 +4824,23 @@ static int bdrv_reopen_parse_file_or_backing(BDRVReopenState *reopen_state, aio_context_acquire(ctx); } + bdrv_graph_wrlock(new_child_bs); + ret = bdrv_set_file_or_backing_noperm(bs, new_child_bs, is_backing, tran, errp); + bdrv_graph_wrunlock(); + if (old_ctx != ctx) { aio_context_release(ctx); aio_context_acquire(old_ctx); } + if (old_child_bs) { + bdrv_drained_end(old_child_bs); + bdrv_unref(old_child_bs); + } + return ret; } @@ -4764,6 +4861,9 @@ static int bdrv_reopen_parse_file_or_backing(BDRVReopenState *reopen_state, * commit() for any other BDS that have been left in a prepare() state * * The caller must hold the AioContext lock of @reopen_state->bs. + * + * After calling this function, the transaction @change_child_tran may only be + * completed while holding a writer lock for the graph. */ static int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue, @@ -5065,9 +5165,11 @@ static void bdrv_close(BlockDriverState *bs) bs->drv = NULL; } + bdrv_graph_wrlock(NULL); QLIST_FOREACH_SAFE(child, &bs->children, next, next) { bdrv_unref_child(bs, child); } + bdrv_graph_wrunlock(); assert(!bs->backing); assert(!bs->file); @@ -5122,7 +5224,7 @@ void bdrv_close_all(void) assert(QTAILQ_EMPTY(&all_bdrv_states)); } -static bool should_update_child(BdrvChild *c, BlockDriverState *to) +static bool GRAPH_RDLOCK should_update_child(BdrvChild *c, BlockDriverState *to) { GQueue *queue; GHashTable *found; @@ -5211,45 +5313,47 @@ static TransactionActionDrv bdrv_remove_child_drv = { .commit = bdrv_remove_child_commit, }; -/* Function doesn't update permissions, caller is responsible for this. */ -static void bdrv_remove_child(BdrvChild *child, Transaction *tran) +/* + * Function doesn't update permissions, caller is responsible for this. + * + * @child->bs (if non-NULL) must be drained. + * + * After calling this function, the transaction @tran may only be completed + * while holding a writer lock for the graph. + */ +static void GRAPH_WRLOCK bdrv_remove_child(BdrvChild *child, Transaction *tran) { if (!child) { return; } if (child->bs) { - BlockDriverState *bs = child->bs; - bdrv_drained_begin(bs); + assert(child->quiesced_parent); bdrv_replace_child_tran(child, NULL, tran); - bdrv_drained_end(bs); } tran_add(tran, &bdrv_remove_child_drv, child); } -static void undrain_on_clean_cb(void *opaque) -{ - bdrv_drained_end(opaque); -} - -static TransactionActionDrv undrain_on_clean = { - .clean = undrain_on_clean_cb, -}; - -static int bdrv_replace_node_noperm(BlockDriverState *from, - BlockDriverState *to, - bool auto_skip, Transaction *tran, - Error **errp) +/* + * Both @from and @to (if non-NULL) must be drained. @to must be kept drained + * until the transaction is completed. + * + * After calling this function, the transaction @tran may only be completed + * while holding a writer lock for the graph. + */ +static int GRAPH_WRLOCK +bdrv_replace_node_noperm(BlockDriverState *from, + BlockDriverState *to, + bool auto_skip, Transaction *tran, + Error **errp) { BdrvChild *c, *next; GLOBAL_STATE_CODE(); - bdrv_drained_begin(from); - bdrv_drained_begin(to); - tran_add(tran, &undrain_on_clean, from); - tran_add(tran, &undrain_on_clean, to); + assert(from->quiesce_counter); + assert(to->quiesce_counter); QLIST_FOREACH_SAFE(c, &from->parents, next_parent, next) { assert(c->bs == from); @@ -5312,6 +5416,9 @@ static int bdrv_replace_node_common(BlockDriverState *from, assert(qemu_get_current_aio_context() == qemu_get_aio_context()); assert(bdrv_get_aio_context(from) == bdrv_get_aio_context(to)); bdrv_drained_begin(from); + bdrv_drained_begin(to); + + bdrv_graph_wrlock(to); /* * Do the replacement without permission update. @@ -5325,6 +5432,7 @@ static int bdrv_replace_node_common(BlockDriverState *from, } if (detach_subchain) { + /* to_cow_parent is already drained because from is drained */ bdrv_remove_child(bdrv_filter_or_cow_child(to_cow_parent), tran); } @@ -5340,7 +5448,9 @@ static int bdrv_replace_node_common(BlockDriverState *from, out: tran_finalize(tran, ret); + bdrv_graph_wrunlock(); + bdrv_drained_end(to); bdrv_drained_end(from); bdrv_unref(from); @@ -5390,6 +5500,22 @@ int bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top, assert(!bs_new->backing); old_context = bdrv_get_aio_context(bs_top); + bdrv_drained_begin(bs_top); + + /* + * bdrv_drained_begin() requires that only the AioContext of the drained + * node is locked, and at this point it can still differ from the AioContext + * of bs_top. + */ + new_context = bdrv_get_aio_context(bs_new); + aio_context_release(old_context); + aio_context_acquire(new_context); + bdrv_drained_begin(bs_new); + aio_context_release(new_context); + aio_context_acquire(old_context); + new_context = NULL; + + bdrv_graph_wrlock(bs_top); child = bdrv_attach_child_noperm(bs_new, bs_top, "backing", &child_of_bds, bdrv_backing_role(bs_new), @@ -5400,10 +5526,9 @@ int bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top, } /* - * bdrv_attach_child_noperm could change the AioContext of bs_top. - * bdrv_replace_node_noperm calls bdrv_drained_begin, so let's temporarily - * hold the new AioContext, since bdrv_drained_begin calls BDRV_POLL_WHILE - * that assumes the new lock is taken. + * bdrv_attach_child_noperm could change the AioContext of bs_top and + * bs_new, but at least they are in the same AioContext now. This is the + * AioContext that we need to lock for the rest of the function. */ new_context = bdrv_get_aio_context(bs_top); @@ -5421,9 +5546,11 @@ int bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top, out: tran_finalize(tran, ret); - bdrv_graph_rdlock_main_loop(); bdrv_refresh_limits(bs_top, NULL, NULL); - bdrv_graph_rdunlock_main_loop(); + bdrv_graph_wrunlock(); + + bdrv_drained_end(bs_top); + bdrv_drained_end(bs_new); if (new_context && old_context != new_context) { aio_context_release(new_context); @@ -5447,6 +5574,7 @@ int bdrv_replace_child_bs(BdrvChild *child, BlockDriverState *new_bs, bdrv_ref(old_bs); bdrv_drained_begin(old_bs); bdrv_drained_begin(new_bs); + bdrv_graph_wrlock(new_bs); bdrv_replace_child_tran(child, new_bs, tran); @@ -5457,6 +5585,7 @@ int bdrv_replace_child_bs(BdrvChild *child, BlockDriverState *new_bs, tran_finalize(tran, ret); + bdrv_graph_wrunlock(); bdrv_drained_end(old_bs); bdrv_drained_end(new_bs); bdrv_unref(old_bs); @@ -5812,9 +5941,11 @@ int bdrv_drop_intermediate(BlockDriverState *top, BlockDriverState *base, backing_file_str = base->filename; } + bdrv_graph_rdlock_main_loop(); QLIST_FOREACH(c, &top->parents, next_parent) { updated_children = g_slist_prepend(updated_children, c); } + bdrv_graph_rdunlock_main_loop(); /* * It seems correct to pass detach_subchain=true here, but it triggers @@ -6737,6 +6868,7 @@ int bdrv_activate(BlockDriverState *bs, Error **errp) BdrvDirtyBitmap *bm; GLOBAL_STATE_CODE(); + GRAPH_RDLOCK_GUARD_MAINLOOP(); if (!bs->drv) { return -ENOMEDIUM; @@ -6867,6 +6999,7 @@ static int bdrv_inactivate_recurse(BlockDriverState *bs) uint64_t cumulative_perms, cumulative_shared_perms; GLOBAL_STATE_CODE(); + GRAPH_RDLOCK_GUARD_MAINLOOP(); if (!bs->drv) { return -ENOMEDIUM; @@ -7045,6 +7178,23 @@ void bdrv_unref(BlockDriverState *bs) } } +/* + * Release a BlockDriverState reference while holding the graph write lock. + * + * Calling bdrv_unref() directly is forbidden while holding the graph lock + * because bdrv_close() both involves polling and taking the graph lock + * internally. bdrv_schedule_unref() instead delays decreasing the refcount and + * possibly closing @bs until the graph lock is released. + */ +void bdrv_schedule_unref(BlockDriverState *bs) +{ + if (!bs) { + return; + } + aio_bh_schedule_oneshot(qemu_get_aio_context(), + (QEMUBHFunc *) bdrv_unref, bs); +} + struct BdrvOpBlocker { Error *reason; QLIST_ENTRY(BdrvOpBlocker) list; @@ -7541,17 +7691,21 @@ static bool bdrv_change_aio_context(BlockDriverState *bs, AioContext *ctx, return true; } + bdrv_graph_rdlock_main_loop(); QLIST_FOREACH(c, &bs->parents, next_parent) { if (!bdrv_parent_change_aio_context(c, ctx, visited, tran, errp)) { + bdrv_graph_rdunlock_main_loop(); return false; } } QLIST_FOREACH(c, &bs->children, next) { if (!bdrv_child_change_aio_context(c, ctx, visited, tran, errp)) { + bdrv_graph_rdunlock_main_loop(); return false; } } + bdrv_graph_rdunlock_main_loop(); state = g_new(BdrvStateSetAioContext, 1); *state = (BdrvStateSetAioContext) { diff --git a/block/blklogwrites.c b/block/blklogwrites.c index 3ea7141cb5..a0d70729bb 100644 --- a/block/blklogwrites.c +++ b/block/blklogwrites.c @@ -251,7 +251,9 @@ static int blk_log_writes_open(BlockDriverState *bs, QDict *options, int flags, ret = 0; fail_log: if (ret < 0) { + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, s->log_file); + bdrv_graph_wrunlock(); s->log_file = NULL; } fail: @@ -263,8 +265,10 @@ static void blk_log_writes_close(BlockDriverState *bs) { BDRVBlkLogWritesState *s = bs->opaque; + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, s->log_file); s->log_file = NULL; + bdrv_graph_wrunlock(); } static int64_t coroutine_fn GRAPH_RDLOCK diff --git a/block/blkverify.c b/block/blkverify.c index 7326461f30..dae9716a26 100644 --- a/block/blkverify.c +++ b/block/blkverify.c @@ -151,8 +151,10 @@ static void blkverify_close(BlockDriverState *bs) { BDRVBlkverifyState *s = bs->opaque; + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, s->test_file); s->test_file = NULL; + bdrv_graph_wrunlock(); } static int64_t coroutine_fn GRAPH_RDLOCK diff --git a/block/block-backend.c b/block/block-backend.c index 4009ed5fed..efe2e7cbf8 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -33,8 +33,6 @@ #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ -static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb); - typedef struct BlockBackendAioNotifier { void (*attached_aio_context)(AioContext *new_context, void *opaque); void (*detach_aio_context)(void *opaque); @@ -103,7 +101,6 @@ typedef struct BlockBackendAIOCB { } BlockBackendAIOCB; static const AIOCBInfo block_backend_aiocb_info = { - .get_aio_context = blk_aiocb_get_aio_context, .aiocb_size = sizeof(BlockBackendAIOCB), }; @@ -121,6 +118,10 @@ static QTAILQ_HEAD(, BlockBackend) block_backends = static QTAILQ_HEAD(, BlockBackend) monitor_block_backends = QTAILQ_HEAD_INITIALIZER(monitor_block_backends); +static int coroutine_mixed_fn GRAPH_RDLOCK +blk_set_perm_locked(BlockBackend *blk, uint64_t perm, uint64_t shared_perm, + Error **errp); + static void blk_root_inherit_options(BdrvChildRole role, bool parent_is_format, int *child_flags, QDict *child_options, int parent_flags, QDict *parent_options) @@ -186,7 +187,7 @@ static void blk_vm_state_changed(void *opaque, bool running, RunState state) * * If an error is returned, the VM cannot be allowed to be resumed. */ -static void blk_root_activate(BdrvChild *child, Error **errp) +static void GRAPH_RDLOCK blk_root_activate(BdrvChild *child, Error **errp) { BlockBackend *blk = child->opaque; Error *local_err = NULL; @@ -207,7 +208,7 @@ static void blk_root_activate(BdrvChild *child, Error **errp) */ saved_shared_perm = blk->shared_perm; - blk_set_perm(blk, blk->perm, BLK_PERM_ALL, &local_err); + blk_set_perm_locked(blk, blk->perm, BLK_PERM_ALL, &local_err); if (local_err) { error_propagate(errp, local_err); blk->disable_perm = true; @@ -226,7 +227,7 @@ static void blk_root_activate(BdrvChild *child, Error **errp) return; } - blk_set_perm(blk, blk->perm, blk->shared_perm, &local_err); + blk_set_perm_locked(blk, blk->perm, blk->shared_perm, &local_err); if (local_err) { error_propagate(errp, local_err); blk->disable_perm = true; @@ -259,7 +260,7 @@ static bool blk_can_inactivate(BlockBackend *blk) return blk->force_allow_inactivate; } -static int blk_root_inactivate(BdrvChild *child) +static int GRAPH_RDLOCK blk_root_inactivate(BdrvChild *child) { BlockBackend *blk = child->opaque; @@ -911,7 +912,10 @@ void blk_remove_bs(BlockBackend *blk) blk_drain(blk); root = blk->root; blk->root = NULL; + + bdrv_graph_wrlock(NULL); bdrv_root_unref_child(root); + bdrv_graph_wrunlock(); } /* @@ -953,8 +957,9 @@ int blk_replace_bs(BlockBackend *blk, BlockDriverState *new_bs, Error **errp) /* * Sets the permission bitmasks that the user of the BlockBackend needs. */ -int blk_set_perm(BlockBackend *blk, uint64_t perm, uint64_t shared_perm, - Error **errp) +static int coroutine_mixed_fn GRAPH_RDLOCK +blk_set_perm_locked(BlockBackend *blk, uint64_t perm, uint64_t shared_perm, + Error **errp) { int ret; GLOBAL_STATE_CODE(); @@ -972,6 +977,15 @@ int blk_set_perm(BlockBackend *blk, uint64_t perm, uint64_t shared_perm, return 0; } +int blk_set_perm(BlockBackend *blk, uint64_t perm, uint64_t shared_perm, + Error **errp) +{ + GLOBAL_STATE_CODE(); + GRAPH_RDLOCK_GUARD_MAINLOOP(); + + return blk_set_perm_locked(blk, perm, shared_perm, errp); +} + void blk_get_perm(BlockBackend *blk, uint64_t *perm, uint64_t *shared_perm) { GLOBAL_STATE_CODE(); @@ -1341,7 +1355,7 @@ blk_co_do_preadv_part(BlockBackend *blk, int64_t offset, int64_t bytes, /* throttling disk I/O */ if (blk->public.throttle_group_member.throttle_state) { throttle_group_co_io_limits_intercept(&blk->public.throttle_group_member, - bytes, false); + bytes, THROTTLE_READ); } ret = bdrv_co_preadv_part(blk->root, offset, bytes, qiov, qiov_offset, @@ -1415,7 +1429,7 @@ blk_co_do_pwritev_part(BlockBackend *blk, int64_t offset, int64_t bytes, /* throttling disk I/O */ if (blk->public.throttle_group_member.throttle_state) { throttle_group_co_io_limits_intercept(&blk->public.throttle_group_member, - bytes, true); + bytes, THROTTLE_WRITE); } if (!blk->enable_write_cache) { @@ -1533,7 +1547,7 @@ BlockAIOCB *blk_abort_aio_request(BlockBackend *blk, acb->blk = blk; acb->ret = ret; - replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(), error_callback_bh, acb); return &acb->common; } @@ -1545,16 +1559,8 @@ typedef struct BlkAioEmAIOCB { bool has_returned; } BlkAioEmAIOCB; -static AioContext *blk_aio_em_aiocb_get_aio_context(BlockAIOCB *acb_) -{ - BlkAioEmAIOCB *acb = container_of(acb_, BlkAioEmAIOCB, common); - - return blk_get_aio_context(acb->rwco.blk); -} - static const AIOCBInfo blk_aio_em_aiocb_info = { .aiocb_size = sizeof(BlkAioEmAIOCB), - .get_aio_context = blk_aio_em_aiocb_get_aio_context, }; static void blk_aio_complete(BlkAioEmAIOCB *acb) @@ -1595,11 +1601,11 @@ static BlockAIOCB *blk_aio_prwv(BlockBackend *blk, int64_t offset, acb->has_returned = false; co = qemu_coroutine_create(co_entry, acb); - aio_co_enter(blk_get_aio_context(blk), co); + aio_co_enter(qemu_get_current_aio_context(), co); acb->has_returned = true; if (acb->rwco.ret != NOT_DONE) { - replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(), blk_aio_complete_bh, acb); } @@ -1901,11 +1907,11 @@ BlockAIOCB *blk_aio_zone_report(BlockBackend *blk, int64_t offset, acb->has_returned = false; co = qemu_coroutine_create(blk_aio_zone_report_entry, acb); - aio_co_enter(blk_get_aio_context(blk), co); + aio_co_enter(qemu_get_current_aio_context(), co); acb->has_returned = true; if (acb->rwco.ret != NOT_DONE) { - replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(), blk_aio_complete_bh, acb); } @@ -1942,11 +1948,11 @@ BlockAIOCB *blk_aio_zone_mgmt(BlockBackend *blk, BlockZoneOp op, acb->has_returned = false; co = qemu_coroutine_create(blk_aio_zone_mgmt_entry, acb); - aio_co_enter(blk_get_aio_context(blk), co); + aio_co_enter(qemu_get_current_aio_context(), co); acb->has_returned = true; if (acb->rwco.ret != NOT_DONE) { - replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(), blk_aio_complete_bh, acb); } @@ -1982,10 +1988,10 @@ BlockAIOCB *blk_aio_zone_append(BlockBackend *blk, int64_t *offset, acb->has_returned = false; co = qemu_coroutine_create(blk_aio_zone_append_entry, acb); - aio_co_enter(blk_get_aio_context(blk), co); + aio_co_enter(qemu_get_current_aio_context(), co); acb->has_returned = true; if (acb->rwco.ret != NOT_DONE) { - replay_bh_schedule_oneshot_event(blk_get_aio_context(blk), + replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(), blk_aio_complete_bh, acb); } @@ -2434,12 +2440,6 @@ AioContext *blk_get_aio_context(BlockBackend *blk) return blk->ctx; } -static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb) -{ - BlockBackendAIOCB *blk_acb = DO_UPCAST(BlockBackendAIOCB, common, acb); - return blk_get_aio_context(blk_acb->blk); -} - int blk_set_aio_context(BlockBackend *blk, AioContext *new_context, Error **errp) { diff --git a/block/copy-before-write.c b/block/copy-before-write.c index 9a0e2b69d9..aeaff3bb82 100644 --- a/block/copy-before-write.c +++ b/block/copy-before-write.c @@ -341,11 +341,11 @@ static void cbw_refresh_filename(BlockDriverState *bs) bs->file->bs->filename); } -static void cbw_child_perm(BlockDriverState *bs, BdrvChild *c, - BdrvChildRole role, - BlockReopenQueue *reopen_queue, - uint64_t perm, uint64_t shared, - uint64_t *nperm, uint64_t *nshared) +static void GRAPH_RDLOCK +cbw_child_perm(BlockDriverState *bs, BdrvChild *c, BdrvChildRole role, + BlockReopenQueue *reopen_queue, + uint64_t perm, uint64_t shared, + uint64_t *nperm, uint64_t *nshared) { if (!(role & BDRV_CHILD_FILTERED)) { /* diff --git a/block/crypto.c b/block/crypto.c index 6ee8d46d30..c9c9a39fa3 100644 --- a/block/crypto.c +++ b/block/crypto.c @@ -777,7 +777,7 @@ block_crypto_get_specific_info_luks(BlockDriverState *bs, Error **errp) return spec_info; } -static int +static int GRAPH_RDLOCK block_crypto_amend_prepare(BlockDriverState *bs, Error **errp) { BlockCrypto *crypto = bs->opaque; @@ -793,7 +793,7 @@ block_crypto_amend_prepare(BlockDriverState *bs, Error **errp) return ret; } -static void +static void GRAPH_RDLOCK block_crypto_amend_cleanup(BlockDriverState *bs) { BlockCrypto *crypto = bs->opaque; @@ -841,6 +841,8 @@ block_crypto_amend_options_luks(BlockDriverState *bs, QCryptoBlockAmendOptions *amend_options = NULL; int ret = -EINVAL; + assume_graph_lock(); /* FIXME */ + assert(crypto); assert(crypto->block); diff --git a/block/file-posix.c b/block/file-posix.c index 4757914ac0..50e2b20d5c 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -1412,11 +1412,9 @@ static void raw_refresh_zoned_limits(BlockDriverState *bs, struct stat *st, BlockZoneModel zoned; int ret; - bs->bl.zoned = BLK_Z_NONE; - ret = get_sysfs_zoned_model(st, &zoned); if (ret < 0 || zoned == BLK_Z_NONE) { - return; + goto no_zoned; } bs->bl.zoned = zoned; @@ -1437,10 +1435,10 @@ static void raw_refresh_zoned_limits(BlockDriverState *bs, struct stat *st, if (ret < 0) { error_setg_errno(errp, -ret, "Unable to read chunk_sectors " "sysfs attribute"); - return; + goto no_zoned; } else if (!ret) { error_setg(errp, "Read 0 from chunk_sectors sysfs attribute"); - return; + goto no_zoned; } bs->bl.zone_size = ret << BDRV_SECTOR_BITS; @@ -1448,10 +1446,10 @@ static void raw_refresh_zoned_limits(BlockDriverState *bs, struct stat *st, if (ret < 0) { error_setg_errno(errp, -ret, "Unable to read nr_zones " "sysfs attribute"); - return; + goto no_zoned; } else if (!ret) { error_setg(errp, "Read 0 from nr_zones sysfs attribute"); - return; + goto no_zoned; } bs->bl.nr_zones = ret; @@ -1472,10 +1470,15 @@ static void raw_refresh_zoned_limits(BlockDriverState *bs, struct stat *st, ret = get_zones_wp(bs, s->fd, 0, bs->bl.nr_zones, 0); if (ret < 0) { error_setg_errno(errp, -ret, "report wps failed"); - bs->wps = NULL; - return; + goto no_zoned; } qemu_co_mutex_init(&bs->wps->colock); + return; + +no_zoned: + bs->bl.zoned = BLK_Z_NONE; + g_free(bs->wps); + bs->wps = NULL; } #else /* !defined(CONFIG_BLKZONED) */ static void raw_refresh_zoned_limits(BlockDriverState *bs, struct stat *st, @@ -2452,9 +2455,10 @@ static int coroutine_fn raw_co_prw(BlockDriverState *bs, uint64_t offset, if (fd_open(bs) < 0) return -EIO; #if defined(CONFIG_BLKZONED) - if ((type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) && bs->wps) { + if ((type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) && + bs->bl.zoned != BLK_Z_NONE) { qemu_co_mutex_lock(&bs->wps->colock); - if (type & QEMU_AIO_ZONE_APPEND && bs->bl.zone_size) { + if (type & QEMU_AIO_ZONE_APPEND) { int index = offset / bs->bl.zone_size; offset = bs->wps->wp[index]; } @@ -2502,11 +2506,10 @@ static int coroutine_fn raw_co_prw(BlockDriverState *bs, uint64_t offset, out: #if defined(CONFIG_BLKZONED) -{ - BlockZoneWps *wps = bs->wps; - if (ret == 0) { - if ((type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) - && wps && bs->bl.zone_size) { + if ((type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) && + bs->bl.zoned != BLK_Z_NONE) { + BlockZoneWps *wps = bs->wps; + if (ret == 0) { uint64_t *wp = &wps->wp[offset / bs->bl.zone_size]; if (!BDRV_ZT_IS_CONV(*wp)) { if (type & QEMU_AIO_ZONE_APPEND) { @@ -2519,17 +2522,12 @@ out: *wp = offset + bytes; } } - } - } else { - if (type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) { + } else { update_zones_wp(bs, s->fd, 0, 1); } - } - if ((type & (QEMU_AIO_WRITE | QEMU_AIO_ZONE_APPEND)) && wps) { qemu_co_mutex_unlock(&wps->colock); } -} #endif return ret; } diff --git a/block/graph-lock.c b/block/graph-lock.c index f357a2c0b1..58a799065f 100644 --- a/block/graph-lock.c +++ b/block/graph-lock.c @@ -163,17 +163,29 @@ void bdrv_graph_wrlock(BlockDriverState *bs) void bdrv_graph_wrunlock(void) { GLOBAL_STATE_CODE(); - QEMU_LOCK_GUARD(&aio_context_list_lock); assert(qatomic_read(&has_writer)); + WITH_QEMU_LOCK_GUARD(&aio_context_list_lock) { + /* + * No need for memory barriers, this works in pair with + * the slow path of rdlock() and both take the lock. + */ + qatomic_store_release(&has_writer, 0); + + /* Wake up all coroutines that are waiting to read the graph */ + qemu_co_enter_all(&reader_queue, &aio_context_list_lock); + } + /* - * No need for memory barriers, this works in pair with - * the slow path of rdlock() and both take the lock. + * Run any BHs that were scheduled during the wrlock section and that + * callers might expect to have finished (in particular, this is important + * for bdrv_schedule_unref()). + * + * Do this only after restarting coroutines so that nested event loops in + * BHs don't deadlock if their condition relies on the coroutine making + * progress. */ - qatomic_store_release(&has_writer, 0); - - /* Wake up all coroutine that are waiting to read the graph */ - qemu_co_enter_all(&reader_queue, &aio_context_list_lock); + aio_bh_poll(qemu_get_aio_context()); } void coroutine_fn bdrv_graph_co_rdlock(void) diff --git a/block/io.c b/block/io.c index ba23a9bcd3..209a6da0c8 100644 --- a/block/io.c +++ b/block/io.c @@ -2950,25 +2950,18 @@ int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, /**************************************************************/ /* async I/Os */ +/** + * Synchronously cancels an acb. Must be called with the BQL held and the acb + * must be processed with the BQL held too (IOThreads are not allowed). + * + * Use bdrv_aio_cancel_async() instead when possible. + */ void bdrv_aio_cancel(BlockAIOCB *acb) { - IO_CODE(); + GLOBAL_STATE_CODE(); qemu_aio_ref(acb); bdrv_aio_cancel_async(acb); - while (acb->refcnt > 1) { - if (acb->aiocb_info->get_aio_context) { - aio_poll(acb->aiocb_info->get_aio_context(acb), true); - } else if (acb->bs) { - /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so - * assert that we're not using an I/O thread. Thread-safe - * code should use bdrv_aio_cancel_async exclusively. - */ - assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); - aio_poll(bdrv_get_aio_context(acb->bs), true); - } else { - abort(); - } - } + AIO_WAIT_WHILE_UNLOCKED(NULL, acb->refcnt > 1); qemu_aio_unref(acb); } diff --git a/block/mirror.c b/block/mirror.c index aae4bebbb6..3cc0757a03 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -702,8 +702,12 @@ static int mirror_exit_common(Job *job) * mirror_top_bs from now on, so keep it drained. */ bdrv_drained_begin(mirror_top_bs); bs_opaque->stop = true; + + bdrv_graph_rdlock_main_loop(); bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing, &error_abort); + bdrv_graph_rdunlock_main_loop(); + if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) { BlockDriverState *backing = s->is_none_mode ? src : s->base; BlockDriverState *unfiltered_target = bdrv_skip_filters(target_bs); @@ -1670,6 +1674,8 @@ static BlockJob *mirror_start_job( uint64_t target_perms, target_shared_perms; int ret; + GLOBAL_STATE_CODE(); + if (granularity == 0) { granularity = bdrv_get_default_bitmap_granularity(target); } @@ -1906,8 +1912,10 @@ fail: } bs_opaque->stop = true; + bdrv_graph_rdlock_main_loop(); bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing, &error_abort); + bdrv_graph_rdunlock_main_loop(); bdrv_replace_node(mirror_top_bs, mirror_top_bs->backing->bs, &error_abort); bdrv_unref(mirror_top_bs); diff --git a/block/parallels.c b/block/parallels.c index 48c32d6821..d026ce9e2f 100644 --- a/block/parallels.c +++ b/block/parallels.c @@ -178,13 +178,82 @@ static void parallels_set_bat_entry(BDRVParallelsState *s, bitmap_set(s->bat_dirty_bmap, bat_entry_off(index) / s->bat_dirty_block, 1); } +static int mark_used(BlockDriverState *bs, unsigned long *bitmap, + uint32_t bitmap_size, int64_t off, uint32_t count) +{ + BDRVParallelsState *s = bs->opaque; + uint32_t cluster_index = host_cluster_index(s, off); + unsigned long next_used; + if (cluster_index + count > bitmap_size) { + return -E2BIG; + } + next_used = find_next_bit(bitmap, bitmap_size, cluster_index); + if (next_used < cluster_index + count) { + return -EBUSY; + } + bitmap_set(bitmap, cluster_index, count); + return 0; +} + +/* + * Collect used bitmap. The image can contain errors, we should fill the + * bitmap anyway, as much as we can. This information will be used for + * error resolution. + */ +static int parallels_fill_used_bitmap(BlockDriverState *bs) +{ + BDRVParallelsState *s = bs->opaque; + int64_t payload_bytes; + uint32_t i; + int err = 0; + + payload_bytes = bdrv_getlength(bs->file->bs); + if (payload_bytes < 0) { + return payload_bytes; + } + payload_bytes -= s->data_start * BDRV_SECTOR_SIZE; + if (payload_bytes < 0) { + return -EINVAL; + } + + s->used_bmap_size = DIV_ROUND_UP(payload_bytes, s->cluster_size); + if (s->used_bmap_size == 0) { + return 0; + } + s->used_bmap = bitmap_try_new(s->used_bmap_size); + if (s->used_bmap == NULL) { + return -ENOMEM; + } + + for (i = 0; i < s->bat_size; i++) { + int err2; + int64_t host_off = bat2sect(s, i) << BDRV_SECTOR_BITS; + if (host_off == 0) { + continue; + } + + err2 = mark_used(bs, s->used_bmap, s->used_bmap_size, host_off, 1); + if (err2 < 0 && err == 0) { + err = err2; + } + } + return err; +} + +static void parallels_free_used_bitmap(BlockDriverState *bs) +{ + BDRVParallelsState *s = bs->opaque; + s->used_bmap_size = 0; + g_free(s->used_bmap); +} + static int64_t coroutine_fn GRAPH_RDLOCK allocate_clusters(BlockDriverState *bs, int64_t sector_num, int nb_sectors, int *pnum) { int ret = 0; BDRVParallelsState *s = bs->opaque; - int64_t pos, space, idx, to_allocate, i, len; + int64_t i, pos, idx, to_allocate, first_free, host_off; pos = block_status(s, sector_num, nb_sectors, pnum); if (pos > 0) { @@ -207,21 +276,21 @@ allocate_clusters(BlockDriverState *bs, int64_t sector_num, */ assert(idx < s->bat_size && idx + to_allocate <= s->bat_size); - space = to_allocate * s->tracks; - len = bdrv_co_getlength(bs->file->bs); - if (len < 0) { - return len; - } - if (s->data_end + space > (len >> BDRV_SECTOR_BITS)) { - space += s->prealloc_size; + first_free = find_first_zero_bit(s->used_bmap, s->used_bmap_size); + if (first_free == s->used_bmap_size) { + uint32_t new_usedsize; + int64_t bytes = to_allocate * s->cluster_size; + bytes += s->prealloc_size * BDRV_SECTOR_SIZE; + + host_off = s->data_end * BDRV_SECTOR_SIZE; + /* * We require the expanded size to read back as zero. If the * user permitted truncation, we try that; but if it fails, we * force the safer-but-slower fallocate. */ if (s->prealloc_mode == PRL_PREALLOC_MODE_TRUNCATE) { - ret = bdrv_co_truncate(bs->file, - (s->data_end + space) << BDRV_SECTOR_BITS, + ret = bdrv_co_truncate(bs->file, host_off + bytes, false, PREALLOC_MODE_OFF, BDRV_REQ_ZERO_WRITE, NULL); if (ret == -ENOTSUP) { @@ -229,13 +298,42 @@ allocate_clusters(BlockDriverState *bs, int64_t sector_num, } } if (s->prealloc_mode == PRL_PREALLOC_MODE_FALLOCATE) { - ret = bdrv_co_pwrite_zeroes(bs->file, - s->data_end << BDRV_SECTOR_BITS, - space << BDRV_SECTOR_BITS, 0); + ret = bdrv_co_pwrite_zeroes(bs->file, host_off, bytes, 0); } if (ret < 0) { return ret; } + + new_usedsize = s->used_bmap_size + bytes / s->cluster_size; + s->used_bmap = bitmap_zero_extend(s->used_bmap, s->used_bmap_size, + new_usedsize); + s->used_bmap_size = new_usedsize; + } else { + int64_t next_used; + next_used = find_next_bit(s->used_bmap, s->used_bmap_size, first_free); + + /* Not enough continuous clusters in the middle, adjust the size */ + if (next_used - first_free < to_allocate) { + to_allocate = next_used - first_free; + *pnum = (idx + to_allocate) * s->tracks - sector_num; + } + + host_off = s->data_start * BDRV_SECTOR_SIZE; + host_off += first_free * s->cluster_size; + + /* + * No need to preallocate if we are using tail area from the above + * branch. In the other case we are likely re-using hole. Preallocate + * the space if required by the prealloc_mode. + */ + if (s->prealloc_mode == PRL_PREALLOC_MODE_FALLOCATE && + host_off < s->data_end * BDRV_SECTOR_SIZE) { + ret = bdrv_co_pwrite_zeroes(bs->file, host_off, + s->cluster_size * to_allocate, 0); + if (ret < 0) { + return ret; + } + } } /* @@ -267,9 +365,18 @@ allocate_clusters(BlockDriverState *bs, int64_t sector_num, } } + ret = mark_used(bs, s->used_bmap, s->used_bmap_size, host_off, to_allocate); + if (ret < 0) { + /* Image consistency is broken. Alarm! */ + return ret; + } for (i = 0; i < to_allocate; i++) { - parallels_set_bat_entry(s, idx + i, s->data_end / s->off_multiplier); - s->data_end += s->tracks; + parallels_set_bat_entry(s, idx + i, + host_off / BDRV_SECTOR_SIZE / s->off_multiplier); + host_off += s->cluster_size; + } + if (host_off > s->data_end * BDRV_SECTOR_SIZE) { + s->data_end = host_off / BDRV_SECTOR_SIZE; } return bat2sect(s, idx) + sector_num % s->tracks; @@ -430,6 +537,64 @@ parallels_co_readv(BlockDriverState *bs, int64_t sector_num, int nb_sectors, return ret; } + +static int coroutine_fn GRAPH_RDLOCK +parallels_co_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes) +{ + int ret = 0; + uint32_t cluster, count; + BDRVParallelsState *s = bs->opaque; + + /* + * The image does not support ZERO mark inside the BAT, which means that + * stale data could be exposed from the backing file. + */ + if (bs->backing) { + return -ENOTSUP; + } + + if (!QEMU_IS_ALIGNED(offset, s->cluster_size)) { + return -ENOTSUP; + } else if (!QEMU_IS_ALIGNED(bytes, s->cluster_size)) { + return -ENOTSUP; + } + + cluster = offset / s->cluster_size; + count = bytes / s->cluster_size; + + qemu_co_mutex_lock(&s->lock); + for (; count > 0; cluster++, count--) { + int64_t host_off = bat2sect(s, cluster) << BDRV_SECTOR_BITS; + if (host_off == 0) { + continue; + } + + ret = bdrv_co_pdiscard(bs->file, host_off, s->cluster_size); + if (ret < 0) { + goto done; + } + + parallels_set_bat_entry(s, cluster, 0); + bitmap_clear(s->used_bmap, host_cluster_index(s, host_off), 1); + } +done: + qemu_co_mutex_unlock(&s->lock); + return ret; +} + +static int coroutine_fn GRAPH_RDLOCK +parallels_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int64_t bytes, + BdrvRequestFlags flags) +{ + /* + * The zero flag is missed in the Parallels format specification. We can + * resort to discard if we have no backing file (this condition is checked + * inside parallels_co_pdiscard(). + */ + return parallels_co_pdiscard(bs, offset, bytes); +} + + static void parallels_check_unclean(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix) @@ -515,7 +680,17 @@ parallels_check_data_off(BlockDriverState *bs, BdrvCheckResult *res, res->corruptions++; if (fix & BDRV_FIX_ERRORS) { + int err; s->header->data_off = cpu_to_le32(data_off); + s->data_start = data_off; + + parallels_free_used_bitmap(bs); + err = parallels_fill_used_bitmap(bs); + if (err == -ENOMEM) { + res->check_errors++; + return err; + } + res->corruptions_fixed++; } @@ -621,7 +796,7 @@ parallels_check_duplicate(BlockDriverState *bs, BdrvCheckResult *res, BDRVParallelsState *s = bs->opaque; int64_t host_off, host_sector, guest_sector; unsigned long *bitmap; - uint32_t i, bitmap_size, cluster_index, bat_entry; + uint32_t i, bitmap_size, bat_entry; int n, ret = 0; uint64_t *buf = NULL; bool fixed = false; @@ -655,10 +830,9 @@ parallels_check_duplicate(BlockDriverState *bs, BdrvCheckResult *res, continue; } - cluster_index = host_cluster_index(s, host_off); - assert(cluster_index < bitmap_size); - if (!test_bit(cluster_index, bitmap)) { - bitmap_set(bitmap, cluster_index, 1); + ret = mark_used(bs, bitmap, bitmap_size, host_off, 1); + assert(ret != -E2BIG); + if (ret == 0) { continue; } @@ -713,11 +887,13 @@ parallels_check_duplicate(BlockDriverState *bs, BdrvCheckResult *res, * consistent for the new allocated clusters too. * * Note, clusters allocated outside the current image are not - * considered, and the bitmap size doesn't change. + * considered, and the bitmap size doesn't change. This specifically + * means that -E2BIG is OK. */ - cluster_index = host_cluster_index(s, host_off); - if (cluster_index < bitmap_size) { - bitmap_set(bitmap, cluster_index, 1); + ret = mark_used(bs, bitmap, bitmap_size, host_off, 1); + if (ret == -EBUSY) { + res->check_errors++; + goto out_repair_bat; } fixed = true; @@ -1025,6 +1201,44 @@ static int parallels_update_header(BlockDriverState *bs) return bdrv_pwrite_sync(bs->file, 0, size, s->header, 0); } + +static int parallels_opts_prealloc(BlockDriverState *bs, QDict *options, + Error **errp) +{ + int err; + char *buf; + int64_t bytes; + BDRVParallelsState *s = bs->opaque; + Error *local_err = NULL; + QemuOpts *opts = qemu_opts_create(¶llels_runtime_opts, NULL, 0, errp); + if (!opts) { + return -ENOMEM; + } + + err = -EINVAL; + if (!qemu_opts_absorb_qdict(opts, options, errp)) { + goto done; + } + + bytes = qemu_opt_get_size_del(opts, PARALLELS_OPT_PREALLOC_SIZE, 0); + s->prealloc_size = bytes >> BDRV_SECTOR_BITS; + buf = qemu_opt_get_del(opts, PARALLELS_OPT_PREALLOC_MODE); + /* prealloc_mode can be downgraded later during allocate_clusters */ + s->prealloc_mode = qapi_enum_parse(&prealloc_mode_lookup, buf, + PRL_PREALLOC_MODE_FALLOCATE, + &local_err); + g_free(buf); + if (local_err != NULL) { + error_propagate(errp, local_err); + goto done; + } + err = 0; + +done: + qemu_opts_del(opts); + return err; +} + static int parallels_open(BlockDriverState *bs, QDict *options, int flags, Error **errp) { @@ -1033,10 +1247,12 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags, int ret, size, i; int64_t file_nb_sectors, sector; uint32_t data_start; - QemuOpts *opts = NULL; - Error *local_err = NULL; - char *buf; - bool data_off_is_correct; + bool need_check = false; + + ret = parallels_opts_prealloc(bs, options, errp); + if (ret < 0) { + return ret; + } ret = bdrv_open_file_child(NULL, options, "file", bs, errp); if (ret < 0) { @@ -1050,7 +1266,7 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags, ret = bdrv_pread(bs->file, 0, sizeof(ph), &ph, 0); if (ret < 0) { - goto fail; + return ret; } bs->total_sectors = le64_to_cpu(ph.nb_sectors); @@ -1070,29 +1286,26 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags, s->tracks = le32_to_cpu(ph.tracks); if (s->tracks == 0) { error_setg(errp, "Invalid image: Zero sectors per track"); - ret = -EINVAL; - goto fail; + return -EINVAL; } if (s->tracks > INT32_MAX/513) { error_setg(errp, "Invalid image: Too big cluster"); - ret = -EFBIG; - goto fail; + return -EFBIG; } + s->prealloc_size = MAX(s->tracks, s->prealloc_size); s->cluster_size = s->tracks << BDRV_SECTOR_BITS; s->bat_size = le32_to_cpu(ph.bat_entries); if (s->bat_size > INT_MAX / sizeof(uint32_t)) { error_setg(errp, "Catalog too large"); - ret = -EFBIG; - goto fail; + return -EFBIG; } size = bat_entry_off(s->bat_size); s->header_size = ROUND_UP(size, bdrv_opt_mem_align(bs->file->bs)); s->header = qemu_try_blockalign(bs->file->bs, s->header_size); if (s->header == NULL) { - ret = -ENOMEM; - goto fail; + return -ENOMEM; } ret = bdrv_pread(bs->file, 0, s->header_size, s->header, 0); @@ -1102,11 +1315,14 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags, s->bat_bitmap = (uint32_t *)(s->header + 1); if (le32_to_cpu(ph.inuse) == HEADER_INUSE_MAGIC) { - s->header_unclean = true; + need_check = s->header_unclean = true; + } + + { + bool ok = parallels_test_data_off(s, file_nb_sectors, &data_start); + need_check = need_check || !ok; } - data_off_is_correct = parallels_test_data_off(s, file_nb_sectors, - &data_start); s->data_start = data_start; s->data_end = s->data_start; if (s->data_end < (s->header_size >> BDRV_SECTOR_BITS)) { @@ -1117,29 +1333,6 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags, s->header_size = size; } - opts = qemu_opts_create(¶llels_runtime_opts, NULL, 0, errp); - if (!opts) { - goto fail_options; - } - - if (!qemu_opts_absorb_qdict(opts, options, errp)) { - goto fail_options; - } - - s->prealloc_size = - qemu_opt_get_size_del(opts, PARALLELS_OPT_PREALLOC_SIZE, 0); - s->prealloc_size = MAX(s->tracks, s->prealloc_size >> BDRV_SECTOR_BITS); - buf = qemu_opt_get_del(opts, PARALLELS_OPT_PREALLOC_MODE); - /* prealloc_mode can be downgraded later during allocate_clusters */ - s->prealloc_mode = qapi_enum_parse(&prealloc_mode_lookup, buf, - PRL_PREALLOC_MODE_FALLOCATE, - &local_err); - g_free(buf); - if (local_err != NULL) { - error_propagate(errp, local_err); - goto fail_options; - } - if (ph.ext_off) { if (flags & BDRV_O_RDWR) { /* @@ -1186,6 +1379,15 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags, s->data_end = sector + s->tracks; } } + need_check = need_check || s->data_end > file_nb_sectors; + + if (!need_check) { + ret = parallels_fill_used_bitmap(bs); + if (ret == -ENOMEM) { + goto fail; + } + need_check = need_check || ret < 0; /* These are correctable errors */ + } /* * We don't repair the image here if it's opened for checks. Also we don't @@ -1195,12 +1397,8 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags, return 0; } - /* - * Repair the image if it's dirty or - * out-of-image corruption was detected. - */ - if (s->data_end > file_nb_sectors || s->header_unclean - || !data_off_is_correct) { + /* Repair the image if corruption was detected. */ + if (need_check) { BdrvCheckResult res; ret = bdrv_check(bs, &res, BDRV_FIX_ERRORS | BDRV_FIX_LEAKS); if (ret < 0) { @@ -1209,18 +1407,19 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags, goto fail; } } - return 0; fail_format: error_setg(errp, "Image not in Parallels format"); -fail_options: - ret = -EINVAL; + return -EINVAL; + fail: /* * "s" object was allocated by g_malloc0 so we can safely * try to free its fields even they were not allocated. */ + parallels_free_used_bitmap(bs); + error_free(s->migration_blocker); g_free(s->bat_dirty_bmap); qemu_vfree(s->header); @@ -1241,6 +1440,8 @@ static void parallels_close(BlockDriverState *bs) PREALLOC_MODE_OFF, 0, NULL); } + parallels_free_used_bitmap(bs); + g_free(s->bat_dirty_bmap); qemu_vfree(s->header); @@ -1248,24 +1449,34 @@ static void parallels_close(BlockDriverState *bs) error_free(s->migration_blocker); } +static bool parallels_is_support_dirty_bitmaps(BlockDriverState *bs) +{ + return 1; +} + static BlockDriver bdrv_parallels = { - .format_name = "parallels", - .instance_size = sizeof(BDRVParallelsState), - .bdrv_probe = parallels_probe, - .bdrv_open = parallels_open, - .bdrv_close = parallels_close, - .bdrv_child_perm = bdrv_default_perms, - .bdrv_co_block_status = parallels_co_block_status, - .bdrv_has_zero_init = bdrv_has_zero_init_1, - .bdrv_co_flush_to_os = parallels_co_flush_to_os, - .bdrv_co_readv = parallels_co_readv, - .bdrv_co_writev = parallels_co_writev, - .is_format = true, - .supports_backing = true, - .bdrv_co_create = parallels_co_create, - .bdrv_co_create_opts = parallels_co_create_opts, - .bdrv_co_check = parallels_co_check, - .create_opts = ¶llels_create_opts, + .format_name = "parallels", + .instance_size = sizeof(BDRVParallelsState), + .create_opts = ¶llels_create_opts, + .is_format = true, + .supports_backing = true, + + .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_supports_persistent_dirty_bitmap = parallels_is_support_dirty_bitmaps, + + .bdrv_probe = parallels_probe, + .bdrv_open = parallels_open, + .bdrv_close = parallels_close, + .bdrv_child_perm = bdrv_default_perms, + .bdrv_co_block_status = parallels_co_block_status, + .bdrv_co_flush_to_os = parallels_co_flush_to_os, + .bdrv_co_readv = parallels_co_readv, + .bdrv_co_writev = parallels_co_writev, + .bdrv_co_create = parallels_co_create, + .bdrv_co_create_opts = parallels_co_create_opts, + .bdrv_co_check = parallels_co_check, + .bdrv_co_pdiscard = parallels_co_pdiscard, + .bdrv_co_pwrite_zeroes = parallels_co_pwrite_zeroes, }; static void bdrv_parallels_init(void) diff --git a/block/parallels.h b/block/parallels.h index 4e53e9572d..6b199443cf 100644 --- a/block/parallels.h +++ b/block/parallels.h @@ -72,6 +72,9 @@ typedef struct BDRVParallelsState { unsigned long *bat_dirty_bmap; unsigned int bat_dirty_block; + unsigned long *used_bmap; + unsigned long used_bmap_size; + uint32_t *bat_bitmap; unsigned int bat_size; diff --git a/block/preallocate.c b/block/preallocate.c index 3d0f621003..bfb638d8b1 100644 --- a/block/preallocate.c +++ b/block/preallocate.c @@ -75,8 +75,14 @@ typedef struct BDRVPreallocateState { * be invalid (< 0) when we don't have both exclusive BLK_PERM_RESIZE and * BLK_PERM_WRITE permissions on file child. */ + + /* Gives up the resize permission on children when parents don't need it */ + QEMUBH *drop_resize_bh; } BDRVPreallocateState; +static int preallocate_drop_resize(BlockDriverState *bs, Error **errp); +static void preallocate_drop_resize_bh(void *opaque); + #define PREALLOCATE_OPT_PREALLOC_ALIGN "prealloc-align" #define PREALLOCATE_OPT_PREALLOC_SIZE "prealloc-size" static QemuOptsList runtime_opts = { @@ -142,6 +148,7 @@ static int preallocate_open(BlockDriverState *bs, QDict *options, int flags, * For this to work, mark them invalid. */ s->file_end = s->zero_start = s->data_end = -EINVAL; + s->drop_resize_bh = qemu_bh_new(preallocate_drop_resize_bh, bs); ret = bdrv_open_file_child(NULL, options, "file", bs, errp); if (ret < 0) { @@ -162,26 +169,42 @@ static int preallocate_open(BlockDriverState *bs, QDict *options, int flags, return 0; } -static void preallocate_close(BlockDriverState *bs) +static int preallocate_truncate_to_real_size(BlockDriverState *bs, Error **errp) { - int ret; BDRVPreallocateState *s = bs->opaque; - - if (s->data_end < 0) { - return; - } + int ret; if (s->file_end < 0) { s->file_end = bdrv_getlength(bs->file->bs); if (s->file_end < 0) { - return; + error_setg_errno(errp, -s->file_end, "Failed to get file length"); + return s->file_end; } } if (s->data_end < s->file_end) { ret = bdrv_truncate(bs->file, s->data_end, true, PREALLOC_MODE_OFF, 0, NULL); - s->file_end = ret < 0 ? ret : s->data_end; + if (ret < 0) { + error_setg_errno(errp, -ret, "Failed to drop preallocation"); + s->file_end = ret; + return ret; + } + s->file_end = s->data_end; + } + + return 0; +} + +static void preallocate_close(BlockDriverState *bs) +{ + BDRVPreallocateState *s = bs->opaque; + + qemu_bh_cancel(s->drop_resize_bh); + qemu_bh_delete(s->drop_resize_bh); + + if (s->data_end >= 0) { + preallocate_truncate_to_real_size(bs, NULL); } } @@ -198,6 +221,7 @@ static int preallocate_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue, Error **errp) { PreallocateOpts *opts = g_new0(PreallocateOpts, 1); + int ret; if (!preallocate_absorb_opts(opts, reopen_state->options, reopen_state->bs->file->bs, errp)) { @@ -205,6 +229,19 @@ static int preallocate_reopen_prepare(BDRVReopenState *reopen_state, return -EINVAL; } + /* + * Drop the preallocation already here if reopening read-only. The child + * might also be reopened read-only and then scheduling a BH during the + * permission update is too late. + */ + if ((reopen_state->flags & BDRV_O_RDWR) == 0) { + ret = preallocate_drop_resize(reopen_state->bs, errp); + if (ret < 0) { + g_free(opts); + return ret; + } + } + reopen_state->opaque = opts; return 0; @@ -462,58 +499,61 @@ preallocate_co_getlength(BlockDriverState *bs) return ret; } -static int preallocate_check_perm(BlockDriverState *bs, - uint64_t perm, uint64_t shared, Error **errp) +static int preallocate_drop_resize(BlockDriverState *bs, Error **errp) { BDRVPreallocateState *s = bs->opaque; + int ret; - if (s->data_end >= 0 && !can_write_resize(perm)) { - /* - * Lose permissions. - * We should truncate in check_perm, as in set_perm bs->file->perm will - * be already changed, and we should not violate it. - */ - if (s->file_end < 0) { - s->file_end = bdrv_getlength(bs->file->bs); - if (s->file_end < 0) { - error_setg(errp, "Failed to get file length"); - return s->file_end; - } - } + if (s->data_end < 0) { + return 0; + } - if (s->data_end < s->file_end) { - int ret = bdrv_truncate(bs->file, s->data_end, true, - PREALLOC_MODE_OFF, 0, NULL); - if (ret < 0) { - error_setg(errp, "Failed to drop preallocation"); - s->file_end = ret; - return ret; - } - s->file_end = s->data_end; - } + /* + * Before switching children to be read-only, truncate them to remove + * the preallocation and let them have the real size. + */ + ret = preallocate_truncate_to_real_size(bs, errp); + if (ret < 0) { + return ret; } + /* + * We'll drop our permissions and will allow other users to take write and + * resize permissions (see preallocate_child_perm). Anyone will be able to + * change the child, so mark all states invalid. We'll regain control if a + * parent requests write access again. + */ + s->data_end = s->file_end = s->zero_start = -EINVAL; + + bdrv_graph_rdlock_main_loop(); + bdrv_child_refresh_perms(bs, bs->file, NULL); + bdrv_graph_rdunlock_main_loop(); + return 0; } +static void preallocate_drop_resize_bh(void *opaque) +{ + /* + * In case of errors, we'll simply keep the exclusive lock on the image + * indefinitely. + */ + preallocate_drop_resize(opaque, NULL); +} + static void preallocate_set_perm(BlockDriverState *bs, uint64_t perm, uint64_t shared) { BDRVPreallocateState *s = bs->opaque; if (can_write_resize(perm)) { + qemu_bh_cancel(s->drop_resize_bh); if (s->data_end < 0) { s->data_end = s->file_end = s->zero_start = - bdrv_getlength(bs->file->bs); + bs->file->bs->total_sectors * BDRV_SECTOR_SIZE; } } else { - /* - * We drop our permissions, as well as allow shared - * permissions (see preallocate_child_perm), anyone will be able to - * change the child, so mark all states invalid. We'll regain control if - * get good permissions back. - */ - s->data_end = s->file_end = s->zero_start = -EINVAL; + qemu_bh_schedule(s->drop_resize_bh); } } @@ -521,10 +561,16 @@ static void preallocate_child_perm(BlockDriverState *bs, BdrvChild *c, BdrvChildRole role, BlockReopenQueue *reopen_queue, uint64_t perm, uint64_t shared, uint64_t *nperm, uint64_t *nshared) { + BDRVPreallocateState *s = bs->opaque; + bdrv_default_perms(bs, c, role, reopen_queue, perm, shared, nperm, nshared); - if (can_write_resize(perm)) { - /* This should come by default, but let's enforce: */ + /* + * We need exclusive write and resize permissions on the child not only when + * the parent can write to it, but also after the parent gave up write + * permissions until preallocate_drop_resize() has completed. + */ + if (can_write_resize(perm) || s->data_end != -EINVAL) { *nperm |= BLK_PERM_WRITE | BLK_PERM_RESIZE; /* @@ -554,7 +600,6 @@ static BlockDriver bdrv_preallocate_filter = { .bdrv_co_flush = preallocate_co_flush, .bdrv_co_truncate = preallocate_co_truncate, - .bdrv_check_perm = preallocate_check_perm, .bdrv_set_perm = preallocate_set_perm, .bdrv_child_perm = preallocate_child_perm, diff --git a/block/qcow.c b/block/qcow.c index 577bd70324..d56d24ab6d 100644 --- a/block/qcow.c +++ b/block/qcow.c @@ -549,7 +549,10 @@ qcow_co_block_status(BlockDriverState *bs, bool want_zero, if (!cluster_offset) { return 0; } - if ((cluster_offset & QCOW_OFLAG_COMPRESSED) || s->crypto) { + if (cluster_offset & QCOW_OFLAG_COMPRESSED) { + return BDRV_BLOCK_DATA | BDRV_BLOCK_COMPRESSED; + } + if (s->crypto) { return BDRV_BLOCK_DATA; } *map = cluster_offset | index_in_cluster; diff --git a/block/qcow2.c b/block/qcow2.c index b48cd9ce63..af43d59d76 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -1880,7 +1880,7 @@ qcow2_do_open(BlockDriverState *bs, QDict *options, int flags, g_free(s->image_data_file); if (open_data_file && has_data_file(bs)) { bdrv_graph_co_rdunlock(); - bdrv_unref_child(bs, s->data_file); + bdrv_co_unref_child(bs, s->data_file); bdrv_graph_co_rdlock(); s->data_file = NULL; } @@ -2162,6 +2162,9 @@ qcow2_co_block_status(BlockDriverState *bs, bool want_zero, int64_t offset, { status |= BDRV_BLOCK_RECURSE; } + if (type == QCOW2_SUBCLUSTER_COMPRESSED) { + status |= BDRV_BLOCK_COMPRESSED; + } return status; } @@ -2790,7 +2793,9 @@ static void qcow2_do_close(BlockDriverState *bs, bool close_data_file) g_free(s->image_backing_format); if (close_data_file && has_data_file(bs)) { + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, s->data_file); + bdrv_graph_wrunlock(); s->data_file = NULL; } diff --git a/block/quorum.c b/block/quorum.c index f28758cf2b..05220cab7f 100644 --- a/block/quorum.c +++ b/block/quorum.c @@ -1037,12 +1037,14 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags, close_exit: /* cleanup on error */ + bdrv_graph_wrlock(NULL); for (i = 0; i < s->num_children; i++) { if (!opened[i]) { continue; } bdrv_unref_child(bs, s->children[i]); } + bdrv_graph_wrunlock(); g_free(s->children); g_free(opened); exit: @@ -1055,15 +1057,17 @@ static void quorum_close(BlockDriverState *bs) BDRVQuorumState *s = bs->opaque; int i; + bdrv_graph_wrlock(NULL); for (i = 0; i < s->num_children; i++) { bdrv_unref_child(bs, s->children[i]); } + bdrv_graph_wrunlock(); g_free(s->children); } -static void quorum_add_child(BlockDriverState *bs, BlockDriverState *child_bs, - Error **errp) +static void GRAPH_WRLOCK +quorum_add_child(BlockDriverState *bs, BlockDriverState *child_bs, Error **errp) { BDRVQuorumState *s = bs->opaque; BdrvChild *child; @@ -1089,8 +1093,6 @@ static void quorum_add_child(BlockDriverState *bs, BlockDriverState *child_bs, } s->next_child_index++; - bdrv_drained_begin(bs); - /* We can safely add the child now */ bdrv_ref(child_bs); @@ -1098,18 +1100,15 @@ static void quorum_add_child(BlockDriverState *bs, BlockDriverState *child_bs, BDRV_CHILD_DATA, errp); if (child == NULL) { s->next_child_index--; - goto out; + return; } s->children = g_renew(BdrvChild *, s->children, s->num_children + 1); s->children[s->num_children++] = child; quorum_refresh_flags(bs); - -out: - bdrv_drained_end(bs); } -static void quorum_del_child(BlockDriverState *bs, BdrvChild *child, - Error **errp) +static void GRAPH_WRLOCK +quorum_del_child(BlockDriverState *bs, BdrvChild *child, Error **errp) { BDRVQuorumState *s = bs->opaque; char indexstr[INDEXSTR_LEN]; @@ -1139,16 +1138,14 @@ static void quorum_del_child(BlockDriverState *bs, BdrvChild *child, s->next_child_index--; } - bdrv_drained_begin(bs); - /* We can safely remove this child now */ memmove(&s->children[i], &s->children[i + 1], (s->num_children - i - 1) * sizeof(BdrvChild *)); s->children = g_renew(BdrvChild *, s->children, --s->num_children); + bdrv_unref_child(bs, child); quorum_refresh_flags(bs); - bdrv_drained_end(bs); } static void quorum_gather_child_options(BlockDriverState *bs, QDict *target, diff --git a/block/replication.c b/block/replication.c index ea4bf1aa80..dd166d2d82 100644 --- a/block/replication.c +++ b/block/replication.c @@ -542,12 +542,15 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, return; } + bdrv_graph_wrlock(bs); + bdrv_ref(hidden_disk->bs); s->hidden_disk = bdrv_attach_child(bs, hidden_disk->bs, "hidden disk", &child_of_bds, BDRV_CHILD_DATA, &local_err); if (local_err) { error_propagate(errp, local_err); + bdrv_graph_wrunlock(); aio_context_release(aio_context); return; } @@ -558,10 +561,13 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, BDRV_CHILD_DATA, &local_err); if (local_err) { error_propagate(errp, local_err); + bdrv_graph_wrunlock(); aio_context_release(aio_context); return; } + bdrv_graph_wrunlock(); + /* start backup job now */ error_setg(&s->blocker, "Block device is in use by internal backup job"); @@ -666,10 +672,13 @@ static void replication_done(void *opaque, int ret) if (ret == 0) { s->stage = BLOCK_REPLICATION_DONE; + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, s->secondary_disk); s->secondary_disk = NULL; bdrv_unref_child(bs, s->hidden_disk); s->hidden_disk = NULL; + bdrv_graph_wrunlock(); + s->error = 0; } else { s->stage = BLOCK_REPLICATION_FAILOVER_FAILED; diff --git a/block/snapshot.c b/block/snapshot.c index e22ac3eac6..b86b5b24ad 100644 --- a/block/snapshot.c +++ b/block/snapshot.c @@ -281,7 +281,9 @@ int bdrv_snapshot_goto(BlockDriverState *bs, } /* .bdrv_open() will re-attach it */ + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, fallback); + bdrv_graph_wrunlock(); ret = bdrv_snapshot_goto(fallback_bs, snapshot_id, errp); open_ret = drv->bdrv_open(bs, options, bs->open_flags, &local_err); diff --git a/block/stream.c b/block/stream.c index e522bbdec5..e4da214f1f 100644 --- a/block/stream.c +++ b/block/stream.c @@ -54,6 +54,7 @@ static int stream_prepare(Job *job) { StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); BlockDriverState *unfiltered_bs = bdrv_skip_filters(s->target_bs); + BlockDriverState *unfiltered_bs_cow = bdrv_cow_bs(unfiltered_bs); BlockDriverState *base; BlockDriverState *unfiltered_base; Error *local_err = NULL; @@ -64,13 +65,18 @@ static int stream_prepare(Job *job) s->cor_filter_bs = NULL; /* - * bdrv_set_backing_hd() requires that unfiltered_bs is drained. Drain - * already here and use bdrv_set_backing_hd_drained() instead because - * the polling during drained_begin() might change the graph, and if we do - * this only later, we may end up working with the wrong base node (or it - * might even have gone away by the time we want to use it). + * bdrv_set_backing_hd() requires that the unfiltered_bs and the COW child + * of unfiltered_bs is drained. Drain already here and use + * bdrv_set_backing_hd_drained() instead because the polling during + * drained_begin() might change the graph, and if we do this only later, we + * may end up working with the wrong base node (or it might even have gone + * away by the time we want to use it). */ bdrv_drained_begin(unfiltered_bs); + if (unfiltered_bs_cow) { + bdrv_ref(unfiltered_bs_cow); + bdrv_drained_begin(unfiltered_bs_cow); + } base = bdrv_filter_or_cow_bs(s->above_base); unfiltered_base = bdrv_skip_filters(base); @@ -100,6 +106,10 @@ static int stream_prepare(Job *job) } out: + if (unfiltered_bs_cow) { + bdrv_drained_end(unfiltered_bs_cow); + bdrv_unref(unfiltered_bs_cow); + } bdrv_drained_end(unfiltered_bs); return ret; } diff --git a/block/throttle-groups.c b/block/throttle-groups.c index fb203c3ced..3eda4c4e3d 100644 --- a/block/throttle-groups.c +++ b/block/throttle-groups.c @@ -37,7 +37,7 @@ static void throttle_group_obj_init(Object *obj); static void throttle_group_obj_complete(UserCreatable *obj, Error **errp); -static void timer_cb(ThrottleGroupMember *tgm, bool is_write); +static void timer_cb(ThrottleGroupMember *tgm, ThrottleDirection direction); /* The ThrottleGroup structure (with its ThrottleState) is shared * among different ThrottleGroupMembers and it's independent from @@ -73,8 +73,8 @@ struct ThrottleGroup { QemuMutex lock; /* This lock protects the following four fields */ ThrottleState ts; QLIST_HEAD(, ThrottleGroupMember) head; - ThrottleGroupMember *tokens[2]; - bool any_timer_armed[2]; + ThrottleGroupMember *tokens[THROTTLE_MAX]; + bool any_timer_armed[THROTTLE_MAX]; QEMUClockType clock_type; /* This field is protected by the global QEMU mutex */ @@ -197,13 +197,13 @@ static ThrottleGroupMember *throttle_group_next_tgm(ThrottleGroupMember *tgm) * This assumes that tg->lock is held. * * @tgm: the ThrottleGroupMember - * @is_write: the type of operation (read/write) + * @direction: the ThrottleDirection * @ret: whether the ThrottleGroupMember has pending requests. */ static inline bool tgm_has_pending_reqs(ThrottleGroupMember *tgm, - bool is_write) + ThrottleDirection direction) { - return tgm->pending_reqs[is_write]; + return tgm->pending_reqs[direction]; } /* Return the next ThrottleGroupMember in the round-robin sequence with pending @@ -212,12 +212,12 @@ static inline bool tgm_has_pending_reqs(ThrottleGroupMember *tgm, * This assumes that tg->lock is held. * * @tgm: the current ThrottleGroupMember - * @is_write: the type of operation (read/write) + * @direction: the ThrottleDirection * @ret: the next ThrottleGroupMember with pending requests, or tgm if * there is none. */ static ThrottleGroupMember *next_throttle_token(ThrottleGroupMember *tgm, - bool is_write) + ThrottleDirection direction) { ThrottleState *ts = tgm->throttle_state; ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts); @@ -227,16 +227,16 @@ static ThrottleGroupMember *next_throttle_token(ThrottleGroupMember *tgm, * it's being drained. Skip the round-robin search and return tgm * immediately if it has pending requests. Otherwise we could be * forcing it to wait for other member's throttled requests. */ - if (tgm_has_pending_reqs(tgm, is_write) && + if (tgm_has_pending_reqs(tgm, direction) && qatomic_read(&tgm->io_limits_disabled)) { return tgm; } - start = token = tg->tokens[is_write]; + start = token = tg->tokens[direction]; /* get next bs round in round robin style */ token = throttle_group_next_tgm(token); - while (token != start && !tgm_has_pending_reqs(token, is_write)) { + while (token != start && !tgm_has_pending_reqs(token, direction)) { token = throttle_group_next_tgm(token); } @@ -244,12 +244,12 @@ static ThrottleGroupMember *next_throttle_token(ThrottleGroupMember *tgm, * then decide the token is the current tgm because chances are * the current tgm got the current request queued. */ - if (token == start && !tgm_has_pending_reqs(token, is_write)) { + if (token == start && !tgm_has_pending_reqs(token, direction)) { token = tgm; } /* Either we return the original TGM, or one with pending requests */ - assert(token == tgm || tgm_has_pending_reqs(token, is_write)); + assert(token == tgm || tgm_has_pending_reqs(token, direction)); return token; } @@ -261,11 +261,11 @@ static ThrottleGroupMember *next_throttle_token(ThrottleGroupMember *tgm, * This assumes that tg->lock is held. * * @tgm: the current ThrottleGroupMember - * @is_write: the type of operation (read/write) + * @direction: the ThrottleDirection * @ret: whether the I/O request needs to be throttled or not */ static bool throttle_group_schedule_timer(ThrottleGroupMember *tgm, - bool is_write) + ThrottleDirection direction) { ThrottleState *ts = tgm->throttle_state; ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts); @@ -277,16 +277,16 @@ static bool throttle_group_schedule_timer(ThrottleGroupMember *tgm, } /* Check if any of the timers in this group is already armed */ - if (tg->any_timer_armed[is_write]) { + if (tg->any_timer_armed[direction]) { return true; } - must_wait = throttle_schedule_timer(ts, tt, is_write); + must_wait = throttle_schedule_timer(ts, tt, direction); /* If a timer just got armed, set tgm as the current token */ if (must_wait) { - tg->tokens[is_write] = tgm; - tg->any_timer_armed[is_write] = true; + tg->tokens[direction] = tgm; + tg->any_timer_armed[direction] = true; } return must_wait; @@ -296,15 +296,15 @@ static bool throttle_group_schedule_timer(ThrottleGroupMember *tgm, * any request was actually pending. * * @tgm: the current ThrottleGroupMember - * @is_write: the type of operation (read/write) + * @direction: the ThrottleDirection */ static bool coroutine_fn throttle_group_co_restart_queue(ThrottleGroupMember *tgm, - bool is_write) + ThrottleDirection direction) { bool ret; qemu_co_mutex_lock(&tgm->throttled_reqs_lock); - ret = qemu_co_queue_next(&tgm->throttled_reqs[is_write]); + ret = qemu_co_queue_next(&tgm->throttled_reqs[direction]); qemu_co_mutex_unlock(&tgm->throttled_reqs_lock); return ret; @@ -315,9 +315,10 @@ static bool coroutine_fn throttle_group_co_restart_queue(ThrottleGroupMember *tg * This assumes that tg->lock is held. * * @tgm: the current ThrottleGroupMember - * @is_write: the type of operation (read/write) + * @direction: the ThrottleDirection */ -static void schedule_next_request(ThrottleGroupMember *tgm, bool is_write) +static void schedule_next_request(ThrottleGroupMember *tgm, + ThrottleDirection direction) { ThrottleState *ts = tgm->throttle_state; ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts); @@ -325,27 +326,27 @@ static void schedule_next_request(ThrottleGroupMember *tgm, bool is_write) ThrottleGroupMember *token; /* Check if there's any pending request to schedule next */ - token = next_throttle_token(tgm, is_write); - if (!tgm_has_pending_reqs(token, is_write)) { + token = next_throttle_token(tgm, direction); + if (!tgm_has_pending_reqs(token, direction)) { return; } /* Set a timer for the request if it needs to be throttled */ - must_wait = throttle_group_schedule_timer(token, is_write); + must_wait = throttle_group_schedule_timer(token, direction); /* If it doesn't have to wait, queue it for immediate execution */ if (!must_wait) { /* Give preference to requests from the current tgm */ if (qemu_in_coroutine() && - throttle_group_co_restart_queue(tgm, is_write)) { + throttle_group_co_restart_queue(tgm, direction)) { token = tgm; } else { ThrottleTimers *tt = &token->throttle_timers; int64_t now = qemu_clock_get_ns(tg->clock_type); - timer_mod(tt->timers[is_write], now); - tg->any_timer_armed[is_write] = true; + timer_mod(tt->timers[direction], now); + tg->any_timer_armed[direction] = true; } - tg->tokens[is_write] = token; + tg->tokens[direction] = token; } } @@ -355,48 +356,49 @@ static void schedule_next_request(ThrottleGroupMember *tgm, bool is_write) * * @tgm: the current ThrottleGroupMember * @bytes: the number of bytes for this I/O - * @is_write: the type of operation (read/write) + * @direction: the ThrottleDirection */ void coroutine_fn throttle_group_co_io_limits_intercept(ThrottleGroupMember *tgm, int64_t bytes, - bool is_write) + ThrottleDirection direction) { bool must_wait; ThrottleGroupMember *token; ThrottleGroup *tg = container_of(tgm->throttle_state, ThrottleGroup, ts); assert(bytes >= 0); + assert(direction < THROTTLE_MAX); qemu_mutex_lock(&tg->lock); /* First we check if this I/O has to be throttled. */ - token = next_throttle_token(tgm, is_write); - must_wait = throttle_group_schedule_timer(token, is_write); + token = next_throttle_token(tgm, direction); + must_wait = throttle_group_schedule_timer(token, direction); /* Wait if there's a timer set or queued requests of this type */ - if (must_wait || tgm->pending_reqs[is_write]) { - tgm->pending_reqs[is_write]++; + if (must_wait || tgm->pending_reqs[direction]) { + tgm->pending_reqs[direction]++; qemu_mutex_unlock(&tg->lock); qemu_co_mutex_lock(&tgm->throttled_reqs_lock); - qemu_co_queue_wait(&tgm->throttled_reqs[is_write], + qemu_co_queue_wait(&tgm->throttled_reqs[direction], &tgm->throttled_reqs_lock); qemu_co_mutex_unlock(&tgm->throttled_reqs_lock); qemu_mutex_lock(&tg->lock); - tgm->pending_reqs[is_write]--; + tgm->pending_reqs[direction]--; } /* The I/O will be executed, so do the accounting */ - throttle_account(tgm->throttle_state, is_write, bytes); + throttle_account(tgm->throttle_state, direction, bytes); /* Schedule the next request */ - schedule_next_request(tgm, is_write); + schedule_next_request(tgm, direction); qemu_mutex_unlock(&tg->lock); } typedef struct { ThrottleGroupMember *tgm; - bool is_write; + ThrottleDirection direction; } RestartData; static void coroutine_fn throttle_group_restart_queue_entry(void *opaque) @@ -405,16 +407,16 @@ static void coroutine_fn throttle_group_restart_queue_entry(void *opaque) ThrottleGroupMember *tgm = data->tgm; ThrottleState *ts = tgm->throttle_state; ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts); - bool is_write = data->is_write; + ThrottleDirection direction = data->direction; bool empty_queue; - empty_queue = !throttle_group_co_restart_queue(tgm, is_write); + empty_queue = !throttle_group_co_restart_queue(tgm, direction); /* If the request queue was empty then we have to take care of * scheduling the next one */ if (empty_queue) { qemu_mutex_lock(&tg->lock); - schedule_next_request(tgm, is_write); + schedule_next_request(tgm, direction); qemu_mutex_unlock(&tg->lock); } @@ -424,18 +426,19 @@ static void coroutine_fn throttle_group_restart_queue_entry(void *opaque) aio_wait_kick(); } -static void throttle_group_restart_queue(ThrottleGroupMember *tgm, bool is_write) +static void throttle_group_restart_queue(ThrottleGroupMember *tgm, + ThrottleDirection direction) { Coroutine *co; RestartData *rd = g_new0(RestartData, 1); rd->tgm = tgm; - rd->is_write = is_write; + rd->direction = direction; /* This function is called when a timer is fired or when * throttle_group_restart_tgm() is called. Either way, there can * be no timer pending on this tgm at this point */ - assert(!timer_pending(tgm->throttle_timers.timers[is_write])); + assert(!timer_pending(tgm->throttle_timers.timers[direction])); qatomic_inc(&tgm->restart_pending); @@ -445,18 +448,18 @@ static void throttle_group_restart_queue(ThrottleGroupMember *tgm, bool is_write void throttle_group_restart_tgm(ThrottleGroupMember *tgm) { - int i; + ThrottleDirection dir; if (tgm->throttle_state) { - for (i = 0; i < 2; i++) { - QEMUTimer *t = tgm->throttle_timers.timers[i]; + for (dir = THROTTLE_READ; dir < THROTTLE_MAX; dir++) { + QEMUTimer *t = tgm->throttle_timers.timers[dir]; if (timer_pending(t)) { /* If there's a pending timer on this tgm, fire it now */ timer_del(t); - timer_cb(tgm, i); + timer_cb(tgm, dir); } else { /* Else run the next request from the queue manually */ - throttle_group_restart_queue(tgm, i); + throttle_group_restart_queue(tgm, dir); } } } @@ -500,30 +503,30 @@ void throttle_group_get_config(ThrottleGroupMember *tgm, ThrottleConfig *cfg) * because it had been throttled. * * @tgm: the ThrottleGroupMember whose request had been throttled - * @is_write: the type of operation (read/write) + * @direction: the ThrottleDirection */ -static void timer_cb(ThrottleGroupMember *tgm, bool is_write) +static void timer_cb(ThrottleGroupMember *tgm, ThrottleDirection direction) { ThrottleState *ts = tgm->throttle_state; ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts); /* The timer has just been fired, so we can update the flag */ qemu_mutex_lock(&tg->lock); - tg->any_timer_armed[is_write] = false; + tg->any_timer_armed[direction] = false; qemu_mutex_unlock(&tg->lock); /* Run the request that was waiting for this timer */ - throttle_group_restart_queue(tgm, is_write); + throttle_group_restart_queue(tgm, direction); } static void read_timer_cb(void *opaque) { - timer_cb(opaque, false); + timer_cb(opaque, THROTTLE_READ); } static void write_timer_cb(void *opaque) { - timer_cb(opaque, true); + timer_cb(opaque, THROTTLE_WRITE); } /* Register a ThrottleGroupMember from the throttling group, also initializing @@ -541,7 +544,7 @@ void throttle_group_register_tgm(ThrottleGroupMember *tgm, const char *groupname, AioContext *ctx) { - int i; + ThrottleDirection dir; ThrottleState *ts = throttle_group_incref(groupname); ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts); @@ -551,10 +554,11 @@ void throttle_group_register_tgm(ThrottleGroupMember *tgm, QEMU_LOCK_GUARD(&tg->lock); /* If the ThrottleGroup is new set this ThrottleGroupMember as the token */ - for (i = 0; i < 2; i++) { - if (!tg->tokens[i]) { - tg->tokens[i] = tgm; + for (dir = THROTTLE_READ; dir < THROTTLE_MAX; dir++) { + if (!tg->tokens[dir]) { + tg->tokens[dir] = tgm; } + qemu_co_queue_init(&tgm->throttled_reqs[dir]); } QLIST_INSERT_HEAD(&tg->head, tgm, round_robin); @@ -566,8 +570,6 @@ void throttle_group_register_tgm(ThrottleGroupMember *tgm, write_timer_cb, tgm); qemu_co_mutex_init(&tgm->throttled_reqs_lock); - qemu_co_queue_init(&tgm->throttled_reqs[0]); - qemu_co_queue_init(&tgm->throttled_reqs[1]); } /* Unregister a ThrottleGroupMember from its group, removing it from the list, @@ -585,7 +587,7 @@ void throttle_group_unregister_tgm(ThrottleGroupMember *tgm) ThrottleState *ts = tgm->throttle_state; ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts); ThrottleGroupMember *token; - int i; + ThrottleDirection dir; if (!ts) { /* Discard already unregistered tgm */ @@ -596,17 +598,17 @@ void throttle_group_unregister_tgm(ThrottleGroupMember *tgm) AIO_WAIT_WHILE(tgm->aio_context, qatomic_read(&tgm->restart_pending) > 0); WITH_QEMU_LOCK_GUARD(&tg->lock) { - for (i = 0; i < 2; i++) { - assert(tgm->pending_reqs[i] == 0); - assert(qemu_co_queue_empty(&tgm->throttled_reqs[i])); - assert(!timer_pending(tgm->throttle_timers.timers[i])); - if (tg->tokens[i] == tgm) { + for (dir = THROTTLE_READ; dir < THROTTLE_MAX; dir++) { + assert(tgm->pending_reqs[dir] == 0); + assert(qemu_co_queue_empty(&tgm->throttled_reqs[dir])); + assert(!timer_pending(tgm->throttle_timers.timers[dir])); + if (tg->tokens[dir] == tgm) { token = throttle_group_next_tgm(tgm); /* Take care of the case where this is the last tgm in the group */ if (token == tgm) { token = NULL; } - tg->tokens[i] = token; + tg->tokens[dir] = token; } } @@ -631,19 +633,20 @@ void throttle_group_detach_aio_context(ThrottleGroupMember *tgm) { ThrottleGroup *tg = container_of(tgm->throttle_state, ThrottleGroup, ts); ThrottleTimers *tt = &tgm->throttle_timers; - int i; + ThrottleDirection dir; /* Requests must have been drained */ - assert(tgm->pending_reqs[0] == 0 && tgm->pending_reqs[1] == 0); - assert(qemu_co_queue_empty(&tgm->throttled_reqs[0])); - assert(qemu_co_queue_empty(&tgm->throttled_reqs[1])); + for (dir = THROTTLE_READ; dir < THROTTLE_MAX; dir++) { + assert(tgm->pending_reqs[dir] == 0); + assert(qemu_co_queue_empty(&tgm->throttled_reqs[dir])); + } /* Kick off next ThrottleGroupMember, if necessary */ WITH_QEMU_LOCK_GUARD(&tg->lock) { - for (i = 0; i < 2; i++) { - if (timer_pending(tt->timers[i])) { - tg->any_timer_armed[i] = false; - schedule_next_request(tgm, i); + for (dir = THROTTLE_READ; dir < THROTTLE_MAX; dir++) { + if (timer_pending(tt->timers[dir])) { + tg->any_timer_armed[dir] = false; + schedule_next_request(tgm, dir); } } } diff --git a/block/throttle.c b/block/throttle.c index 3aaef18d4e..1098a4ae9a 100644 --- a/block/throttle.c +++ b/block/throttle.c @@ -118,7 +118,7 @@ throttle_co_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes, { ThrottleGroupMember *tgm = bs->opaque; - throttle_group_co_io_limits_intercept(tgm, bytes, false); + throttle_group_co_io_limits_intercept(tgm, bytes, THROTTLE_READ); return bdrv_co_preadv(bs->file, offset, bytes, qiov, flags); } @@ -128,7 +128,7 @@ throttle_co_pwritev(BlockDriverState *bs, int64_t offset, int64_t bytes, QEMUIOVector *qiov, BdrvRequestFlags flags) { ThrottleGroupMember *tgm = bs->opaque; - throttle_group_co_io_limits_intercept(tgm, bytes, true); + throttle_group_co_io_limits_intercept(tgm, bytes, THROTTLE_WRITE); return bdrv_co_pwritev(bs->file, offset, bytes, qiov, flags); } @@ -138,7 +138,7 @@ throttle_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int64_t bytes, BdrvRequestFlags flags) { ThrottleGroupMember *tgm = bs->opaque; - throttle_group_co_io_limits_intercept(tgm, bytes, true); + throttle_group_co_io_limits_intercept(tgm, bytes, THROTTLE_WRITE); return bdrv_co_pwrite_zeroes(bs->file, offset, bytes, flags); } @@ -147,7 +147,7 @@ static int coroutine_fn GRAPH_RDLOCK throttle_co_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes) { ThrottleGroupMember *tgm = bs->opaque; - throttle_group_co_io_limits_intercept(tgm, bytes, true); + throttle_group_co_io_limits_intercept(tgm, bytes, THROTTLE_WRITE); return bdrv_co_pdiscard(bs->file, offset, bytes); } diff --git a/block/vmdk.c b/block/vmdk.c index 58ce290e9c..e90649c8bf 100644 --- a/block/vmdk.c +++ b/block/vmdk.c @@ -272,6 +272,7 @@ static void vmdk_free_extents(BlockDriverState *bs) BDRVVmdkState *s = bs->opaque; VmdkExtent *e; + bdrv_graph_wrlock(NULL); for (i = 0; i < s->num_extents; i++) { e = &s->extents[i]; g_free(e->l1_table); @@ -282,6 +283,8 @@ static void vmdk_free_extents(BlockDriverState *bs) bdrv_unref_child(bs, e->file); } } + bdrv_graph_wrunlock(); + g_free(s->extents); } @@ -1220,7 +1223,9 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, ret = vmdk_add_extent(bs, extent_file, true, sectors, 0, 0, 0, 0, 0, &extent, errp); if (ret < 0) { + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, extent_file); + bdrv_graph_wrunlock(); goto out; } extent->flat_start_offset = flat_offset << 9; @@ -1235,20 +1240,26 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, } g_free(buf); if (ret) { + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, extent_file); + bdrv_graph_wrunlock(); goto out; } extent = &s->extents[s->num_extents - 1]; } else if (!strcmp(type, "SESPARSE")) { ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp); if (ret) { + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, extent_file); + bdrv_graph_wrunlock(); goto out; } extent = &s->extents[s->num_extents - 1]; } else { error_setg(errp, "Unsupported extent type '%s'", type); + bdrv_graph_wrlock(NULL); bdrv_unref_child(bs, extent_file); + bdrv_graph_wrunlock(); ret = -ENOTSUP; goto out; } @@ -1309,6 +1320,8 @@ static int vmdk_open(BlockDriverState *bs, QDict *options, int flags, BDRVVmdkState *s = bs->opaque; uint32_t magic; + GRAPH_RDLOCK_GUARD_MAINLOOP(); + ret = bdrv_open_file_child(NULL, options, "file", bs, errp); if (ret < 0) { return ret; @@ -1770,6 +1783,8 @@ vmdk_co_block_status(BlockDriverState *bs, bool want_zero, if (extent->flat) { ret |= BDRV_BLOCK_RECURSE; } + } else { + ret |= BDRV_BLOCK_COMPRESSED; } *file = extent->file->bs; break; diff --git a/blockdev.c b/blockdev.c index e6eba61484..325b7a3bef 100644 --- a/blockdev.c +++ b/blockdev.c @@ -1378,6 +1378,9 @@ static void external_snapshot_action(TransactionAction *action, AioContext *aio_context; uint64_t perm, shared; + /* TODO We'll eventually have to take a writer lock in this function */ + GRAPH_RDLOCK_GUARD_MAINLOOP(); + tran_add(tran, &external_snapshot_drv, state); /* 'blockdev-snapshot' and 'blockdev-snapshot-sync' have similar @@ -2521,6 +2524,9 @@ void qmp_block_commit(const char *job_id, const char *device, int job_flags = JOB_DEFAULT; uint64_t top_perm, top_shared; + /* TODO We'll eventually have to take a writer lock in this function */ + GRAPH_RDLOCK_GUARD_MAINLOOP(); + if (!has_speed) { speed = 0; } @@ -3539,8 +3545,8 @@ out: aio_context_release(aio_context); } -static BdrvChild *bdrv_find_child(BlockDriverState *parent_bs, - const char *child_name) +static BdrvChild * GRAPH_RDLOCK +bdrv_find_child(BlockDriverState *parent_bs, const char *child_name) { BdrvChild *child; @@ -3559,9 +3565,11 @@ void qmp_x_blockdev_change(const char *parent, const char *child, BlockDriverState *parent_bs, *new_bs = NULL; BdrvChild *p_child; + bdrv_graph_wrlock(NULL); + parent_bs = bdrv_lookup_bs(parent, parent, errp); if (!parent_bs) { - return; + goto out; } if (!child == !node) { @@ -3570,7 +3578,7 @@ void qmp_x_blockdev_change(const char *parent, const char *child, } else { error_setg(errp, "Either child or node must be specified"); } - return; + goto out; } if (child) { @@ -3578,7 +3586,7 @@ void qmp_x_blockdev_change(const char *parent, const char *child, if (!p_child) { error_setg(errp, "Node '%s' does not have child '%s'", parent, child); - return; + goto out; } bdrv_del_child(parent_bs, p_child, errp); } @@ -3587,10 +3595,13 @@ void qmp_x_blockdev_change(const char *parent, const char *child, new_bs = bdrv_find_node(node); if (!new_bs) { error_setg(errp, "Node '%s' not found", node); - return; + goto out; } bdrv_add_child(parent_bs, new_bs, errp); } + +out: + bdrv_graph_wrunlock(); } BlockJobInfoList *qmp_query_block_jobs(Error **errp) diff --git a/blockjob.c b/blockjob.c index 25fe8e625d..58c5d64539 100644 --- a/blockjob.c +++ b/blockjob.c @@ -198,6 +198,7 @@ void block_job_remove_all_bdrv(BlockJob *job) * one to make sure that such a concurrent access does not attempt * to process an already freed BdrvChild. */ + bdrv_graph_wrlock(NULL); while (job->nodes) { GSList *l = job->nodes; BdrvChild *c = l->data; @@ -209,6 +210,7 @@ void block_job_remove_all_bdrv(BlockJob *job) g_slist_free_1(l); } + bdrv_graph_wrunlock(); } bool block_job_has_bdrv(BlockJob *job, BlockDriverState *bs) diff --git a/bsd-user/errno_defs.h b/bsd-user/errno_defs.h index f3e8ac3488..abe70119d9 100644 --- a/bsd-user/errno_defs.h +++ b/bsd-user/errno_defs.h @@ -149,7 +149,7 @@ #define TARGET_ELAST 90 /* Must be equal largest errno */ /* Internal errors: */ -#define TARGET_EJUSTRETURN 254 /* Just return without modifing regs */ +#define TARGET_EJUSTRETURN 254 /* Just return without modifying regs */ #define TARGET_ERESTART 255 /* Restart syscall */ #include "special-errno.h" diff --git a/bsd-user/freebsd/target_os_siginfo.h b/bsd-user/freebsd/target_os_siginfo.h index 4573738752..6c282d8502 100644 --- a/bsd-user/freebsd/target_os_siginfo.h +++ b/bsd-user/freebsd/target_os_siginfo.h @@ -72,7 +72,7 @@ typedef struct target_siginfo { int32_t _mqd; } _mesgp; - /* SIGPOLL -- Not really genreated in FreeBSD ??? */ + /* SIGPOLL -- Not really generated in FreeBSD ??? */ struct { int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ } _poll; diff --git a/bsd-user/freebsd/target_os_stack.h b/bsd-user/freebsd/target_os_stack.h index 0590133291..d15fc3263f 100644 --- a/bsd-user/freebsd/target_os_stack.h +++ b/bsd-user/freebsd/target_os_stack.h @@ -25,7 +25,7 @@ #include "qemu/guest-random.h" /* - * The inital FreeBSD stack is as follows: + * The initial FreeBSD stack is as follows: * (see kern/kern_exec.c exec_copyout_strings() ) * * Hi Address -> char **ps_argvstr (struct ps_strings for ps, w, etc.) @@ -59,7 +59,7 @@ static inline int setup_initial_stack(struct bsd_binprm *bprm, /* Save some space for ps_strings. */ p -= sizeof(struct target_ps_strings); - /* Add machine depedent sigcode. */ + /* Add machine dependent sigcode. */ p -= TARGET_SZSIGCODE; if (setup_sigtramp(p, (unsigned)offsetof(struct target_sigframe, sf_uc), TARGET_FREEBSD_NR_sigreturn)) { diff --git a/bsd-user/freebsd/target_os_user.h b/bsd-user/freebsd/target_os_user.h index f036a32343..1ca7b5ab17 100644 --- a/bsd-user/freebsd/target_os_user.h +++ b/bsd-user/freebsd/target_os_user.h @@ -26,7 +26,7 @@ struct target_priority { uint8_t pri_class; /* Scheduling class. */ uint8_t pri_level; /* Normal priority level. */ - uint8_t pri_native; /* Priority before propogation. */ + uint8_t pri_native; /* Priority before propagation. */ uint8_t pri_user; /* User priority based on p_cpu and p_nice. */ }; diff --git a/bsd-user/qemu.h b/bsd-user/qemu.h index d3158bc2ed..d9507137cc 100644 --- a/bsd-user/qemu.h +++ b/bsd-user/qemu.h @@ -116,7 +116,7 @@ extern const char *qemu_uname_release; /* * TARGET_ARG_MAX defines the number of bytes allocated for arguments * and envelope for the new program. 256k should suffice for a reasonable - * maxiumum env+arg in 32-bit environments, bump it up to 512k for !ILP32 + * maximum env+arg in 32-bit environments, bump it up to 512k for !ILP32 * platforms. */ #if TARGET_ABI_BITS > 32 diff --git a/bsd-user/signal-common.h b/bsd-user/signal-common.h index 6f90345bb2..c044e81165 100644 --- a/bsd-user/signal-common.h +++ b/bsd-user/signal-common.h @@ -49,11 +49,11 @@ void target_to_host_sigset(sigset_t *d, const target_sigset_t *s); * union in target_siginfo is valid. This only applies between * host_to_target_siginfo_noswap() and tswap_siginfo(); it does not appear * either within host siginfo_t or in target_siginfo structures which we get - * from the guest userspace program. Linux kenrels use this internally, but BSD + * from the guest userspace program. Linux kernels use this internally, but BSD * kernels don't do this, but its a useful abstraction. * * The linux-user version of this uses the top 16 bits, but FreeBSD's SI_USER - * and other signal indepenent SI_ codes have bit 16 set, so we only use the top + * and other signal independent SI_ codes have bit 16 set, so we only use the top * byte instead. * * For FreeBSD, we have si_pid, si_uid, si_status, and si_addr always. Linux and diff --git a/bsd-user/signal.c b/bsd-user/signal.c index 4db85a3485..b6beab659e 100644 --- a/bsd-user/signal.c +++ b/bsd-user/signal.c @@ -44,7 +44,7 @@ static inline int sas_ss_flags(TaskState *ts, unsigned long sp) } /* - * The BSD ABIs use the same singal numbers across all the CPU architectures, so + * The BSD ABIs use the same signal numbers across all the CPU architectures, so * (unlike Linux) these functions are just the identity mapping. This might not * be true for XyzBSD running on AbcBSD, which doesn't currently work. */ @@ -241,7 +241,7 @@ static inline void host_to_target_siginfo_noswap(target_siginfo_t *tinfo, #endif /* * Unsure that this can actually be generated, and our support for - * capsicum is somewhere between weak and non-existant, but if we get + * capsicum is somewhere between weak and non-existent, but if we get * one, then we know what to save. */ #ifdef QEMU_SI_CAPSICUM @@ -319,7 +319,7 @@ int block_signals(void) /* * It's OK to block everything including SIGSEGV, because we won't run any * further guest code before unblocking signals in - * process_pending_signals(). We depend on the FreeBSD behaivor here where + * process_pending_signals(). We depend on the FreeBSD behavior here where * this will only affect this thread's signal mask. We don't use * pthread_sigmask which might seem more correct because that routine also * does odd things with SIGCANCEL to implement pthread_cancel(). diff --git a/contrib/elf2dmp/addrspace.c b/contrib/elf2dmp/addrspace.c index 0b04cba00e..64b5d680ad 100644 --- a/contrib/elf2dmp/addrspace.c +++ b/contrib/elf2dmp/addrspace.c @@ -14,7 +14,7 @@ static struct pa_block *pa_space_find_block(struct pa_space *ps, uint64_t pa) for (i = 0; i < ps->block_nr; i++) { if (ps->block[i].paddr <= pa && - pa <= ps->block[i].paddr + ps->block[i].size) { + pa < ps->block[i].paddr + ps->block[i].size) { return ps->block + i; } } @@ -33,6 +33,30 @@ static uint8_t *pa_space_resolve(struct pa_space *ps, uint64_t pa) return block->addr + (pa - block->paddr); } +static void pa_block_align(struct pa_block *b) +{ + uint64_t low_align = ((b->paddr - 1) | ELF2DMP_PAGE_MASK) + 1 - b->paddr; + uint64_t high_align = (b->paddr + b->size) & ELF2DMP_PAGE_MASK; + + if (low_align == 0 && high_align == 0) { + return; + } + + if (low_align + high_align < b->size) { + printf("Block 0x%"PRIx64"+:0x%"PRIx64" will be aligned to " + "0x%"PRIx64"+:0x%"PRIx64"\n", b->paddr, b->size, + b->paddr + low_align, b->size - low_align - high_align); + b->size -= low_align + high_align; + } else { + printf("Block 0x%"PRIx64"+:0x%"PRIx64" is too small to align\n", + b->paddr, b->size); + b->size = 0; + } + + b->addr += low_align; + b->paddr += low_align; +} + int pa_space_create(struct pa_space *ps, QEMU_Elf *qemu_elf) { Elf64_Half phdr_nr = elf_getphdrnum(qemu_elf->map); @@ -60,10 +84,13 @@ int pa_space_create(struct pa_space *ps, QEMU_Elf *qemu_elf) .paddr = phdr[i].p_paddr, .size = phdr[i].p_filesz, }; - block_i++; + pa_block_align(&ps->block[block_i]); + block_i = ps->block[block_i].size ? (block_i + 1) : block_i; } } + ps->block_nr = block_i; + return 0; } diff --git a/contrib/elf2dmp/addrspace.h b/contrib/elf2dmp/addrspace.h index 00b44c1218..039c70c5b0 100644 --- a/contrib/elf2dmp/addrspace.h +++ b/contrib/elf2dmp/addrspace.h @@ -12,6 +12,7 @@ #define ELF2DMP_PAGE_BITS 12 #define ELF2DMP_PAGE_SIZE (1ULL << ELF2DMP_PAGE_BITS) +#define ELF2DMP_PAGE_MASK (ELF2DMP_PAGE_SIZE - 1) #define ELF2DMP_PFN_MASK (~(ELF2DMP_PAGE_SIZE - 1)) #define INVALID_PA UINT64_MAX diff --git a/contrib/elf2dmp/main.c b/contrib/elf2dmp/main.c index 6d4d18501a..5db163bdbe 100644 --- a/contrib/elf2dmp/main.c +++ b/contrib/elf2dmp/main.c @@ -20,6 +20,7 @@ #define PE_NAME "ntoskrnl.exe" #define INITIAL_MXCSR 0x1f80 +#define MAX_NUMBER_OF_RUNS 42 typedef struct idt_desc { uint16_t offset1; /* offset bits 0..15 */ @@ -234,6 +235,42 @@ static int fix_dtb(struct va_space *vs, QEMU_Elf *qe) return 1; } +static void try_merge_runs(struct pa_space *ps, + WinDumpPhyMemDesc64 *PhysicalMemoryBlock) +{ + unsigned int merge_cnt = 0, run_idx = 0; + + PhysicalMemoryBlock->NumberOfRuns = 0; + + for (size_t idx = 0; idx < ps->block_nr; idx++) { + struct pa_block *blk = ps->block + idx; + struct pa_block *next = blk + 1; + + PhysicalMemoryBlock->NumberOfPages += blk->size / ELF2DMP_PAGE_SIZE; + + if (idx + 1 != ps->block_nr && blk->paddr + blk->size == next->paddr) { + printf("Block #%zu 0x%"PRIx64"+:0x%"PRIx64" and %u previous will be" + " merged\n", idx, blk->paddr, blk->size, merge_cnt); + merge_cnt++; + } else { + struct pa_block *first_merged = blk - merge_cnt; + + printf("Block #%zu 0x%"PRIx64"+:0x%"PRIx64" and %u previous will be" + " merged to 0x%"PRIx64"+:0x%"PRIx64" (run #%u)\n", + idx, blk->paddr, blk->size, merge_cnt, first_merged->paddr, + blk->paddr + blk->size - first_merged->paddr, run_idx); + PhysicalMemoryBlock->Run[run_idx] = (WinDumpPhyMemRun64) { + .BasePage = first_merged->paddr / ELF2DMP_PAGE_SIZE, + .PageCount = (blk->paddr + blk->size - first_merged->paddr) / + ELF2DMP_PAGE_SIZE, + }; + PhysicalMemoryBlock->NumberOfRuns++; + run_idx++; + merge_cnt = 0; + } + } +} + static int fill_header(WinDumpHeader64 *hdr, struct pa_space *ps, struct va_space *vs, uint64_t KdDebuggerDataBlock, KDDEBUGGER_DATA64 *kdbg, uint64_t KdVersionBlock, int nr_cpus) @@ -244,7 +281,6 @@ static int fill_header(WinDumpHeader64 *hdr, struct pa_space *ps, KUSD_OFFSET_PRODUCT_TYPE); DBGKD_GET_VERSION64 kvb; WinDumpHeader64 h; - size_t i; QEMU_BUILD_BUG_ON(KUSD_OFFSET_SUITE_MASK >= ELF2DMP_PAGE_SIZE); QEMU_BUILD_BUG_ON(KUSD_OFFSET_PRODUCT_TYPE >= ELF2DMP_PAGE_SIZE); @@ -282,13 +318,17 @@ static int fill_header(WinDumpHeader64 *hdr, struct pa_space *ps, .RequiredDumpSpace = sizeof(h), }; - for (i = 0; i < ps->block_nr; i++) { - h.PhysicalMemoryBlock.NumberOfPages += - ps->block[i].size / ELF2DMP_PAGE_SIZE; - h.PhysicalMemoryBlock.Run[i] = (WinDumpPhyMemRun64) { - .BasePage = ps->block[i].paddr / ELF2DMP_PAGE_SIZE, - .PageCount = ps->block[i].size / ELF2DMP_PAGE_SIZE, - }; + if (h.PhysicalMemoryBlock.NumberOfRuns <= MAX_NUMBER_OF_RUNS) { + for (size_t idx = 0; idx < ps->block_nr; idx++) { + h.PhysicalMemoryBlock.NumberOfPages += + ps->block[idx].size / ELF2DMP_PAGE_SIZE; + h.PhysicalMemoryBlock.Run[idx] = (WinDumpPhyMemRun64) { + .BasePage = ps->block[idx].paddr / ELF2DMP_PAGE_SIZE, + .PageCount = ps->block[idx].size / ELF2DMP_PAGE_SIZE, + }; + } + } else { + try_merge_runs(ps, &h.PhysicalMemoryBlock); } h.RequiredDumpSpace += @@ -400,9 +440,10 @@ static int write_dump(struct pa_space *ps, for (i = 0; i < ps->block_nr; i++) { struct pa_block *b = &ps->block[i]; - printf("Writing block #%zu/%zu to file...\n", i, ps->block_nr); + printf("Writing block #%zu/%zu of %"PRIu64" bytes to file...\n", i, + ps->block_nr, b->size); if (fwrite(b->addr, b->size, 1, dmp_file) != 1) { - eprintf("Failed to write dump header\n"); + eprintf("Failed to write block\n"); fclose(dmp_file); return 1; } @@ -411,89 +452,64 @@ static int write_dump(struct pa_space *ps, return fclose(dmp_file); } -static bool pe_check_export_name(uint64_t base, void *start_addr, - struct va_space *vs) -{ - IMAGE_EXPORT_DIRECTORY export_dir; - const char *pe_name; - - if (pe_get_data_dir_entry(base, start_addr, IMAGE_FILE_EXPORT_DIRECTORY, - &export_dir, sizeof(export_dir), vs)) { - return false; - } - - pe_name = va_space_resolve(vs, base + export_dir.Name); - if (!pe_name) { - return false; - } - - return !strcmp(pe_name, PE_NAME); -} - -static int pe_get_pdb_symstore_hash(uint64_t base, void *start_addr, - char *hash, struct va_space *vs) +static bool pe_check_pdb_name(uint64_t base, void *start_addr, + struct va_space *vs, OMFSignatureRSDS *rsds) { const char sign_rsds[4] = "RSDS"; IMAGE_DEBUG_DIRECTORY debug_dir; - OMFSignatureRSDS rsds; - char *pdb_name; - size_t pdb_name_sz; - size_t i; + char pdb_name[sizeof(PDB_NAME)]; if (pe_get_data_dir_entry(base, start_addr, IMAGE_FILE_DEBUG_DIRECTORY, &debug_dir, sizeof(debug_dir), vs)) { eprintf("Failed to get Debug Directory\n"); - return 1; + return false; } if (debug_dir.Type != IMAGE_DEBUG_TYPE_CODEVIEW) { - return 1; + eprintf("Debug Directory type is not CodeView\n"); + return false; } if (va_space_rw(vs, base + debug_dir.AddressOfRawData, - &rsds, sizeof(rsds), 0)) { - return 1; + rsds, sizeof(*rsds), 0)) { + eprintf("Failed to resolve OMFSignatureRSDS\n"); + return false; } - printf("CodeView signature is \'%.4s\'\n", rsds.Signature); - - if (memcmp(&rsds.Signature, sign_rsds, sizeof(sign_rsds))) { - return 1; + if (memcmp(&rsds->Signature, sign_rsds, sizeof(sign_rsds))) { + eprintf("CodeView signature is \'%.4s\', \'%s\' expected\n", + rsds->Signature, sign_rsds); + return false; } - pdb_name_sz = debug_dir.SizeOfData - sizeof(rsds); - pdb_name = malloc(pdb_name_sz); - if (!pdb_name) { - return 1; + if (debug_dir.SizeOfData - sizeof(*rsds) != sizeof(PDB_NAME)) { + eprintf("PDB name size doesn't match\n"); + return false; } if (va_space_rw(vs, base + debug_dir.AddressOfRawData + - offsetof(OMFSignatureRSDS, name), pdb_name, pdb_name_sz, 0)) { - free(pdb_name); - return 1; + offsetof(OMFSignatureRSDS, name), pdb_name, sizeof(PDB_NAME), + 0)) { + eprintf("Failed to resolve PDB name\n"); + return false; } printf("PDB name is \'%s\', \'%s\' expected\n", pdb_name, PDB_NAME); - if (strcmp(pdb_name, PDB_NAME)) { - eprintf("Unexpected PDB name, it seems the kernel isn't found\n"); - free(pdb_name); - return 1; - } - - free(pdb_name); + return !strcmp(pdb_name, PDB_NAME); +} - sprintf(hash, "%.08x%.04x%.04x%.02x%.02x", rsds.guid.a, rsds.guid.b, - rsds.guid.c, rsds.guid.d[0], rsds.guid.d[1]); +static void pe_get_pdb_symstore_hash(OMFSignatureRSDS *rsds, char *hash) +{ + sprintf(hash, "%.08x%.04x%.04x%.02x%.02x", rsds->guid.a, rsds->guid.b, + rsds->guid.c, rsds->guid.d[0], rsds->guid.d[1]); hash += 20; - for (i = 0; i < 6; i++, hash += 2) { - sprintf(hash, "%.02x", rsds.guid.e[i]); + for (unsigned int i = 0; i < 6; i++, hash += 2) { + sprintf(hash, "%.02x", rsds->guid.e[i]); } - sprintf(hash, "%.01x", rsds.age); - - return 0; + sprintf(hash, "%.01x", rsds->age); } int main(int argc, char *argv[]) @@ -515,6 +531,7 @@ int main(int argc, char *argv[]) KDDEBUGGER_DATA64 *kdbg; uint64_t KdVersionBlock; bool kernel_found = false; + OMFSignatureRSDS rsds; if (argc != 3) { eprintf("usage:\n\t%s elf_file dmp_file\n", argv[0]); @@ -562,7 +579,8 @@ int main(int argc, char *argv[]) } if (*(uint16_t *)nt_start_addr == 0x5a4d) { /* MZ */ - if (pe_check_export_name(KernBase, nt_start_addr, &vs)) { + printf("Checking candidate KernBase = 0x%016"PRIx64"\n", KernBase); + if (pe_check_pdb_name(KernBase, nt_start_addr, &vs, &rsds)) { kernel_found = true; break; } @@ -578,11 +596,7 @@ int main(int argc, char *argv[]) printf("KernBase = 0x%016"PRIx64", signature is \'%.2s\'\n", KernBase, (char *)nt_start_addr); - if (pe_get_pdb_symstore_hash(KernBase, nt_start_addr, pdb_hash, &vs)) { - eprintf("Failed to get PDB symbol store hash\n"); - err = 1; - goto out_ps; - } + pe_get_pdb_symstore_hash(&rsds, pdb_hash); sprintf(pdb_url, "%s%s/%s/%s", SYM_URL_BASE, PDB_NAME, pdb_hash, PDB_NAME); printf("PDB URL is %s\n", pdb_url); diff --git a/contrib/elf2dmp/pdb.c b/contrib/elf2dmp/pdb.c index adcfa7e154..6ca5086f02 100644 --- a/contrib/elf2dmp/pdb.c +++ b/contrib/elf2dmp/pdb.c @@ -160,7 +160,7 @@ static void *pdb_ds_read_file(struct pdb_reader* r, uint32_t file_number) static int pdb_init_segments(struct pdb_reader *r) { char *segs; - unsigned stream_idx = r->sidx.segments; + unsigned stream_idx = r->segments; segs = pdb_ds_read_file(r, stream_idx); if (!segs) { @@ -177,9 +177,6 @@ static int pdb_init_symbols(struct pdb_reader *r) { int err = 0; PDB_SYMBOLS *symbols; - PDB_STREAM_INDEXES *sidx = &r->sidx; - - memset(sidx, -1, sizeof(*sidx)); symbols = pdb_ds_read_file(r, 3); if (!symbols) { @@ -188,15 +185,11 @@ static int pdb_init_symbols(struct pdb_reader *r) r->symbols = symbols; - if (symbols->stream_index_size != sizeof(PDB_STREAM_INDEXES)) { - err = 1; - goto out_symbols; - } - - memcpy(sidx, (const char *)symbols + sizeof(PDB_SYMBOLS) + + r->segments = *(uint16_t *)((const char *)symbols + sizeof(PDB_SYMBOLS) + symbols->module_size + symbols->offset_size + symbols->hash_size + symbols->srcmodule_size + - symbols->pdbimport_size + symbols->unknown2_size, sizeof(*sidx)); + symbols->pdbimport_size + symbols->unknown2_size + + offsetof(PDB_STREAM_INDEXES, segments)); /* Read global symbol table */ r->modimage = pdb_ds_read_file(r, symbols->gsym_file); diff --git a/contrib/elf2dmp/pdb.h b/contrib/elf2dmp/pdb.h index 4ea8925ee8..2a50da56ac 100644 --- a/contrib/elf2dmp/pdb.h +++ b/contrib/elf2dmp/pdb.h @@ -227,7 +227,7 @@ struct pdb_reader { } ds; uint32_t file_used[1024]; PDB_SYMBOLS *symbols; - PDB_STREAM_INDEXES sidx; + uint16_t segments; uint8_t *modimage; char *segs; size_t segs_size; diff --git a/contrib/elf2dmp/qemu_elf.c b/contrib/elf2dmp/qemu_elf.c index ebda60dcb8..de6ad744c6 100644 --- a/contrib/elf2dmp/qemu_elf.c +++ b/contrib/elf2dmp/qemu_elf.c @@ -165,10 +165,40 @@ static bool check_ehdr(QEMU_Elf *qe) return true; } -int QEMU_Elf_init(QEMU_Elf *qe, const char *filename) +static int QEMU_Elf_map(QEMU_Elf *qe, const char *filename) { +#ifdef CONFIG_LINUX + struct stat st; + int fd; + + printf("Using Linux mmap\n"); + + fd = open(filename, O_RDONLY, 0); + if (fd == -1) { + eprintf("Failed to open ELF dump file \'%s\'\n", filename); + return 1; + } + + if (fstat(fd, &st)) { + eprintf("Failed to get size of ELF dump file\n"); + close(fd); + return 1; + } + qe->size = st.st_size; + + qe->map = mmap(NULL, qe->size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_NORESERVE, fd, 0); + if (qe->map == MAP_FAILED) { + eprintf("Failed to map ELF file\n"); + close(fd); + return 1; + } + + close(fd); +#else GError *gerr = NULL; - int err = 0; + + printf("Using GLib mmap\n"); qe->gmf = g_mapped_file_new(filename, TRUE, &gerr); if (gerr) { @@ -179,29 +209,43 @@ int QEMU_Elf_init(QEMU_Elf *qe, const char *filename) qe->map = g_mapped_file_get_contents(qe->gmf); qe->size = g_mapped_file_get_length(qe->gmf); +#endif + + return 0; +} + +static void QEMU_Elf_unmap(QEMU_Elf *qe) +{ +#ifdef CONFIG_LINUX + munmap(qe->map, qe->size); +#else + g_mapped_file_unref(qe->gmf); +#endif +} + +int QEMU_Elf_init(QEMU_Elf *qe, const char *filename) +{ + if (QEMU_Elf_map(qe, filename)) { + return 1; + } if (!check_ehdr(qe)) { eprintf("Input file has the wrong format\n"); - err = 1; - goto out_unmap; + QEMU_Elf_unmap(qe); + return 1; } if (init_states(qe)) { eprintf("Failed to extract QEMU CPU states\n"); - err = 1; - goto out_unmap; + QEMU_Elf_unmap(qe); + return 1; } return 0; - -out_unmap: - g_mapped_file_unref(qe->gmf); - - return err; } void QEMU_Elf_exit(QEMU_Elf *qe) { exit_states(qe); - g_mapped_file_unref(qe->gmf); + QEMU_Elf_unmap(qe); } diff --git a/contrib/elf2dmp/qemu_elf.h b/contrib/elf2dmp/qemu_elf.h index b2f0d9cbc9..afa75f10b2 100644 --- a/contrib/elf2dmp/qemu_elf.h +++ b/contrib/elf2dmp/qemu_elf.h @@ -32,7 +32,9 @@ typedef struct QEMUCPUState { int is_system(QEMUCPUState *s); typedef struct QEMU_Elf { +#ifndef CONFIG_LINUX GMappedFile *gmf; +#endif size_t size; void *map; QEMUCPUState **state; diff --git a/crypto/clmul.c b/crypto/clmul.c new file mode 100644 index 0000000000..9e3e61a77d --- /dev/null +++ b/crypto/clmul.c @@ -0,0 +1,111 @@ +/* + * Carry-less multiply operations. + * SPDX-License-Identifier: GPL-2.0-or-later + * + * Copyright (C) 2023 Linaro, Ltd. + */ + +#include "qemu/osdep.h" +#include "crypto/clmul.h" + +uint64_t clmul_8x8_low(uint64_t n, uint64_t m) +{ + uint64_t r = 0; + + for (int i = 0; i < 8; ++i) { + uint64_t mask = (n & 0x0101010101010101ull) * 0xff; + r ^= m & mask; + m = (m << 1) & 0xfefefefefefefefeull; + n >>= 1; + } + return r; +} + +static uint64_t clmul_8x4_even_int(uint64_t n, uint64_t m) +{ + uint64_t r = 0; + + for (int i = 0; i < 8; ++i) { + uint64_t mask = (n & 0x0001000100010001ull) * 0xffff; + r ^= m & mask; + n >>= 1; + m <<= 1; + } + return r; +} + +uint64_t clmul_8x4_even(uint64_t n, uint64_t m) +{ + n &= 0x00ff00ff00ff00ffull; + m &= 0x00ff00ff00ff00ffull; + return clmul_8x4_even_int(n, m); +} + +uint64_t clmul_8x4_odd(uint64_t n, uint64_t m) +{ + return clmul_8x4_even(n >> 8, m >> 8); +} + +static uint64_t unpack_8_to_16(uint64_t x) +{ + return (x & 0x000000ff) + | ((x & 0x0000ff00) << 8) + | ((x & 0x00ff0000) << 16) + | ((x & 0xff000000) << 24); +} + +uint64_t clmul_8x4_packed(uint32_t n, uint32_t m) +{ + return clmul_8x4_even_int(unpack_8_to_16(n), unpack_8_to_16(m)); +} + +uint64_t clmul_16x2_even(uint64_t n, uint64_t m) +{ + uint64_t r = 0; + + n &= 0x0000ffff0000ffffull; + m &= 0x0000ffff0000ffffull; + + for (int i = 0; i < 16; ++i) { + uint64_t mask = (n & 0x0000000100000001ull) * 0xffffffffull; + r ^= m & mask; + n >>= 1; + m <<= 1; + } + return r; +} + +uint64_t clmul_16x2_odd(uint64_t n, uint64_t m) +{ + return clmul_16x2_even(n >> 16, m >> 16); +} + +uint64_t clmul_32(uint32_t n, uint32_t m32) +{ + uint64_t r = 0; + uint64_t m = m32; + + for (int i = 0; i < 32; ++i) { + r ^= n & 1 ? m : 0; + n >>= 1; + m <<= 1; + } + return r; +} + +Int128 clmul_64_gen(uint64_t n, uint64_t m) +{ + uint64_t rl = 0, rh = 0; + + /* Bit 0 can only influence the low 64-bit result. */ + if (n & 1) { + rl = m; + } + + for (int i = 1; i < 64; ++i) { + uint64_t mask = -((n >> i) & 1); + rl ^= (m << i) & mask; + rh ^= (m >> (64 - i)) & mask; + } + return int128_make128(rl, rh); +} diff --git a/crypto/meson.build b/crypto/meson.build index 5f03a30d34..9ac1a89802 100644 --- a/crypto/meson.build +++ b/crypto/meson.build @@ -48,9 +48,12 @@ if have_afalg endif crypto_ss.add(when: gnutls, if_true: files('tls-cipher-suites.c')) -util_ss.add(files('sm4.c')) -util_ss.add(files('aes.c')) -util_ss.add(files('init.c')) +util_ss.add(files( + 'aes.c', + 'clmul.c', + 'init.c', + 'sm4.c', +)) if gnutls.found() util_ss.add(gnutls) endif diff --git a/docs/devel/loads-stores.rst b/docs/devel/loads-stores.rst index dab6dfa0ac..ec627aa9c0 100644 --- a/docs/devel/loads-stores.rst +++ b/docs/devel/loads-stores.rst @@ -63,12 +63,12 @@ which stores ``val`` to ``ptr`` as an ``{endian}`` order value of size ``sz`` bytes. -Regexes for git grep +Regexes for git grep: - ``\<ld[us]\?[bwlq]\(_[hbl]e\)\?_p\>`` - ``\<st[bwlq]\(_[hbl]e\)\?_p\>`` - ``\<st24\(_[hbl]e\)\?_p\>`` - - ``\<ldn_\([hbl]e\)?_p\>`` - - ``\<stn_\([hbl]e\)?_p\>`` + - ``\<ldn_\([hbl]e\)\?_p\>`` + - ``\<stn_\([hbl]e\)\?_p\>`` ``cpu_{ld,st}*_mmu`` ~~~~~~~~~~~~~~~~~~~~ @@ -121,8 +121,8 @@ store: ``cpu_st{size}{end}_mmu(env, ptr, val, oi, retaddr)`` - ``_le`` : little endian Regexes for git grep: - - ``\<cpu_ld[bwlq](_[bl]e)\?_mmu\>`` - - ``\<cpu_st[bwlq](_[bl]e)\?_mmu\>`` + - ``\<cpu_ld[bwlq]\(_[bl]e\)\?_mmu\>`` + - ``\<cpu_st[bwlq]\(_[bl]e\)\?_mmu\>`` ``cpu_{ld,st}*_mmuidx_ra`` @@ -155,8 +155,8 @@ store: ``cpu_st{size}{end}_mmuidx_ra(env, ptr, val, mmuidx, retaddr)`` - ``_le`` : little endian Regexes for git grep: - - ``\<cpu_ld[us]\?[bwlq](_[bl]e)\?_mmuidx_ra\>`` - - ``\<cpu_st[bwlq](_[bl]e)\?_mmuidx_ra\>`` + - ``\<cpu_ld[us]\?[bwlq]\(_[bl]e\)\?_mmuidx_ra\>`` + - ``\<cpu_st[bwlq]\(_[bl]e\)\?_mmuidx_ra\>`` ``cpu_{ld,st}*_data_ra`` ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -193,8 +193,8 @@ store: ``cpu_st{size}{end}_data_ra(env, ptr, val, ra)`` - ``_le`` : little endian Regexes for git grep: - - ``\<cpu_ld[us]\?[bwlq](_[bl]e)\?_data_ra\>`` - - ``\<cpu_st[bwlq](_[bl]e)\?_data_ra\>`` + - ``\<cpu_ld[us]\?[bwlq]\(_[bl]e\)\?_data_ra\>`` + - ``\<cpu_st[bwlq]\(_[bl]e\)\?_data_ra\>`` ``cpu_{ld,st}*_data`` ~~~~~~~~~~~~~~~~~~~~~ @@ -231,9 +231,9 @@ store: ``cpu_st{size}{end}_data(env, ptr, val)`` - ``_be`` : big endian - ``_le`` : little endian -Regexes for git grep - - ``\<cpu_ld[us]\?[bwlq](_[bl]e)\?_data\>`` - - ``\<cpu_st[bwlq](_[bl]e)\?_data\+\>`` +Regexes for git grep: + - ``\<cpu_ld[us]\?[bwlq]\(_[bl]e\)\?_data\>`` + - ``\<cpu_st[bwlq]\(_[bl]e\)\?_data\+\>`` ``cpu_ld*_code`` ~~~~~~~~~~~~~~~~ @@ -296,7 +296,7 @@ swap: ``translator_ld{sign}{size}_swap(env, ptr, swap)`` - ``l`` : 32 bits - ``q`` : 64 bits -Regexes for git grep +Regexes for git grep: - ``\<translator_ld[us]\?[bwlq]\(_swap\)\?\>`` ``helper_{ld,st}*_mmu`` @@ -325,7 +325,7 @@ store: ``helper_{size}_mmu(env, addr, val, opindex, retaddr)`` - ``l`` : 32 bits - ``q`` : 64 bits -Regexes for git grep +Regexes for git grep: - ``\<helper_ld[us]\?[bwlq]_mmu\>`` - ``\<helper_st[bwlq]_mmu\>`` @@ -382,7 +382,7 @@ succeeded using a MemTxResult return code. The ``_{endian}`` suffix is omitted for byte accesses. -Regexes for git grep +Regexes for git grep: - ``\<address_space_\(read\|write\|rw\)\>`` - ``\<address_space_ldu\?[bwql]\(_[lb]e\)\?\>`` - ``\<address_space_st[bwql]\(_[lb]e\)\?\>`` @@ -400,7 +400,7 @@ Note that portions of the write which attempt to write data to a device will be silently ignored -- only real RAM and ROM will be written to. -Regexes for git grep +Regexes for git grep: - ``address_space_write_rom`` ``{ld,st}*_phys`` @@ -438,7 +438,7 @@ device doing the access has no way to report such an error. The ``_{endian}_`` infix is omitted for byte accesses. -Regexes for git grep +Regexes for git grep: - ``\<ldu\?[bwlq]\(_[bl]e\)\?_phys\>`` - ``\<st[bwlq]\(_[bl]e\)\?_phys\>`` @@ -462,7 +462,7 @@ For new code they are better avoided: ``cpu_physical_memory_rw`` -Regexes for git grep +Regexes for git grep: - ``\<cpu_physical_memory_\(read\|write\|rw\)\>`` ``cpu_memory_rw_debug`` @@ -497,7 +497,7 @@ make sure our existing code is doing things correctly. ``dma_memory_rw`` -Regexes for git grep +Regexes for git grep: - ``\<dma_memory_\(read\|write\|rw\)\>`` - ``\<ldu\?[bwlq]\(_[bl]e\)\?_dma\>`` - ``\<st[bwlq]\(_[bl]e\)\?_dma\>`` @@ -538,7 +538,7 @@ correct address space for that device. The ``_{endian}_`` infix is omitted for byte accesses. -Regexes for git grep +Regexes for git grep: - ``\<pci_dma_\(read\|write\|rw\)\>`` - ``\<ldu\?[bwlq]\(_[bl]e\)\?_pci_dma\>`` - ``\<st[bwlq]\(_[bl]e\)\?_pci_dma\>`` diff --git a/docs/devel/multi-process.rst b/docs/devel/multi-process.rst index e4801751f2..4ef539c0b0 100644 --- a/docs/devel/multi-process.rst +++ b/docs/devel/multi-process.rst @@ -409,8 +409,9 @@ the initial messages sent to the emulation process is a guest memory table. Each entry in this table consists of a file descriptor and size that the emulation process can ``mmap()`` to directly access guest memory, similar to ``vhost_user_set_mem_table()``. Note guest memory -must be backed by file descriptors, such as when QEMU is given the -*-mem-path* command line option. +must be backed by shared file-backed memory, for example, using +*-object memory-backend-file,share=on* and setting that memory backend +as RAM for the machine. IOMMU operations ^^^^^^^^^^^^^^^^ diff --git a/docs/devel/reset.rst b/docs/devel/reset.rst index 7cc6a6b314..38ed1790f7 100644 --- a/docs/devel/reset.rst +++ b/docs/devel/reset.rst @@ -184,21 +184,20 @@ in reset. { MyDevClass *myclass = MYDEV_CLASS(class); ResettableClass *rc = RESETTABLE_CLASS(class); - resettable_class_set_parent_reset_phases(rc, - mydev_reset_enter, - mydev_reset_hold, - mydev_reset_exit, - &myclass->parent_phases); + resettable_class_set_parent_phases(rc, + mydev_reset_enter, + mydev_reset_hold, + mydev_reset_exit, + &myclass->parent_phases); } In the above example, we override all three phases. It is possible to override only some of them by passing NULL instead of a function pointer to -``resettable_class_set_parent_reset_phases()``. For example, the following will +``resettable_class_set_parent_phases()``. For example, the following will only override the *enter* phase and leave *hold* and *exit* untouched:: - resettable_class_set_parent_reset_phases(rc, mydev_reset_enter, - NULL, NULL, - &myclass->parent_phases); + resettable_class_set_parent_phases(rc, mydev_reset_enter, NULL, NULL, + &myclass->parent_phases); This is equivalent to providing a trivial implementation of the hold and exit phases which does nothing but call the parent class's implementation of the diff --git a/docs/system/arm/emulation.rst b/docs/system/arm/emulation.rst index 3df936fc35..965cbf84c5 100644 --- a/docs/system/arm/emulation.rst +++ b/docs/system/arm/emulation.rst @@ -42,6 +42,7 @@ the following architecture extensions: - FEAT_FlagM2 (Enhancements to flag manipulation instructions) - FEAT_GTG (Guest translation granule size) - FEAT_HAFDBS (Hardware management of the access flag and dirty bit state) +- FEAT_HBC (Hinted conditional branches) - FEAT_HCX (Support for the HCRX_EL2 register) - FEAT_HPDS (Hierarchical permission disables) - FEAT_HPDS2 (Translation table page-based hardware attributes) @@ -57,6 +58,7 @@ the following architecture extensions: - FEAT_LSE (Large System Extensions) - FEAT_LSE2 (Large System Extensions v2) - FEAT_LVA (Large Virtual Address space) +- FEAT_MOPS (Standardization of memory operations) - FEAT_MTE (Memory Tagging Extension) - FEAT_MTE2 (Memory Tagging Extension) - FEAT_MTE3 (MTE Asymmetric Fault Handling) diff --git a/docs/system/devices/cxl.rst b/docs/system/devices/cxl.rst index f12011e230..6ab5f72473 100644 --- a/docs/system/devices/cxl.rst +++ b/docs/system/devices/cxl.rst @@ -157,7 +157,7 @@ responsible for allocating appropriate ranges from within the CFMWs and exposing those via normal memory configurations as would be done for system RAM. -Example system Topology. x marks the match in each decoder level:: +Example system topology. x marks the match in each decoder level:: |<------------------SYSTEM PHYSICAL ADDRESS MAP (1)----------------->| | __________ __________________________________ __________ | @@ -187,8 +187,8 @@ Example system Topology. x marks the match in each decoder level:: ___________|___ __________|__ __|_________ ___|_________ (3)| Root Port 0 | | Root Port 1 | | Root Port 2| | Root Port 3 | | Appears in | | Appears in | | Appears in | | Appear in | - | PCI topology | | PCI Topology| | PCI Topo | | PCI Topo | - | As 0c:00.0 | | as 0c:01.0 | | as de:00.0 | | as de:01.0 | + | PCI topology | | PCI topology| | PCI topo | | PCI topo | + | as 0c:00.0 | | as 0c:01.0 | | as de:00.0 | | as de:01.0 | |_______________| |_____________| |____________| |_____________| | | | | | | | | @@ -272,7 +272,7 @@ Example topology involving a switch:: | Root Port 0 | | Appears in | | PCI topology | - | As 0c:00.0 | + | as 0c:00.0 | |___________x___| | | @@ -313,7 +313,7 @@ A very simple setup with just one directly attached CXL Type 3 Persistent Memory A very simple setup with just one directly attached CXL Type 3 Volatile Memory device:: - qemu-system-aarch64 -M virt,gic-version=3,cxl=on -m 4g,maxmem=8G,slots=8 -cpu max \ + qemu-system-x86_64 -M q35,cxl=on -m 4G,maxmem=8G,slots=8 -smp 4 \ ... -object memory-backend-ram,id=vmem0,share=on,size=256M \ -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \ @@ -323,7 +323,7 @@ A very simple setup with just one directly attached CXL Type 3 Volatile Memory d The same volatile setup may optionally include an LSA region:: - qemu-system-aarch64 -M virt,gic-version=3,cxl=on -m 4g,maxmem=8G,slots=8 -cpu max \ + qemu-system-x86_64 -M q35,cxl=on -m 4G,maxmem=8G,slots=8 -smp 4 \ ... -object memory-backend-ram,id=vmem0,share=on,size=256M \ -object memory-backend-file,id=cxl-lsa0,share=on,mem-path=/tmp/lsa.raw,size=256M \ diff --git a/docs/system/index.rst b/docs/system/index.rst index 45bf1f19e7..c21065e519 100644 --- a/docs/system/index.rst +++ b/docs/system/index.rst @@ -38,3 +38,4 @@ or Hypervisor.Framework. security multi-process confidential-guest-support + vm-templating diff --git a/docs/system/vm-templating.rst b/docs/system/vm-templating.rst new file mode 100644 index 0000000000..28905a1eeb --- /dev/null +++ b/docs/system/vm-templating.rst @@ -0,0 +1,125 @@ +QEMU VM templating +================== + +This document explains how to use VM templating in QEMU. + +For now, the focus is on VM memory aspects, and not about how to save and +restore other VM state (i.e., migrate-to-file with ``x-ignore-shared``). + +Overview +-------- + +With VM templating, a single template VM serves as the starting point for +new VMs. This allows for fast and efficient replication of VMs, resulting +in fast startup times and reduced memory consumption. + +Conceptually, the VM state is frozen, to then be used as a basis for new +VMs. The Copy-On-Write mechanism in the operating systems makes sure that +new VMs are able to read template VM memory; however, any modifications +stay private and don't modify the original template VM or any other +created VM. + +!!! Security Alert !!! +---------------------- + +When effectively cloning VMs by VM templating, hardware identifiers +(such as UUIDs and NIC MAC addresses), and similar data in the guest OS +(such as machine IDs, SSH keys, certificates) that are supposed to be +*unique* are no longer unique, which can be a security concern. + +Please be aware of these implications and how to mitigate them for your +use case, which might involve vmgenid, hot(un)plug of NIC, etc.. + +Memory configuration +-------------------- + +In order to create the template VM, we have to make sure that VM memory +ends up in a file, from where it can be reused for the new VMs: + +Supply VM RAM via memory-backend-file, with ``share=on`` (modifications go +to the file) and ``readonly=off`` (open the file writable). Note that +``readonly=off`` is implicit. + +In the following command-line example, a 2GB VM is created, whereby VM RAM +is to be stored in the ``template`` file. + +.. parsed-literal:: + + |qemu_system| [...] -m 2g \\ + -object memory-backend-file,id=pc.ram,mem-path=template,size=2g,share=on,... \\ + -machine q35,memory-backend=pc.ram + +If multiple memory backends are used (vNUMA, DIMMs), configure all +memory backends accordingly. + +Once the VM is in the desired state, stop the VM and save other VM state, +leaving the current state of VM RAM reside in the file. + +In order to have a new VM be based on a template VM, we have to +configure VM RAM to be based on a template VM RAM file; however, the VM +should not be able to modify file content. + +Supply VM RAM via memory-backend-file, with ``share=off`` (modifications +stay private), ``readonly=on`` (open the file readonly) and ``rom=off`` +(don't make the memory readonly for the VM). Note that ``share=off`` is +implicit and that other VM state has to be restored separately. + +In the following command-line example, a 2GB VM is created based on the +existing 2GB file ``template``. + +.. parsed-literal:: + + |qemu_system| [...] -m 2g \\ + -object memory-backend-file,id=pc.ram,mem-path=template,size=2g,readonly=on,rom=off,... \\ + -machine q35,memory-backend=pc.ram + +If multiple memory backends are used (vNUMA, DIMMs), configure all +memory backends accordingly. + +Note that ``-mem-path`` cannot be used for VM templating when creating the +template VM or when starting new VMs based on a template VM. + +Incompatible features +--------------------- + +Some features are incompatible with VM templating, as the underlying file +cannot be modified to discard VM RAM, or to actually share memory with +another process. + +vhost-user and multi-process QEMU +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +vhost-user and multi-process QEMU are incompatible with VM templating. +These technologies rely on shared memory, however, the template VMs +don't actually share memory (``share=off``), even though they are +file-based. + +virtio-balloon +~~~~~~~~~~~~~~ + +virtio-balloon inflation and "free page reporting" cannot discard VM RAM +and will repeatedly report errors. While virtio-balloon can be used +for template VMs (e.g., report VM RAM stats), "free page reporting" +should be disabled and the balloon should not be inflated. + +virtio-mem +~~~~~~~~~~ + +virtio-mem cannot discard VM RAM that is managed by the virtio-mem +device. virtio-mem will fail early when realizing the device. To use +VM templating with virtio-mem, either hotplug virtio-mem devices to the +new VM, or don't supply any memory to the template VM using virtio-mem +(requested-size=0), not using a template VM file as memory backend for the +virtio-mem device. + +VM migration +~~~~~~~~~~~~ + +For VM migration, "x-release-ram" similarly relies on discarding of VM +RAM on the migration source to free up migrated RAM, and will +repeatedly report errors. + +Postcopy live migration fails discarding VM RAM on the migration +destination early and refuses to activate postcopy live migration. Note +that postcopy live migration usually only works on selected filesystems +(shmem/tmpfs, hugetlbfs) either way. diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index 527e15e6ab..a44649f4f4 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -118,7 +118,8 @@ static void partsN(canonicalize)(FloatPartsN *p, float_status *status, } else { int shift = frac_normalize(p); p->cls = float_class_normal; - p->exp = fmt->frac_shift - fmt->exp_bias - shift + 1; + p->exp = fmt->frac_shift - fmt->exp_bias + - shift + !fmt->m68k_denormal; } } else if (likely(p->exp < fmt->exp_max) || fmt->arm_althp) { p->cls = float_class_normal; @@ -256,7 +257,7 @@ static void partsN(uncanon_normal)(FloatPartsN *p, float_status *s, is_tiny = !frac_addi(&discard, p, inc); } - frac_shrjam(p, 1 - exp); + frac_shrjam(p, !fmt->m68k_denormal - exp); if (p->frac_lo & round_mask) { /* Need to recompute round-to-even/round-to-odd. */ @@ -287,7 +288,7 @@ static void partsN(uncanon_normal)(FloatPartsN *p, float_status *s, p->frac_lo &= ~round_mask; } - exp = (p->frac_hi & DECOMPOSED_IMPLICIT_BIT) != 0; + exp = (p->frac_hi & DECOMPOSED_IMPLICIT_BIT) && !fmt->m68k_denormal; frac_shr(p, frac_shift); if (is_tiny && (flags & float_flag_inexact)) { diff --git a/fpu/softfloat.c b/fpu/softfloat.c index 0cc130ae9b..027a8e576d 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -517,6 +517,7 @@ typedef struct { * round_mask: bits below lsb which must be rounded * The following optional modifiers are available: * arm_althp: handle ARM Alternative Half Precision + * m68k_denormal: explicit integer bit for extended precision may be 1 */ typedef struct { int exp_size; @@ -526,6 +527,7 @@ typedef struct { int frac_size; int frac_shift; bool arm_althp; + bool m68k_denormal; uint64_t round_mask; } FloatFmt; @@ -576,7 +578,12 @@ static const FloatFmt float128_params = { static const FloatFmt floatx80_params[3] = { [floatx80_precision_s] = { FLOATX80_PARAMS(23) }, [floatx80_precision_d] = { FLOATX80_PARAMS(52) }, - [floatx80_precision_x] = { FLOATX80_PARAMS(64) }, + [floatx80_precision_x] = { + FLOATX80_PARAMS(64), +#ifdef TARGET_M68K + .m68k_denormal = true, +#endif + }, }; /* Unpack a float to parts, but do not canonicalize. */ @@ -3126,6 +3133,15 @@ int64_t float64_to_int64_scalbn(float64 a, FloatRoundMode rmode, int scale, return parts_float_to_sint(&p, rmode, scale, INT64_MIN, INT64_MAX, s); } +int8_t bfloat16_to_int8_scalbn(bfloat16 a, FloatRoundMode rmode, int scale, + float_status *s) +{ + FloatParts64 p; + + bfloat16_unpack_canonical(&p, a, s); + return parts_float_to_sint(&p, rmode, scale, INT8_MIN, INT8_MAX, s); +} + int16_t bfloat16_to_int16_scalbn(bfloat16 a, FloatRoundMode rmode, int scale, float_status *s) { @@ -3392,6 +3408,11 @@ int64_t floatx80_to_int64_round_to_zero(floatx80 a, float_status *s) return floatx80_to_int64_scalbn(a, float_round_to_zero, 0, s); } +int8_t bfloat16_to_int8(bfloat16 a, float_status *s) +{ + return bfloat16_to_int8_scalbn(a, s->float_rounding_mode, 0, s); +} + int16_t bfloat16_to_int16(bfloat16 a, float_status *s) { return bfloat16_to_int16_scalbn(a, s->float_rounding_mode, 0, s); @@ -3407,6 +3428,11 @@ int64_t bfloat16_to_int64(bfloat16 a, float_status *s) return bfloat16_to_int64_scalbn(a, s->float_rounding_mode, 0, s); } +int8_t bfloat16_to_int8_round_to_zero(bfloat16 a, float_status *s) +{ + return bfloat16_to_int8_scalbn(a, float_round_to_zero, 0, s); +} + int16_t bfloat16_to_int16_round_to_zero(bfloat16 a, float_status *s) { return bfloat16_to_int16_scalbn(a, float_round_to_zero, 0, s); @@ -3534,6 +3560,15 @@ uint64_t float64_to_uint64_scalbn(float64 a, FloatRoundMode rmode, int scale, return parts_float_to_uint(&p, rmode, scale, UINT64_MAX, s); } +uint8_t bfloat16_to_uint8_scalbn(bfloat16 a, FloatRoundMode rmode, + int scale, float_status *s) +{ + FloatParts64 p; + + bfloat16_unpack_canonical(&p, a, s); + return parts_float_to_uint(&p, rmode, scale, UINT8_MAX, s); +} + uint16_t bfloat16_to_uint16_scalbn(bfloat16 a, FloatRoundMode rmode, int scale, float_status *s) { @@ -3759,6 +3794,11 @@ Int128 float128_to_uint128_round_to_zero(float128 a, float_status *s) return float128_to_uint128_scalbn(a, float_round_to_zero, 0, s); } +uint8_t bfloat16_to_uint8(bfloat16 a, float_status *s) +{ + return bfloat16_to_uint8_scalbn(a, s->float_rounding_mode, 0, s); +} + uint16_t bfloat16_to_uint16(bfloat16 a, float_status *s) { return bfloat16_to_uint16_scalbn(a, s->float_rounding_mode, 0, s); @@ -3774,6 +3814,11 @@ uint64_t bfloat16_to_uint64(bfloat16 a, float_status *s) return bfloat16_to_uint64_scalbn(a, s->float_rounding_mode, 0, s); } +uint8_t bfloat16_to_uint8_round_to_zero(bfloat16 a, float_status *s) +{ + return bfloat16_to_uint8_scalbn(a, float_round_to_zero, 0, s); +} + uint16_t bfloat16_to_uint16_round_to_zero(bfloat16 a, float_status *s) { return bfloat16_to_uint16_scalbn(a, float_round_to_zero, 0, s); @@ -3929,6 +3974,11 @@ bfloat16 int16_to_bfloat16_scalbn(int16_t a, int scale, float_status *status) return int64_to_bfloat16_scalbn(a, scale, status); } +bfloat16 int8_to_bfloat16_scalbn(int8_t a, int scale, float_status *status) +{ + return int64_to_bfloat16_scalbn(a, scale, status); +} + bfloat16 int64_to_bfloat16(int64_t a, float_status *status) { return int64_to_bfloat16_scalbn(a, 0, status); @@ -3944,6 +3994,11 @@ bfloat16 int16_to_bfloat16(int16_t a, float_status *status) return int64_to_bfloat16_scalbn(a, 0, status); } +bfloat16 int8_to_bfloat16(int8_t a, float_status *status) +{ + return int64_to_bfloat16_scalbn(a, 0, status); +} + float128 int128_to_float128(Int128 a, float_status *status) { FloatParts128 p = { }; @@ -4139,6 +4194,11 @@ bfloat16 uint16_to_bfloat16_scalbn(uint16_t a, int scale, float_status *status) return uint64_to_bfloat16_scalbn(a, scale, status); } +bfloat16 uint8_to_bfloat16_scalbn(uint8_t a, int scale, float_status *status) +{ + return uint64_to_bfloat16_scalbn(a, scale, status); +} + bfloat16 uint64_to_bfloat16(uint64_t a, float_status *status) { return uint64_to_bfloat16_scalbn(a, 0, status); @@ -4154,6 +4214,11 @@ bfloat16 uint16_to_bfloat16(uint16_t a, float_status *status) return uint64_to_bfloat16_scalbn(a, 0, status); } +bfloat16 uint8_to_bfloat16(uint8_t a, float_status *status) +{ + return uint64_to_bfloat16_scalbn(a, 0, status); +} + float128 uint64_to_float128(uint64_t a, float_status *status) { FloatParts128 p; diff --git a/fsdev/qemu-fsdev-throttle.c b/fsdev/qemu-fsdev-throttle.c index 5c83a1cc09..d912da906d 100644 --- a/fsdev/qemu-fsdev-throttle.c +++ b/fsdev/qemu-fsdev-throttle.c @@ -94,20 +94,22 @@ void fsdev_throttle_init(FsThrottle *fst) } } -void coroutine_fn fsdev_co_throttle_request(FsThrottle *fst, bool is_write, +void coroutine_fn fsdev_co_throttle_request(FsThrottle *fst, + ThrottleDirection direction, struct iovec *iov, int iovcnt) { + assert(direction < THROTTLE_MAX); if (throttle_enabled(&fst->cfg)) { - if (throttle_schedule_timer(&fst->ts, &fst->tt, is_write) || - !qemu_co_queue_empty(&fst->throttled_reqs[is_write])) { - qemu_co_queue_wait(&fst->throttled_reqs[is_write], NULL); + if (throttle_schedule_timer(&fst->ts, &fst->tt, direction) || + !qemu_co_queue_empty(&fst->throttled_reqs[direction])) { + qemu_co_queue_wait(&fst->throttled_reqs[direction], NULL); } - throttle_account(&fst->ts, is_write, iov_size(iov, iovcnt)); + throttle_account(&fst->ts, direction, iov_size(iov, iovcnt)); - if (!qemu_co_queue_empty(&fst->throttled_reqs[is_write]) && - !throttle_schedule_timer(&fst->ts, &fst->tt, is_write)) { - qemu_co_queue_next(&fst->throttled_reqs[is_write]); + if (!qemu_co_queue_empty(&fst->throttled_reqs[direction]) && + !throttle_schedule_timer(&fst->ts, &fst->tt, direction)) { + qemu_co_queue_next(&fst->throttled_reqs[direction]); } } } diff --git a/fsdev/qemu-fsdev-throttle.h b/fsdev/qemu-fsdev-throttle.h index a21aecddc7..daa8ca2494 100644 --- a/fsdev/qemu-fsdev-throttle.h +++ b/fsdev/qemu-fsdev-throttle.h @@ -23,14 +23,14 @@ typedef struct FsThrottle { ThrottleState ts; ThrottleTimers tt; ThrottleConfig cfg; - CoQueue throttled_reqs[2]; + CoQueue throttled_reqs[THROTTLE_MAX]; } FsThrottle; int fsdev_throttle_parse_opts(QemuOpts *, FsThrottle *, Error **); void fsdev_throttle_init(FsThrottle *); -void coroutine_fn fsdev_co_throttle_request(FsThrottle *, bool , +void coroutine_fn fsdev_co_throttle_request(FsThrottle *, ThrottleDirection , struct iovec *, int); void fsdev_throttle_cleanup(FsThrottle *); diff --git a/hmp-commands.hx b/hmp-commands.hx index 2cbd0f77a0..63eac22734 100644 --- a/hmp-commands.hx +++ b/hmp-commands.hx @@ -1296,6 +1296,9 @@ ERST .name = "netdev_add", .args_type = "netdev:O", .params = "[user|tap|socket|stream|dgram|vde|bridge|hubport|netmap|vhost-user" +#ifdef CONFIG_AF_XDP + "|af-xdp" +#endif #ifdef CONFIG_VMNET "|vmnet-host|vmnet-shared|vmnet-bridged" #endif diff --git a/host/include/aarch64/host/cpuinfo.h b/host/include/aarch64/host/cpuinfo.h index 769626b098..fe671534e4 100644 --- a/host/include/aarch64/host/cpuinfo.h +++ b/host/include/aarch64/host/cpuinfo.h @@ -10,6 +10,8 @@ #define CPUINFO_LSE (1u << 1) #define CPUINFO_LSE2 (1u << 2) #define CPUINFO_AES (1u << 3) +#define CPUINFO_PMULL (1u << 4) +#define CPUINFO_BTI (1u << 5) /* Initialized with a constructor. */ extern unsigned cpuinfo; diff --git a/host/include/aarch64/host/crypto/clmul.h b/host/include/aarch64/host/crypto/clmul.h new file mode 100644 index 0000000000..bb516d8b2f --- /dev/null +++ b/host/include/aarch64/host/crypto/clmul.h @@ -0,0 +1,41 @@ +/* + * AArch64 specific clmul acceleration. + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef AARCH64_HOST_CRYPTO_CLMUL_H +#define AARCH64_HOST_CRYPTO_CLMUL_H + +#include "host/cpuinfo.h" +#include <arm_neon.h> + +/* + * 64x64->128 pmull is available with FEAT_PMULL. + * Both FEAT_AES and FEAT_PMULL are covered under the same macro. + */ +#ifdef __ARM_FEATURE_AES +# define HAVE_CLMUL_ACCEL true +#else +# define HAVE_CLMUL_ACCEL likely(cpuinfo & CPUINFO_PMULL) +#endif +#if !defined(__ARM_FEATURE_AES) && defined(CONFIG_ARM_AES_BUILTIN) +# define ATTR_CLMUL_ACCEL __attribute__((target("+crypto"))) +#else +# define ATTR_CLMUL_ACCEL +#endif + +static inline Int128 ATTR_CLMUL_ACCEL +clmul_64_accel(uint64_t n, uint64_t m) +{ + union { poly128_t v; Int128 s; } u; + +#ifdef CONFIG_ARM_AES_BUILTIN + u.v = vmull_p64((poly64_t)n, (poly64_t)m); +#else + asm(".arch_extension aes\n\t" + "pmull %0.1q, %1.1d, %2.1d" : "=w"(u.v) : "w"(n), "w"(m)); +#endif + return u.s; +} + +#endif /* AARCH64_HOST_CRYPTO_CLMUL_H */ diff --git a/host/include/generic/host/crypto/clmul.h b/host/include/generic/host/crypto/clmul.h new file mode 100644 index 0000000000..915bfb88d3 --- /dev/null +++ b/host/include/generic/host/crypto/clmul.h @@ -0,0 +1,15 @@ +/* + * No host specific carry-less multiply acceleration. + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef GENERIC_HOST_CRYPTO_CLMUL_H +#define GENERIC_HOST_CRYPTO_CLMUL_H + +#define HAVE_CLMUL_ACCEL false +#define ATTR_CLMUL_ACCEL + +Int128 clmul_64_accel(uint64_t, uint64_t) + QEMU_ERROR("unsupported accel"); + +#endif /* GENERIC_HOST_CRYPTO_CLMUL_H */ diff --git a/host/include/i386/host/cpuinfo.h b/host/include/i386/host/cpuinfo.h index 073d0a426f..b89e6d2e55 100644 --- a/host/include/i386/host/cpuinfo.h +++ b/host/include/i386/host/cpuinfo.h @@ -1,6 +1,6 @@ /* * SPDX-License-Identifier: GPL-2.0-or-later - * Host specific cpu indentification for x86. + * Host specific cpu identification for x86. */ #ifndef HOST_CPUINFO_H @@ -27,6 +27,7 @@ #define CPUINFO_ATOMIC_VMOVDQA (1u << 16) #define CPUINFO_ATOMIC_VMOVDQU (1u << 17) #define CPUINFO_AES (1u << 18) +#define CPUINFO_PCLMUL (1u << 19) /* Initialized with a constructor. */ extern unsigned cpuinfo; diff --git a/host/include/i386/host/crypto/clmul.h b/host/include/i386/host/crypto/clmul.h new file mode 100644 index 0000000000..dc3c814797 --- /dev/null +++ b/host/include/i386/host/crypto/clmul.h @@ -0,0 +1,29 @@ +/* + * x86 specific clmul acceleration. + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef X86_HOST_CRYPTO_CLMUL_H +#define X86_HOST_CRYPTO_CLMUL_H + +#include "host/cpuinfo.h" +#include <immintrin.h> + +#if defined(__PCLMUL__) +# define HAVE_CLMUL_ACCEL true +# define ATTR_CLMUL_ACCEL +#else +# define HAVE_CLMUL_ACCEL likely(cpuinfo & CPUINFO_PCLMUL) +# define ATTR_CLMUL_ACCEL __attribute__((target("pclmul"))) +#endif + +static inline Int128 ATTR_CLMUL_ACCEL +clmul_64_accel(uint64_t n, uint64_t m) +{ + union { __m128i v; Int128 s; } u; + + u.v = _mm_clmulepi64_si128(_mm_set_epi64x(0, n), _mm_set_epi64x(0, m), 0); + return u.s; +} + +#endif /* X86_HOST_CRYPTO_CLMUL_H */ diff --git a/host/include/ppc/host/cpuinfo.h b/host/include/ppc/host/cpuinfo.h index 29ee7f9ef8..38b8eabe2a 100644 --- a/host/include/ppc/host/cpuinfo.h +++ b/host/include/ppc/host/cpuinfo.h @@ -1,6 +1,6 @@ /* * SPDX-License-Identifier: GPL-2.0-or-later - * Host specific cpu indentification for ppc. + * Host specific cpu identification for ppc. */ #ifndef HOST_CPUINFO_H diff --git a/host/include/x86_64/host/crypto/clmul.h b/host/include/x86_64/host/crypto/clmul.h new file mode 100644 index 0000000000..f25eced416 --- /dev/null +++ b/host/include/x86_64/host/crypto/clmul.h @@ -0,0 +1 @@ +#include "host/include/i386/host/crypto/clmul.h" diff --git a/hw/9pfs/cofile.c b/hw/9pfs/cofile.c index 9c5344039e..71174c3e4a 100644 --- a/hw/9pfs/cofile.c +++ b/hw/9pfs/cofile.c @@ -252,7 +252,7 @@ int coroutine_fn v9fs_co_pwritev(V9fsPDU *pdu, V9fsFidState *fidp, if (v9fs_request_cancelled(pdu)) { return -EINTR; } - fsdev_co_throttle_request(s->ctx.fst, true, iov, iovcnt); + fsdev_co_throttle_request(s->ctx.fst, THROTTLE_WRITE, iov, iovcnt); v9fs_co_run_in_worker( { err = s->ops->pwritev(&s->ctx, &fidp->fs, iov, iovcnt, offset); @@ -272,7 +272,7 @@ int coroutine_fn v9fs_co_preadv(V9fsPDU *pdu, V9fsFidState *fidp, if (v9fs_request_cancelled(pdu)) { return -EINTR; } - fsdev_co_throttle_request(s->ctx.fst, false, iov, iovcnt); + fsdev_co_throttle_request(s->ctx.fst, THROTTLE_READ, iov, iovcnt); v9fs_co_run_in_worker( { err = s->ops->preadv(&s->ctx, &fidp->fs, iov, iovcnt, offset); diff --git a/hw/acpi/aml-build.c b/hw/acpi/aml-build.c index ea331a20d1..af66bde0f5 100644 --- a/hw/acpi/aml-build.c +++ b/hw/acpi/aml-build.c @@ -312,7 +312,7 @@ build_prepend_package_length(GArray *package, unsigned length, bool incl_self) /* * PkgLength is the length of the inclusive length of the data * and PkgLength's length itself when used for terms with - * explitit length. + * explicit length. */ length += length_bytes; } @@ -680,7 +680,7 @@ Aml *aml_store(Aml *val, Aml *target) * "Op Operand Operand Target" * pattern. * - * Returns: The newly allocated and composed according to patter Aml object. + * Returns: The newly allocated and composed according to pattern Aml object. */ static Aml * build_opcode_2arg_dst(uint8_t op, Aml *arg1, Aml *arg2, Aml *dst) @@ -2159,7 +2159,7 @@ void build_fadt(GArray *tbl, BIOSLinker *linker, const AcpiFadtData *f, /* FADT Minor Version */ build_append_int_noprefix(tbl, f->minor_ver, 1); } else { - build_append_int_noprefix(tbl, 0, 3); /* Reserved upto ACPI 5.0 */ + build_append_int_noprefix(tbl, 0, 3); /* Reserved up to ACPI 5.0 */ } build_append_int_noprefix(tbl, 0, 8); /* X_FIRMWARE_CTRL */ diff --git a/hw/acpi/hmat.c b/hw/acpi/hmat.c index 3a6d51282a..2d5e199ba9 100644 --- a/hw/acpi/hmat.c +++ b/hw/acpi/hmat.c @@ -82,7 +82,7 @@ static void build_hmat_lb(GArray *table_data, HMAT_LB_Info *hmat_lb, uint32_t base; /* Length in bytes for entire structure */ uint32_t lb_length - = 32 /* Table length upto and including Entry Base Unit */ + = 32 /* Table length up to and including Entry Base Unit */ + 4 * num_initiator /* Initiator Proximity Domain List */ + 4 * num_target /* Target Proximity Domain List */ + 2 * num_initiator * num_target; /* Latency or Bandwidth Entries */ diff --git a/hw/acpi/nvdimm.c b/hw/acpi/nvdimm.c index a3b25a92f3..9ba90806f2 100644 --- a/hw/acpi/nvdimm.c +++ b/hw/acpi/nvdimm.c @@ -670,7 +670,8 @@ static void nvdimm_dsm_label_size(NVDIMMDevice *nvdimm, hwaddr dsm_mem_addr) } static uint32_t nvdimm_rw_label_data_check(NVDIMMDevice *nvdimm, - uint32_t offset, uint32_t length) + uint32_t offset, uint32_t length, + bool is_write) { uint32_t ret = NVDIMM_DSM_RET_STATUS_INVALID; @@ -690,6 +691,10 @@ static uint32_t nvdimm_rw_label_data_check(NVDIMMDevice *nvdimm, return ret; } + if (is_write && nvdimm->readonly) { + return NVDIMM_DSM_RET_STATUS_UNSUPPORT; + } + return NVDIMM_DSM_RET_STATUS_SUCCESS; } @@ -713,7 +718,7 @@ static void nvdimm_dsm_get_label_data(NVDIMMDevice *nvdimm, NvdimmDsmIn *in, get_label_data->length); status = nvdimm_rw_label_data_check(nvdimm, get_label_data->offset, - get_label_data->length); + get_label_data->length, false); if (status != NVDIMM_DSM_RET_STATUS_SUCCESS) { nvdimm_dsm_no_payload(status, dsm_mem_addr); return; @@ -752,7 +757,7 @@ static void nvdimm_dsm_set_label_data(NVDIMMDevice *nvdimm, NvdimmDsmIn *in, set_label_data->length); status = nvdimm_rw_label_data_check(nvdimm, set_label_data->offset, - set_label_data->length); + set_label_data->length, true); if (status != NVDIMM_DSM_RET_STATUS_SUCCESS) { nvdimm_dsm_no_payload(status, dsm_mem_addr); return; @@ -1097,7 +1102,7 @@ static void nvdimm_build_common_dsm(Aml *dev, * be treated as an integer. Moreover, the integer size depends on * DSDT tables revision number. If revision number is < 2, integer * size is 32 bits, otherwise it is 64 bits. - * Because of this CreateField() canot be used if RLEN < Integer Size. + * Because of this CreateField() cannot be used if RLEN < Integer Size. * * Also please note that APCI ASL operator SizeOf() doesn't support * Integer and there isn't any other way to figure out the Integer diff --git a/hw/arm/boot.c b/hw/arm/boot.c index 720f22531a..24fa169060 100644 --- a/hw/arm/boot.c +++ b/hw/arm/boot.c @@ -761,6 +761,10 @@ static void do_cpu_reset(void *opaque) if (cpu_isar_feature(aa64_hcx, cpu)) { env->cp15.scr_el3 |= SCR_HXEN; } + if (cpu_isar_feature(aa64_fgt, cpu)) { + env->cp15.scr_el3 |= SCR_FGTEN; + } + /* AArch64 kernels never boot in secure mode */ assert(!info->secure_boot); /* This hook is only supported for AArch32 currently: diff --git a/hw/arm/sbsa-ref.c b/hw/arm/sbsa-ref.c index bc89eb4806..3c7dfcd6dc 100644 --- a/hw/arm/sbsa-ref.c +++ b/hw/arm/sbsa-ref.c @@ -61,6 +61,7 @@ #define ARCH_TIMER_S_EL1_IRQ 13 #define ARCH_TIMER_NS_EL1_IRQ 14 #define ARCH_TIMER_NS_EL2_IRQ 10 +#define ARCH_TIMER_NS_EL2_VIRT_IRQ 12 enum { SBSA_FLASH, @@ -489,6 +490,7 @@ static void create_gic(SBSAMachineState *sms, MemoryRegion *mem) [GTIMER_VIRT] = ARCH_TIMER_VIRT_IRQ, [GTIMER_HYP] = ARCH_TIMER_NS_EL2_IRQ, [GTIMER_SEC] = ARCH_TIMER_S_EL1_IRQ, + [GTIMER_HYPVIRT] = ARCH_TIMER_NS_EL2_VIRT_IRQ, }; for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) { diff --git a/hw/block/hd-geometry.c b/hw/block/hd-geometry.c index dae13ab14d..2b0af4430f 100644 --- a/hw/block/hd-geometry.c +++ b/hw/block/hd-geometry.c @@ -50,7 +50,7 @@ struct partition { uint32_t nr_sects; /* nr of sectors in partition */ } QEMU_PACKED; -/* try to guess the disk logical geometry from the MSDOS partition table. +/* try to guess the disk logical geometry from the MS-DOS partition table. Return 0 if OK, -1 if could not guess */ static int guess_disk_lchs(BlockBackend *blk, int *pcylinders, int *pheads, int *psectors) @@ -66,7 +66,7 @@ static int guess_disk_lchs(BlockBackend *blk, if (blk_pread(blk, 0, BDRV_SECTOR_SIZE, buf, 0) < 0) { return -1; } - /* test msdos magic */ + /* test MS-DOS magic */ if (buf[510] != 0x55 || buf[511] != 0xaa) { return -1; } diff --git a/hw/block/pflash_cfi01.c b/hw/block/pflash_cfi01.c index 3c066e3405..62056b1d74 100644 --- a/hw/block/pflash_cfi01.c +++ b/hw/block/pflash_cfi01.c @@ -891,7 +891,7 @@ static Property pflash_cfi01_properties[] = { /* num-blocks is the number of blocks actually visible to the guest, * ie the total size of the device divided by the sector length. * If we're emulating flash devices wired in parallel the actual - * number of blocks per indvidual device will differ. + * number of blocks per individual device will differ. */ DEFINE_PROP_UINT32("num-blocks", PFlashCFI01, nb_blocs, 0), DEFINE_PROP_UINT64("sector-length", PFlashCFI01, sector_len, 0), diff --git a/hw/char/cadence_uart.c b/hw/char/cadence_uart.c index eff0304a18..a2ac062b1e 100644 --- a/hw/char/cadence_uart.c +++ b/hw/char/cadence_uart.c @@ -575,7 +575,7 @@ static int cadence_uart_pre_load(void *opaque) { CadenceUARTState *s = opaque; - /* the frequency will be overriden if the refclk field is present */ + /* the frequency will be overridden if the refclk field is present */ clock_set_hz(s->refclk, UART_DEFAULT_REF_CLK); return 0; } diff --git a/hw/char/imx_serial.c b/hw/char/imx_serial.c index 1b75a89588..377d1d9773 100644 --- a/hw/char/imx_serial.c +++ b/hw/char/imx_serial.c @@ -112,7 +112,7 @@ static void imx_serial_reset_at_boot(DeviceState *dev) imx_serial_reset(s); /* - * enable the uart on boot, so messages from the linux decompresser + * enable the uart on boot, so messages from the linux decompressor * are visible. On real hardware this is done by the boot rom * before anything else is loaded. */ diff --git a/hw/char/serial.c b/hw/char/serial.c index f3094f860f..a32eb25f58 100644 --- a/hw/char/serial.c +++ b/hw/char/serial.c @@ -54,7 +54,7 @@ #define UART_IIR_RLSI 0x06 /* Receiver line status interrupt */ #define UART_IIR_CTI 0x0C /* Character Timeout Indication */ -#define UART_IIR_FENF 0x80 /* Fifo enabled, but not functionning */ +#define UART_IIR_FENF 0x80 /* Fifo enabled, but not functioning */ #define UART_IIR_FE 0xC0 /* Fifo enabled */ /* diff --git a/hw/core/generic-loader.c b/hw/core/generic-loader.c index 4f4d77908d..d4b5c501d8 100644 --- a/hw/core/generic-loader.c +++ b/hw/core/generic-loader.c @@ -24,7 +24,7 @@ * callback that does the memory operations. * This device allows the user to monkey patch memory. To be able to do - * this it needs a backend to manage the datas, the same as other + * this it needs a backend to manage the data, the same as other * memory-related devices. In this case as the backend is so trivial we * have merged it with the frontend instead of creating and maintaining a * separate backend. @@ -166,7 +166,7 @@ static void generic_loader_realize(DeviceState *dev, Error **errp) } } - /* Convert the data endiannes */ + /* Convert the data endianness */ if (s->data_be) { s->data = cpu_to_be64(s->data); } else { diff --git a/hw/core/machine.c b/hw/core/machine.c index da699cf4e1..cb38b8cf4c 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -38,6 +38,7 @@ #include "exec/confidential-guest-support.h" #include "hw/virtio/virtio.h" #include "hw/virtio/virtio-pci.h" +#include "hw/virtio/virtio-net.h" GlobalProperty hw_compat_8_1[] = {}; const size_t hw_compat_8_1_len = G_N_ELEMENTS(hw_compat_8_1); @@ -45,6 +46,9 @@ const size_t hw_compat_8_1_len = G_N_ELEMENTS(hw_compat_8_1); GlobalProperty hw_compat_8_0[] = { { "migration", "multifd-flush-after-each-section", "on"}, { TYPE_PCI_DEVICE, "x-pcie-ari-nextfn-1", "on" }, + { TYPE_VIRTIO_NET, "host_uso", "off"}, + { TYPE_VIRTIO_NET, "guest_uso4", "off"}, + { TYPE_VIRTIO_NET, "guest_uso6", "off"}, }; const size_t hw_compat_8_0_len = G_N_ELEMENTS(hw_compat_8_0); @@ -1355,6 +1359,7 @@ out: void machine_run_board_init(MachineState *machine, const char *mem_path, Error **errp) { + ERRP_GUARD(); MachineClass *machine_class = MACHINE_GET_CLASS(machine); ObjectClass *oc = object_class_by_name(machine->cpu_type); CPUClass *cc; @@ -1383,9 +1388,13 @@ void machine_run_board_init(MachineState *machine, const char *mem_path, Error * numa_uses_legacy_mem()) { if (object_property_find(object_get_objects_root(), machine_class->default_ram_id)) { - error_setg(errp, "object name '%s' is reserved for the default" - " RAM backend, it can't be used for any other purposes." - " Change the object's 'id' to something else", + error_setg(errp, "object's id '%s' is reserved for the default" + " RAM backend, it can't be used for any other purposes", + machine_class->default_ram_id); + error_append_hint(errp, + "Change the object's 'id' to something else or disable" + " automatic creation of the default RAM backend by setting" + " 'memory-backend=%s' with '-machine'.\n", machine_class->default_ram_id); return; } @@ -1417,7 +1426,7 @@ void machine_run_board_init(MachineState *machine, const char *mem_path, Error * for (i = 0; machine_class->valid_cpu_types[i]; i++) { if (object_class_dynamic_cast(oc, machine_class->valid_cpu_types[i])) { - /* The user specificed CPU is in the valid field, we are + /* The user specified CPU is in the valid field, we are * good to go. */ break; diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c index 6d5d43eda2..41b7e682c7 100644 --- a/hw/core/qdev-properties-system.c +++ b/hw/core/qdev-properties-system.c @@ -107,7 +107,7 @@ static void set_drive_helper(Object *obj, Visitor *v, const char *name, } if (*ptr) { - /* BlockBackend alread exists. So, we want to change attached node */ + /* BlockBackend already exists. So, we want to change attached node */ blk = *ptr; ctx = blk_get_aio_context(blk); bs = bdrv_lookup_bs(NULL, str, errp); diff --git a/hw/cpu/a15mpcore.c b/hw/cpu/a15mpcore.c index 774ca9987a..bfd8aa5644 100644 --- a/hw/cpu/a15mpcore.c +++ b/hw/cpu/a15mpcore.c @@ -161,7 +161,7 @@ static void a15mp_priv_class_init(ObjectClass *klass, void *data) dc->realize = a15mp_priv_realize; device_class_set_props(dc, a15mp_priv_properties); - /* We currently have no savable state */ + /* We currently have no saveable state */ } static const TypeInfo a15mp_priv_info = { diff --git a/hw/cxl/cxl-events.c b/hw/cxl/cxl-events.c index d161d57456..3ddd6369ad 100644 --- a/hw/cxl/cxl-events.c +++ b/hw/cxl/cxl-events.c @@ -197,7 +197,7 @@ CXLRetCode cxl_event_clear_records(CXLDeviceState *cxlds, CXLClearEventPayload * QEMU_LOCK_GUARD(&log->lock); /* - * Must itterate the queue twice. + * Must iterate the queue twice. * "The device shall verify the event record handles specified in the input * payload are in temporal order. If the device detects an older event * record that will not be cleared when Clear Event Records is executed, diff --git a/hw/cxl/cxl-host.c b/hw/cxl/cxl-host.c index 034c7805b3..f0920da956 100644 --- a/hw/cxl/cxl-host.c +++ b/hw/cxl/cxl-host.c @@ -39,12 +39,6 @@ static void cxl_fixed_memory_window_config(CXLState *cxl_state, return; } - fw->targets = g_malloc0_n(fw->num_targets, sizeof(*fw->targets)); - for (i = 0, target = object->targets; target; i++, target = target->next) { - /* This link cannot be resolved yet, so stash the name for now */ - fw->targets[i] = g_strdup(target->value); - } - if (object->size % (256 * MiB)) { error_setg(errp, "Size of a CXL fixed memory window must be a multiple of 256MiB"); @@ -64,6 +58,12 @@ static void cxl_fixed_memory_window_config(CXLState *cxl_state, fw->enc_int_gran = 0; } + fw->targets = g_malloc0_n(fw->num_targets, sizeof(*fw->targets)); + for (i = 0, target = object->targets; target; i++, target = target->next) { + /* This link cannot be resolved yet, so stash the name for now */ + fw->targets[i] = g_strdup(target->value); + } + cxl_state->fixed_windows = g_list_append(cxl_state->fixed_windows, g_steal_pointer(&fw)); diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c index 02f9b5a870..434ccc5f6e 100644 --- a/hw/cxl/cxl-mailbox-utils.c +++ b/hw/cxl/cxl-mailbox-utils.c @@ -39,7 +39,7 @@ * fill the output data into cmd->payload (overwriting what was there), * setting the length, and returning a valid return code. * - * XXX: The handler need not worry about endianess. The payload is read out of + * XXX: The handler need not worry about endianness. The payload is read out of * a register interface that already deals with it. */ @@ -501,7 +501,7 @@ static CXLRetCode cmd_media_get_poison_list(struct cxl_cmd *cmd, uint16_t out_pl_len; query_start = ldq_le_p(&in->pa); - /* 64 byte alignemnt required */ + /* 64 byte alignment required */ if (query_start & 0x3f) { return CXL_MBOX_INVALID_INPUT; } diff --git a/hw/dma/omap_dma.c b/hw/dma/omap_dma.c index c6e35ba4b8..77797a67b5 100644 --- a/hw/dma/omap_dma.c +++ b/hw/dma/omap_dma.c @@ -247,7 +247,7 @@ static void omap_dma_deactivate_channel(struct omap_dma_s *s, return; } - /* Don't deactive the channel if it is synchronized and the DMA request is + /* Don't deactivate the channel if it is synchronized and the DMA request is active */ if (ch->sync && ch->enable && (s->dma->drqbmp & (1ULL << ch->sync))) return; @@ -422,7 +422,7 @@ static void omap_dma_transfer_generic(struct soc_dma_ch_s *dma) if (ch->fs && ch->bs) { a->pck_element ++; - /* Check if a full packet has beed transferred. */ + /* Check if a full packet has been transferred. */ if (a->pck_element == a->pck_elements) { a->pck_element = 0; diff --git a/hw/hppa/machine.c b/hw/hppa/machine.c index 866e11d208..cf28cb9586 100644 --- a/hw/hppa/machine.c +++ b/hw/hppa/machine.c @@ -133,14 +133,10 @@ static FWCfgState *create_fw_cfg(MachineState *ms) fw_cfg_add_file(fw_cfg, "/etc/firmware-min-version", g_memdup(&val, sizeof(val)), sizeof(val)); - val = cpu_to_le64(HPPA_TLB_ENTRIES); + val = cpu_to_le64(HPPA_TLB_ENTRIES - HPPA_BTLB_ENTRIES); fw_cfg_add_file(fw_cfg, "/etc/cpu/tlb_entries", g_memdup(&val, sizeof(val)), sizeof(val)); - val = cpu_to_le64(HPPA_BTLB_ENTRIES); - fw_cfg_add_file(fw_cfg, "/etc/cpu/btlb_entries", - g_memdup(&val, sizeof(val)), sizeof(val)); - val = cpu_to_le64(HPA_POWER_BUTTON); fw_cfg_add_file(fw_cfg, "/etc/power-button-addr", g_memdup(&val, sizeof(val)), sizeof(val)); @@ -433,6 +429,10 @@ static void hppa_machine_reset(MachineState *ms, ShutdownCause reason) cs->exception_index = -1; cs->halted = 0; + + /* clear any existing TLB and BTLB entries */ + memset(cpu[i]->env.tlb, 0, sizeof(cpu[i]->env.tlb)); + cpu[i]->env.tlb_last = HPPA_BTLB_ENTRIES; } /* already initialized by machine_hppa_init()? */ diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index bb12b0ad43..4d2d40bab5 100644 --- a/hw/i386/acpi-build.c +++ b/hw/i386/acpi-build.c @@ -779,7 +779,7 @@ static Aml *initialize_route(Aml *route, const char *link_name, * * Returns an array of 128 routes, one for each device, * based on device location. - * The main goal is to equaly distribute the interrupts + * The main goal is to equally distribute the interrupts * over the 4 existing ACPI links (works only for i440fx). * The hash function is (slot + pin) & 3 -> "LNK[D|A|B|C]". * @@ -2079,7 +2079,7 @@ build_srat(GArray *table_data, BIOSLinker *linker, MachineState *machine) } /* - * Insert DMAR scope for PCI bridges and endpoint devcie + * Insert DMAR scope for PCI bridges and endpoint devices */ static void insert_scope(PCIBus *bus, PCIDevice *dev, void *opaque) diff --git a/hw/i386/amd_iommu.c b/hw/i386/amd_iommu.c index 9c77304438..c98a3c6e11 100644 --- a/hw/i386/amd_iommu.c +++ b/hw/i386/amd_iommu.c @@ -259,7 +259,7 @@ static void amdvi_log_command_error(AMDVIState *s, hwaddr addr) pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS, PCI_STATUS_SIG_TARGET_ABORT); } -/* log an illegal comand event +/* log an illegal command event * @addr : address of illegal command */ static void amdvi_log_illegalcom_error(AMDVIState *s, uint16_t info, @@ -767,7 +767,7 @@ static void amdvi_mmio_write(void *opaque, hwaddr addr, uint64_t val, break; case AMDVI_MMIO_COMMAND_BASE: amdvi_mmio_reg_write(s, size, val, addr); - /* FIXME - make sure System Software has finished writing incase + /* FIXME - make sure System Software has finished writing in case * it writes in chucks less than 8 bytes in a robust way.As for * now, this hacks works for the linux driver */ diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c index c9961ef752..c0ce896668 100644 --- a/hw/i386/intel_iommu.c +++ b/hw/i386/intel_iommu.c @@ -52,7 +52,7 @@ /* * PCI bus number (or SID) is not reliable since the device is usaully - * initalized before guest can configure the PCI bridge + * initialized before guest can configure the PCI bridge * (SECONDARY_BUS_NUMBER). */ struct vtd_as_key { @@ -1694,7 +1694,7 @@ static bool vtd_switch_address_space(VTDAddressSpace *as) * """ * * We enable per as memory region (iommu_ir_fault) for catching - * the tranlsation for interrupt range through PASID + PT. + * the translation for interrupt range through PASID + PT. */ if (pt && as->pasid != PCI_NO_PASID) { memory_region_set_enabled(&as->iommu_ir_fault, true); diff --git a/hw/i386/kvm/xen_xenstore.c b/hw/i386/kvm/xen_xenstore.c index 133d89e953..660d0b72f9 100644 --- a/hw/i386/kvm/xen_xenstore.c +++ b/hw/i386/kvm/xen_xenstore.c @@ -1156,7 +1156,7 @@ static unsigned int copy_to_ring(XenXenstoreState *s, uint8_t *ptr, /* * This matches the barrier in copy_to_ring() (or the guest's - * equivalent) betweem writing the data to the ring and updating + * equivalent) between writing the data to the ring and updating * rsp_prod. It protects against the pathological case (which * again I think never happened except on Alpha) where our * subsequent writes to the ring could *cross* the read of diff --git a/hw/i386/kvm/xenstore_impl.c b/hw/i386/kvm/xenstore_impl.c index d9732b567e..1d134a6866 100644 --- a/hw/i386/kvm/xenstore_impl.c +++ b/hw/i386/kvm/xenstore_impl.c @@ -1436,7 +1436,7 @@ static void save_node(gpointer key, gpointer value, gpointer opaque) /* * If we already wrote this node, refer to the previous copy. * There's no rename/move in XenStore, so all we need to find - * it is the tx_id of the transation in which it exists. Which + * it is the tx_id of the transaction in which it exists. Which * may be the root tx. */ if (n->serialized_tx != XBT_NULL) { diff --git a/hw/i386/pc.c b/hw/i386/pc.c index 54838c0c41..3db0743f31 100644 --- a/hw/i386/pc.c +++ b/hw/i386/pc.c @@ -436,7 +436,7 @@ static uint64_t ioport80_read(void *opaque, hwaddr addr, unsigned size) return 0xffffffffffffffffULL; } -/* MSDOS compatibility mode FPU exception support */ +/* MS-DOS compatibility mode FPU exception support */ static void ioportF0_write(void *opaque, hwaddr addr, uint64_t data, unsigned size) { @@ -1746,16 +1746,16 @@ static void pc_machine_set_max_fw_size(Object *obj, Visitor *v, } /* - * We don't have a theoretically justifiable exact lower bound on the base - * address of any flash mapping. In practice, the IO-APIC MMIO range is - * [0xFEE00000..0xFEE01000] -- see IO_APIC_DEFAULT_ADDRESS --, leaving free - * only 18MB-4KB below 4G. For now, restrict the cumulative mapping to 8MB in - * size. - */ + * We don't have a theoretically justifiable exact lower bound on the base + * address of any flash mapping. In practice, the IO-APIC MMIO range is + * [0xFEE00000..0xFEE01000] -- see IO_APIC_DEFAULT_ADDRESS --, leaving free + * only 18MiB-4KiB below 4GiB. For now, restrict the cumulative mapping to + * 16MiB in size. + */ if (value > 16 * MiB) { error_setg(errp, "User specified max allowed firmware size %" PRIu64 " is " - "greater than 16MiB. If combined firwmare size exceeds " + "greater than 16MiB. If combined firmware size exceeds " "16MiB the system may not boot, or experience intermittent" "stability issues.", value); diff --git a/hw/input/hid.c b/hw/input/hid.c index e7ecebdf8f..a9c7dd1ce1 100644 --- a/hw/input/hid.c +++ b/hw/input/hid.c @@ -209,7 +209,7 @@ static void hid_pointer_sync(DeviceState *dev) prev->dz += curr->dz; curr->dz = 0; } else { - /* prepate next (clear rel, copy abs + btns) */ + /* prepare next (clear rel, copy abs + btns) */ if (hs->kind == HID_MOUSE) { next->xdx = 0; next->ydy = 0; diff --git a/hw/input/tsc2005.c b/hw/input/tsc2005.c index 555b677173..db2b80e35f 100644 --- a/hw/input/tsc2005.c +++ b/hw/input/tsc2005.c @@ -157,14 +157,14 @@ static uint16_t tsc2005_read(TSC2005State *s, int reg) s->reset = true; return ret; - case 0x8: /* AUX high treshold */ + case 0x8: /* AUX high threshold */ return s->aux_thr[1]; - case 0x9: /* AUX low treshold */ + case 0x9: /* AUX low threshold */ return s->aux_thr[0]; - case 0xa: /* TEMP high treshold */ + case 0xa: /* TEMP high threshold */ return s->temp_thr[1]; - case 0xb: /* TEMP low treshold */ + case 0xb: /* TEMP low threshold */ return s->temp_thr[0]; case 0xc: /* CFR0 */ @@ -186,17 +186,17 @@ static uint16_t tsc2005_read(TSC2005State *s, int reg) static void tsc2005_write(TSC2005State *s, int reg, uint16_t data) { switch (reg) { - case 0x8: /* AUX high treshold */ + case 0x8: /* AUX high threshold */ s->aux_thr[1] = data; break; - case 0x9: /* AUX low treshold */ + case 0x9: /* AUX low threshold */ s->aux_thr[0] = data; break; - case 0xa: /* TEMP high treshold */ + case 0xa: /* TEMP high threshold */ s->temp_thr[1] = data; break; - case 0xb: /* TEMP low treshold */ + case 0xb: /* TEMP low threshold */ s->temp_thr[0] = data; break; diff --git a/hw/intc/loongarch_extioi.c b/hw/intc/loongarch_extioi.c index af75460643..24fb3af8cc 100644 --- a/hw/intc/loongarch_extioi.c +++ b/hw/intc/loongarch_extioi.c @@ -191,7 +191,7 @@ static MemTxResult extioi_writew(void *opaque, hwaddr addr, cpu = attrs.requester_id; old_data = s->coreisr[cpu][index]; s->coreisr[cpu][index] = old_data & ~val; - /* write 1 to clear interrrupt */ + /* write 1 to clear interrupt */ old_data &= val; irq = ctz32(old_data); while (irq != 32) { diff --git a/hw/intc/loongson_liointc.c b/hw/intc/loongson_liointc.c index cc11b544cb..c10fb97a06 100644 --- a/hw/intc/loongson_liointc.c +++ b/hw/intc/loongson_liointc.c @@ -1,5 +1,5 @@ /* - * QEMU Loongson Local I/O interrupt controler. + * QEMU Loongson Local I/O interrupt controller. * * Copyright (c) 2020 Huacai Chen <chenhc@lemote.com> * Copyright (c) 2020 Jiaxun Yang <jiaxun.yang@flygoat.com> diff --git a/hw/intc/omap_intc.c b/hw/intc/omap_intc.c index 647bf324a8..435c47600f 100644 --- a/hw/intc/omap_intc.c +++ b/hw/intc/omap_intc.c @@ -68,7 +68,7 @@ static void omap_inth_sir_update(OMAPIntcState *s, int is_fiq) p_intr = 255; /* Find the interrupt line with the highest dynamic priority. - * Note: 0 denotes the hightest priority. + * Note: 0 denotes the highest priority. * If all interrupts have the same priority, the default order is IRQ_N, * IRQ_N-1,...,IRQ_0. */ for (j = 0; j < s->nbanks; ++j) { diff --git a/hw/intc/pnv_xive.c b/hw/intc/pnv_xive.c index 9b10e90519..da10deceb8 100644 --- a/hw/intc/pnv_xive.c +++ b/hw/intc/pnv_xive.c @@ -210,7 +210,7 @@ static uint64_t pnv_xive_vst_addr_remote(PnvXive *xive, uint32_t type, return 0; } - remote_addr |= idx << xive->pc_shift; + remote_addr |= ((uint64_t)idx) << xive->pc_shift; vst_addr = address_space_ldq_be(&address_space_memory, remote_addr, MEMTXATTRS_UNSPECIFIED, &result); @@ -988,7 +988,7 @@ static void pnv_xive_ic_reg_write(void *opaque, hwaddr offset, */ case VC_SBC_CONFIG: /* Store EOI configuration */ /* - * Configure store EOI if required by firwmare (skiboot has removed + * Configure store EOI if required by firmware (skiboot has removed * support recently though) */ if (val & (VC_SBC_CONF_CPLX_CIST | VC_SBC_CONF_CIST_BOTH)) { diff --git a/hw/intc/spapr_xive.c b/hw/intc/spapr_xive.c index 8bcab2846c..7f701d414b 100644 --- a/hw/intc/spapr_xive.c +++ b/hw/intc/spapr_xive.c @@ -27,7 +27,7 @@ #include "trace.h" /* - * XIVE Virtualization Controller BAR and Thread Managment BAR that we + * XIVE Virtualization Controller BAR and Thread Management BAR that we * use for the ESB pages and the TIMA pages */ #define SPAPR_XIVE_VC_BASE 0x0006010000000000ull diff --git a/hw/intc/spapr_xive_kvm.c b/hw/intc/spapr_xive_kvm.c index 61fe7bd2d3..5789062379 100644 --- a/hw/intc/spapr_xive_kvm.c +++ b/hw/intc/spapr_xive_kvm.c @@ -485,7 +485,7 @@ static int kvmppc_xive_get_queues(SpaprXive *xive, Error **errp) * * Whenever the VM is stopped, the VM change handler sets the source * PQs to PENDING to stop the flow of events and to possibly catch a - * triggered interrupt occuring while the VM is stopped. The previous + * triggered interrupt occurring while the VM is stopped. The previous * state is saved in anticipation of a migration. The XIVE controller * is then synced through KVM to flush any in-flight event * notification and stabilize the EQs. @@ -551,7 +551,7 @@ static void kvmppc_xive_change_state_handler(void *opaque, bool running, /* * PQ is set to PENDING to possibly catch a triggered - * interrupt occuring while the VM is stopped (hotplug event + * interrupt occurring while the VM is stopped (hotplug event * for instance) . */ if (pq != XIVE_ESB_OFF) { @@ -633,7 +633,7 @@ int kvmppc_xive_post_load(SpaprXive *xive, int version_id) /* The KVM XIVE device should be in use */ assert(xive->fd != -1); - /* Restore the ENDT first. The targetting depends on it. */ + /* Restore the ENDT first. The targeting depends on it. */ for (i = 0; i < xive->nr_ends; i++) { if (!xive_end_is_valid(&xive->endt[i])) { continue; diff --git a/hw/intc/xive.c b/hw/intc/xive.c index df3ee0496f..a3585593d8 100644 --- a/hw/intc/xive.c +++ b/hw/intc/xive.c @@ -1608,7 +1608,7 @@ int xive_presenter_tctx_match(XivePresenter *xptr, XiveTCTX *tctx, * * It receives notification requests sent by the IVRE to find one * matching NVT (or more) dispatched on the processor threads. In case - * of a single NVT notification, the process is abreviated and the + * of a single NVT notification, the process is abbreviated and the * thread is signaled if a match is found. In case of a logical server * notification (bits ignored at the end of the NVT identifier), the * IVPE and IVRE select a winning thread using different filters. This diff --git a/hw/intc/xive2.c b/hw/intc/xive2.c index c37ef25d44..98c0d8ba44 100644 --- a/hw/intc/xive2.c +++ b/hw/intc/xive2.c @@ -542,7 +542,7 @@ static void xive2_router_realize(DeviceState *dev, Error **errp) /* * Notification using the END ESe/ESn bit (Event State Buffer for - * escalation and notification). Profide futher coalescing in the + * escalation and notification). Profide further coalescing in the * Router. */ static bool xive2_router_end_es_notify(Xive2Router *xrtr, uint8_t end_blk, @@ -621,7 +621,7 @@ static void xive2_router_end_notify(Xive2Router *xrtr, uint8_t end_blk, /* * Check the END ESn (Event State Buffer for notification) for - * even futher coalescing in the Router + * even further coalescing in the Router */ if (!xive2_end_is_notify(&end)) { /* ESn[Q]=1 : end of notification */ @@ -702,7 +702,7 @@ do_escalation: /* * Check the END ESe (Event State Buffer for escalation) for even - * futher coalescing in the Router + * further coalescing in the Router */ if (!xive2_end_is_uncond_escalation(&end)) { /* ESe[Q]=1 : end of escalation notification */ diff --git a/hw/ipmi/ipmi_bmc_extern.c b/hw/ipmi/ipmi_bmc_extern.c index acf2bab35f..e232d35ba2 100644 --- a/hw/ipmi/ipmi_bmc_extern.c +++ b/hw/ipmi/ipmi_bmc_extern.c @@ -301,7 +301,7 @@ static void handle_msg(IPMIBmcExtern *ibe) ipmi_debug("msg checksum failure\n"); return; } else { - ibe->inpos--; /* Remove checkum */ + ibe->inpos--; /* Remove checksum */ } timer_del(ibe->extern_timer); diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c index 4e314748d3..4cdcb3f7e7 100644 --- a/hw/mem/cxl_type3.c +++ b/hw/mem/cxl_type3.c @@ -1,3 +1,14 @@ +/* + * CXL Type 3 (memory expander) device + * + * Copyright(C) 2020 Intel Corporation. + * + * This work is licensed under the terms of the GNU GPL, version 2. See the + * COPYING file in the top-level directory. + * + * SPDX-License-Identifier: GPL-v2-only + */ + #include "qemu/osdep.h" #include "qemu/units.h" #include "qemu/error-report.h" @@ -538,7 +549,7 @@ static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value, FIRST_ERROR_POINTER, cxl_err->type); } else { /* - * If no more errors, then follow recomendation of PCI spec + * If no more errors, then follow recommendation of PCI spec * r6.0 6.2.4.2 to set the first error pointer to a status * bit that will never be used. */ @@ -697,7 +708,7 @@ static void ct3_realize(PCIDevice *pci_dev, Error **errp) PCI_BASE_ADDRESS_MEM_TYPE_64, &ct3d->cxl_dstate.device_registers); - /* MSI(-X) Initailization */ + /* MSI(-X) Initialization */ rc = msix_init_exclusive_bar(pci_dev, msix_num, 4, NULL); if (rc) { goto err_address_space_free; @@ -706,7 +717,7 @@ static void ct3_realize(PCIDevice *pci_dev, Error **errp) msix_vector_use(pci_dev, i); } - /* DOE Initailization */ + /* DOE Initialization */ pcie_doe_init(pci_dev, &ct3d->doe_cdat, 0x190, doe_cdat_prot, true, 0); cxl_cstate->cdat.build_cdat_table = ct3_build_cdat_table; diff --git a/hw/mem/cxl_type3_stubs.c b/hw/mem/cxl_type3_stubs.c index f3e4a9fa72..8ba5d3d1f7 100644 --- a/hw/mem/cxl_type3_stubs.c +++ b/hw/mem/cxl_type3_stubs.c @@ -1,3 +1,13 @@ +/* + * CXL Type 3 (memory expander) device QMP stubs + * + * Copyright(C) 2020 Intel Corporation. + * + * This work is licensed under the terms of the GNU GPL, version 2. See the + * COPYING file in the top-level directory. + * + * SPDX-License-Identifier: GPL-v2-only + */ #include "qemu/osdep.h" #include "qapi/error.h" diff --git a/hw/mem/nvdimm.c b/hw/mem/nvdimm.c index 31080c22c9..1631a7d13f 100644 --- a/hw/mem/nvdimm.c +++ b/hw/mem/nvdimm.c @@ -154,6 +154,9 @@ static void nvdimm_prepare_memory_region(NVDIMMDevice *nvdimm, Error **errp) object_get_canonical_path_component(OBJECT(hostmem))); return; } + if (memory_region_is_rom(mr)) { + nvdimm->readonly = true; + } nvdimm->nvdimm_mr = g_new(MemoryRegion, 1); memory_region_init_alias(nvdimm->nvdimm_mr, OBJECT(dimm), @@ -207,15 +210,16 @@ static void nvdimm_unrealize(PCDIMMDevice *dimm) * label read/write functions. */ static void nvdimm_validate_rw_label_data(NVDIMMDevice *nvdimm, uint64_t size, - uint64_t offset) + uint64_t offset, bool is_write) { assert((nvdimm->label_size >= size + offset) && (offset + size > offset)); + assert(!is_write || !nvdimm->readonly); } static void nvdimm_read_label_data(NVDIMMDevice *nvdimm, void *buf, uint64_t size, uint64_t offset) { - nvdimm_validate_rw_label_data(nvdimm, size, offset); + nvdimm_validate_rw_label_data(nvdimm, size, offset, false); memcpy(buf, nvdimm->label_data + offset, size); } @@ -229,7 +233,7 @@ static void nvdimm_write_label_data(NVDIMMDevice *nvdimm, const void *buf, "pmem", NULL); uint64_t backend_offset; - nvdimm_validate_rw_label_data(nvdimm, size, offset); + nvdimm_validate_rw_label_data(nvdimm, size, offset, true); if (!is_pmem) { memcpy(nvdimm->label_data + offset, buf, size); diff --git a/hw/misc/imx7_ccm.c b/hw/misc/imx7_ccm.c index f135ec7b7e..7539f7fb45 100644 --- a/hw/misc/imx7_ccm.c +++ b/hw/misc/imx7_ccm.c @@ -227,7 +227,7 @@ static uint32_t imx7_ccm_get_clock_frequency(IMXCCMState *dev, IMXClk clock) * have fixed frequencies and we can provide requested frequency * easily. However for CCM provided clocks (like IPG) each GPT * timer can have its own clock root. - * This means we need additionnal information when calling this + * This means we need additional information when calling this * function to know the requester's identity. */ uint32_t freq = 0; diff --git a/hw/misc/mac_via.c b/hw/misc/mac_via.c index 0787a0268d..f84cc68849 100644 --- a/hw/misc/mac_via.c +++ b/hw/misc/mac_via.c @@ -246,7 +246,7 @@ #define vT2CL 0x1000 /* [VIA only] Timer two counter low. */ #define vT2CH 0x1200 /* [VIA only] Timer two counter high. */ #define vSR 0x1400 /* [VIA only] Shift register. */ -#define vACR 0x1600 /* [VIA only] Auxilary control register. */ +#define vACR 0x1600 /* [VIA only] Auxiliary control register. */ #define vPCR 0x1800 /* [VIA only] Peripheral control register. */ /* * CHRP sez never ever to *write* this. diff --git a/hw/misc/stm32f2xx_syscfg.c b/hw/misc/stm32f2xx_syscfg.c index 04c22c2850..19c1e86424 100644 --- a/hw/misc/stm32f2xx_syscfg.c +++ b/hw/misc/stm32f2xx_syscfg.c @@ -94,12 +94,12 @@ static void stm32f2xx_syscfg_write(void *opaque, hwaddr addr, switch (addr) { case SYSCFG_MEMRMP: qemu_log_mask(LOG_UNIMP, - "%s: Changeing the memory mapping isn't supported " \ + "%s: Changing the memory mapping isn't supported " \ "in QEMU\n", __func__); return; case SYSCFG_PMC: qemu_log_mask(LOG_UNIMP, - "%s: Changeing the memory mapping isn't supported " \ + "%s: Changing the memory mapping isn't supported " \ "in QEMU\n", __func__); return; case SYSCFG_EXTICR1: diff --git a/hw/misc/trace-events b/hw/misc/trace-events index e8b2be14c0..bc87cd3670 100644 --- a/hw/misc/trace-events +++ b/hw/misc/trace-events @@ -155,7 +155,7 @@ stm32f4xx_syscfg_read(uint64_t addr) "reg read: addr: 0x%" PRIx64 " " stm32f4xx_syscfg_write(uint64_t addr, uint64_t data) "reg write: addr: 0x%" PRIx64 " val: 0x%" PRIx64 "" # stm32f4xx_exti.c -stm32f4xx_exti_set_irq(int irq, int leve) "Set EXTI: %d to %d" +stm32f4xx_exti_set_irq(int irq, int level) "Set EXTI: %d to %d" stm32f4xx_exti_read(uint64_t addr) "reg read: addr: 0x%" PRIx64 " " stm32f4xx_exti_write(uint64_t addr, uint64_t data) "reg write: addr: 0x%" PRIx64 " val: 0x%" PRIx64 "" diff --git a/hw/misc/zynq_slcr.c b/hw/misc/zynq_slcr.c index 8b70285961..41f38a98e9 100644 --- a/hw/misc/zynq_slcr.c +++ b/hw/misc/zynq_slcr.c @@ -285,7 +285,7 @@ static void zynq_slcr_compute_clocks_internal(ZynqSLCRState *s, uint64_t ps_clk) } /** - * Compute and set the ouputs clocks periods. + * Compute and set the outputs clocks periods. * But do not propagate them further. Connected clocks * will not receive any updates (See zynq_slcr_compute_clocks()) */ diff --git a/hw/net/cadence_gem.c b/hw/net/cadence_gem.c index 42ea2411a2..f445d8bb5e 100644 --- a/hw/net/cadence_gem.c +++ b/hw/net/cadence_gem.c @@ -81,8 +81,8 @@ #define GEM_IPGSTRETCH (0x000000BC / 4) /* IPG Stretch reg */ #define GEM_SVLAN (0x000000C0 / 4) /* Stacked VLAN reg */ #define GEM_MODID (0x000000FC / 4) /* Module ID reg */ -#define GEM_OCTTXLO (0x00000100 / 4) /* Octects transmitted Low reg */ -#define GEM_OCTTXHI (0x00000104 / 4) /* Octects transmitted High reg */ +#define GEM_OCTTXLO (0x00000100 / 4) /* Octets transmitted Low reg */ +#define GEM_OCTTXHI (0x00000104 / 4) /* Octets transmitted High reg */ #define GEM_TXCNT (0x00000108 / 4) /* Error-free Frames transmitted */ #define GEM_TXBCNT (0x0000010C / 4) /* Error-free Broadcast Frames */ #define GEM_TXMCNT (0x00000110 / 4) /* Error-free Multicast Frame */ @@ -101,8 +101,8 @@ #define GEM_LATECOLLCNT (0x00000144 / 4) /* Late Collision Frames */ #define GEM_DEFERTXCNT (0x00000148 / 4) /* Deferred Transmission Frames */ #define GEM_CSENSECNT (0x0000014C / 4) /* Carrier Sense Error Counter */ -#define GEM_OCTRXLO (0x00000150 / 4) /* Octects Received register Low */ -#define GEM_OCTRXHI (0x00000154 / 4) /* Octects Received register High */ +#define GEM_OCTRXLO (0x00000150 / 4) /* Octets Received register Low */ +#define GEM_OCTRXHI (0x00000154 / 4) /* Octets Received register High */ #define GEM_RXCNT (0x00000158 / 4) /* Error-free Frames Received */ #define GEM_RXBROADCNT (0x0000015C / 4) /* Error-free Broadcast Frames RX */ #define GEM_RXMULTICNT (0x00000160 / 4) /* Error-free Multicast Frames RX */ @@ -954,7 +954,7 @@ static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size) /* Is this destination MAC address "for us" ? */ maf = gem_mac_address_filter(s, buf); if (maf == GEM_RX_REJECT) { - return size; /* no, drop siliently b/c it's not an error */ + return size; /* no, drop silently b/c it's not an error */ } /* Discard packets with receive length error enabled ? */ diff --git a/hw/net/dp8393x.c b/hw/net/dp8393x.c index a596f7fbc6..c6f5fb7dce 100644 --- a/hw/net/dp8393x.c +++ b/hw/net/dp8393x.c @@ -551,7 +551,7 @@ static uint64_t dp8393x_read(void *opaque, hwaddr addr, unsigned int size) val = s->cam[s->regs[SONIC_CEP] & 0xf][SONIC_CAP0 - reg]; } break; - /* All other registers have no special contraints */ + /* All other registers have no special constraints */ default: val = s->regs[reg]; } diff --git a/hw/net/e1000_regs.h b/hw/net/e1000_regs.h index 8a4ce82034..39f4882510 100644 --- a/hw/net/e1000_regs.h +++ b/hw/net/e1000_regs.h @@ -130,7 +130,7 @@ #define E1000_GCR2 0x05B64 /* 3GIO Control Register 2 */ #define E1000_FFLT_DBG 0x05F04 /* Debug Register */ -#define E1000_HICR 0x08F00 /* Host Inteface Control */ +#define E1000_HICR 0x08F00 /* Host Interface Control */ #define E1000_RXMTRL 0x0B634 /* Time sync Rx EtherType and Msg Type - RW */ #define E1000_RXUDP 0x0B638 /* Time Sync Rx UDP Port - RW */ diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c index f8aeafa16b..e324c02dd5 100644 --- a/hw/net/e1000e_core.c +++ b/hw/net/e1000e_core.c @@ -810,24 +810,24 @@ e1000e_txdesc_writeback(E1000ECore *core, dma_addr_t base, return e1000e_tx_wb_interrupt_cause(core, queue_idx); } -typedef struct E1000E_RingInfo_st { +typedef struct E1000ERingInfo { int dbah; int dbal; int dlen; int dh; int dt; int idx; -} E1000E_RingInfo; +} E1000ERingInfo; static inline bool -e1000e_ring_empty(E1000ECore *core, const E1000E_RingInfo *r) +e1000e_ring_empty(E1000ECore *core, const E1000ERingInfo *r) { return core->mac[r->dh] == core->mac[r->dt] || core->mac[r->dt] >= core->mac[r->dlen] / E1000_RING_DESC_LEN; } static inline uint64_t -e1000e_ring_base(E1000ECore *core, const E1000E_RingInfo *r) +e1000e_ring_base(E1000ECore *core, const E1000ERingInfo *r) { uint64_t bah = core->mac[r->dbah]; uint64_t bal = core->mac[r->dbal]; @@ -836,13 +836,13 @@ e1000e_ring_base(E1000ECore *core, const E1000E_RingInfo *r) } static inline uint64_t -e1000e_ring_head_descr(E1000ECore *core, const E1000E_RingInfo *r) +e1000e_ring_head_descr(E1000ECore *core, const E1000ERingInfo *r) { return e1000e_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh]; } static inline void -e1000e_ring_advance(E1000ECore *core, const E1000E_RingInfo *r, uint32_t count) +e1000e_ring_advance(E1000ECore *core, const E1000ERingInfo *r, uint32_t count) { core->mac[r->dh] += count; @@ -852,7 +852,7 @@ e1000e_ring_advance(E1000ECore *core, const E1000E_RingInfo *r, uint32_t count) } static inline uint32_t -e1000e_ring_free_descr_num(E1000ECore *core, const E1000E_RingInfo *r) +e1000e_ring_free_descr_num(E1000ECore *core, const E1000ERingInfo *r) { trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen], core->mac[r->dh], core->mac[r->dt]); @@ -871,19 +871,19 @@ e1000e_ring_free_descr_num(E1000ECore *core, const E1000E_RingInfo *r) } static inline bool -e1000e_ring_enabled(E1000ECore *core, const E1000E_RingInfo *r) +e1000e_ring_enabled(E1000ECore *core, const E1000ERingInfo *r) { return core->mac[r->dlen] > 0; } static inline uint32_t -e1000e_ring_len(E1000ECore *core, const E1000E_RingInfo *r) +e1000e_ring_len(E1000ECore *core, const E1000ERingInfo *r) { return core->mac[r->dlen]; } typedef struct E1000E_TxRing_st { - const E1000E_RingInfo *i; + const E1000ERingInfo *i; struct e1000e_tx *tx; } E1000E_TxRing; @@ -896,7 +896,7 @@ e1000e_mq_queue_idx(int base_reg_idx, int reg_idx) static inline void e1000e_tx_ring_init(E1000ECore *core, E1000E_TxRing *txr, int idx) { - static const E1000E_RingInfo i[E1000E_NUM_QUEUES] = { + static const E1000ERingInfo i[E1000E_NUM_QUEUES] = { { TDBAH, TDBAL, TDLEN, TDH, TDT, 0 }, { TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 } }; @@ -908,13 +908,13 @@ e1000e_tx_ring_init(E1000ECore *core, E1000E_TxRing *txr, int idx) } typedef struct E1000E_RxRing_st { - const E1000E_RingInfo *i; + const E1000ERingInfo *i; } E1000E_RxRing; static inline void e1000e_rx_ring_init(E1000ECore *core, E1000E_RxRing *rxr, int idx) { - static const E1000E_RingInfo i[E1000E_NUM_QUEUES] = { + static const E1000ERingInfo i[E1000E_NUM_QUEUES] = { { RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 }, { RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 } }; @@ -930,7 +930,7 @@ e1000e_start_xmit(E1000ECore *core, const E1000E_TxRing *txr) dma_addr_t base; struct e1000_tx_desc desc; bool ide = false; - const E1000E_RingInfo *txi = txr->i; + const E1000ERingInfo *txi = txr->i; uint32_t cause = E1000_ICS_TXQE; if (!(core->mac[TCTL] & E1000_TCTL_EN)) { @@ -960,7 +960,7 @@ e1000e_start_xmit(E1000ECore *core, const E1000E_TxRing *txr) } static bool -e1000e_has_rxbufs(E1000ECore *core, const E1000E_RingInfo *r, +e1000e_has_rxbufs(E1000ECore *core, const E1000ERingInfo *r, size_t total_size) { uint32_t bufs = e1000e_ring_free_descr_num(core, r); @@ -1397,17 +1397,17 @@ e1000e_pci_dma_write_rx_desc(E1000ECore *core, dma_addr_t addr, } } -typedef struct e1000e_ba_state_st { +typedef struct E1000EBAState { uint16_t written[MAX_PS_BUFFERS]; uint8_t cur_idx; -} e1000e_ba_state; +} E1000EBAState; static inline void -e1000e_write_hdr_to_rx_buffers(E1000ECore *core, - hwaddr ba[MAX_PS_BUFFERS], - e1000e_ba_state *bastate, - const char *data, - dma_addr_t data_len) +e1000e_write_hdr_frag_to_rx_buffers(E1000ECore *core, + hwaddr ba[MAX_PS_BUFFERS], + E1000EBAState *bastate, + const char *data, + dma_addr_t data_len) { assert(data_len <= core->rxbuf_sizes[0] - bastate->written[0]); @@ -1418,11 +1418,11 @@ e1000e_write_hdr_to_rx_buffers(E1000ECore *core, } static void -e1000e_write_to_rx_buffers(E1000ECore *core, - hwaddr ba[MAX_PS_BUFFERS], - e1000e_ba_state *bastate, - const char *data, - dma_addr_t data_len) +e1000e_write_payload_frag_to_rx_buffers(E1000ECore *core, + hwaddr ba[MAX_PS_BUFFERS], + E1000EBAState *bastate, + const char *data, + dma_addr_t data_len) { while (data_len > 0) { uint32_t cur_buf_len = core->rxbuf_sizes[bastate->cur_idx]; @@ -1460,7 +1460,7 @@ e1000e_update_rx_stats(E1000ECore *core, size_t pkt_size, size_t pkt_fcs_size) } static inline bool -e1000e_rx_descr_threshold_hit(E1000ECore *core, const E1000E_RingInfo *rxi) +e1000e_rx_descr_threshold_hit(E1000ECore *core, const E1000ERingInfo *rxi) { return e1000e_ring_free_descr_num(core, rxi) == e1000e_ring_len(core, rxi) >> core->rxbuf_min_shift; @@ -1521,7 +1521,7 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, struct iovec *iov = net_rx_pkt_get_iovec(pkt); size_t size = net_rx_pkt_get_total_len(pkt); size_t total_size = size + e1000x_fcs_len(core->mac); - const E1000E_RingInfo *rxi; + const E1000ERingInfo *rxi; size_t ps_hdr_len = 0; bool do_ps = e1000e_do_ps(core, pkt, &ps_hdr_len); bool is_first = true; @@ -1530,7 +1530,7 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, do { hwaddr ba[MAX_PS_BUFFERS]; - e1000e_ba_state bastate = { { 0 } }; + E1000EBAState bastate = { { 0 } }; bool is_last = false; desc_size = total_size - desc_offset; @@ -1568,8 +1568,10 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, iov_copy = MIN(ps_hdr_len - ps_hdr_copied, iov->iov_len - iov_ofs); - e1000e_write_hdr_to_rx_buffers(core, ba, &bastate, - iov->iov_base, iov_copy); + e1000e_write_hdr_frag_to_rx_buffers(core, ba, + &bastate, + iov->iov_base, + iov_copy); copy_size -= iov_copy; ps_hdr_copied += iov_copy; @@ -1585,8 +1587,8 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, } else { /* Leave buffer 0 of each descriptor except first */ /* empty as per spec 7.1.5.1 */ - e1000e_write_hdr_to_rx_buffers(core, ba, &bastate, - NULL, 0); + e1000e_write_hdr_frag_to_rx_buffers(core, ba, &bastate, + NULL, 0); } } @@ -1594,8 +1596,10 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, while (copy_size) { iov_copy = MIN(copy_size, iov->iov_len - iov_ofs); - e1000e_write_to_rx_buffers(core, ba, &bastate, - iov->iov_base + iov_ofs, iov_copy); + e1000e_write_payload_frag_to_rx_buffers(core, ba, &bastate, + iov->iov_base + + iov_ofs, + iov_copy); copy_size -= iov_copy; iov_ofs += iov_copy; @@ -1607,7 +1611,7 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, if (desc_offset + desc_size >= total_size) { /* Simulate FCS checksum presence in the last descriptor */ - e1000e_write_to_rx_buffers(core, ba, &bastate, + e1000e_write_payload_frag_to_rx_buffers(core, ba, &bastate, (const char *) &fcs_pad, e1000x_fcs_len(core->mac)); } } @@ -2852,7 +2856,7 @@ e1000e_update_rx_offloads(E1000ECore *core) if (core->has_vnet) { qemu_set_offload(qemu_get_queue(core->owner_nic)->peer, - cso_state, 0, 0, 0, 0); + cso_state, 0, 0, 0, 0, 0, 0); } } diff --git a/hw/net/e1000x_regs.h b/hw/net/e1000x_regs.h index 13760c66d3..cd896fc0ca 100644 --- a/hw/net/e1000x_regs.h +++ b/hw/net/e1000x_regs.h @@ -839,7 +839,7 @@ union e1000_rx_desc_packet_split { #define E1000_RXD_STAT_EOP 0x02 /* End of Packet */ #define E1000_RXD_STAT_IXSM 0x04 /* Ignore checksum */ #define E1000_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ -#define E1000_RXD_STAT_UDPCS 0x10 /* UDP xsum caculated */ +#define E1000_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ #define E1000_RXD_STAT_TCPCS 0x20 /* TCP xsum calculated */ #define E1000_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ #define E1000_RXD_STAT_PIF 0x80 /* passed in-exact filter */ diff --git a/hw/net/fsl_etsec/rings.c b/hw/net/fsl_etsec/rings.c index 788463f1b6..42216de6c9 100644 --- a/hw/net/fsl_etsec/rings.c +++ b/hw/net/fsl_etsec/rings.c @@ -365,13 +365,19 @@ void etsec_walk_tx_ring(eTSEC *etsec, int ring_nbr) } while (TRUE); /* Save the Buffer Descriptor Pointers to last bd that was not - * succesfully closed */ + * successfully closed */ etsec->regs[TBPTR0 + ring_nbr].value = bd_addr; /* Set transmit halt THLTx */ etsec->regs[TSTAT].value |= 1 << (31 - ring_nbr); } +/* + * rx_init_frame() ensures we never do more padding than this + * (checksum plus minimum data packet size) + */ +#define MAX_RX_PADDING 64 + static void fill_rx_bd(eTSEC *etsec, eTSEC_rxtx_bd *bd, const uint8_t **buf, @@ -380,9 +386,11 @@ static void fill_rx_bd(eTSEC *etsec, uint16_t to_write; hwaddr bufptr = bd->bufptr + ((hwaddr)(etsec->regs[TBDBPH].value & 0xF) << 32); - uint8_t padd[etsec->rx_padding]; + uint8_t padd[MAX_RX_PADDING]; uint8_t rem; + assert(etsec->rx_padding <= MAX_RX_PADDING); + RING_DEBUG("eTSEC fill Rx buffer @ 0x%016" HWADDR_PRIx " size:%zu(padding + crc:%u) + fcb:%u\n", bufptr, *size, etsec->rx_padding, etsec->rx_fcb_size); @@ -426,7 +434,7 @@ static void fill_rx_bd(eTSEC *etsec, rem = MIN(etsec->regs[MRBLR].value - bd->length, etsec->rx_padding); if (rem > 0) { - memset(padd, 0x0, sizeof(padd)); + memset(padd, 0x0, rem); etsec->rx_padding -= rem; *size -= rem; bd->length += rem; diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c index 8b6b75c522..f6a5e2327b 100644 --- a/hw/net/igb_core.c +++ b/hw/net/igb_core.c @@ -267,6 +267,29 @@ igb_rx_use_legacy_descriptor(IGBCore *core) return false; } +typedef struct E1000ERingInfo { + int dbah; + int dbal; + int dlen; + int dh; + int dt; + int idx; +} E1000ERingInfo; + +static uint32_t +igb_rx_queue_desctyp_get(IGBCore *core, const E1000ERingInfo *r) +{ + return core->mac[E1000_SRRCTL(r->idx) >> 2] & E1000_SRRCTL_DESCTYPE_MASK; +} + +static bool +igb_rx_use_ps_descriptor(IGBCore *core, const E1000ERingInfo *r) +{ + uint32_t desctyp = igb_rx_queue_desctyp_get(core, r); + return desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT || + desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; +} + static inline bool igb_rss_enabled(IGBCore *core) { @@ -694,24 +717,15 @@ static uint32_t igb_rx_wb_eic(IGBCore *core, int queue_idx) return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0; } -typedef struct E1000E_RingInfo_st { - int dbah; - int dbal; - int dlen; - int dh; - int dt; - int idx; -} E1000E_RingInfo; - static inline bool -igb_ring_empty(IGBCore *core, const E1000E_RingInfo *r) +igb_ring_empty(IGBCore *core, const E1000ERingInfo *r) { return core->mac[r->dh] == core->mac[r->dt] || core->mac[r->dt] >= core->mac[r->dlen] / E1000_RING_DESC_LEN; } static inline uint64_t -igb_ring_base(IGBCore *core, const E1000E_RingInfo *r) +igb_ring_base(IGBCore *core, const E1000ERingInfo *r) { uint64_t bah = core->mac[r->dbah]; uint64_t bal = core->mac[r->dbal]; @@ -720,13 +734,13 @@ igb_ring_base(IGBCore *core, const E1000E_RingInfo *r) } static inline uint64_t -igb_ring_head_descr(IGBCore *core, const E1000E_RingInfo *r) +igb_ring_head_descr(IGBCore *core, const E1000ERingInfo *r) { return igb_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh]; } static inline void -igb_ring_advance(IGBCore *core, const E1000E_RingInfo *r, uint32_t count) +igb_ring_advance(IGBCore *core, const E1000ERingInfo *r, uint32_t count) { core->mac[r->dh] += count; @@ -736,7 +750,7 @@ igb_ring_advance(IGBCore *core, const E1000E_RingInfo *r, uint32_t count) } static inline uint32_t -igb_ring_free_descr_num(IGBCore *core, const E1000E_RingInfo *r) +igb_ring_free_descr_num(IGBCore *core, const E1000ERingInfo *r) { trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen], core->mac[r->dh], core->mac[r->dt]); @@ -755,13 +769,13 @@ igb_ring_free_descr_num(IGBCore *core, const E1000E_RingInfo *r) } static inline bool -igb_ring_enabled(IGBCore *core, const E1000E_RingInfo *r) +igb_ring_enabled(IGBCore *core, const E1000ERingInfo *r) { return core->mac[r->dlen] > 0; } typedef struct IGB_TxRing_st { - const E1000E_RingInfo *i; + const E1000ERingInfo *i; struct igb_tx *tx; } IGB_TxRing; @@ -774,7 +788,7 @@ igb_mq_queue_idx(int base_reg_idx, int reg_idx) static inline void igb_tx_ring_init(IGBCore *core, IGB_TxRing *txr, int idx) { - static const E1000E_RingInfo i[IGB_NUM_QUEUES] = { + static const E1000ERingInfo i[IGB_NUM_QUEUES] = { { TDBAH0, TDBAL0, TDLEN0, TDH0, TDT0, 0 }, { TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 }, { TDBAH2, TDBAL2, TDLEN2, TDH2, TDT2, 2 }, @@ -800,13 +814,13 @@ igb_tx_ring_init(IGBCore *core, IGB_TxRing *txr, int idx) } typedef struct E1000E_RxRing_st { - const E1000E_RingInfo *i; + const E1000ERingInfo *i; } E1000E_RxRing; static inline void igb_rx_ring_init(IGBCore *core, E1000E_RxRing *rxr, int idx) { - static const E1000E_RingInfo i[IGB_NUM_QUEUES] = { + static const E1000ERingInfo i[IGB_NUM_QUEUES] = { { RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 }, { RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 }, { RDBAH2, RDBAL2, RDLEN2, RDH2, RDT2, 2 }, @@ -833,7 +847,7 @@ igb_rx_ring_init(IGBCore *core, E1000E_RxRing *rxr, int idx) static uint32_t igb_txdesc_writeback(IGBCore *core, dma_addr_t base, union e1000_adv_tx_desc *tx_desc, - const E1000E_RingInfo *txi) + const E1000ERingInfo *txi) { PCIDevice *d; uint32_t cmd_type_len = le32_to_cpu(tx_desc->read.cmd_type_len); @@ -866,7 +880,7 @@ igb_txdesc_writeback(IGBCore *core, dma_addr_t base, } static inline bool -igb_tx_enabled(IGBCore *core, const E1000E_RingInfo *txi) +igb_tx_enabled(IGBCore *core, const E1000ERingInfo *txi) { bool vmdq = core->mac[MRQC] & 1; uint16_t qn = txi->idx; @@ -883,7 +897,7 @@ igb_start_xmit(IGBCore *core, const IGB_TxRing *txr) PCIDevice *d; dma_addr_t base; union e1000_adv_tx_desc desc; - const E1000E_RingInfo *txi = txr->i; + const E1000ERingInfo *txi = txr->i; uint32_t eic = 0; if (!igb_tx_enabled(core, txi)) { @@ -918,7 +932,7 @@ igb_start_xmit(IGBCore *core, const IGB_TxRing *txr) } static uint32_t -igb_rxbufsize(IGBCore *core, const E1000E_RingInfo *r) +igb_rxbufsize(IGBCore *core, const E1000ERingInfo *r) { uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2]; uint32_t bsizepkt = srrctl & E1000_SRRCTL_BSIZEPKT_MASK; @@ -930,7 +944,7 @@ igb_rxbufsize(IGBCore *core, const E1000E_RingInfo *r) } static bool -igb_has_rxbufs(IGBCore *core, const E1000E_RingInfo *r, size_t total_size) +igb_has_rxbufs(IGBCore *core, const E1000ERingInfo *r, size_t total_size) { uint32_t bufs = igb_ring_free_descr_num(core, r); uint32_t bufsize = igb_rxbufsize(core, r); @@ -941,6 +955,14 @@ igb_has_rxbufs(IGBCore *core, const E1000E_RingInfo *r, size_t total_size) bufsize; } +static uint32_t +igb_rxhdrbufsize(IGBCore *core, const E1000ERingInfo *r) +{ + uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2]; + return (srrctl & E1000_SRRCTL_BSIZEHDRSIZE_MASK) >> + E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; +} + void igb_start_recv(IGBCore *core) { @@ -1225,21 +1247,77 @@ igb_read_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc, } static inline void -igb_read_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc, - hwaddr *buff_addr) +igb_read_adv_rx_single_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc, + hwaddr *buff_addr) { *buff_addr = le64_to_cpu(desc->read.pkt_addr); } static inline void -igb_read_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc, - hwaddr *buff_addr) +igb_read_adv_rx_split_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc, + hwaddr *buff_addr) { + buff_addr[0] = le64_to_cpu(desc->read.hdr_addr); + buff_addr[1] = le64_to_cpu(desc->read.pkt_addr); +} + +typedef struct IGBBAState { + uint16_t written[IGB_MAX_PS_BUFFERS]; + uint8_t cur_idx; +} IGBBAState; + +typedef struct IGBSplitDescriptorData { + bool sph; + bool hbo; + size_t hdr_len; +} IGBSplitDescriptorData; + +typedef struct IGBPacketRxDMAState { + size_t size; + size_t total_size; + size_t ps_hdr_len; + size_t desc_size; + size_t desc_offset; + uint32_t rx_desc_packet_buf_size; + uint32_t rx_desc_header_buf_size; + struct iovec *iov; + size_t iov_ofs; + bool do_ps; + bool is_first; + IGBBAState bastate; + hwaddr ba[IGB_MAX_PS_BUFFERS]; + IGBSplitDescriptorData ps_desc_data; +} IGBPacketRxDMAState; + +static inline void +igb_read_rx_descr(IGBCore *core, + union e1000_rx_desc_union *desc, + IGBPacketRxDMAState *pdma_st, + const E1000ERingInfo *r) +{ + uint32_t desc_type; + if (igb_rx_use_legacy_descriptor(core)) { - igb_read_lgcy_rx_descr(core, &desc->legacy, buff_addr); - } else { - igb_read_adv_rx_descr(core, &desc->adv, buff_addr); + igb_read_lgcy_rx_descr(core, &desc->legacy, &pdma_st->ba[1]); + pdma_st->ba[0] = 0; + return; } + + /* advanced header split descriptor */ + if (igb_rx_use_ps_descriptor(core, r)) { + igb_read_adv_rx_split_buf_descr(core, &desc->adv, &pdma_st->ba[0]); + return; + } + + /* descriptor replication modes not supported */ + desc_type = igb_rx_queue_desctyp_get(core, r); + if (desc_type != E1000_SRRCTL_DESCTYPE_ADV_ONEBUF) { + trace_igb_wrn_rx_desc_modes_not_supp(desc_type); + } + + /* advanced single buffer descriptor */ + igb_read_adv_rx_single_buf_descr(core, &desc->adv, &pdma_st->ba[1]); + pdma_st->ba[0] = 0; } static void @@ -1281,15 +1359,11 @@ igb_verify_csum_in_sw(IGBCore *core, } static void -igb_build_rx_metadata(IGBCore *core, - struct NetRxPkt *pkt, - bool is_eop, - const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts, - uint16_t *pkt_info, uint16_t *hdr_info, - uint32_t *rss, - uint32_t *status_flags, - uint16_t *ip_id, - uint16_t *vlan_tag) +igb_build_rx_metadata_common(IGBCore *core, + struct NetRxPkt *pkt, + bool is_eop, + uint32_t *status_flags, + uint16_t *vlan_tag) { struct virtio_net_hdr *vhdr; bool hasip4, hasip6, csum_valid; @@ -1298,7 +1372,6 @@ igb_build_rx_metadata(IGBCore *core, *status_flags = E1000_RXD_STAT_DD; /* No additional metadata needed for non-EOP descriptors */ - /* TODO: EOP apply only to status so don't skip whole function. */ if (!is_eop) { goto func_exit; } @@ -1315,64 +1388,6 @@ igb_build_rx_metadata(IGBCore *core, trace_e1000e_rx_metadata_vlan(*vlan_tag); } - /* Packet parsing results */ - if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) { - if (rss_info->enabled) { - *rss = cpu_to_le32(rss_info->hash); - trace_igb_rx_metadata_rss(*rss); - } - } else if (hasip4) { - *status_flags |= E1000_RXD_STAT_IPIDV; - *ip_id = cpu_to_le16(net_rx_pkt_get_ip_id(pkt)); - trace_e1000e_rx_metadata_ip_id(*ip_id); - } - - if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && net_rx_pkt_is_tcp_ack(pkt)) { - *status_flags |= E1000_RXD_STAT_ACK; - trace_e1000e_rx_metadata_ack(); - } - - if (pkt_info) { - *pkt_info = rss_info->enabled ? rss_info->type : 0; - - if (etqf < 8) { - *pkt_info |= (BIT(11) | etqf) << 4; - } else { - if (hasip4) { - *pkt_info |= E1000_ADVRXD_PKT_IP4; - } - - if (hasip6) { - *pkt_info |= E1000_ADVRXD_PKT_IP6; - } - - switch (l4hdr_proto) { - case ETH_L4_HDR_PROTO_TCP: - *pkt_info |= E1000_ADVRXD_PKT_TCP; - break; - - case ETH_L4_HDR_PROTO_UDP: - *pkt_info |= E1000_ADVRXD_PKT_UDP; - break; - - case ETH_L4_HDR_PROTO_SCTP: - *pkt_info |= E1000_ADVRXD_PKT_SCTP; - break; - - default: - break; - } - } - } - - if (hdr_info) { - *hdr_info = 0; - } - - if (ts) { - *status_flags |= BIT(16); - } - /* RX CSO information */ if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) { trace_e1000e_rx_metadata_ipv6_sum_disabled(); @@ -1428,56 +1443,168 @@ func_exit: static inline void igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc, struct NetRxPkt *pkt, - const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts, + const E1000E_RSSInfo *rss_info, uint16_t length) { - uint32_t status_flags, rss; - uint16_t ip_id; + uint32_t status_flags; assert(!rss_info->enabled); + + memset(desc, 0, sizeof(*desc)); desc->length = cpu_to_le16(length); - desc->csum = 0; + igb_build_rx_metadata_common(core, pkt, pkt != NULL, + &status_flags, + &desc->special); - igb_build_rx_metadata(core, pkt, pkt != NULL, - rss_info, etqf, ts, - NULL, NULL, &rss, - &status_flags, &ip_id, - &desc->special); desc->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24); desc->status = (uint8_t) le32_to_cpu(status_flags); } +static bool +igb_rx_ps_descriptor_split_always(IGBCore *core, const E1000ERingInfo *r) +{ + uint32_t desctyp = igb_rx_queue_desctyp_get(core, r); + return desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; +} + +static uint16_t +igb_rx_desc_get_packet_type(IGBCore *core, struct NetRxPkt *pkt, uint16_t etqf) +{ + uint16_t pkt_type; + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; + + if (etqf < 8) { + pkt_type = BIT(11) | etqf; + return pkt_type; + } + + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + + if (hasip6 && !(core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) { + eth_ip6_hdr_info *ip6hdr_info = net_rx_pkt_get_ip6_info(pkt); + pkt_type = ip6hdr_info->has_ext_hdrs ? E1000_ADVRXD_PKT_IP6E : + E1000_ADVRXD_PKT_IP6; + } else if (hasip4) { + pkt_type = E1000_ADVRXD_PKT_IP4; + } else { + pkt_type = 0; + } + + switch (l4hdr_proto) { + case ETH_L4_HDR_PROTO_TCP: + pkt_type |= E1000_ADVRXD_PKT_TCP; + break; + case ETH_L4_HDR_PROTO_UDP: + pkt_type |= E1000_ADVRXD_PKT_UDP; + break; + case ETH_L4_HDR_PROTO_SCTP: + pkt_type |= E1000_ADVRXD_PKT_SCTP; + break; + default: + break; + } + + return pkt_type; +} + static inline void igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc, struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts, uint16_t length) { + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; + uint16_t rss_type = 0, pkt_type; + bool eop = (pkt != NULL); + uint32_t adv_desc_status_error = 0; memset(&desc->wb, 0, sizeof(desc->wb)); desc->wb.upper.length = cpu_to_le16(length); + igb_build_rx_metadata_common(core, pkt, eop, + &desc->wb.upper.status_error, + &desc->wb.upper.vlan); + + if (!eop) { + return; + } + + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + + if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) { + if (rss_info->enabled) { + desc->wb.lower.hi_dword.rss = cpu_to_le32(rss_info->hash); + rss_type = rss_info->type; + trace_igb_rx_metadata_rss(desc->wb.lower.hi_dword.rss, rss_type); + } + } else if (hasip4) { + adv_desc_status_error |= E1000_RXD_STAT_IPIDV; + desc->wb.lower.hi_dword.csum_ip.ip_id = + cpu_to_le16(net_rx_pkt_get_ip_id(pkt)); + trace_e1000e_rx_metadata_ip_id( + desc->wb.lower.hi_dword.csum_ip.ip_id); + } + + if (ts) { + adv_desc_status_error |= BIT(16); + } + + pkt_type = igb_rx_desc_get_packet_type(core, pkt, etqf); + trace_e1000e_rx_metadata_pkt_type(pkt_type); + desc->wb.lower.lo_dword.pkt_info = cpu_to_le16(rss_type | (pkt_type << 4)); + desc->wb.upper.status_error |= cpu_to_le32(adv_desc_status_error); +} + +static inline void +igb_write_adv_ps_rx_descr(IGBCore *core, + union e1000_adv_rx_desc *desc, + struct NetRxPkt *pkt, + const E1000E_RSSInfo *rss_info, + const E1000ERingInfo *r, + uint16_t etqf, + bool ts, + IGBPacketRxDMAState *pdma_st) +{ + size_t pkt_len; + uint16_t hdr_info = 0; - igb_build_rx_metadata(core, pkt, pkt != NULL, - rss_info, etqf, ts, - &desc->wb.lower.lo_dword.pkt_info, - &desc->wb.lower.lo_dword.hdr_info, - &desc->wb.lower.hi_dword.rss, - &desc->wb.upper.status_error, - &desc->wb.lower.hi_dword.csum_ip.ip_id, - &desc->wb.upper.vlan); + if (pdma_st->do_ps) { + pkt_len = pdma_st->bastate.written[1]; + } else { + pkt_len = pdma_st->bastate.written[0] + pdma_st->bastate.written[1]; + } + + igb_write_adv_rx_descr(core, desc, pkt, rss_info, etqf, ts, pkt_len); + + hdr_info = (pdma_st->ps_desc_data.hdr_len << E1000_ADVRXD_HDR_LEN_OFFSET) & + E1000_ADVRXD_ADV_HDR_LEN_MASK; + hdr_info |= pdma_st->ps_desc_data.sph ? E1000_ADVRXD_HDR_SPH : 0; + desc->wb.lower.lo_dword.hdr_info = cpu_to_le16(hdr_info); + + desc->wb.upper.status_error |= cpu_to_le32( + pdma_st->ps_desc_data.hbo ? E1000_ADVRXD_ST_ERR_HBO_OFFSET : 0); } static inline void -igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc, - struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, - uint16_t etqf, bool ts, uint16_t length) +igb_write_rx_descr(IGBCore *core, + union e1000_rx_desc_union *desc, + struct NetRxPkt *pkt, + const E1000E_RSSInfo *rss_info, + uint16_t etqf, + bool ts, + IGBPacketRxDMAState *pdma_st, + const E1000ERingInfo *r) { if (igb_rx_use_legacy_descriptor(core)) { igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, - etqf, ts, length); + pdma_st->bastate.written[1]); + } else if (igb_rx_use_ps_descriptor(core, r)) { + igb_write_adv_ps_rx_descr(core, &desc->adv, pkt, rss_info, r, etqf, ts, + pdma_st); } else { igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info, - etqf, ts, length); + etqf, ts, pdma_st->bastate.written[1]); } } @@ -1514,20 +1641,7 @@ igb_pci_dma_write_rx_desc(IGBCore *core, PCIDevice *dev, dma_addr_t addr, } static void -igb_write_to_rx_buffers(IGBCore *core, - PCIDevice *d, - hwaddr ba, - uint16_t *written, - const char *data, - dma_addr_t data_len) -{ - trace_igb_rx_desc_buff_write(ba, *written, data, data_len); - pci_dma_write(d, ba + *written, data, data_len); - *written += data_len; -} - -static void -igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi, +igb_update_rx_stats(IGBCore *core, const E1000ERingInfo *rxi, size_t pkt_size, size_t pkt_fcs_size) { eth_pkt_types_e pkt_type = net_rx_pkt_get_packet_type(core->rx_pkt); @@ -1545,12 +1659,256 @@ igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi, } static inline bool -igb_rx_descr_threshold_hit(IGBCore *core, const E1000E_RingInfo *rxi) +igb_rx_descr_threshold_hit(IGBCore *core, const E1000ERingInfo *rxi) { return igb_ring_free_descr_num(core, rxi) == ((core->mac[E1000_SRRCTL(rxi->idx) >> 2] >> 20) & 31) * 16; } +static bool +igb_do_ps(IGBCore *core, + const E1000ERingInfo *r, + struct NetRxPkt *pkt, + IGBPacketRxDMAState *pdma_st) +{ + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; + bool fragment; + bool split_always; + size_t bheader_size; + size_t total_pkt_len; + + if (!igb_rx_use_ps_descriptor(core, r)) { + return false; + } + + total_pkt_len = net_rx_pkt_get_total_len(pkt); + bheader_size = igb_rxhdrbufsize(core, r); + split_always = igb_rx_ps_descriptor_split_always(core, r); + if (split_always && total_pkt_len <= bheader_size) { + pdma_st->ps_hdr_len = total_pkt_len; + pdma_st->ps_desc_data.hdr_len = total_pkt_len; + return true; + } + + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + + if (hasip4) { + fragment = net_rx_pkt_get_ip4_info(pkt)->fragment; + } else if (hasip6) { + fragment = net_rx_pkt_get_ip6_info(pkt)->fragment; + } else { + pdma_st->ps_desc_data.hdr_len = bheader_size; + goto header_not_handled; + } + + if (fragment && (core->mac[RFCTL] & E1000_RFCTL_IPFRSP_DIS)) { + pdma_st->ps_desc_data.hdr_len = bheader_size; + goto header_not_handled; + } + + /* no header splitting for SCTP */ + if (!fragment && (l4hdr_proto == ETH_L4_HDR_PROTO_UDP || + l4hdr_proto == ETH_L4_HDR_PROTO_TCP)) { + pdma_st->ps_hdr_len = net_rx_pkt_get_l5_hdr_offset(pkt); + } else { + pdma_st->ps_hdr_len = net_rx_pkt_get_l4_hdr_offset(pkt); + } + + pdma_st->ps_desc_data.sph = true; + pdma_st->ps_desc_data.hdr_len = pdma_st->ps_hdr_len; + + if (pdma_st->ps_hdr_len > bheader_size) { + pdma_st->ps_desc_data.hbo = true; + goto header_not_handled; + } + + return true; + +header_not_handled: + if (split_always) { + pdma_st->ps_hdr_len = bheader_size; + return true; + } + + return false; +} + +static void +igb_truncate_to_descriptor_size(IGBPacketRxDMAState *pdma_st, size_t *size) +{ + if (pdma_st->do_ps && pdma_st->is_first) { + if (*size > pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len) { + *size = pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len; + } + } else { + if (*size > pdma_st->rx_desc_packet_buf_size) { + *size = pdma_st->rx_desc_packet_buf_size; + } + } +} + +static inline void +igb_write_hdr_frag_to_rx_buffers(IGBCore *core, + PCIDevice *d, + IGBPacketRxDMAState *pdma_st, + const char *data, + dma_addr_t data_len) +{ + assert(data_len <= pdma_st->rx_desc_header_buf_size - + pdma_st->bastate.written[0]); + pci_dma_write(d, + pdma_st->ba[0] + pdma_st->bastate.written[0], + data, data_len); + pdma_st->bastate.written[0] += data_len; + pdma_st->bastate.cur_idx = 1; +} + +static void +igb_write_header_to_rx_buffers(IGBCore *core, + struct NetRxPkt *pkt, + PCIDevice *d, + IGBPacketRxDMAState *pdma_st, + size_t *copy_size) +{ + size_t iov_copy; + size_t ps_hdr_copied = 0; + + if (!pdma_st->is_first) { + /* Leave buffer 0 of each descriptor except first */ + /* empty */ + pdma_st->bastate.cur_idx = 1; + return; + } + + do { + iov_copy = MIN(pdma_st->ps_hdr_len - ps_hdr_copied, + pdma_st->iov->iov_len - pdma_st->iov_ofs); + + igb_write_hdr_frag_to_rx_buffers(core, d, pdma_st, + pdma_st->iov->iov_base, + iov_copy); + + *copy_size -= iov_copy; + ps_hdr_copied += iov_copy; + + pdma_st->iov_ofs += iov_copy; + if (pdma_st->iov_ofs == pdma_st->iov->iov_len) { + pdma_st->iov++; + pdma_st->iov_ofs = 0; + } + } while (ps_hdr_copied < pdma_st->ps_hdr_len); + + pdma_st->is_first = false; +} + +static void +igb_write_payload_frag_to_rx_buffers(IGBCore *core, + PCIDevice *d, + IGBPacketRxDMAState *pdma_st, + const char *data, + dma_addr_t data_len) +{ + while (data_len > 0) { + assert(pdma_st->bastate.cur_idx < IGB_MAX_PS_BUFFERS); + + uint32_t cur_buf_bytes_left = + pdma_st->rx_desc_packet_buf_size - + pdma_st->bastate.written[pdma_st->bastate.cur_idx]; + uint32_t bytes_to_write = MIN(data_len, cur_buf_bytes_left); + + trace_igb_rx_desc_buff_write( + pdma_st->bastate.cur_idx, + pdma_st->ba[pdma_st->bastate.cur_idx], + pdma_st->bastate.written[pdma_st->bastate.cur_idx], + data, + bytes_to_write); + + pci_dma_write(d, + pdma_st->ba[pdma_st->bastate.cur_idx] + + pdma_st->bastate.written[pdma_st->bastate.cur_idx], + data, bytes_to_write); + + pdma_st->bastate.written[pdma_st->bastate.cur_idx] += bytes_to_write; + data += bytes_to_write; + data_len -= bytes_to_write; + + if (pdma_st->bastate.written[pdma_st->bastate.cur_idx] == + pdma_st->rx_desc_packet_buf_size) { + pdma_st->bastate.cur_idx++; + } + } +} + +static void +igb_write_payload_to_rx_buffers(IGBCore *core, + struct NetRxPkt *pkt, + PCIDevice *d, + IGBPacketRxDMAState *pdma_st, + size_t *copy_size) +{ + static const uint32_t fcs_pad; + size_t iov_copy; + + /* Copy packet payload */ + while (*copy_size) { + iov_copy = MIN(*copy_size, pdma_st->iov->iov_len - pdma_st->iov_ofs); + igb_write_payload_frag_to_rx_buffers(core, d, + pdma_st, + pdma_st->iov->iov_base + + pdma_st->iov_ofs, + iov_copy); + + *copy_size -= iov_copy; + pdma_st->iov_ofs += iov_copy; + if (pdma_st->iov_ofs == pdma_st->iov->iov_len) { + pdma_st->iov++; + pdma_st->iov_ofs = 0; + } + } + + if (pdma_st->desc_offset + pdma_st->desc_size >= pdma_st->total_size) { + /* Simulate FCS checksum presence in the last descriptor */ + igb_write_payload_frag_to_rx_buffers(core, d, + pdma_st, + (const char *) &fcs_pad, + e1000x_fcs_len(core->mac)); + } +} + +static void +igb_write_to_rx_buffers(IGBCore *core, + struct NetRxPkt *pkt, + PCIDevice *d, + IGBPacketRxDMAState *pdma_st) +{ + size_t copy_size; + + if (!(pdma_st->ba)[1] || (pdma_st->do_ps && !(pdma_st->ba[0]))) { + /* as per intel docs; skip descriptors with null buf addr */ + trace_e1000e_rx_null_descriptor(); + return; + } + + if (pdma_st->desc_offset >= pdma_st->size) { + return; + } + + pdma_st->desc_size = pdma_st->total_size - pdma_st->desc_offset; + igb_truncate_to_descriptor_size(pdma_st, &pdma_st->desc_size); + copy_size = pdma_st->size - pdma_st->desc_offset; + igb_truncate_to_descriptor_size(pdma_st, ©_size); + + /* For PS mode copy the packet header first */ + if (pdma_st->do_ps) { + igb_write_header_to_rx_buffers(core, pkt, d, pdma_st, ©_size); + } else { + pdma_st->bastate.cur_idx = 1; + } + + igb_write_payload_to_rx_buffers(core, pkt, d, pdma_st, ©_size); +} + static void igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt, const E1000E_RxRing *rxr, @@ -1560,95 +1918,61 @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt, PCIDevice *d; dma_addr_t base; union e1000_rx_desc_union desc; - size_t desc_size; - size_t desc_offset = 0; - size_t iov_ofs = 0; - - struct iovec *iov = net_rx_pkt_get_iovec(pkt); - size_t size = net_rx_pkt_get_total_len(pkt); - size_t total_size = size + e1000x_fcs_len(core->mac); - const E1000E_RingInfo *rxi = rxr->i; - size_t bufsize = igb_rxbufsize(core, rxi); - + const E1000ERingInfo *rxi; + size_t rx_desc_len; + + IGBPacketRxDMAState pdma_st = {0}; + pdma_st.is_first = true; + pdma_st.size = net_rx_pkt_get_total_len(pkt); + pdma_st.total_size = pdma_st.size + e1000x_fcs_len(core->mac); + + rxi = rxr->i; + rx_desc_len = core->rx_desc_len; + pdma_st.rx_desc_packet_buf_size = igb_rxbufsize(core, rxi); + pdma_st.rx_desc_header_buf_size = igb_rxhdrbufsize(core, rxi); + pdma_st.iov = net_rx_pkt_get_iovec(pkt); d = pcie_sriov_get_vf_at_index(core->owner, rxi->idx % 8); if (!d) { d = core->owner; } + pdma_st.do_ps = igb_do_ps(core, rxi, pkt, &pdma_st); + do { - hwaddr ba; - uint16_t written = 0; + memset(&pdma_st.bastate, 0, sizeof(IGBBAState)); bool is_last = false; - desc_size = total_size - desc_offset; - - if (desc_size > bufsize) { - desc_size = bufsize; - } - if (igb_ring_empty(core, rxi)) { return; } base = igb_ring_head_descr(core, rxi); + pci_dma_read(d, base, &desc, rx_desc_len); + trace_e1000e_rx_descr(rxi->idx, base, rx_desc_len); - pci_dma_read(d, base, &desc, core->rx_desc_len); - - trace_e1000e_rx_descr(rxi->idx, base, core->rx_desc_len); - - igb_read_rx_descr(core, &desc, &ba); - - if (ba) { - if (desc_offset < size) { - static const uint32_t fcs_pad; - size_t iov_copy; - size_t copy_size = size - desc_offset; - if (copy_size > bufsize) { - copy_size = bufsize; - } - - /* Copy packet payload */ - while (copy_size) { - iov_copy = MIN(copy_size, iov->iov_len - iov_ofs); + igb_read_rx_descr(core, &desc, &pdma_st, rxi); - igb_write_to_rx_buffers(core, d, ba, &written, - iov->iov_base + iov_ofs, iov_copy); - - copy_size -= iov_copy; - iov_ofs += iov_copy; - if (iov_ofs == iov->iov_len) { - iov++; - iov_ofs = 0; - } - } - - if (desc_offset + desc_size >= total_size) { - /* Simulate FCS checksum presence in the last descriptor */ - igb_write_to_rx_buffers(core, d, ba, &written, - (const char *) &fcs_pad, e1000x_fcs_len(core->mac)); - } - } - } else { /* as per intel docs; skip descriptors with null buf addr */ - trace_e1000e_rx_null_descriptor(); - } - desc_offset += desc_size; - if (desc_offset >= total_size) { + igb_write_to_rx_buffers(core, pkt, d, &pdma_st); + pdma_st.desc_offset += pdma_st.desc_size; + if (pdma_st.desc_offset >= pdma_st.total_size) { is_last = true; } - igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL, - rss_info, etqf, ts, written); - igb_pci_dma_write_rx_desc(core, d, base, &desc, core->rx_desc_len); - - igb_ring_advance(core, rxi, core->rx_desc_len / E1000_MIN_RX_DESC_LEN); - - } while (desc_offset < total_size); + igb_write_rx_descr(core, &desc, + is_last ? pkt : NULL, + rss_info, + etqf, ts, + &pdma_st, + rxi); + igb_pci_dma_write_rx_desc(core, d, base, &desc, rx_desc_len); + igb_ring_advance(core, rxi, rx_desc_len / E1000_MIN_RX_DESC_LEN); + } while (pdma_st.desc_offset < pdma_st.total_size); - igb_update_rx_stats(core, rxi, size, total_size); + igb_update_rx_stats(core, rxi, pdma_st.size, pdma_st.total_size); } static bool -igb_rx_strip_vlan(IGBCore *core, const E1000E_RingInfo *rxi) +igb_rx_strip_vlan(IGBCore *core, const E1000ERingInfo *rxi) { if (core->mac[MRQC] & 1) { uint16_t pool = rxi->idx % IGB_NUM_VM_POOLS; @@ -2753,7 +3077,7 @@ igb_update_rx_offloads(IGBCore *core) if (core->has_vnet) { qemu_set_offload(qemu_get_queue(core->owner_nic)->peer, - cso_state, 0, 0, 0, 0); + cso_state, 0, 0, 0, 0, 0, 0); } } diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h index 82ff195dfc..e5a47eab64 100644 --- a/hw/net/igb_regs.h +++ b/hw/net/igb_regs.h @@ -364,7 +364,7 @@ union e1000_adv_rx_desc { /* Indicates that VF is still clear to send requests */ #define E1000_VT_MSGTYPE_CTS 0x20000000 #define E1000_VT_MSGINFO_SHIFT 16 -/* bits 23:16 are used for exra info for certain messages */ +/* bits 23:16 are used for extra info for certain messages */ #define E1000_VT_MSGINFO_MASK (0xFF << E1000_VT_MSGINFO_SHIFT) #define E1000_VF_RESET 0x01 /* VF requests reset */ @@ -452,6 +452,7 @@ union e1000_adv_rx_desc { #define E1000_SRRCTL_BSIZEHDRSIZE_MASK 0x00000F00 #define E1000_SRRCTL_BSIZEHDRSIZE_SHIFT 2 /* Shift _left_ */ #define E1000_SRRCTL_DESCTYPE_ADV_ONEBUF 0x02000000 +#define E1000_SRRCTL_DESCTYPE_HDR_SPLIT 0x04000000 #define E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS 0x0A000000 #define E1000_SRRCTL_DESCTYPE_MASK 0x0E000000 #define E1000_SRRCTL_DROP_EN 0x80000000 @@ -490,7 +491,7 @@ union e1000_adv_rx_desc { #define E1000_VF_MBX_INIT_DELAY 500 /* usec delay between retries */ #define E1000_VT_MSGINFO_SHIFT 16 -/* bits 23:16 are used for exra info for certain messages */ +/* bits 23:16 are used for extra info for certain messages */ #define E1000_VT_MSGINFO_MASK (0xFF << E1000_VT_MSGINFO_SHIFT) #define E1000_VF_RESET 0x01 /* VF requests reset */ @@ -692,11 +693,20 @@ union e1000_adv_rx_desc { #define E1000_STATUS_NUM_VFS_SHIFT 14 -#define E1000_ADVRXD_PKT_IP4 BIT(4) -#define E1000_ADVRXD_PKT_IP6 BIT(6) -#define E1000_ADVRXD_PKT_TCP BIT(8) -#define E1000_ADVRXD_PKT_UDP BIT(9) -#define E1000_ADVRXD_PKT_SCTP BIT(10) +#define E1000_ADVRXD_PKT_IP4 BIT(0) +#define E1000_ADVRXD_PKT_IP6 BIT(2) +#define E1000_ADVRXD_PKT_IP6E BIT(3) +#define E1000_ADVRXD_PKT_TCP BIT(4) +#define E1000_ADVRXD_PKT_UDP BIT(5) +#define E1000_ADVRXD_PKT_SCTP BIT(6) + +#define IGB_MAX_PS_BUFFERS 2 + +#define E1000_ADVRXD_HDR_LEN_OFFSET (21 - 16) +#define E1000_ADVRXD_ADV_HDR_LEN_MASK ((BIT(10) - 1) << \ + E1000_ADVRXD_HDR_LEN_OFFSET) +#define E1000_ADVRXD_HDR_SPH BIT(15) +#define E1000_ADVRXD_ST_ERR_HBO_OFFSET BIT(3 + 20) static inline uint8_t igb_ivar_entry_rx(uint8_t i) { diff --git a/hw/net/mcf_fec.c b/hw/net/mcf_fec.c index 8aa27bd322..ec3ddf520a 100644 --- a/hw/net/mcf_fec.c +++ b/hw/net/mcf_fec.c @@ -571,7 +571,7 @@ static ssize_t mcf_fec_receive(NetClientState *nc, const uint8_t *buf, size_t si size += 4; crc = cpu_to_be32(crc32(~0, buf, size)); crc_ptr = (uint8_t *)&crc; - /* Huge frames are truncted. */ + /* Huge frames are truncated. */ if (size > FEC_MAX_FRAME_SIZE) { size = FEC_MAX_FRAME_SIZE; flags |= FEC_BD_TR | FEC_BD_LG; diff --git a/hw/net/rocker/rocker_fp.c b/hw/net/rocker/rocker_fp.c index cbeed65bd5..9afd0c5e3f 100644 --- a/hw/net/rocker/rocker_fp.c +++ b/hw/net/rocker/rocker_fp.c @@ -134,7 +134,7 @@ static ssize_t fp_port_receive_iov(NetClientState *nc, const struct iovec *iov, FpPort *port = qemu_get_nic_opaque(nc); /* If the port is disabled, we want to drop this pkt - * now rather than queing it for later. We don't want + * now rather than queueing it for later. We don't want * any stale pkts getting into the device when the port * transitions to enabled. */ diff --git a/hw/net/rocker/rocker_of_dpa.c b/hw/net/rocker/rocker_of_dpa.c index dfe4754469..5e16056be6 100644 --- a/hw/net/rocker/rocker_of_dpa.c +++ b/hw/net/rocker/rocker_of_dpa.c @@ -1043,7 +1043,7 @@ static void of_dpa_flow_ig_tbl(OfDpaFlowContext *fc, uint32_t tbl_id) static ssize_t of_dpa_ig(World *world, uint32_t pport, const struct iovec *iov, int iovcnt) { - struct iovec iov_copy[iovcnt + 2]; + g_autofree struct iovec *iov_copy = g_new(struct iovec, iovcnt + 2); OfDpaFlowContext fc = { .of_dpa = world_private(world), .in_pport = pport, diff --git a/hw/net/rtl8139.c b/hw/net/rtl8139.c index b4df75b2c9..4525fda383 100644 --- a/hw/net/rtl8139.c +++ b/hw/net/rtl8139.c @@ -100,7 +100,7 @@ enum RTL8139_registers { MAC0 = 0, /* Ethernet hardware address. */ MAR0 = 8, /* Multicast filter. */ TxStatus0 = 0x10,/* Transmit status (Four 32bit registers). C mode only */ - /* Dump Tally Conter control register(64bit). C+ mode only */ + /* Dump Tally Counter control register(64bit). C+ mode only */ TxAddr0 = 0x20, /* Tx descriptors (also four 32bit). */ RxBuf = 0x30, ChipCmd = 0x37, diff --git a/hw/net/smc91c111.c b/hw/net/smc91c111.c index ad778cd8fc..ddbceda967 100644 --- a/hw/net/smc91c111.c +++ b/hw/net/smc91c111.c @@ -361,7 +361,7 @@ static void smc91c111_writeb(void *opaque, hwaddr offset, case 4: case 5: case 6: case 7: case 8: case 9: /* IA */ /* Not implemented. */ return; - case 10: /* Genral Purpose */ + case 10: /* General Purpose */ SET_LOW(gpr, value); return; case 11: diff --git a/hw/net/sungem.c b/hw/net/sungem.c index 510b370e5f..c2e2c90668 100644 --- a/hw/net/sungem.c +++ b/hw/net/sungem.c @@ -1228,7 +1228,7 @@ static void sungem_mmio_mif_write(void *opaque, hwaddr addr, uint64_t val, case MIF_SMACHINE: return; /* No actual write */ case MIF_CFG: - /* Maintain the RO MDI bits to advertize an MDIO PHY on MDI0 */ + /* Maintain the RO MDI bits to advertise an MDIO PHY on MDI0 */ val &= ~MIF_CFG_MDI1; val |= MIF_CFG_MDI0; break; diff --git a/hw/net/sunhme.c b/hw/net/sunhme.c index 391d26fb82..64d4ea5850 100644 --- a/hw/net/sunhme.c +++ b/hw/net/sunhme.c @@ -901,7 +901,7 @@ static void sunhme_reset(DeviceState *ds) /* Configure internal transceiver */ s->mifregs[HME_MIFI_CFG >> 2] |= HME_MIF_CFG_MDI0; - /* Advetise auto, 100Mbps FD */ + /* Advertise auto, 100Mbps FD */ s->miiregs[MII_ANAR] = MII_ANAR_TXFD; s->miiregs[MII_BMSR] = MII_BMSR_AUTONEG | MII_BMSR_100TX_FD | MII_BMSR_AN_COMP; diff --git a/hw/net/trace-events b/hw/net/trace-events index 6b5ba669a2..3abfd65e5b 100644 --- a/hw/net/trace-events +++ b/hw/net/trace-events @@ -278,9 +278,9 @@ igb_core_mdic_write_unhandled(uint32_t addr) "MDIC WRITE: PHY[%u] UNHANDLED" igb_link_set_ext_params(bool asd_check, bool speed_select_bypass, bool pfrstd) "Set extended link params: ASD check: %d, Speed select bypass: %d, PF reset done: %d" igb_rx_desc_buff_size(uint32_t b) "buffer size: %u" -igb_rx_desc_buff_write(uint64_t addr, uint16_t offset, const void* source, uint32_t len) "addr: 0x%"PRIx64", offset: %u, from: %p, length: %u" +igb_rx_desc_buff_write(uint8_t idx, uint64_t addr, uint16_t offset, const void* source, uint32_t len) "buffer %u, addr: 0x%"PRIx64", offset: %u, from: %p, length: %u" -igb_rx_metadata_rss(uint32_t rss) "RSS data: 0x%X" +igb_rx_metadata_rss(uint32_t rss, uint16_t rss_pkt_type) "RSS data: rss: 0x%X, rss_pkt_type: 0x%X" igb_irq_icr_clear_gpie_nsicr(void) "Clearing ICR on read due to GPIE.NSICR enabled" igb_irq_set_iam(uint32_t icr) "Update IAM: 0x%x" @@ -295,6 +295,8 @@ igb_irq_eitr_set(uint32_t eitr_num, uint32_t val) "EITR[%u] = 0x%x" igb_set_pfmailbox(uint32_t vf_num, uint32_t val) "PFMailbox[%d]: 0x%x" igb_set_vfmailbox(uint32_t vf_num, uint32_t val) "VFMailbox[%d]: 0x%x" +igb_wrn_rx_desc_modes_not_supp(int desc_type) "Not supported descriptor type: %d" + # igbvf.c igbvf_wrn_io_addr_unknown(uint64_t addr) "IO unknown register 0x%"PRIx64 diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c index 6b958d6363..57427a3997 100644 --- a/hw/net/vhost_net.c +++ b/hw/net/vhost_net.c @@ -78,6 +78,9 @@ static const int user_feature_bits[] = { VIRTIO_F_RING_RESET, VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, + VIRTIO_NET_F_GUEST_USO4, + VIRTIO_NET_F_GUEST_USO6, + VIRTIO_NET_F_HOST_USO, /* This bit implies RARP isn't sent by QEMU out of band */ VIRTIO_NET_F_GUEST_ANNOUNCE, diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c index 7102ec4817..5a0201c423 100644 --- a/hw/net/virtio-net.c +++ b/hw/net/virtio-net.c @@ -659,6 +659,15 @@ static int peer_has_ufo(VirtIONet *n) return n->has_ufo; } +static int peer_has_uso(VirtIONet *n) +{ + if (!peer_has_vnet_hdr(n)) { + return 0; + } + + return qemu_has_uso(qemu_get_queue(n->nic)->peer); +} + static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs, int version_1, int hash_report) { @@ -796,6 +805,10 @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6); virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN); + virtio_clear_feature(&features, VIRTIO_NET_F_HOST_USO); + virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO4); + virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO6); + virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT); } @@ -804,6 +817,12 @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, virtio_clear_feature(&features, VIRTIO_NET_F_HOST_UFO); } + if (!peer_has_uso(n)) { + virtio_clear_feature(&features, VIRTIO_NET_F_HOST_USO); + virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO4); + virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO6); + } + if (!get_vhost_net(nc->peer)) { return features; } @@ -859,17 +878,21 @@ static void virtio_net_apply_guest_offloads(VirtIONet *n) !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)), !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)), !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)), - !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO))); + !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO)), + !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO4)), + !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO6))); } -static uint64_t virtio_net_guest_offloads_by_features(uint32_t features) +static uint64_t virtio_net_guest_offloads_by_features(uint64_t features) { static const uint64_t guest_offloads_mask = (1ULL << VIRTIO_NET_F_GUEST_CSUM) | (1ULL << VIRTIO_NET_F_GUEST_TSO4) | (1ULL << VIRTIO_NET_F_GUEST_TSO6) | (1ULL << VIRTIO_NET_F_GUEST_ECN) | - (1ULL << VIRTIO_NET_F_GUEST_UFO); + (1ULL << VIRTIO_NET_F_GUEST_UFO) | + (1ULL << VIRTIO_NET_F_GUEST_USO4) | + (1ULL << VIRTIO_NET_F_GUEST_USO6); return guest_offloads_mask & features; } @@ -1307,7 +1330,7 @@ static void virtio_net_detach_epbf_rss(VirtIONet *n) static bool virtio_net_load_ebpf(VirtIONet *n) { if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) { - /* backend does't support steering ebpf */ + /* backend doesn't support steering ebpf */ return false; } @@ -2046,7 +2069,7 @@ static void virtio_net_rsc_extract_unit6(VirtioNetRscChain *chain, + sizeof(struct ip6_header)); unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; - /* There is a difference between payload lenght in ipv4 and v6, + /* There is a difference between payload length in ipv4 and v6, ip header is excluded in ipv6 */ unit->payload = htons(*unit->ip_plen) - unit->tcp_hdrlen; } @@ -3795,7 +3818,7 @@ static void virtio_net_instance_init(Object *obj) /* * The default config_size is sizeof(struct virtio_net_config). - * Can be overriden with virtio_net_set_config_size. + * Can be overridden with virtio_net_set_config_size. */ n->config_size = sizeof(struct virtio_net_config); device_add_bootindex_property(obj, &n->nic_conf.bootindex, @@ -3922,6 +3945,12 @@ static Property virtio_net_properties[] = { DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN), DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str), DEFINE_PROP_BOOL("failover", VirtIONet, failover, false), + DEFINE_PROP_BIT64("guest_uso4", VirtIONet, host_features, + VIRTIO_NET_F_GUEST_USO4, true), + DEFINE_PROP_BIT64("guest_uso6", VirtIONet, host_features, + VIRTIO_NET_F_GUEST_USO6, true), + DEFINE_PROP_BIT64("host_uso", VirtIONet, host_features, + VIRTIO_NET_F_HOST_USO, true), DEFINE_PROP_END_OF_LIST(), }; diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c index 3fb108751a..1b48d7743e 100644 --- a/hw/net/vmxnet3.c +++ b/hw/net/vmxnet3.c @@ -1341,6 +1341,8 @@ static void vmxnet3_update_features(VMXNET3State *s) s->lro_supported, s->lro_supported, 0, + 0, + 0, 0); } } @@ -1887,7 +1889,7 @@ vmxnet3_io_bar1_read(void *opaque, hwaddr addr, unsigned size) break; default: - VMW_CBPRN("Unknow read BAR1[%" PRIx64 "], %d bytes", addr, size); + VMW_CBPRN("Unknown read BAR1[%" PRIx64 "], %d bytes", addr, size); break; } diff --git a/hw/net/vmxnet3.h b/hw/net/vmxnet3.h index bf4f6de74a..f9283f9e7b 100644 --- a/hw/net/vmxnet3.h +++ b/hw/net/vmxnet3.h @@ -733,7 +733,7 @@ struct Vmxnet3_TxQueueDesc { struct Vmxnet3_RxQueueDesc { struct Vmxnet3_RxQueueCtrl ctrl; struct Vmxnet3_RxQueueConf conf; - /* Driver read after a GET commad */ + /* Driver read after a GET command */ struct Vmxnet3_QueueStatus status; struct UPT1_RxStats stats; u8 __pad[88]; /* 128 aligned */ diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c index 90687b168a..f026245d1e 100644 --- a/hw/nvme/ctrl.c +++ b/hw/nvme/ctrl.c @@ -17,7 +17,7 @@ * Notes on coding style * --------------------- * While QEMU coding style prefers lowercase hexadecimals in constants, the - * NVMe subsystem use thes format from the NVMe specifications in the comments + * NVMe subsystem use this format from the NVMe specifications in the comments * (i.e. 'h' suffix instead of '0x' prefix). * * Usage @@ -730,7 +730,7 @@ static inline void nvme_sg_unmap(NvmeSg *sg) } /* - * When metadata is transfered as extended LBAs, the DPTR mapped into `sg` + * When metadata is transferred as extended LBAs, the DPTR mapped into `sg` * holds both data and metadata. This function splits the data and metadata * into two separate QSG/IOVs. */ @@ -2130,11 +2130,6 @@ static inline bool nvme_is_write(NvmeRequest *req) rw->opcode == NVME_CMD_WRITE_ZEROES; } -static AioContext *nvme_get_aio_context(BlockAIOCB *acb) -{ - return qemu_get_aio_context(); -} - static void nvme_misc_cb(void *opaque, int ret) { NvmeRequest *req = opaque; @@ -3302,7 +3297,6 @@ static void nvme_flush_cancel(BlockAIOCB *acb) static const AIOCBInfo nvme_flush_aiocb_info = { .aiocb_size = sizeof(NvmeFlushAIOCB), .cancel_async = nvme_flush_cancel, - .get_aio_context = nvme_get_aio_context, }; static void nvme_do_flush(NvmeFlushAIOCB *iocb); @@ -6478,7 +6472,6 @@ static void nvme_format_cancel(BlockAIOCB *aiocb) static const AIOCBInfo nvme_format_aiocb_info = { .aiocb_size = sizeof(NvmeFormatAIOCB), .cancel_async = nvme_format_cancel, - .get_aio_context = nvme_get_aio_context, }; static void nvme_format_set(NvmeNamespace *ns, uint8_t lbaf, uint8_t mset, @@ -7594,7 +7587,7 @@ static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val) /* * NVM Express v1.3d, Section 4.1 state: "If host software writes * an invalid value to the Submission Queue Tail Doorbell or - * Completion Queue Head Doorbell regiter and an Asynchronous Event + * Completion Queue Head Doorbell register and an Asynchronous Event * Request command is outstanding, then an asynchronous event is * posted to the Admin Completion Queue with a status code of * Invalid Doorbell Write Value." diff --git a/hw/nvram/eeprom_at24c.c b/hw/nvram/eeprom_at24c.c index 613c4929e3..3272068663 100644 --- a/hw/nvram/eeprom_at24c.c +++ b/hw/nvram/eeprom_at24c.c @@ -51,7 +51,7 @@ struct EEPROMState { bool writable; /* cells changed since last START? */ bool changed; - /* during WRITE, # of address bytes transfered */ + /* during WRITE, # of address bytes transferred */ uint8_t haveaddr; uint8_t *mem; diff --git a/hw/nvram/fw_cfg.c b/hw/nvram/fw_cfg.c index 29a5bef1d5..4e4524673a 100644 --- a/hw/nvram/fw_cfg.c +++ b/hw/nvram/fw_cfg.c @@ -877,7 +877,7 @@ static struct { /* * Any sub-page size update to these table MRs will be lost during migration, * as we use aligned size in ram_load_precopy() -> qemu_ram_resize() path. - * In order to avoid the inconsistency in sizes save them seperately and + * In order to avoid the inconsistency in sizes save them separately and * migrate over in vmstate post_load(). */ static void fw_cfg_acpi_mr_save(FWCfgState *s, const char *filename, size_t len) diff --git a/hw/pci-bridge/cxl_downstream.c b/hw/pci-bridge/cxl_downstream.c index 54f507318f..5a2b749c8e 100644 --- a/hw/pci-bridge/cxl_downstream.c +++ b/hw/pci-bridge/cxl_downstream.c @@ -42,7 +42,7 @@ static void latch_registers(CXLDownstreamPort *dsp) CXL2_DOWNSTREAM_PORT); } -/* TODO: Look at sharing this code acorss all CXL port types */ +/* TODO: Look at sharing this code across all CXL port types */ static void cxl_dsp_dvsec_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len) { diff --git a/hw/pci-bridge/cxl_upstream.c b/hw/pci-bridge/cxl_upstream.c index 9159f48a8c..2b9cf0cc97 100644 --- a/hw/pci-bridge/cxl_upstream.c +++ b/hw/pci-bridge/cxl_upstream.c @@ -262,7 +262,7 @@ static int build_cdat_table(CDATSubHeader ***cdat_table, void *priv) .length = sslbis_size, }, .data_type = HMATLB_DATA_TYPE_ACCESS_BANDWIDTH, - .entry_base_unit = 1000, + .entry_base_unit = 1024, }, }; diff --git a/hw/pci-bridge/pci_expander_bridge.c b/hw/pci-bridge/pci_expander_bridge.c index 613857b601..535889f7c2 100644 --- a/hw/pci-bridge/pci_expander_bridge.c +++ b/hw/pci-bridge/pci_expander_bridge.c @@ -263,7 +263,7 @@ static int pxb_map_irq_fn(PCIDevice *pci_dev, int pin) /* * First carry out normal swizzle to handle - * multple root ports on a pxb instance. + * multiple root ports on a pxb instance. */ pin = pci_swizzle_map_irq_fn(pci_dev, pin); diff --git a/hw/pci-host/bonito.c b/hw/pci-host/bonito.c index 4701481b9b..ee6cb85e97 100644 --- a/hw/pci-host/bonito.c +++ b/hw/pci-host/bonito.c @@ -62,7 +62,7 @@ #define DPRINTF(fmt, ...) #endif -/* from linux soure code. include/asm-mips/mips-boards/bonito64.h*/ +/* from linux source code. include/asm-mips/mips-boards/bonito64.h*/ #define BONITO_BOOT_BASE 0x1fc00000 #define BONITO_BOOT_SIZE 0x00100000 #define BONITO_BOOT_TOP (BONITO_BOOT_BASE + BONITO_BOOT_SIZE - 1) diff --git a/hw/pci-host/designware.c b/hw/pci-host/designware.c index 388d252ee2..6f5442f108 100644 --- a/hw/pci-host/designware.c +++ b/hw/pci-host/designware.c @@ -488,7 +488,7 @@ static void designware_pcie_root_realize(PCIDevice *dev, Error **errp) /* * If no inbound iATU windows are configured, HW defaults to - * letting inbound TLPs to pass in. We emulate that by exlicitly + * letting inbound TLPs to pass in. We emulate that by explicitly * configuring first inbound window to cover all of target's * address space. * @@ -503,7 +503,7 @@ static void designware_pcie_root_realize(PCIDevice *dev, Error **errp) &designware_pci_host_msi_ops, root, "pcie-msi", 0x4); /* - * We initially place MSI interrupt I/O region a adress 0 and + * We initially place MSI interrupt I/O region at address 0 and * disable it. It'll be later moved to correct offset and enabled * in designware_pcie_root_update_msi_mapping() as a part of * initialization done by guest OS diff --git a/hw/pci-host/dino.c b/hw/pci-host/dino.c index e8eaebca54..82503229fa 100644 --- a/hw/pci-host/dino.c +++ b/hw/pci-host/dino.c @@ -1,5 +1,5 @@ /* - * HP-PARISC Dino PCI chipset emulation, as in B160L and similiar machines + * HP-PARISC Dino PCI chipset emulation, as in B160L and similar machines * * (C) 2017-2019 by Helge Deller <deller@gmx.de> * diff --git a/hw/pci-host/gpex-acpi.c b/hw/pci-host/gpex-acpi.c index 7c7316bc96..1092dc3b70 100644 --- a/hw/pci-host/gpex-acpi.c +++ b/hw/pci-host/gpex-acpi.c @@ -177,7 +177,7 @@ void acpi_dsdt_add_gpex(Aml *scope, struct GPEXConfig *cfg) acpi_dsdt_add_pci_route_table(dev, cfg->irq); /* - * Resources defined for PXBs are composed by the folling parts: + * Resources defined for PXBs are composed of the following parts: * 1. The resources the pci-brige/pcie-root-port need. * 2. The resources the devices behind pxb need. */ diff --git a/hw/pci-host/gt64120.c b/hw/pci-host/gt64120.c index 82c15edb46..143bf053d7 100644 --- a/hw/pci-host/gt64120.c +++ b/hw/pci-host/gt64120.c @@ -331,9 +331,9 @@ static void gt64120_update_pci_cfgdata_mapping(GT64120State *s) /* * The setting of the MByteSwap bit and MWordSwap bit in the PCI Internal * Command Register determines how data transactions from the CPU to/from - * PCI are handled along with the setting of the Endianess bit in the CPU + * PCI are handled along with the setting of the Endianness bit in the CPU * Configuration Register. See: - * - Table 16: 32-bit PCI Transaction Endianess + * - Table 16: 32-bit PCI Transaction Endianness * - Table 158: PCI_0 Command, Offset: 0xc00 */ diff --git a/hw/pci-host/pnv_phb.c b/hw/pci-host/pnv_phb.c index 82332d7a05..157c00782c 100644 --- a/hw/pci-host/pnv_phb.c +++ b/hw/pci-host/pnv_phb.c @@ -25,7 +25,7 @@ * state associated with the child has an id, use it as QOM id. * Otherwise use object_typename[index] as QOM id. * - * This helper does both operations at the same time because seting + * This helper does both operations at the same time because setting * a new QOM child will erase the bus parent of the device. This happens * because object_unparent() will call object_property_del_child(), * which in turn calls the property release callback prop->release if diff --git a/hw/pci-host/pnv_phb3.c b/hw/pci-host/pnv_phb3.c index 7a21497cf8..c5e58f4086 100644 --- a/hw/pci-host/pnv_phb3.c +++ b/hw/pci-host/pnv_phb3.c @@ -757,7 +757,7 @@ static void pnv_phb3_translate_tve(PnvPhb3DMASpace *ds, hwaddr addr, * We only support non-translate in top window. * * TODO: Venice/Murano support it on bottom window above 4G and - * Naples suports it on everything + * Naples supports it on everything */ if (!(tve & PPC_BIT(51))) { phb3_error(phb, "xlate for invalid non-translate TVE"); diff --git a/hw/pci-host/pnv_phb3_msi.c b/hw/pci-host/pnv_phb3_msi.c index 41e63b066f..dc8d8637f2 100644 --- a/hw/pci-host/pnv_phb3_msi.c +++ b/hw/pci-host/pnv_phb3_msi.c @@ -281,7 +281,7 @@ static void phb3_msi_instance_init(Object *obj) object_property_allow_set_link, OBJ_PROP_LINK_STRONG); - /* Will be overriden later */ + /* Will be overridden later */ ics->offset = 0; } diff --git a/hw/pci-host/pnv_phb4.c b/hw/pci-host/pnv_phb4.c index 6232cbeee1..29cb11a5d9 100644 --- a/hw/pci-host/pnv_phb4.c +++ b/hw/pci-host/pnv_phb4.c @@ -207,7 +207,7 @@ static void pnv_phb4_check_mbt(PnvPHB4 *phb, uint32_t index) start = base | (phb->regs[PHB_M64_UPPER_BITS >> 3]); } - /* TODO: Figure out how to implemet/decode AOMASK */ + /* TODO: Figure out how to implement/decode AOMASK */ /* Check if it matches an enabled MMIO region in the PEC stack */ if (memory_region_is_mapped(&phb->mmbar0) && @@ -391,7 +391,7 @@ static void pnv_phb4_ioda_write(PnvPHB4 *phb, uint64_t val) case IODA3_TBL_MBT: *tptr = val; - /* Copy accross the valid bit to the other half */ + /* Copy across the valid bit to the other half */ phb->ioda_MBT[idx ^ 1] &= 0x7fffffffffffffffull; phb->ioda_MBT[idx ^ 1] |= 0x8000000000000000ull & val; @@ -1408,7 +1408,7 @@ static void pnv_phb4_msi_write(void *opaque, hwaddr addr, return; } - /* TODO: check PE/MSI assignement */ + /* TODO: check PE/MSI assignment */ qemu_irq_pulse(phb->qirqs[src]); } diff --git a/hw/pci/pcie_aer.c b/hw/pci/pcie_aer.c index 374d593ead..b68c7ecb49 100644 --- a/hw/pci/pcie_aer.c +++ b/hw/pci/pcie_aer.c @@ -324,7 +324,7 @@ static void pcie_aer_msg_root_port(PCIDevice *dev, const PCIEAERMsg *msg) * it isn't implemented in qemu right now. * So just discard the error for now. * OS which cares of aer would receive errors via - * native aer mechanims, so this wouldn't matter. + * native aer mechanisms, so this wouldn't matter. */ } diff --git a/hw/pci/shpc.c b/hw/pci/shpc.c index e7bc7192f1..df7f370111 100644 --- a/hw/pci/shpc.c +++ b/hw/pci/shpc.c @@ -615,7 +615,7 @@ int shpc_init(PCIDevice *d, PCIBus *sec_bus, MemoryRegion *bar, } if (nslots > SHPC_MAX_SLOTS || SHPC_IDX_TO_PCI(nslots) > PCI_SLOT_MAX) { - /* TODO: report an error mesage that makes sense. */ + /* TODO: report an error message that makes sense. */ return -EINVAL; } shpc->nslots = nslots; diff --git a/hw/ppc/meson.build b/hw/ppc/meson.build index a313d4b964..7c2c52434a 100644 --- a/hw/ppc/meson.build +++ b/hw/ppc/meson.build @@ -36,7 +36,6 @@ ppc_ss.add(when: ['CONFIG_PSERIES', 'CONFIG_TCG'], if_true: files( ppc_ss.add(when: 'CONFIG_SPAPR_RNG', if_true: files('spapr_rng.c')) ppc_ss.add(when: ['CONFIG_PSERIES', 'CONFIG_LINUX'], if_true: files( 'spapr_pci_vfio.c', - 'spapr_pci_nvlink2.c' )) # IBM PowerNV diff --git a/hw/ppc/ppc.c b/hw/ppc/ppc.c index aeb116d919..be167710a3 100644 --- a/hw/ppc/ppc.c +++ b/hw/ppc/ppc.c @@ -738,7 +738,7 @@ static target_ulong _cpu_ppc_load_decr(CPUPPCState *env, int64_t now) decr = __cpu_ppc_load_decr(env, now, tb_env->decr_next); /* - * If large decrementer is enabled then the decrementer is signed extened + * If large decrementer is enabled then the decrementer is signed extended * to 64 bits, otherwise it is a 32 bit value. */ if (env->spr[SPR_LPCR] & LPCR_LD) { diff --git a/hw/ppc/prep_systemio.c b/hw/ppc/prep_systemio.c index 5a56f155f5..c96cefb13d 100644 --- a/hw/ppc/prep_systemio.c +++ b/hw/ppc/prep_systemio.c @@ -39,7 +39,7 @@ #define TYPE_PREP_SYSTEMIO "prep-systemio" OBJECT_DECLARE_SIMPLE_TYPE(PrepSystemIoState, PREP_SYSTEMIO) -/* Bit as defined in PowerPC Reference Plaform v1.1, sect. 6.1.5, p. 132 */ +/* Bit as defined in PowerPC Reference Platform v1.1, sect. 6.1.5, p. 132 */ #define PREP_BIT(n) (1 << (7 - (n))) struct PrepSystemIoState { diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index f7cc6a890f..1f1aa2a6d4 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -2573,7 +2573,7 @@ static void spapr_set_vsmt_mode(SpaprMachineState *spapr, Error **errp) return; } - /* Detemine the VSMT mode to use: */ + /* Determine the VSMT mode to use: */ if (vsmt_user) { if (spapr->vsmt < smp_threads) { error_setg(errp, "Cannot support VSMT mode %d" @@ -2875,8 +2875,6 @@ static void spapr_machine_init(MachineState *machine) /* init CPUs */ spapr_init_cpus(spapr); - spapr->gpu_numa_id = spapr_numa_initial_nvgpu_numa_id(machine); - /* Init numa_assoc_array */ spapr_numa_associativity_init(spapr, machine); @@ -3109,7 +3107,7 @@ static int spapr_kvm_type(MachineState *machine, const char *vm_type) { /* * The use of g_ascii_strcasecmp() for 'hv' and 'pr' is to - * accomodate the 'HV' and 'PV' formats that exists in the + * accommodate the 'HV' and 'PV' formats that exists in the * wild. The 'auto' mode is being introduced already as * lower-case, thus we don't need to bother checking for * "AUTO". @@ -4134,7 +4132,6 @@ static bool spapr_phb_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, &sphb->buid, &sphb->io_win_addr, &sphb->mem_win_addr, &sphb->mem64_win_addr, windows_supported, sphb->dma_liobn, - &sphb->nv2_gpa_win_addr, &sphb->nv2_atsd_win_addr, errp); } @@ -4343,7 +4340,7 @@ spapr_cpu_index_to_props(MachineState *machine, unsigned cpu_index) CPUArchId *core_slot; MachineClass *mc = MACHINE_GET_CLASS(machine); - /* make sure possible_cpu are intialized */ + /* make sure possible_cpu are initialized */ mc->possible_cpu_arch_ids(machine); /* get CPU core slot containing thread that matches cpu_index */ core_slot = spapr_find_cpu_slot(machine, cpu_index, NULL); @@ -4397,8 +4394,7 @@ static const CPUArchIdList *spapr_possible_cpu_arch_ids(MachineState *machine) static bool spapr_phb_placement(SpaprMachineState *spapr, uint32_t index, uint64_t *buid, hwaddr *pio, hwaddr *mmio32, hwaddr *mmio64, - unsigned n_dma, uint32_t *liobns, - hwaddr *nv2gpa, hwaddr *nv2atsd, Error **errp) + unsigned n_dma, uint32_t *liobns, Error **errp) { /* * New-style PHB window placement. @@ -4443,9 +4439,6 @@ static bool spapr_phb_placement(SpaprMachineState *spapr, uint32_t index, *pio = SPAPR_PCI_BASE + index * SPAPR_PCI_IO_WIN_SIZE; *mmio32 = SPAPR_PCI_BASE + (index + 1) * SPAPR_PCI_MEM32_WIN_SIZE; *mmio64 = SPAPR_PCI_BASE + (index + 1) * SPAPR_PCI_MEM64_WIN_SIZE; - - *nv2gpa = SPAPR_PCI_NV2RAM64_WIN_BASE + index * SPAPR_PCI_NV2RAM64_WIN_SIZE; - *nv2atsd = SPAPR_PCI_NV2ATSD_WIN_BASE + index * SPAPR_PCI_NV2ATSD_WIN_SIZE; return true; } @@ -4963,16 +4956,12 @@ DEFINE_SPAPR_MACHINE(4_1, "4.1", false); static bool phb_placement_4_0(SpaprMachineState *spapr, uint32_t index, uint64_t *buid, hwaddr *pio, hwaddr *mmio32, hwaddr *mmio64, - unsigned n_dma, uint32_t *liobns, - hwaddr *nv2gpa, hwaddr *nv2atsd, Error **errp) + unsigned n_dma, uint32_t *liobns, Error **errp) { if (!spapr_phb_placement(spapr, index, buid, pio, mmio32, mmio64, n_dma, - liobns, nv2gpa, nv2atsd, errp)) { + liobns, errp)) { return false; } - - *nv2gpa = 0; - *nv2atsd = 0; return true; } static void spapr_machine_4_0_class_options(MachineClass *mc) @@ -5045,7 +5034,7 @@ static void spapr_machine_2_12_class_options(MachineClass *mc) /* We depend on kvm_enabled() to choose a default value for the * hpt-max-page-size capability. Of course we can't do it here - * because this is too early and the HW accelerator isn't initialzed + * because this is too early and the HW accelerator isn't initialized * yet. Postpone this to machine init (see default_caps_with_cpu()). */ smc->default_caps.caps[SPAPR_CAP_HPT_MAXPAGESIZE] = 0; @@ -5137,8 +5126,7 @@ DEFINE_SPAPR_MACHINE(2_8, "2.8", false); static bool phb_placement_2_7(SpaprMachineState *spapr, uint32_t index, uint64_t *buid, hwaddr *pio, hwaddr *mmio32, hwaddr *mmio64, - unsigned n_dma, uint32_t *liobns, - hwaddr *nv2gpa, hwaddr *nv2atsd, Error **errp) + unsigned n_dma, uint32_t *liobns, Error **errp) { /* Legacy PHB placement for pseries-2.7 and earlier machine types */ const uint64_t base_buid = 0x800000020000000ULL; @@ -5183,8 +5171,6 @@ static bool phb_placement_2_7(SpaprMachineState *spapr, uint32_t index, * window into contiguous 32-bit and 64-bit windows */ - *nv2gpa = 0; - *nv2atsd = 0; return true; } diff --git a/hw/ppc/spapr_hcall.c b/hw/ppc/spapr_hcall.c index b7dc388f2f..522a2396c7 100644 --- a/hw/ppc/spapr_hcall.c +++ b/hw/ppc/spapr_hcall.c @@ -1615,7 +1615,7 @@ static void hypercall_register_types(void) spapr_register_hypercall(H_GET_CPU_CHARACTERISTICS, h_get_cpu_characteristics); - /* "debugger" hcalls (also used by SLOF). Note: We do -not- differenciate + /* "debugger" hcalls (also used by SLOF). Note: We do -not- differentiate * here between the "CI" and the "CACHE" variants, they will use whatever * mapping attributes qemu is using. When using KVM, the kernel will * enforce the attributes more strongly diff --git a/hw/ppc/spapr_numa.c b/hw/ppc/spapr_numa.c index a64098c375..ea6762d3d2 100644 --- a/hw/ppc/spapr_numa.c +++ b/hw/ppc/spapr_numa.c @@ -109,20 +109,6 @@ static bool spapr_numa_is_symmetrical(MachineState *ms) } /* - * NVLink2-connected GPU RAM needs to be placed on a separate NUMA node. - * We assign a new numa ID per GPU in spapr_pci_collect_nvgpu() which is - * called from vPHB reset handler so we initialize the counter here. - * If no NUMA is configured from the QEMU side, we start from 1 as GPU RAM - * must be equally distant from any other node. - * The final value of spapr->gpu_numa_id is going to be written to - * max-associativity-domains in spapr_build_fdt(). - */ -unsigned int spapr_numa_initial_nvgpu_numa_id(MachineState *machine) -{ - return MAX(1, machine->numa_state->num_nodes); -} - -/* * This function will translate the user distances into * what the kernel understand as possible values: 10 * (local distance), 20, 40, 80 and 160, and return the equivalent @@ -277,7 +263,7 @@ static void spapr_numa_FORM1_affinity_init(SpaprMachineState *spapr, { SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr); int nb_numa_nodes = machine->numa_state->num_nodes; - int i, j, max_nodes_with_gpus; + int i, j; /* * For all associativity arrays: first position is the size, @@ -293,17 +279,7 @@ static void spapr_numa_FORM1_affinity_init(SpaprMachineState *spapr, spapr->FORM1_assoc_array[i][FORM1_DIST_REF_POINTS] = cpu_to_be32(i); } - /* - * Initialize NVLink GPU associativity arrays. We know that - * the first GPU will take the first available NUMA id, and - * we'll have a maximum of NVGPU_MAX_NUM GPUs in the machine. - * At this point we're not sure if there are GPUs or not, but - * let's initialize the associativity arrays and allow NVLink - * GPUs to be handled like regular NUMA nodes later on. - */ - max_nodes_with_gpus = nb_numa_nodes + NVGPU_MAX_NUM; - - for (i = nb_numa_nodes; i < max_nodes_with_gpus; i++) { + for (i = nb_numa_nodes; i < nb_numa_nodes; i++) { spapr->FORM1_assoc_array[i][0] = cpu_to_be32(FORM1_DIST_REF_POINTS); for (j = 1; j < FORM1_DIST_REF_POINTS; j++) { @@ -345,10 +321,6 @@ static void spapr_numa_FORM2_affinity_init(SpaprMachineState *spapr) * CPUs will write an additional 'vcpu_id' on top of the arrays * being initialized here. 'numa_id' is represented by the * index 'i' of the loop. - * - * Given that this initialization is also valid for GPU associativity - * arrays, handle everything in one single step by populating the - * arrays up to NUMA_NODES_MAX_NUM. */ for (i = 0; i < NUMA_NODES_MAX_NUM; i++) { spapr->FORM2_assoc_array[i][0] = cpu_to_be32(1); @@ -461,8 +433,6 @@ static void spapr_numa_FORM1_write_rtas_dt(SpaprMachineState *spapr, { MachineState *ms = MACHINE(spapr); SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr); - uint32_t number_nvgpus_nodes = spapr->gpu_numa_id - - spapr_numa_initial_nvgpu_numa_id(ms); uint32_t refpoints[] = { cpu_to_be32(0x4), cpu_to_be32(0x3), @@ -470,7 +440,7 @@ static void spapr_numa_FORM1_write_rtas_dt(SpaprMachineState *spapr, cpu_to_be32(0x1), }; uint32_t nr_refpoints = ARRAY_SIZE(refpoints); - uint32_t maxdomain = ms->numa_state->num_nodes + number_nvgpus_nodes; + uint32_t maxdomain = ms->numa_state->num_nodes; uint32_t maxdomains[] = { cpu_to_be32(4), cpu_to_be32(maxdomain), @@ -486,13 +456,12 @@ static void spapr_numa_FORM1_write_rtas_dt(SpaprMachineState *spapr, cpu_to_be32(0x4), cpu_to_be32(0x2), }; - uint32_t legacy_maxdomain = spapr->gpu_numa_id > 1 ? 1 : 0; uint32_t legacy_maxdomains[] = { cpu_to_be32(4), - cpu_to_be32(legacy_maxdomain), - cpu_to_be32(legacy_maxdomain), - cpu_to_be32(legacy_maxdomain), - cpu_to_be32(spapr->gpu_numa_id), + cpu_to_be32(0), + cpu_to_be32(0), + cpu_to_be32(0), + cpu_to_be32(maxdomain ? maxdomain : 1), }; G_STATIC_ASSERT(sizeof(legacy_refpoints) <= sizeof(refpoints)); @@ -581,8 +550,6 @@ static void spapr_numa_FORM2_write_rtas_dt(SpaprMachineState *spapr, void *fdt, int rtas) { MachineState *ms = MACHINE(spapr); - uint32_t number_nvgpus_nodes = spapr->gpu_numa_id - - spapr_numa_initial_nvgpu_numa_id(ms); /* * In FORM2, ibm,associativity-reference-points will point to @@ -596,7 +563,7 @@ static void spapr_numa_FORM2_write_rtas_dt(SpaprMachineState *spapr, */ uint32_t refpoints[] = { cpu_to_be32(1) }; - uint32_t maxdomain = ms->numa_state->num_nodes + number_nvgpus_nodes; + uint32_t maxdomain = ms->numa_state->num_nodes; uint32_t maxdomains[] = { cpu_to_be32(1), cpu_to_be32(maxdomain) }; _FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points", diff --git a/hw/ppc/spapr_nvdimm.c b/hw/ppc/spapr_nvdimm.c index a8688243a6..b2f009c816 100644 --- a/hw/ppc/spapr_nvdimm.c +++ b/hw/ppc/spapr_nvdimm.c @@ -320,7 +320,8 @@ static target_ulong h_scm_write_metadata(PowerPCCPU *cpu, nvdimm = NVDIMM(drc->dev); if ((offset + len < offset) || - (nvdimm->label_size < len + offset)) { + (nvdimm->label_size < len + offset) || + nvdimm->readonly) { return H_P2; } @@ -377,7 +378,7 @@ static target_ulong h_scm_bind_mem(PowerPCCPU *cpu, SpaprMachineState *spapr, /* * Currently continue token should be zero qemu has already bound - * everything and this hcall doesnt return H_BUSY. + * everything and this hcall doesn't return H_BUSY. */ if (continue_token > 0) { return H_P5; @@ -588,7 +589,7 @@ void spapr_nvdimm_finish_flushes(void) * Called on reset path, the main loop thread which calls * the pending BHs has gotten out running in the reset path, * finally reaching here. Other code path being guest - * h_client_architecture_support, thats early boot up. + * h_client_architecture_support, that's early boot up. */ nvdimms = nvdimm_get_device_list(); for (list = nvdimms; list; list = list->next) { diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c index 75aacda65a..ce14959317 100644 --- a/hw/ppc/spapr_pci.c +++ b/hw/ppc/spapr_pci.c @@ -1443,8 +1443,6 @@ static int spapr_dt_pci_device(SpaprPhbState *sphb, PCIDevice *dev, _FDT(fdt_setprop_cell(fdt, offset, "ibm,pci-config-space-type", 0x1)); } - spapr_phb_nvgpu_populate_pcidev_dt(dev, fdt, offset, sphb); - if (!IS_PCI_BRIDGE(dev)) { /* Properties only for non-bridges */ uint32_t min_grant = pci_default_read_config(dev, PCI_MIN_GNT, 1); @@ -1757,8 +1755,6 @@ static void spapr_phb_unrealize(DeviceState *dev) int i; const unsigned windows_supported = spapr_phb_windows_supported(sphb); - spapr_phb_nvgpu_free(sphb); - if (sphb->msi) { g_hash_table_unref(sphb->msi); sphb->msi = NULL; @@ -2069,14 +2065,8 @@ void spapr_phb_dma_reset(SpaprPhbState *sphb) static void spapr_phb_reset(DeviceState *qdev) { SpaprPhbState *sphb = SPAPR_PCI_HOST_BRIDGE(qdev); - Error *err = NULL; spapr_phb_dma_reset(sphb); - spapr_phb_nvgpu_free(sphb); - spapr_phb_nvgpu_setup(sphb, &err); - if (err) { - error_report_err(err); - } /* Reset the IOMMU state */ object_child_foreach(OBJECT(qdev), spapr_phb_children_reset, NULL); @@ -2112,8 +2102,6 @@ static Property spapr_phb_properties[] = { pre_2_8_migration, false), DEFINE_PROP_BOOL("pcie-extended-configuration-space", SpaprPhbState, pcie_ecs, true), - DEFINE_PROP_UINT64("gpa", SpaprPhbState, nv2_gpa_win_addr, 0), - DEFINE_PROP_UINT64("atsd", SpaprPhbState, nv2_atsd_win_addr, 0), DEFINE_PROP_BOOL("pre-5.1-associativity", SpaprPhbState, pre_5_1_assoc, false), DEFINE_PROP_END_OF_LIST(), @@ -2362,7 +2350,6 @@ int spapr_dt_phb(SpaprMachineState *spapr, SpaprPhbState *phb, }; SpaprTceTable *tcet; SpaprDrc *drc; - Error *err = NULL; /* Start populating the FDT */ _FDT(bus_off = fdt_add_subnode(fdt, 0, phb->dtbusname)); @@ -2443,12 +2430,6 @@ int spapr_dt_phb(SpaprMachineState *spapr, SpaprPhbState *phb, return ret; } - spapr_phb_nvgpu_populate_dt(phb, fdt, bus_off, &err); - if (err) { - error_report_err(err); - } - spapr_phb_nvgpu_ram_populate_dt(phb, fdt); - return 0; } diff --git a/hw/ppc/spapr_pci_nvlink2.c b/hw/ppc/spapr_pci_nvlink2.c deleted file mode 100644 index 2a8a11be1d..0000000000 --- a/hw/ppc/spapr_pci_nvlink2.c +++ /dev/null @@ -1,442 +0,0 @@ -/* - * QEMU sPAPR PCI for NVLink2 pass through - * - * Copyright (c) 2019 Alexey Kardashevskiy, IBM Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include "qemu/osdep.h" -#include "qapi/error.h" -#include "hw/pci/pci.h" -#include "hw/pci-host/spapr.h" -#include "hw/ppc/spapr_numa.h" -#include "qemu/error-report.h" -#include "hw/ppc/fdt.h" -#include "hw/pci/pci_bridge.h" - -#define PHANDLE_PCIDEV(phb, pdev) (0x12000000 | \ - (((phb)->index) << 16) | ((pdev)->devfn)) -#define PHANDLE_GPURAM(phb, n) (0x110000FF | ((n) << 8) | \ - (((phb)->index) << 16)) -#define PHANDLE_NVLINK(phb, gn, nn) (0x00130000 | (((phb)->index) << 8) | \ - ((gn) << 4) | (nn)) - -typedef struct SpaprPhbPciNvGpuSlot { - uint64_t tgt; - uint64_t gpa; - unsigned numa_id; - PCIDevice *gpdev; - int linknum; - struct { - uint64_t atsd_gpa; - PCIDevice *npdev; - uint32_t link_speed; - } links[NVGPU_MAX_LINKS]; -} SpaprPhbPciNvGpuSlot; - -struct SpaprPhbPciNvGpuConfig { - uint64_t nv2_ram_current; - uint64_t nv2_atsd_current; - int num; /* number of non empty (i.e. tgt!=0) entries in slots[] */ - SpaprPhbPciNvGpuSlot slots[NVGPU_MAX_NUM]; - Error *err; -}; - -static SpaprPhbPciNvGpuSlot * -spapr_nvgpu_get_slot(SpaprPhbPciNvGpuConfig *nvgpus, uint64_t tgt) -{ - int i; - - /* Search for partially collected "slot" */ - for (i = 0; i < nvgpus->num; ++i) { - if (nvgpus->slots[i].tgt == tgt) { - return &nvgpus->slots[i]; - } - } - - if (nvgpus->num == ARRAY_SIZE(nvgpus->slots)) { - return NULL; - } - - i = nvgpus->num; - nvgpus->slots[i].tgt = tgt; - ++nvgpus->num; - - return &nvgpus->slots[i]; -} - -static void spapr_pci_collect_nvgpu(SpaprPhbPciNvGpuConfig *nvgpus, - PCIDevice *pdev, uint64_t tgt, - MemoryRegion *mr, Error **errp) -{ - MachineState *machine = MACHINE(qdev_get_machine()); - SpaprMachineState *spapr = SPAPR_MACHINE(machine); - SpaprPhbPciNvGpuSlot *nvslot = spapr_nvgpu_get_slot(nvgpus, tgt); - - if (!nvslot) { - error_setg(errp, "Found too many GPUs per vPHB"); - return; - } - g_assert(!nvslot->gpdev); - nvslot->gpdev = pdev; - - nvslot->gpa = nvgpus->nv2_ram_current; - nvgpus->nv2_ram_current += memory_region_size(mr); - nvslot->numa_id = spapr->gpu_numa_id; - ++spapr->gpu_numa_id; -} - -static void spapr_pci_collect_nvnpu(SpaprPhbPciNvGpuConfig *nvgpus, - PCIDevice *pdev, uint64_t tgt, - MemoryRegion *mr, Error **errp) -{ - SpaprPhbPciNvGpuSlot *nvslot = spapr_nvgpu_get_slot(nvgpus, tgt); - int j; - - if (!nvslot) { - error_setg(errp, "Found too many NVLink bridges per vPHB"); - return; - } - - j = nvslot->linknum; - if (j == ARRAY_SIZE(nvslot->links)) { - error_setg(errp, "Found too many NVLink bridges per GPU"); - return; - } - ++nvslot->linknum; - - g_assert(!nvslot->links[j].npdev); - nvslot->links[j].npdev = pdev; - nvslot->links[j].atsd_gpa = nvgpus->nv2_atsd_current; - nvgpus->nv2_atsd_current += memory_region_size(mr); - nvslot->links[j].link_speed = - object_property_get_uint(OBJECT(pdev), "nvlink2-link-speed", NULL); -} - -static void spapr_phb_pci_collect_nvgpu(PCIBus *bus, PCIDevice *pdev, - void *opaque) -{ - PCIBus *sec_bus; - Object *po = OBJECT(pdev); - uint64_t tgt = object_property_get_uint(po, "nvlink2-tgt", NULL); - - if (tgt) { - Error *local_err = NULL; - SpaprPhbPciNvGpuConfig *nvgpus = opaque; - Object *mr_gpu = object_property_get_link(po, "nvlink2-mr[0]", NULL); - Object *mr_npu = object_property_get_link(po, "nvlink2-atsd-mr[0]", - NULL); - - g_assert(mr_gpu || mr_npu); - if (mr_gpu) { - spapr_pci_collect_nvgpu(nvgpus, pdev, tgt, MEMORY_REGION(mr_gpu), - &local_err); - } else { - spapr_pci_collect_nvnpu(nvgpus, pdev, tgt, MEMORY_REGION(mr_npu), - &local_err); - } - error_propagate(&nvgpus->err, local_err); - } - if ((pci_default_read_config(pdev, PCI_HEADER_TYPE, 1) != - PCI_HEADER_TYPE_BRIDGE)) { - return; - } - - sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(pdev)); - if (!sec_bus) { - return; - } - - pci_for_each_device_under_bus(sec_bus, spapr_phb_pci_collect_nvgpu, opaque); -} - -void spapr_phb_nvgpu_setup(SpaprPhbState *sphb, Error **errp) -{ - int i, j, valid_gpu_num; - PCIBus *bus; - - /* Search for GPUs and NPUs */ - if (!sphb->nv2_gpa_win_addr || !sphb->nv2_atsd_win_addr) { - return; - } - - sphb->nvgpus = g_new0(SpaprPhbPciNvGpuConfig, 1); - sphb->nvgpus->nv2_ram_current = sphb->nv2_gpa_win_addr; - sphb->nvgpus->nv2_atsd_current = sphb->nv2_atsd_win_addr; - - bus = PCI_HOST_BRIDGE(sphb)->bus; - pci_for_each_device_under_bus(bus, spapr_phb_pci_collect_nvgpu, - sphb->nvgpus); - - if (sphb->nvgpus->err) { - error_propagate(errp, sphb->nvgpus->err); - sphb->nvgpus->err = NULL; - goto cleanup_exit; - } - - /* Add found GPU RAM and ATSD MRs if found */ - for (i = 0, valid_gpu_num = 0; i < sphb->nvgpus->num; ++i) { - Object *nvmrobj; - SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i]; - - if (!nvslot->gpdev) { - continue; - } - nvmrobj = object_property_get_link(OBJECT(nvslot->gpdev), - "nvlink2-mr[0]", NULL); - /* ATSD is pointless without GPU RAM MR so skip those */ - if (!nvmrobj) { - continue; - } - - ++valid_gpu_num; - memory_region_add_subregion(get_system_memory(), nvslot->gpa, - MEMORY_REGION(nvmrobj)); - - for (j = 0; j < nvslot->linknum; ++j) { - Object *atsdmrobj; - - atsdmrobj = object_property_get_link(OBJECT(nvslot->links[j].npdev), - "nvlink2-atsd-mr[0]", NULL); - if (!atsdmrobj) { - continue; - } - memory_region_add_subregion(get_system_memory(), - nvslot->links[j].atsd_gpa, - MEMORY_REGION(atsdmrobj)); - } - } - - if (valid_gpu_num) { - return; - } - /* We did not find any interesting GPU */ -cleanup_exit: - g_free(sphb->nvgpus); - sphb->nvgpus = NULL; -} - -void spapr_phb_nvgpu_free(SpaprPhbState *sphb) -{ - int i, j; - - if (!sphb->nvgpus) { - return; - } - - for (i = 0; i < sphb->nvgpus->num; ++i) { - SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i]; - Object *nv_mrobj = object_property_get_link(OBJECT(nvslot->gpdev), - "nvlink2-mr[0]", NULL); - - if (nv_mrobj) { - memory_region_del_subregion(get_system_memory(), - MEMORY_REGION(nv_mrobj)); - } - for (j = 0; j < nvslot->linknum; ++j) { - PCIDevice *npdev = nvslot->links[j].npdev; - Object *atsd_mrobj; - atsd_mrobj = object_property_get_link(OBJECT(npdev), - "nvlink2-atsd-mr[0]", NULL); - if (atsd_mrobj) { - memory_region_del_subregion(get_system_memory(), - MEMORY_REGION(atsd_mrobj)); - } - } - } - g_free(sphb->nvgpus); - sphb->nvgpus = NULL; -} - -void spapr_phb_nvgpu_populate_dt(SpaprPhbState *sphb, void *fdt, int bus_off, - Error **errp) -{ - int i, j, atsdnum = 0; - uint64_t atsd[8]; /* The existing limitation of known guests */ - - if (!sphb->nvgpus) { - return; - } - - for (i = 0; (i < sphb->nvgpus->num) && (atsdnum < ARRAY_SIZE(atsd)); ++i) { - SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i]; - - if (!nvslot->gpdev) { - continue; - } - for (j = 0; j < nvslot->linknum; ++j) { - if (!nvslot->links[j].atsd_gpa) { - continue; - } - - if (atsdnum == ARRAY_SIZE(atsd)) { - error_report("Only %"PRIuPTR" ATSD registers supported", - ARRAY_SIZE(atsd)); - break; - } - atsd[atsdnum] = cpu_to_be64(nvslot->links[j].atsd_gpa); - ++atsdnum; - } - } - - if (!atsdnum) { - error_setg(errp, "No ATSD registers found"); - return; - } - - if (!spapr_phb_eeh_available(sphb)) { - /* - * ibm,mmio-atsd contains ATSD registers; these belong to an NPU PHB - * which we do not emulate as a separate device. Instead we put - * ibm,mmio-atsd to the vPHB with GPU and make sure that we do not - * put GPUs from different IOMMU groups to the same vPHB to ensure - * that the guest will use ATSDs from the corresponding NPU. - */ - error_setg(errp, "ATSD requires separate vPHB per GPU IOMMU group"); - return; - } - - _FDT((fdt_setprop(fdt, bus_off, "ibm,mmio-atsd", atsd, - atsdnum * sizeof(atsd[0])))); -} - -void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt) -{ - int i, j, linkidx, npuoff; - g_autofree char *npuname = NULL; - - if (!sphb->nvgpus) { - return; - } - - npuname = g_strdup_printf("npuphb%d", sphb->index); - npuoff = fdt_add_subnode(fdt, 0, npuname); - _FDT(npuoff); - _FDT(fdt_setprop_cell(fdt, npuoff, "#address-cells", 1)); - _FDT(fdt_setprop_cell(fdt, npuoff, "#size-cells", 0)); - /* Advertise NPU as POWER9 so the guest can enable NPU2 contexts */ - _FDT((fdt_setprop_string(fdt, npuoff, "compatible", "ibm,power9-npu"))); - - for (i = 0, linkidx = 0; i < sphb->nvgpus->num; ++i) { - for (j = 0; j < sphb->nvgpus->slots[i].linknum; ++j) { - g_autofree char *linkname = g_strdup_printf("link@%d", linkidx); - int off = fdt_add_subnode(fdt, npuoff, linkname); - - _FDT(off); - /* _FDT((fdt_setprop_cell(fdt, off, "reg", linkidx))); */ - _FDT((fdt_setprop_string(fdt, off, "compatible", - "ibm,npu-link"))); - _FDT((fdt_setprop_cell(fdt, off, "phandle", - PHANDLE_NVLINK(sphb, i, j)))); - _FDT((fdt_setprop_cell(fdt, off, "ibm,npu-link-index", linkidx))); - ++linkidx; - } - } - - /* Add memory nodes for GPU RAM and mark them unusable */ - for (i = 0; i < sphb->nvgpus->num; ++i) { - SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i]; - Object *nv_mrobj = object_property_get_link(OBJECT(nvslot->gpdev), - "nvlink2-mr[0]", - &error_abort); - uint64_t size = object_property_get_uint(nv_mrobj, "size", NULL); - uint64_t mem_reg[2] = { cpu_to_be64(nvslot->gpa), cpu_to_be64(size) }; - g_autofree char *mem_name = g_strdup_printf("memory@%"PRIx64, - nvslot->gpa); - int off = fdt_add_subnode(fdt, 0, mem_name); - - _FDT(off); - _FDT((fdt_setprop_string(fdt, off, "device_type", "memory"))); - _FDT((fdt_setprop(fdt, off, "reg", mem_reg, sizeof(mem_reg)))); - - spapr_numa_write_associativity_dt(SPAPR_MACHINE(qdev_get_machine()), - fdt, off, nvslot->numa_id); - - _FDT((fdt_setprop_string(fdt, off, "compatible", - "ibm,coherent-device-memory"))); - - mem_reg[1] = cpu_to_be64(0); - _FDT((fdt_setprop(fdt, off, "linux,usable-memory", mem_reg, - sizeof(mem_reg)))); - _FDT((fdt_setprop_cell(fdt, off, "phandle", - PHANDLE_GPURAM(sphb, i)))); - } - -} - -void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset, - SpaprPhbState *sphb) -{ - int i, j; - - if (!sphb->nvgpus) { - return; - } - - for (i = 0; i < sphb->nvgpus->num; ++i) { - SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i]; - - /* Skip "slot" without attached GPU */ - if (!nvslot->gpdev) { - continue; - } - if (dev == nvslot->gpdev) { - g_autofree uint32_t *npus = g_new(uint32_t, nvslot->linknum); - - for (j = 0; j < nvslot->linknum; ++j) { - PCIDevice *npdev = nvslot->links[j].npdev; - - npus[j] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev)); - } - _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus, - j * sizeof(npus[0]))); - _FDT((fdt_setprop_cell(fdt, offset, "phandle", - PHANDLE_PCIDEV(sphb, dev)))); - continue; - } - - for (j = 0; j < nvslot->linknum; ++j) { - if (dev != nvslot->links[j].npdev) { - continue; - } - - _FDT((fdt_setprop_cell(fdt, offset, "phandle", - PHANDLE_PCIDEV(sphb, dev)))); - _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu", - PHANDLE_PCIDEV(sphb, nvslot->gpdev))); - _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink", - PHANDLE_NVLINK(sphb, i, j)))); - /* - * If we ever want to emulate GPU RAM at the same location as on - * the host - here is the encoding GPA->TGT: - * - * gta = ((sphb->nv2_gpa >> 42) & 0x1) << 42; - * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43; - * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45; - * gta |= sphb->nv2_gpa & ((1UL << 43) - 1); - */ - _FDT(fdt_setprop_cell(fdt, offset, "memory-region", - PHANDLE_GPURAM(sphb, i))); - _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr", - nvslot->tgt)); - _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed", - nvslot->links[j].link_speed)); - } - } -} diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c index d8aeee0b7e..9016720547 100644 --- a/hw/ppc/spapr_pci_vfio.c +++ b/hw/ppc/spapr_pci_vfio.c @@ -78,7 +78,7 @@ int spapr_phb_vfio_eeh_set_option(SpaprPhbState *sphb, * call. Now we just need to check the validity of the PCI * pass-through devices (vfio-pci) under this sphb bus. * We have already validated that all the devices under this sphb - * are from same iommu group (within same PE) before comming here. + * are from same iommu group (within same PE) before coming here. * * Prior to linux commit 98ba956f6a389 ("powerpc/pseries/eeh: * Rework device EEH PE determination") kernel would call diff --git a/hw/rtc/exynos4210_rtc.c b/hw/rtc/exynos4210_rtc.c index 2b8a38a296..cc7101c530 100644 --- a/hw/rtc/exynos4210_rtc.c +++ b/hw/rtc/exynos4210_rtc.c @@ -202,7 +202,7 @@ static void exynos4210_rtc_update_freq(Exynos4210RTCState *s, uint32_t freq; freq = s->freq; - /* set frequncy for time generator */ + /* set frequency for time generator */ s->freq = RTC_BASE_FREQ / (1 << TICCKSEL(reg_value)); if (freq != s->freq) { diff --git a/hw/rx/rx62n.c b/hw/rx/rx62n.c index 3e887a0fc7..d00fcb0ef0 100644 --- a/hw/rx/rx62n.c +++ b/hw/rx/rx62n.c @@ -114,7 +114,7 @@ static const uint8_t ipr_table[NR_IRQS] = { }; /* - * Level triggerd IRQ list + * Level triggered IRQ list * Not listed IRQ is Edge trigger. * See "11.3.1 Interrupt Vector Table" in hardware manual. */ diff --git a/hw/scsi/lsi53c895a.c b/hw/scsi/lsi53c895a.c index f7d45b0b20..634ed49c2e 100644 --- a/hw/scsi/lsi53c895a.c +++ b/hw/scsi/lsi53c895a.c @@ -1321,7 +1321,7 @@ again: } trace_lsi_execute_script_io_selected(id, insn & (1 << 3) ? " ATN" : ""); - /* ??? Linux drivers compain when this is set. Maybe + /* ??? Linux drivers complain when this is set. Maybe it only applies in low-level mode (unimplemented). lsi_script_scsi_interrupt(s, LSI_SIST0_CMP, 0); */ s->select_tag = id << 8; diff --git a/hw/scsi/mfi.h b/hw/scsi/mfi.h index 0b4ee53dfc..cf7a2d775b 100644 --- a/hw/scsi/mfi.h +++ b/hw/scsi/mfi.h @@ -65,7 +65,7 @@ #define MFI_IQPH 0xc4 /* Inbound queue port (high bytes) */ #define MFI_DIAG 0xf8 /* Host diag */ #define MFI_SEQ 0xfc /* Sequencer offset */ -#define MFI_1078_EIM 0x80000004 /* 1078 enable intrrupt mask */ +#define MFI_1078_EIM 0x80000004 /* 1078 enable interrupt mask */ #define MFI_RMI 0x2 /* reply message interrupt */ #define MFI_1078_RM 0x80000000 /* reply 1078 message interrupt */ #define MFI_ODC 0x4 /* outbound doorbell change interrupt */ diff --git a/hw/sh4/sh7750_regs.h b/hw/sh4/sh7750_regs.h index 94043431e6..edb5d18f00 100644 --- a/hw/sh4/sh7750_regs.h +++ b/hw/sh4/sh7750_regs.h @@ -113,7 +113,7 @@ #define SH7750_TTB SH7750_P4_REG32(SH7750_TTB_REGOFS) #define SH7750_TTB_A7 SH7750_A7_REG32(SH7750_TTB_REGOFS) -/* TLB exeption address register - TEA */ +/* TLB exception address register - TEA */ #define SH7750_TEA_REGOFS 0x00000c /* offset */ #define SH7750_TEA SH7750_P4_REG32(SH7750_TEA_REGOFS) #define SH7750_TEA_A7 SH7750_A7_REG32(SH7750_TEA_REGOFS) @@ -183,19 +183,19 @@ #define SH7750_TRA_IMM 0x000003fd /* Immediate data operand */ #define SH7750_TRA_IMM_S 2 -/* Exeption event register - EXPEVT */ +/* Exception event register - EXPEVT */ #define SH7750_EXPEVT_REGOFS 0x000024 #define SH7750_EXPEVT SH7750_P4_REG32(SH7750_EXPEVT_REGOFS) #define SH7750_EXPEVT_A7 SH7750_A7_REG32(SH7750_EXPEVT_REGOFS) -#define SH7750_EXPEVT_EX 0x00000fff /* Exeption code */ +#define SH7750_EXPEVT_EX 0x00000fff /* Exception code */ #define SH7750_EXPEVT_EX_S 0 /* Interrupt event register */ #define SH7750_INTEVT_REGOFS 0x000028 #define SH7750_INTEVT SH7750_P4_REG32(SH7750_INTEVT_REGOFS) #define SH7750_INTEVT_A7 SH7750_A7_REG32(SH7750_INTEVT_REGOFS) -#define SH7750_INTEVT_EX 0x00000fff /* Exeption code */ +#define SH7750_INTEVT_EX 0x00000fff /* Exception code */ #define SH7750_INTEVT_EX_S 0 /* @@ -1274,15 +1274,15 @@ /* * User Break Controller registers */ -#define SH7750_BARA 0x200000 /* Break address regiser A */ -#define SH7750_BAMRA 0x200004 /* Break address mask regiser A */ -#define SH7750_BBRA 0x200008 /* Break bus cycle regiser A */ -#define SH7750_BARB 0x20000c /* Break address regiser B */ -#define SH7750_BAMRB 0x200010 /* Break address mask regiser B */ -#define SH7750_BBRB 0x200014 /* Break bus cycle regiser B */ -#define SH7750_BASRB 0x000018 /* Break ASID regiser B */ -#define SH7750_BDRB 0x200018 /* Break data regiser B */ -#define SH7750_BDMRB 0x20001c /* Break data mask regiser B */ +#define SH7750_BARA 0x200000 /* Break address register A */ +#define SH7750_BAMRA 0x200004 /* Break address mask register A */ +#define SH7750_BBRA 0x200008 /* Break bus cycle register A */ +#define SH7750_BARB 0x20000c /* Break address register B */ +#define SH7750_BAMRB 0x200010 /* Break address mask register B */ +#define SH7750_BBRB 0x200014 /* Break bus cycle register B */ +#define SH7750_BASRB 0x000018 /* Break ASID register B */ +#define SH7750_BDRB 0x200018 /* Break data register B */ +#define SH7750_BDMRB 0x20001c /* Break data mask register B */ #define SH7750_BRCR 0x200020 /* Break control register */ #define SH7750_BRCR_UDBE 0x0001 /* User break debug enable bit */ diff --git a/hw/smbios/smbios.c b/hw/smbios/smbios.c index 10cd22f610..b753705856 100644 --- a/hw/smbios/smbios.c +++ b/hw/smbios/smbios.c @@ -1110,7 +1110,7 @@ void smbios_get_tables(MachineState *ms, dimm_cnt = QEMU_ALIGN_UP(current_machine->ram_size, MAX_DIMM_SZ) / MAX_DIMM_SZ; /* - * The offset determines if we need to keep additional space betweeen + * The offset determines if we need to keep additional space between * table 17 and table 19 header handle numbers so that they do * not overlap. For example, for a VM with larger than 8 TB guest * memory and DIMM like chunks of 16 GiB, the default space between diff --git a/hw/ssi/xilinx_spips.c b/hw/ssi/xilinx_spips.c index 97009d3a5d..a3955c6c50 100644 --- a/hw/ssi/xilinx_spips.c +++ b/hw/ssi/xilinx_spips.c @@ -163,7 +163,7 @@ FIELD(GQSPI_CNFG, ENDIAN, 26, 1) /* Poll timeout not implemented */ FIELD(GQSPI_CNFG, EN_POLL_TIMEOUT, 20, 1) - /* QEMU doesnt care about any of these last three */ + /* QEMU doesn't care about any of these last three */ FIELD(GQSPI_CNFG, BR, 3, 3) FIELD(GQSPI_CNFG, CPH, 2, 1) FIELD(GQSPI_CNFG, CPL, 1, 1) @@ -469,7 +469,7 @@ static void xlnx_zynqmp_qspips_flush_fifo_g(XlnxZynqMPQSPIPS *s) imm = ARRAY_FIELD_EX32(s->regs, GQSPI_GF_SNAPSHOT, IMMEDIATE_DATA); if (!ARRAY_FIELD_EX32(s->regs, GQSPI_GF_SNAPSHOT, DATA_XFER)) { - /* immedate transfer */ + /* immediate transfer */ if (ARRAY_FIELD_EX32(s->regs, GQSPI_GF_SNAPSHOT, TRANSMIT) || ARRAY_FIELD_EX32(s->regs, GQSPI_GF_SNAPSHOT, RECIEVE)) { s->regs[R_GQSPI_DATA_STS] = 1; @@ -768,7 +768,7 @@ static void xilinx_spips_check_zero_pump(XilinxSPIPS *s) */ while (s->regs[R_TRANSFER_SIZE] && s->rx_fifo.num + s->tx_fifo.num < RXFF_A_Q - 3) { - /* endianess just doesn't matter when zero pumping */ + /* endianness just doesn't matter when zero pumping */ tx_data_bytes(&s->tx_fifo, 0, 4, false); s->regs[R_TRANSFER_SIZE] &= ~0x03ull; s->regs[R_TRANSFER_SIZE] -= 4; diff --git a/hw/ssi/xlnx-versal-ospi.c b/hw/ssi/xlnx-versal-ospi.c index c762e0b367..1a61679c2f 100644 --- a/hw/ssi/xlnx-versal-ospi.c +++ b/hw/ssi/xlnx-versal-ospi.c @@ -837,7 +837,7 @@ static void ospi_do_ind_read(XlnxVersalOspi *s) /* Continue to read flash until we run out of space in sram */ while (!ospi_ind_op_completed(op) && !fifo8_is_full(&s->rx_sram)) { - /* Read reqested number of bytes, max bytes limited to size of sram */ + /* Read requested number of bytes, max bytes limited to size of sram */ next_b = ind_op_next_byte(op); end_b = next_b + fifo8_num_free(&s->rx_sram); end_b = MIN(end_b, ind_op_end_byte(op)); diff --git a/hw/timer/etraxfs_timer.c b/hw/timer/etraxfs_timer.c index 2d6d92ef93..f035b74560 100644 --- a/hw/timer/etraxfs_timer.c +++ b/hw/timer/etraxfs_timer.c @@ -236,7 +236,7 @@ static void watchdog_hit(void *opaque) { ETRAXTimerState *t = opaque; if (t->wd_hits == 0) { - /* real hw gives a single tick before reseting but we are + /* real hw gives a single tick before resetting but we are a bit friendlier to compensate for our slower execution. */ ptimer_set_count(t->ptimer_wd, 10); ptimer_run(t->ptimer_wd, 1); diff --git a/hw/timer/renesas_tmr.c b/hw/timer/renesas_tmr.c index c15f654738..43b31213bc 100644 --- a/hw/timer/renesas_tmr.c +++ b/hw/timer/renesas_tmr.c @@ -115,7 +115,7 @@ static int elapsed_time(RTMRState *tmr, int ch, int64_t delta) et = tmr->div_round[ch] / divrate; tmr->div_round[ch] %= divrate; } else { - /* disble clock. so no update */ + /* disable clock. so no update */ et = 0; } return et; diff --git a/hw/tpm/tpm_tis.h b/hw/tpm/tpm_tis.h index 6f29a508dd..6f14896b97 100644 --- a/hw/tpm/tpm_tis.h +++ b/hw/tpm/tpm_tis.h @@ -19,7 +19,7 @@ * specification. * * TPM TIS for TPM 2 implementation following TCG PC Client Platform - * TPM Profile (PTP) Specification, Familiy 2.0, Revision 00.43 + * TPM Profile (PTP) Specification, Family 2.0, Revision 00.43 */ #ifndef TPM_TPM_TIS_H #define TPM_TPM_TIS_H diff --git a/hw/tpm/tpm_tis_common.c b/hw/tpm/tpm_tis_common.c index c07c179dbc..279ce436b5 100644 --- a/hw/tpm/tpm_tis_common.c +++ b/hw/tpm/tpm_tis_common.c @@ -20,7 +20,7 @@ * specification. * * TPM TIS for TPM 2 implementation following TCG PC Client Platform - * TPM Profile (PTP) Specification, Familiy 2.0, Revision 00.43 + * TPM Profile (PTP) Specification, Family 2.0, Revision 00.43 */ #include "qemu/osdep.h" #include "hw/irq.h" diff --git a/hw/tpm/tpm_tis_i2c.c b/hw/tpm/tpm_tis_i2c.c index b695fd3a46..4ecea7fa3e 100644 --- a/hw/tpm/tpm_tis_i2c.c +++ b/hw/tpm/tpm_tis_i2c.c @@ -13,7 +13,7 @@ * Family 2.0, Level 00, Revision 1.00 * * TPM TIS for TPM 2 implementation following TCG PC Client Platform - * TPM Profile (PTP) Specification, Familiy 2.0, Revision 00.43 + * TPM Profile (PTP) Specification, Family 2.0, Revision 00.43 * */ @@ -507,7 +507,7 @@ static void tpm_tis_i2c_realizefn(DeviceState *dev, Error **errp) } /* - * Get the backend pointer. It is not initialized propery during + * Get the backend pointer. It is not initialized properly during * device_class_set_props */ s->be_driver = qemu_find_tpm_be("tpm0"); diff --git a/hw/tpm/tpm_tis_isa.c b/hw/tpm/tpm_tis_isa.c index 91e3792248..0367401586 100644 --- a/hw/tpm/tpm_tis_isa.c +++ b/hw/tpm/tpm_tis_isa.c @@ -19,7 +19,7 @@ * specification. * * TPM TIS for TPM 2 implementation following TCG PC Client Platform - * TPM Profile (PTP) Specification, Familiy 2.0, Revision 00.43 + * TPM Profile (PTP) Specification, Family 2.0, Revision 00.43 */ #include "qemu/osdep.h" diff --git a/hw/tpm/tpm_tis_sysbus.c b/hw/tpm/tpm_tis_sysbus.c index 6724b3d4f6..2fc550f119 100644 --- a/hw/tpm/tpm_tis_sysbus.c +++ b/hw/tpm/tpm_tis_sysbus.c @@ -19,7 +19,7 @@ * specification. * * TPM TIS for TPM 2 implementation following TCG PC Client Platform - * TPM Profile (PTP) Specification, Familiy 2.0, Revision 00.43 + * TPM Profile (PTP) Specification, Family 2.0, Revision 00.43 */ #include "qemu/osdep.h" diff --git a/hw/vfio/pci-quirks.c b/hw/vfio/pci-quirks.c index f4ff836805..84b1a7b948 100644 --- a/hw/vfio/pci-quirks.c +++ b/hw/vfio/pci-quirks.c @@ -1610,121 +1610,6 @@ static int vfio_add_nv_gpudirect_cap(VFIOPCIDevice *vdev, Error **errp) return 0; } -int vfio_pci_nvidia_v100_ram_init(VFIOPCIDevice *vdev, Error **errp) -{ - int ret; - void *p; - struct vfio_region_info *nv2reg = NULL; - struct vfio_info_cap_header *hdr; - struct vfio_region_info_cap_nvlink2_ssatgt *cap; - VFIOQuirk *quirk; - - ret = vfio_get_dev_region_info(&vdev->vbasedev, - VFIO_REGION_TYPE_PCI_VENDOR_TYPE | - PCI_VENDOR_ID_NVIDIA, - VFIO_REGION_SUBTYPE_NVIDIA_NVLINK2_RAM, - &nv2reg); - if (ret) { - return ret; - } - - hdr = vfio_get_region_info_cap(nv2reg, VFIO_REGION_INFO_CAP_NVLINK2_SSATGT); - if (!hdr) { - ret = -ENODEV; - goto free_exit; - } - cap = (void *) hdr; - - p = mmap(NULL, nv2reg->size, PROT_READ | PROT_WRITE, - MAP_SHARED, vdev->vbasedev.fd, nv2reg->offset); - if (p == MAP_FAILED) { - ret = -errno; - goto free_exit; - } - - quirk = vfio_quirk_alloc(1); - memory_region_init_ram_ptr(&quirk->mem[0], OBJECT(vdev), "nvlink2-mr", - nv2reg->size, p); - QLIST_INSERT_HEAD(&vdev->bars[0].quirks, quirk, next); - - object_property_add_uint64_ptr(OBJECT(vdev), "nvlink2-tgt", - (uint64_t *) &cap->tgt, - OBJ_PROP_FLAG_READ); - trace_vfio_pci_nvidia_gpu_setup_quirk(vdev->vbasedev.name, cap->tgt, - nv2reg->size); -free_exit: - g_free(nv2reg); - - return ret; -} - -int vfio_pci_nvlink2_init(VFIOPCIDevice *vdev, Error **errp) -{ - int ret; - void *p; - struct vfio_region_info *atsdreg = NULL; - struct vfio_info_cap_header *hdr; - struct vfio_region_info_cap_nvlink2_ssatgt *captgt; - struct vfio_region_info_cap_nvlink2_lnkspd *capspeed; - VFIOQuirk *quirk; - - ret = vfio_get_dev_region_info(&vdev->vbasedev, - VFIO_REGION_TYPE_PCI_VENDOR_TYPE | - PCI_VENDOR_ID_IBM, - VFIO_REGION_SUBTYPE_IBM_NVLINK2_ATSD, - &atsdreg); - if (ret) { - return ret; - } - - hdr = vfio_get_region_info_cap(atsdreg, - VFIO_REGION_INFO_CAP_NVLINK2_SSATGT); - if (!hdr) { - ret = -ENODEV; - goto free_exit; - } - captgt = (void *) hdr; - - hdr = vfio_get_region_info_cap(atsdreg, - VFIO_REGION_INFO_CAP_NVLINK2_LNKSPD); - if (!hdr) { - ret = -ENODEV; - goto free_exit; - } - capspeed = (void *) hdr; - - /* Some NVLink bridges may not have assigned ATSD */ - if (atsdreg->size) { - p = mmap(NULL, atsdreg->size, PROT_READ | PROT_WRITE, - MAP_SHARED, vdev->vbasedev.fd, atsdreg->offset); - if (p == MAP_FAILED) { - ret = -errno; - goto free_exit; - } - - quirk = vfio_quirk_alloc(1); - memory_region_init_ram_device_ptr(&quirk->mem[0], OBJECT(vdev), - "nvlink2-atsd-mr", atsdreg->size, p); - QLIST_INSERT_HEAD(&vdev->bars[0].quirks, quirk, next); - } - - object_property_add_uint64_ptr(OBJECT(vdev), "nvlink2-tgt", - (uint64_t *) &captgt->tgt, - OBJ_PROP_FLAG_READ); - trace_vfio_pci_nvlink2_setup_quirk_ssatgt(vdev->vbasedev.name, captgt->tgt, - atsdreg->size); - - object_property_add_uint32_ptr(OBJECT(vdev), "nvlink2-link-speed", - &capspeed->link_speed, - OBJ_PROP_FLAG_READ); - trace_vfio_pci_nvlink2_setup_quirk_lnkspd(vdev->vbasedev.name, - capspeed->link_speed); -free_exit: - g_free(atsdreg); - - return ret; -} - /* * The VMD endpoint provides a real PCIe domain to the guest and the guest * kernel performs enumeration of the VMD sub-device domain. Guest transactions diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index a205c6b113..3b2ca3c24c 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -3271,20 +3271,6 @@ static void vfio_realize(PCIDevice *pdev, Error **errp) } } - if (vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID)) { - ret = vfio_pci_nvidia_v100_ram_init(vdev, errp); - if (ret && ret != -ENODEV) { - error_report("Failed to setup NVIDIA V100 GPU RAM"); - } - } - - if (vfio_pci_is(vdev, PCI_VENDOR_ID_IBM, PCI_ANY_ID)) { - ret = vfio_pci_nvlink2_init(vdev, errp); - if (ret && ret != -ENODEV) { - error_report("Failed to setup NVlink2 bridge"); - } - } - if (!pdev->failover_pair_id) { if (!vfio_migration_realize(vbasedev, errp)) { goto out_deregister; diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h index a2771b9ff3..2d836093a8 100644 --- a/hw/vfio/pci.h +++ b/hw/vfio/pci.h @@ -221,8 +221,6 @@ int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp); int vfio_pci_igd_opregion_init(VFIOPCIDevice *vdev, struct vfio_region_info *info, Error **errp); -int vfio_pci_nvidia_v100_ram_init(VFIOPCIDevice *vdev, Error **errp); -int vfio_pci_nvlink2_init(VFIOPCIDevice *vdev, Error **errp); void vfio_display_reset(VFIOPCIDevice *vdev); int vfio_display_probe(VFIOPCIDevice *vdev, Error **errp); diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events index 81ec7c7a95..e64ca4a019 100644 --- a/hw/vfio/trace-events +++ b/hw/vfio/trace-events @@ -82,10 +82,6 @@ vfio_ioeventfd_handler(const char *name, uint64_t addr, unsigned size, uint64_t vfio_ioeventfd_init(const char *name, uint64_t addr, unsigned size, uint64_t data, bool vfio) "%s+0x%"PRIx64"[%d]:0x%"PRIx64" vfio:%d" vfio_pci_igd_opregion_enabled(const char *name) "%s" -vfio_pci_nvidia_gpu_setup_quirk(const char *name, uint64_t tgt, uint64_t size) "%s tgt=0x%"PRIx64" size=0x%"PRIx64 -vfio_pci_nvlink2_setup_quirk_ssatgt(const char *name, uint64_t tgt, uint64_t size) "%s tgt=0x%"PRIx64" size=0x%"PRIx64 -vfio_pci_nvlink2_setup_quirk_lnkspd(const char *name, uint32_t link_speed) "%s link_speed=0x%x" - # igd.c vfio_pci_igd_bar4_write(const char *name, uint32_t index, uint32_t data, uint32_t base) "%s [0x%03x] 0x%08x -> 0x%08x" vfio_pci_igd_bdsm_enabled(const char *name, int size) "%s %dMB" diff --git a/hw/virtio/virtio-crypto.c b/hw/virtio/virtio-crypto.c index 13aec771e1..0e2cc8d5a8 100644 --- a/hw/virtio/virtio-crypto.c +++ b/hw/virtio/virtio-crypto.c @@ -655,7 +655,7 @@ virtio_crypto_sym_op_helper(VirtIODevice *vdev, op_info->len_to_hash = len_to_hash; op_info->cipher_start_src_offset = cipher_start_src_offset; op_info->len_to_cipher = len_to_cipher; - /* Handle the initilization vector */ + /* Handle the initialization vector */ if (op_info->iv_len > 0) { DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len); op_info->iv = op_info->data + curr_size; @@ -1278,7 +1278,7 @@ static void virtio_crypto_instance_init(Object *obj) /* * The default config_size is sizeof(struct virtio_crypto_config). - * Can be overriden with virtio_crypto_set_config_size. + * Can be overridden with virtio_crypto_set_config_size. */ vcrypto->config_size = sizeof(struct virtio_crypto_config); } diff --git a/hw/virtio/virtio-mem.c b/hw/virtio/virtio-mem.c index b6e781741e..da5b09cefc 100644 --- a/hw/virtio/virtio-mem.c +++ b/hw/virtio/virtio-mem.c @@ -1119,7 +1119,7 @@ static int virtio_mem_mig_sanity_checks_post_load(void *opaque, int version_id) return -EINVAL; } /* - * Note: Preparation for resizeable memory regions. The maximum size + * Note: Preparation for resizable memory regions. The maximum size * of the memory region must not change during migration. */ if (tmp->region_size != new_region_size) { diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index 969c25f4cf..4577f3f5b3 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -2096,7 +2096,7 @@ void virtio_queue_enable(VirtIODevice *vdev, uint32_t queue_index) * being converted to LOG_GUEST_ERROR. * if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { - error_report("queue_enable is only suppported in devices of virtio " + error_report("queue_enable is only supported in devices of virtio " "1.0 or later."); } */ diff --git a/include/block/aio.h b/include/block/aio.h index 32042e8905..f08b358077 100644 --- a/include/block/aio.h +++ b/include/block/aio.h @@ -31,7 +31,6 @@ typedef void BlockCompletionFunc(void *opaque, int ret); typedef struct AIOCBInfo { void (*cancel_async)(BlockAIOCB *acb); - AioContext *(*get_aio_context)(BlockAIOCB *acb); size_t aiocb_size; } AIOCBInfo; @@ -468,7 +467,7 @@ void aio_dispatch(AioContext *ctx); * or more AIO events have completed, to ensure something has moved * before returning. */ -bool aio_poll(AioContext *ctx, bool blocking); +bool no_coroutine_fn aio_poll(AioContext *ctx, bool blocking); /* Register a file descriptor and associated callbacks. Behaves very similarly * to qemu_set_fd_handler. Unlike qemu_set_fd_handler, these callbacks will diff --git a/include/block/block-common.h b/include/block/block-common.h index df5ffc8d09..2d2af7230d 100644 --- a/include/block/block-common.h +++ b/include/block/block-common.h @@ -66,10 +66,14 @@ * function. The coroutine yields after scheduling the BH and is reentered when * the wrapped function returns. * + * A no_co_wrapper_bdrv_wrlock function is a no_co_wrapper function that + * automatically takes the graph wrlock when calling the wrapped function. + * * If the first parameter of the function is a BlockDriverState, BdrvChild or * BlockBackend pointer, the AioContext lock for it is taken in the wrapper. */ #define no_co_wrapper +#define no_co_wrapper_bdrv_wrlock #include "block/blockjob.h" @@ -287,6 +291,8 @@ typedef enum { * layer rather than any backing, set by block layer * BDRV_BLOCK_EOF: the returned pnum covers through end of file for this * layer, set by block layer + * BDRV_BLOCK_COMPRESSED: the underlying data is compressed; only valid for + * the formats supporting compression: qcow, qcow2 * * Internal flags: * BDRV_BLOCK_RAW: for use by passthrough drivers, such as raw, to request @@ -322,6 +328,7 @@ typedef enum { #define BDRV_BLOCK_ALLOCATED 0x10 #define BDRV_BLOCK_EOF 0x20 #define BDRV_BLOCK_RECURSE 0x40 +#define BDRV_BLOCK_COMPRESSED 0x80 typedef QTAILQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue; diff --git a/include/block/block-global-state.h b/include/block/block-global-state.h index f347199bff..6061220a6c 100644 --- a/include/block/block-global-state.h +++ b/include/block/block-global-state.h @@ -185,6 +185,8 @@ void bdrv_drain_all_begin_nopoll(void); void bdrv_drain_all_end(void); void bdrv_drain_all(void); +void bdrv_aio_cancel(BlockAIOCB *acb); + int bdrv_has_zero_init_1(BlockDriverState *bs); int bdrv_has_zero_init(BlockDriverState *bs); BlockDriverState *bdrv_find_node(const char *node_name); @@ -224,13 +226,21 @@ void bdrv_img_create(const char *filename, const char *fmt, void bdrv_ref(BlockDriverState *bs); void no_coroutine_fn bdrv_unref(BlockDriverState *bs); void coroutine_fn no_co_wrapper bdrv_co_unref(BlockDriverState *bs); -void bdrv_unref_child(BlockDriverState *parent, BdrvChild *child); -BdrvChild *bdrv_attach_child(BlockDriverState *parent_bs, - BlockDriverState *child_bs, - const char *child_name, - const BdrvChildClass *child_class, - BdrvChildRole child_role, - Error **errp); +void GRAPH_WRLOCK bdrv_schedule_unref(BlockDriverState *bs); + +void GRAPH_WRLOCK +bdrv_unref_child(BlockDriverState *parent, BdrvChild *child); + +void coroutine_fn no_co_wrapper_bdrv_wrlock +bdrv_co_unref_child(BlockDriverState *parent, BdrvChild *child); + +BdrvChild * GRAPH_WRLOCK +bdrv_attach_child(BlockDriverState *parent_bs, + BlockDriverState *child_bs, + const char *child_name, + const BdrvChildClass *child_class, + BdrvChildRole child_role, + Error **errp); bool bdrv_op_is_blocked(BlockDriverState *bs, BlockOpType op, Error **errp); void bdrv_op_block(BlockDriverState *bs, BlockOpType op, Error *reason); @@ -268,9 +278,11 @@ int bdrv_try_change_aio_context(BlockDriverState *bs, AioContext *ctx, int bdrv_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz); int bdrv_probe_geometry(BlockDriverState *bs, HDGeometry *geo); -void bdrv_add_child(BlockDriverState *parent, BlockDriverState *child, - Error **errp); -void bdrv_del_child(BlockDriverState *parent, BdrvChild *child, Error **errp); +void GRAPH_WRLOCK +bdrv_add_child(BlockDriverState *parent, BlockDriverState *child, Error **errp); + +void GRAPH_WRLOCK +bdrv_del_child(BlockDriverState *parent, BdrvChild *child, Error **errp); /** * diff --git a/include/block/block-io.h b/include/block/block-io.h index 6db48f2d35..f1c796a1ce 100644 --- a/include/block/block-io.h +++ b/include/block/block-io.h @@ -101,7 +101,6 @@ bdrv_co_delete_file_noerr(BlockDriverState *bs); /* async block I/O */ -void bdrv_aio_cancel(BlockAIOCB *acb); void bdrv_aio_cancel_async(BlockAIOCB *acb); /* sg packet commands */ diff --git a/include/block/block_int-common.h b/include/block/block_int-common.h index 85be256c09..2ca3758cb8 100644 --- a/include/block/block_int-common.h +++ b/include/block/block_int-common.h @@ -311,7 +311,7 @@ struct BlockDriver { */ void (*bdrv_cancel_in_flight)(BlockDriverState *bs); - int (*bdrv_inactivate)(BlockDriverState *bs); + int GRAPH_RDLOCK_PTR (*bdrv_inactivate)(BlockDriverState *bs); int (*bdrv_snapshot_create)(BlockDriverState *bs, QEMUSnapshotInfo *sn_info); @@ -393,10 +393,11 @@ struct BlockDriver { */ int (*bdrv_probe_geometry)(BlockDriverState *bs, HDGeometry *geo); - void (*bdrv_add_child)(BlockDriverState *parent, BlockDriverState *child, - Error **errp); - void (*bdrv_del_child)(BlockDriverState *parent, BdrvChild *child, - Error **errp); + void GRAPH_WRLOCK_PTR (*bdrv_add_child)( + BlockDriverState *parent, BlockDriverState *child, Error **errp); + + void GRAPH_WRLOCK_PTR (*bdrv_del_child)( + BlockDriverState *parent, BdrvChild *child, Error **errp); /** * Informs the block driver that a permission change is intended. The @@ -413,8 +414,8 @@ struct BlockDriver { * If both conditions are met, 0 is returned. Otherwise, -errno is returned * and errp is set to an error describing the conflict. */ - int (*bdrv_check_perm)(BlockDriverState *bs, uint64_t perm, - uint64_t shared, Error **errp); + int GRAPH_RDLOCK_PTR (*bdrv_check_perm)(BlockDriverState *bs, uint64_t perm, + uint64_t shared, Error **errp); /** * Called to inform the driver that the set of cumulative set of used @@ -426,7 +427,8 @@ struct BlockDriver { * This function is only invoked after bdrv_check_perm(), so block drivers * may rely on preparations made in their .bdrv_check_perm implementation. */ - void (*bdrv_set_perm)(BlockDriverState *bs, uint64_t perm, uint64_t shared); + void GRAPH_RDLOCK_PTR (*bdrv_set_perm)( + BlockDriverState *bs, uint64_t perm, uint64_t shared); /* * Called to inform the driver that after a previous bdrv_check_perm() @@ -436,7 +438,7 @@ struct BlockDriver { * This function can be called even for nodes that never saw a * bdrv_check_perm() call. It is a no-op then. */ - void (*bdrv_abort_perm_update)(BlockDriverState *bs); + void GRAPH_RDLOCK_PTR (*bdrv_abort_perm_update)(BlockDriverState *bs); /** * Returns in @nperm and @nshared the permissions that the driver for @bs @@ -450,11 +452,11 @@ struct BlockDriver { * permissions, but those that will be needed after applying the * @reopen_queue. */ - void (*bdrv_child_perm)(BlockDriverState *bs, BdrvChild *c, - BdrvChildRole role, - BlockReopenQueue *reopen_queue, - uint64_t parent_perm, uint64_t parent_shared, - uint64_t *nperm, uint64_t *nshared); + void GRAPH_RDLOCK_PTR (*bdrv_child_perm)( + BlockDriverState *bs, BdrvChild *c, BdrvChildRole role, + BlockReopenQueue *reopen_queue, + uint64_t parent_perm, uint64_t parent_shared, + uint64_t *nperm, uint64_t *nshared); /** * Register/unregister a buffer for I/O. For example, when the driver is @@ -944,8 +946,8 @@ struct BdrvChildClass { * when migration is completing) and it can start/stop requesting * permissions and doing I/O on it. */ - void (*activate)(BdrvChild *child, Error **errp); - int (*inactivate)(BdrvChild *child); + void GRAPH_RDLOCK_PTR (*activate)(BdrvChild *child, Error **errp); + int GRAPH_RDLOCK_PTR (*inactivate)(BdrvChild *child); void GRAPH_WRLOCK_PTR (*attach)(BdrvChild *child); void GRAPH_WRLOCK_PTR (*detach)(BdrvChild *child); diff --git a/include/block/block_int-global-state.h b/include/block/block_int-global-state.h index da5fb31089..074b677838 100644 --- a/include/block/block_int-global-state.h +++ b/include/block/block_int-global-state.h @@ -202,18 +202,19 @@ BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs, BdrvChildRole child_role, uint64_t perm, uint64_t shared_perm, void *opaque, Error **errp); -void bdrv_root_unref_child(BdrvChild *child); +void GRAPH_WRLOCK bdrv_root_unref_child(BdrvChild *child); -void bdrv_get_cumulative_perm(BlockDriverState *bs, uint64_t *perm, - uint64_t *shared_perm); +void GRAPH_RDLOCK bdrv_get_cumulative_perm(BlockDriverState *bs, uint64_t *perm, + uint64_t *shared_perm); /** * Sets a BdrvChild's permissions. Avoid if the parent is a BDS; use * bdrv_child_refresh_perms() instead and make the parent's * .bdrv_child_perm() implementation return the correct values. */ -int bdrv_child_try_set_perm(BdrvChild *c, uint64_t perm, uint64_t shared, - Error **errp); +int GRAPH_RDLOCK +bdrv_child_try_set_perm(BdrvChild *c, uint64_t perm, uint64_t shared, + Error **errp); /** * Calls bs->drv->bdrv_child_perm() and updates the child's permission @@ -223,7 +224,8 @@ int bdrv_child_try_set_perm(BdrvChild *c, uint64_t perm, uint64_t shared, * values than before, but which will not result in the block layer * automatically refreshing the permissions. */ -int bdrv_child_refresh_perms(BlockDriverState *bs, BdrvChild *c, Error **errp); +int GRAPH_RDLOCK +bdrv_child_refresh_perms(BlockDriverState *bs, BdrvChild *c, Error **errp); bool GRAPH_RDLOCK bdrv_recurse_can_replace(BlockDriverState *bs, BlockDriverState *to_replace); diff --git a/include/block/throttle-groups.h b/include/block/throttle-groups.h index ff282fc0f8..2355e8d9de 100644 --- a/include/block/throttle-groups.h +++ b/include/block/throttle-groups.h @@ -37,7 +37,7 @@ typedef struct ThrottleGroupMember { AioContext *aio_context; /* throttled_reqs_lock protects the CoQueues for throttled requests. */ CoMutex throttled_reqs_lock; - CoQueue throttled_reqs[2]; + CoQueue throttled_reqs[THROTTLE_MAX]; /* Nonzero if the I/O limits are currently being ignored; generally * it is zero. Accessed with atomic operations. @@ -54,7 +54,7 @@ typedef struct ThrottleGroupMember { * throttle_state tells us if I/O limits are configured. */ ThrottleState *throttle_state; ThrottleTimers throttle_timers; - unsigned pending_reqs[2]; + unsigned pending_reqs[THROTTLE_MAX]; QLIST_ENTRY(ThrottleGroupMember) round_robin; } ThrottleGroupMember; @@ -78,7 +78,7 @@ void throttle_group_restart_tgm(ThrottleGroupMember *tgm); void coroutine_fn throttle_group_co_io_limits_intercept(ThrottleGroupMember *tgm, int64_t bytes, - bool is_write); + ThrottleDirection direction); void throttle_group_attach_aio_context(ThrottleGroupMember *tgm, AioContext *new_context); void throttle_group_detach_aio_context(ThrottleGroupMember *tgm); diff --git a/include/crypto/clmul.h b/include/crypto/clmul.h new file mode 100644 index 0000000000..446931fe05 --- /dev/null +++ b/include/crypto/clmul.h @@ -0,0 +1,83 @@ +/* + * Carry-less multiply operations. + * SPDX-License-Identifier: GPL-2.0-or-later + * + * Copyright (C) 2023 Linaro, Ltd. + */ + +#ifndef CRYPTO_CLMUL_H +#define CRYPTO_CLMUL_H + +#include "qemu/int128.h" +#include "host/crypto/clmul.h" + +/** + * clmul_8x8_low: + * + * Perform eight 8x8->8 carry-less multiplies. + */ +uint64_t clmul_8x8_low(uint64_t, uint64_t); + +/** + * clmul_8x4_even: + * + * Perform four 8x8->16 carry-less multiplies. + * The odd bytes of the inputs are ignored. + */ +uint64_t clmul_8x4_even(uint64_t, uint64_t); + +/** + * clmul_8x4_odd: + * + * Perform four 8x8->16 carry-less multiplies. + * The even bytes of the inputs are ignored. + */ +uint64_t clmul_8x4_odd(uint64_t, uint64_t); + +/** + * clmul_8x4_packed: + * + * Perform four 8x8->16 carry-less multiplies. + */ +uint64_t clmul_8x4_packed(uint32_t, uint32_t); + +/** + * clmul_16x2_even: + * + * Perform two 16x16->32 carry-less multiplies. + * The odd words of the inputs are ignored. + */ +uint64_t clmul_16x2_even(uint64_t, uint64_t); + +/** + * clmul_16x2_odd: + * + * Perform two 16x16->32 carry-less multiplies. + * The even words of the inputs are ignored. + */ +uint64_t clmul_16x2_odd(uint64_t, uint64_t); + +/** + * clmul_32: + * + * Perform a 32x32->64 carry-less multiply. + */ +uint64_t clmul_32(uint32_t, uint32_t); + +/** + * clmul_64: + * + * Perform a 64x64->128 carry-less multiply. + */ +Int128 clmul_64_gen(uint64_t, uint64_t); + +static inline Int128 clmul_64(uint64_t a, uint64_t b) +{ + if (HAVE_CLMUL_ACCEL) { + return clmul_64_accel(a, b); + } else { + return clmul_64_gen(a, b); + } +} + +#endif /* CRYPTO_CLMUL_H */ diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index fb4c8d480f..350287852e 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -100,12 +100,12 @@ typedef struct CPUTLBEntryFull { /* * @xlat_section contains: - * - in the lower TARGET_PAGE_BITS, a physical section number - * - with the lower TARGET_PAGE_BITS masked off, an offset which - * must be added to the virtual address to obtain: - * + the ram_addr_t of the target RAM (if the physical section - * number is PHYS_SECTION_NOTDIRTY or PHYS_SECTION_ROM) - * + the offset within the target MemoryRegion (otherwise) + * - For ram, an offset which must be added to the virtual address + * to obtain the ram_addr_t of the target RAM + * - For other memory regions, + * + in the lower TARGET_PAGE_BITS, the physical section number + * + with the TARGET_PAGE_BITS masked off, the offset within + * the target MemoryRegion */ hwaddr xlat_section; diff --git a/include/exec/memory.h b/include/exec/memory.h index 68284428f8..ef23d65afc 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -235,6 +235,12 @@ typedef struct IOMMUTLBEvent { /* RAM is an mmap-ed named file */ #define RAM_NAMED_FILE (1 << 9) +/* RAM is mmap-ed read-only */ +#define RAM_READONLY (1 << 10) + +/* RAM FD is opened read-only */ +#define RAM_READONLY_FD (1 << 11) + static inline void iommu_notifier_init(IOMMUNotifier *n, IOMMUNotify fn, IOMMUNotifierFlag flags, hwaddr start, hwaddr end, @@ -1089,6 +1095,7 @@ struct AddressSpace { struct FlatView *current_map; int ioeventfd_nb; + int ioeventfd_notifiers; struct MemoryRegionIoeventfd *ioeventfds; QTAILQ_HEAD(, MemoryListener) listeners; QTAILQ_ENTRY(AddressSpace) address_spaces_link; @@ -1331,10 +1338,10 @@ void memory_region_init_resizeable_ram(MemoryRegion *mr, * @align: alignment of the region base address; if 0, the default alignment * (getpagesize()) will be used. * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM, - * RAM_NORESERVE, + * RAM_NORESERVE, RAM_PROTECTED, RAM_NAMED_FILE, RAM_READONLY, + * RAM_READONLY_FD * @path: the path in which to allocate the RAM. * @offset: offset within the file referenced by path - * @readonly: true to open @path for reading, false for read/write. * @errp: pointer to Error*, to store an error if it happens. * * Note that this function does not do anything to cause the data in the @@ -1348,7 +1355,6 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, uint32_t ram_flags, const char *path, ram_addr_t offset, - bool readonly, Error **errp); /** @@ -1360,7 +1366,8 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, * @name: the name of the region. * @size: size of the region. * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM, - * RAM_NORESERVE, RAM_PROTECTED. + * RAM_NORESERVE, RAM_PROTECTED, RAM_NAMED_FILE, RAM_READONLY, + * RAM_READONLY_FD * @fd: the fd to mmap. * @offset: offset within the file referenced by fd * @errp: pointer to Error*, to store an error if it happens. diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 9f2e3893f5..90676093f5 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -108,10 +108,10 @@ long qemu_maxrampagesize(void); * @size: the size in bytes of the ram block * @mr: the memory region where the ram block is * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM, - * RAM_NORESERVE. + * RAM_NORESERVE, RAM_PROTECTED, RAM_NAMED_FILE, RAM_READONLY, + * RAM_READONLY_FD * @mem_path or @fd: specify the backing file or device * @offset: Offset into target file - * @readonly: true to open @path for reading, false for read/write. * @errp: pointer to Error*, to store an error if it happens * * Return: @@ -120,10 +120,10 @@ long qemu_maxrampagesize(void); */ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, const char *mem_path, - off_t offset, bool readonly, Error **errp); + off_t offset, Error **errp); RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, int fd, off_t offset, - bool readonly, Error **errp); + Error **errp); RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, MemoryRegion *mr, Error **errp); diff --git a/include/exec/user/thunk.h b/include/exec/user/thunk.h index 6eedef48d8..2ebfecf58e 100644 --- a/include/exec/user/thunk.h +++ b/include/exec/user/thunk.h @@ -111,8 +111,7 @@ static inline int thunk_type_size(const argtype *type_ptr, int is_host) if (is_host) { #if defined(HOST_X86_64) return 8; -#elif defined(HOST_ALPHA) || defined(HOST_IA64) || defined(HOST_MIPS) || \ - defined(HOST_PARISC) || defined(HOST_SPARC64) +#elif defined(HOST_MIPS) || defined(HOST_SPARC64) return 4; #elif defined(HOST_PPC) return sizeof(void *); diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h index cd130564d8..eb64075b9c 100644 --- a/include/fpu/softfloat.h +++ b/include/fpu/softfloat.h @@ -366,6 +366,8 @@ float32 bfloat16_to_float32(bfloat16, float_status *status); bfloat16 float64_to_bfloat16(float64 a, float_status *status); float64 bfloat16_to_float64(bfloat16 a, float_status *status); +int8_t bfloat16_to_int8_scalbn(bfloat16, FloatRoundMode, + int, float_status *status); int16_t bfloat16_to_int16_scalbn(bfloat16, FloatRoundMode, int, float_status *status); int32_t bfloat16_to_int32_scalbn(bfloat16, FloatRoundMode, @@ -373,14 +375,18 @@ int32_t bfloat16_to_int32_scalbn(bfloat16, FloatRoundMode, int64_t bfloat16_to_int64_scalbn(bfloat16, FloatRoundMode, int, float_status *status); +int8_t bfloat16_to_int8(bfloat16, float_status *status); int16_t bfloat16_to_int16(bfloat16, float_status *status); int32_t bfloat16_to_int32(bfloat16, float_status *status); int64_t bfloat16_to_int64(bfloat16, float_status *status); +int8_t bfloat16_to_int8_round_to_zero(bfloat16, float_status *status); int16_t bfloat16_to_int16_round_to_zero(bfloat16, float_status *status); int32_t bfloat16_to_int32_round_to_zero(bfloat16, float_status *status); int64_t bfloat16_to_int64_round_to_zero(bfloat16, float_status *status); +uint8_t bfloat16_to_uint8_scalbn(bfloat16 a, FloatRoundMode, + int, float_status *status); uint16_t bfloat16_to_uint16_scalbn(bfloat16 a, FloatRoundMode, int, float_status *status); uint32_t bfloat16_to_uint32_scalbn(bfloat16 a, FloatRoundMode, @@ -388,24 +394,30 @@ uint32_t bfloat16_to_uint32_scalbn(bfloat16 a, FloatRoundMode, uint64_t bfloat16_to_uint64_scalbn(bfloat16 a, FloatRoundMode, int, float_status *status); +uint8_t bfloat16_to_uint8(bfloat16 a, float_status *status); uint16_t bfloat16_to_uint16(bfloat16 a, float_status *status); uint32_t bfloat16_to_uint32(bfloat16 a, float_status *status); uint64_t bfloat16_to_uint64(bfloat16 a, float_status *status); +uint8_t bfloat16_to_uint8_round_to_zero(bfloat16 a, float_status *status); uint16_t bfloat16_to_uint16_round_to_zero(bfloat16 a, float_status *status); uint32_t bfloat16_to_uint32_round_to_zero(bfloat16 a, float_status *status); uint64_t bfloat16_to_uint64_round_to_zero(bfloat16 a, float_status *status); +bfloat16 int8_to_bfloat16_scalbn(int8_t a, int, float_status *status); bfloat16 int16_to_bfloat16_scalbn(int16_t a, int, float_status *status); bfloat16 int32_to_bfloat16_scalbn(int32_t a, int, float_status *status); bfloat16 int64_to_bfloat16_scalbn(int64_t a, int, float_status *status); +bfloat16 uint8_to_bfloat16_scalbn(uint8_t a, int, float_status *status); bfloat16 uint16_to_bfloat16_scalbn(uint16_t a, int, float_status *status); bfloat16 uint32_to_bfloat16_scalbn(uint32_t a, int, float_status *status); bfloat16 uint64_to_bfloat16_scalbn(uint64_t a, int, float_status *status); +bfloat16 int8_to_bfloat16(int8_t a, float_status *status); bfloat16 int16_to_bfloat16(int16_t a, float_status *status); bfloat16 int32_to_bfloat16(int32_t a, float_status *status); bfloat16 int64_to_bfloat16(int64_t a, float_status *status); +bfloat16 uint8_to_bfloat16(uint8_t a, float_status *status); bfloat16 uint16_to_bfloat16(uint16_t a, float_status *status); bfloat16 uint32_to_bfloat16(uint32_t a, float_status *status); bfloat16 uint64_to_bfloat16(uint64_t a, float_status *status); diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index 92a4234439..648b5b3586 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -227,17 +227,6 @@ struct CPUWatchpoint { QTAILQ_ENTRY(CPUWatchpoint) entry; }; -#ifdef CONFIG_PLUGIN -/* - * For plugins we sometime need to save the resolved iotlb data before - * the memory regions get moved around by io_writex. - */ -typedef struct SavedIOTLB { - MemoryRegionSection *section; - hwaddr mr_offset; -} SavedIOTLB; -#endif - struct KVMState; struct kvm_run; @@ -409,8 +398,6 @@ struct CPUState { #ifdef CONFIG_PLUGIN GArray *plugin_mem_cbs; - /* saved iotlb data from io_writex */ - SavedIOTLB saved_iotlb; #endif /* TODO Move common fields from CPUArchState here. */ diff --git a/include/hw/cxl/cxl.h b/include/hw/cxl/cxl.h index 56c9e7676e..4944725849 100644 --- a/include/hw/cxl/cxl.h +++ b/include/hw/cxl/cxl.h @@ -29,7 +29,7 @@ typedef struct PXBCXLDev PXBCXLDev; typedef struct CXLFixedWindow { uint64_t size; char **targets; - PXBCXLDev *target_hbs[8]; + PXBCXLDev *target_hbs[16]; uint8_t num_targets; uint8_t enc_int_ways; uint8_t enc_int_gran; diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h index f717e3f384..51cd0d9ce3 100644 --- a/include/hw/cxl/cxl_device.h +++ b/include/hw/cxl/cxl_device.h @@ -300,7 +300,7 @@ REG64(CXL_MEM_DEV_STS, 0) typedef struct CXLError { QTAILQ_ENTRY(CXLError) node; int type; /* Error code as per FE definition */ - uint32_t header[32]; + uint32_t header[CXL_RAS_ERR_HEADER_NUM]; } CXLError; typedef QTAILQ_HEAD(, CXLError) CXLErrorList; diff --git a/include/hw/i386/topology.h b/include/hw/i386/topology.h index 81573f6cfd..380cb27ded 100644 --- a/include/hw/i386/topology.h +++ b/include/hw/i386/topology.h @@ -31,7 +31,7 @@ * * This code should be compatible with AMD's "Extended Method" described at: * AMD CPUID Specification (Publication #25481) - * Section 3: Multiple Core Calcuation + * Section 3: Multiple Core Calculation * as long as: * nr_threads is set to 1; * OFFSET_IDX is assumed to be 0; diff --git a/include/hw/mem/nvdimm.h b/include/hw/mem/nvdimm.h index acf887c83d..d3b763453a 100644 --- a/include/hw/mem/nvdimm.h +++ b/include/hw/mem/nvdimm.h @@ -78,6 +78,12 @@ struct NVDIMMDevice { bool unarmed; /* + * Whether our DIMM is backed by ROM, and even label data cannot be + * written. If set, implies that "unarmed" is also set. + */ + bool readonly; + + /* * The PPC64 - spapr requires each nvdimm device have a uuid. */ QemuUUID uuid; diff --git a/include/hw/pci-host/spapr.h b/include/hw/pci-host/spapr.h index 5b03a7b0eb..3778aac27b 100644 --- a/include/hw/pci-host/spapr.h +++ b/include/hw/pci-host/spapr.h @@ -47,8 +47,6 @@ typedef struct SpaprPciLsi { uint32_t irq; } SpaprPciLsi; -typedef struct SpaprPhbPciNvGpuConfig SpaprPhbPciNvGpuConfig; - struct SpaprPhbState { PCIHostState parent_obj; @@ -90,9 +88,6 @@ struct SpaprPhbState { uint32_t mig_liobn; hwaddr mig_mem_win_addr, mig_mem_win_size; hwaddr mig_io_win_addr, mig_io_win_size; - hwaddr nv2_gpa_win_addr; - hwaddr nv2_atsd_win_addr; - SpaprPhbPciNvGpuConfig *nvgpus; bool pre_5_1_assoc; }; @@ -112,20 +107,6 @@ struct SpaprPhbState { #define SPAPR_PCI_MSI_WINDOW 0x40000000000ULL -#define SPAPR_PCI_NV2RAM64_WIN_BASE SPAPR_PCI_LIMIT -#define SPAPR_PCI_NV2RAM64_WIN_SIZE (2 * TiB) /* For up to 6 GPUs 256GB each */ - -/* Max number of NVLinks per GPU in any physical box */ -#define NVGPU_MAX_LINKS 3 - -/* - * GPU RAM starts at 64TiB so huge DMA window to cover it all ends at 128TiB - * which is enough. We do not need DMA for ATSD so we put them at 128TiB. - */ -#define SPAPR_PCI_NV2ATSD_WIN_BASE (128 * TiB) -#define SPAPR_PCI_NV2ATSD_WIN_SIZE (NVGPU_MAX_NUM * NVGPU_MAX_LINKS * \ - 64 * KiB) - int spapr_dt_phb(SpaprMachineState *spapr, SpaprPhbState *phb, uint32_t intc_phandle, void *fdt, int *node_offset); @@ -149,13 +130,6 @@ int spapr_phb_vfio_eeh_get_state(SpaprPhbState *sphb, int *state); int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option); int spapr_phb_vfio_eeh_configure(SpaprPhbState *sphb); void spapr_phb_vfio_reset(DeviceState *qdev); -void spapr_phb_nvgpu_setup(SpaprPhbState *sphb, Error **errp); -void spapr_phb_nvgpu_free(SpaprPhbState *sphb); -void spapr_phb_nvgpu_populate_dt(SpaprPhbState *sphb, void *fdt, int bus_off, - Error **errp); -void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt); -void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset, - SpaprPhbState *sphb); #else static inline bool spapr_phb_eeh_available(SpaprPhbState *sphb) { @@ -182,25 +156,6 @@ static inline int spapr_phb_vfio_eeh_configure(SpaprPhbState *sphb) static inline void spapr_phb_vfio_reset(DeviceState *qdev) { } -static inline void spapr_phb_nvgpu_setup(SpaprPhbState *sphb, Error **errp) -{ -} -static inline void spapr_phb_nvgpu_free(SpaprPhbState *sphb) -{ -} -static inline void spapr_phb_nvgpu_populate_dt(SpaprPhbState *sphb, void *fdt, - int bus_off, Error **errp) -{ -} -static inline void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, - void *fdt) -{ -} -static inline void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, - int offset, - SpaprPhbState *sphb) -{ -} #endif void spapr_phb_dma_reset(SpaprPhbState *sphb); diff --git a/include/hw/ppc/openpic.h b/include/hw/ppc/openpic.h index bae8dafe16..9c6af8e207 100644 --- a/include/hw/ppc/openpic.h +++ b/include/hw/ppc/openpic.h @@ -14,7 +14,7 @@ enum { OPENPIC_OUTPUT_INT = 0, /* IRQ */ OPENPIC_OUTPUT_CINT, /* critical IRQ */ OPENPIC_OUTPUT_MCK, /* Machine check event */ - OPENPIC_OUTPUT_DEBUG, /* Inconditional debug event */ + OPENPIC_OUTPUT_DEBUG, /* Unconditional debug event */ OPENPIC_OUTPUT_RESET, /* Core reset event */ OPENPIC_OUTPUT_NB, }; diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h index f4bd204d86..e91791a1a9 100644 --- a/include/hw/ppc/spapr.h +++ b/include/hw/ppc/spapr.h @@ -103,11 +103,8 @@ typedef enum { #define FDT_MAX_SIZE 0x200000 -/* Max number of GPUs per system */ -#define NVGPU_MAX_NUM 6 - /* Max number of NUMA nodes */ -#define NUMA_NODES_MAX_NUM (MAX_NODES + NVGPU_MAX_NUM) +#define NUMA_NODES_MAX_NUM (MAX_NODES) /* * NUMA FORM1 macros. FORM1_DIST_REF_POINTS was taken from @@ -160,8 +157,7 @@ struct SpaprMachineClass { bool (*phb_placement)(SpaprMachineState *spapr, uint32_t index, uint64_t *buid, hwaddr *pio, hwaddr *mmio32, hwaddr *mmio64, - unsigned n_dma, uint32_t *liobns, hwaddr *nv2gpa, - hwaddr *nv2atsd, Error **errp); + unsigned n_dma, uint32_t *liobns, Error **errp); SpaprResizeHpt resize_hpt_default; SpaprCapabilities default_caps; SpaprIrq *irq; @@ -197,7 +193,7 @@ struct SpaprMachineState { SpaprResizeHpt resize_hpt; void *htab; uint32_t htab_shift; - uint64_t patb_entry; /* Process tbl registed in H_REGISTER_PROC_TBL */ + uint64_t patb_entry; /* Process tbl registered in H_REGISTER_PROC_TBL */ SpaprPendingHpt *pending_hpt; /* in-progress resize */ hwaddr rma_size; @@ -276,7 +272,6 @@ struct SpaprMachineState { bool cmd_line_caps[SPAPR_CAP_NUM]; SpaprCapabilities def, eff, mig; - unsigned gpu_numa_id; SpaprTpmProxy *tpm_proxy; uint32_t FORM1_assoc_array[NUMA_NODES_MAX_NUM][FORM1_NUMA_ASSOC_SIZE]; diff --git a/include/net/net.h b/include/net/net.h index 1448d00afb..330d285930 100644 --- a/include/net/net.h +++ b/include/net/net.h @@ -54,11 +54,12 @@ typedef void (LinkStatusChanged)(NetClientState *); typedef void (NetClientDestructor)(NetClientState *); typedef RxFilterInfo *(QueryRxFilter)(NetClientState *); typedef bool (HasUfo)(NetClientState *); +typedef bool (HasUso)(NetClientState *); typedef bool (HasVnetHdr)(NetClientState *); typedef bool (HasVnetHdrLen)(NetClientState *, int); typedef bool (GetUsingVnetHdr)(NetClientState *); typedef void (UsingVnetHdr)(NetClientState *, bool); -typedef void (SetOffload)(NetClientState *, int, int, int, int, int); +typedef void (SetOffload)(NetClientState *, int, int, int, int, int, int, int); typedef int (GetVnetHdrLen)(NetClientState *); typedef void (SetVnetHdrLen)(NetClientState *, int); typedef int (SetVnetLE)(NetClientState *, bool); @@ -84,6 +85,7 @@ typedef struct NetClientInfo { QueryRxFilter *query_rx_filter; NetPoll *poll; HasUfo *has_ufo; + HasUso *has_uso; HasVnetHdr *has_vnet_hdr; HasVnetHdrLen *has_vnet_hdr_len; GetUsingVnetHdr *get_using_vnet_hdr; @@ -187,12 +189,13 @@ void qemu_set_info_str(NetClientState *nc, const char *fmt, ...) G_GNUC_PRINTF(2, 3); void qemu_format_nic_info_str(NetClientState *nc, uint8_t macaddr[6]); bool qemu_has_ufo(NetClientState *nc); +bool qemu_has_uso(NetClientState *nc); bool qemu_has_vnet_hdr(NetClientState *nc); bool qemu_has_vnet_hdr_len(NetClientState *nc, int len); bool qemu_get_using_vnet_hdr(NetClientState *nc); void qemu_using_vnet_hdr(NetClientState *nc, bool enable); void qemu_set_offload(NetClientState *nc, int csum, int tso4, int tso6, - int ecn, int ufo); + int ecn, int ufo, int uso4, int uso6); int qemu_get_vnet_hdr_len(NetClientState *nc); void qemu_set_vnet_hdr_len(NetClientState *nc, int len); int qemu_set_vnet_le(NetClientState *nc, bool is_le); diff --git a/include/qemu/cpuid.h b/include/qemu/cpuid.h index 35325f1995..b11161555b 100644 --- a/include/qemu/cpuid.h +++ b/include/qemu/cpuid.h @@ -25,6 +25,9 @@ #endif /* Leaf 1, %ecx */ +#ifndef bit_PCLMUL +#define bit_PCLMUL (1 << 1) +#endif #ifndef bit_SSE4_1 #define bit_SSE4_1 (1 << 19) #endif diff --git a/include/qemu/plugin-memory.h b/include/qemu/plugin-memory.h index 43165f2452..71c1123308 100644 --- a/include/qemu/plugin-memory.h +++ b/include/qemu/plugin-memory.h @@ -15,15 +15,8 @@ struct qemu_plugin_hwaddr { bool is_io; bool is_store; - union { - struct { - MemoryRegionSection *section; - hwaddr offset; - } io; - struct { - void *hostaddr; - } ram; - } v; + hwaddr phys_addr; + MemoryRegion *mr; }; /** diff --git a/include/qemu/throttle.h b/include/qemu/throttle.h index 05f6346137..181245d29b 100644 --- a/include/qemu/throttle.h +++ b/include/qemu/throttle.h @@ -99,13 +99,18 @@ typedef struct ThrottleState { int64_t previous_leak; /* timestamp of the last leak done */ } ThrottleState; +typedef enum { + THROTTLE_READ = 0, + THROTTLE_WRITE, + THROTTLE_MAX +} ThrottleDirection; + typedef struct ThrottleTimers { - QEMUTimer *timers[2]; /* timers used to do the throttling */ + QEMUTimer *timers[THROTTLE_MAX]; /* timers used to do the throttling */ QEMUClockType clock_type; /* the clock used */ /* Callbacks */ - QEMUTimerCB *read_timer_cb; - QEMUTimerCB *write_timer_cb; + QEMUTimerCB *timer_cb[THROTTLE_MAX]; void *timer_opaque; } ThrottleTimers; @@ -149,9 +154,10 @@ void throttle_config_init(ThrottleConfig *cfg); /* usage */ bool throttle_schedule_timer(ThrottleState *ts, ThrottleTimers *tt, - bool is_write); + ThrottleDirection direction); -void throttle_account(ThrottleState *ts, bool is_write, uint64_t size); +void throttle_account(ThrottleState *ts, ThrottleDirection direction, + uint64_t size); void throttle_limits_to_config(ThrottleLimits *arg, ThrottleConfig *cfg, Error **errp); void throttle_config_to_limits(ThrottleConfig *cfg, ThrottleLimits *var); diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h index 834b0e47a0..5abdbc3874 100644 --- a/include/qemu/typedefs.h +++ b/include/qemu/typedefs.h @@ -129,7 +129,6 @@ typedef struct QString QString; typedef struct RAMBlock RAMBlock; typedef struct Range Range; typedef struct ReservedRegion ReservedRegion; -typedef struct SavedIOTLB SavedIOTLB; typedef struct SHPCDevice SHPCDevice; typedef struct SSIBus SSIBus; typedef struct TCGHelperInfo TCGHelperInfo; diff --git a/include/sysemu/block-backend-global-state.h b/include/sysemu/block-backend-global-state.h index 184e667ebd..d5f675493a 100644 --- a/include/sysemu/block-backend-global-state.h +++ b/include/sysemu/block-backend-global-state.h @@ -61,8 +61,8 @@ int blk_insert_bs(BlockBackend *blk, BlockDriverState *bs, Error **errp); int blk_replace_bs(BlockBackend *blk, BlockDriverState *new_bs, Error **errp); bool bdrv_has_blk(BlockDriverState *bs); bool bdrv_is_root_node(BlockDriverState *bs); -int blk_set_perm(BlockBackend *blk, uint64_t perm, uint64_t shared_perm, - Error **errp); +int GRAPH_UNLOCKED blk_set_perm(BlockBackend *blk, uint64_t perm, + uint64_t shared_perm, Error **errp); void blk_get_perm(BlockBackend *blk, uint64_t *perm, uint64_t *shared_perm); void blk_iostatus_enable(BlockBackend *blk); diff --git a/include/tcg/tcg-op-gvec-common.h b/include/tcg/tcg-op-gvec-common.h index e2683d487f..4db8a58c14 100644 --- a/include/tcg/tcg-op-gvec-common.h +++ b/include/tcg/tcg-op-gvec-common.h @@ -374,6 +374,12 @@ void tcg_gen_gvec_rotrv(unsigned vece, uint32_t dofs, uint32_t aofs, void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_cmpi(TCGCond cond, unsigned vece, uint32_t dofs, + uint32_t aofs, int64_t c, + uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_cmps(TCGCond cond, unsigned vece, uint32_t dofs, + uint32_t aofs, TCGv_i64 c, + uint32_t oprsz, uint32_t maxsz); /* * Perform vector bit select: d = (b & a) | (c & ~a). diff --git a/linux-user/elfload.c b/linux-user/elfload.c index a5b28fa3e7..db75cd4b33 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -402,6 +402,12 @@ enum ARM_HWCAP_ARM_VFPD32 = 1 << 19, ARM_HWCAP_ARM_LPAE = 1 << 20, ARM_HWCAP_ARM_EVTSTRM = 1 << 21, + ARM_HWCAP_ARM_FPHP = 1 << 22, + ARM_HWCAP_ARM_ASIMDHP = 1 << 23, + ARM_HWCAP_ARM_ASIMDDP = 1 << 24, + ARM_HWCAP_ARM_ASIMDFHM = 1 << 25, + ARM_HWCAP_ARM_ASIMDBF16 = 1 << 26, + ARM_HWCAP_ARM_I8MM = 1 << 27, }; enum { @@ -410,6 +416,8 @@ enum { ARM_HWCAP2_ARM_SHA1 = 1 << 2, ARM_HWCAP2_ARM_SHA2 = 1 << 3, ARM_HWCAP2_ARM_CRC32 = 1 << 4, + ARM_HWCAP2_ARM_SB = 1 << 5, + ARM_HWCAP2_ARM_SSBS = 1 << 6, }; /* The commpage only exists for 32 bit kernels */ @@ -498,6 +506,16 @@ uint32_t get_elf_hwcap(void) } } GET_FEATURE_ID(aa32_simdfmac, ARM_HWCAP_ARM_VFPv4); + /* + * MVFR1.FPHP and .SIMDHP must be in sync, and QEMU uses the same + * isar_feature function for both. The kernel reports them as two hwcaps. + */ + GET_FEATURE_ID(aa32_fp16_arith, ARM_HWCAP_ARM_FPHP); + GET_FEATURE_ID(aa32_fp16_arith, ARM_HWCAP_ARM_ASIMDHP); + GET_FEATURE_ID(aa32_dp, ARM_HWCAP_ARM_ASIMDDP); + GET_FEATURE_ID(aa32_fhm, ARM_HWCAP_ARM_ASIMDFHM); + GET_FEATURE_ID(aa32_bf16, ARM_HWCAP_ARM_ASIMDBF16); + GET_FEATURE_ID(aa32_i8mm, ARM_HWCAP_ARM_I8MM); return hwcaps; } @@ -512,6 +530,8 @@ uint32_t get_elf_hwcap2(void) GET_FEATURE_ID(aa32_sha1, ARM_HWCAP2_ARM_SHA1); GET_FEATURE_ID(aa32_sha2, ARM_HWCAP2_ARM_SHA2); GET_FEATURE_ID(aa32_crc32, ARM_HWCAP2_ARM_CRC32); + GET_FEATURE_ID(aa32_sb, ARM_HWCAP2_ARM_SB); + GET_FEATURE_ID(aa32_ssbs, ARM_HWCAP2_ARM_SSBS); return hwcaps; } @@ -540,6 +560,12 @@ const char *elf_hwcap_str(uint32_t bit) [__builtin_ctz(ARM_HWCAP_ARM_VFPD32 )] = "vfpd32", [__builtin_ctz(ARM_HWCAP_ARM_LPAE )] = "lpae", [__builtin_ctz(ARM_HWCAP_ARM_EVTSTRM )] = "evtstrm", + [__builtin_ctz(ARM_HWCAP_ARM_FPHP )] = "fphp", + [__builtin_ctz(ARM_HWCAP_ARM_ASIMDHP )] = "asimdhp", + [__builtin_ctz(ARM_HWCAP_ARM_ASIMDDP )] = "asimddp", + [__builtin_ctz(ARM_HWCAP_ARM_ASIMDFHM )] = "asimdfhm", + [__builtin_ctz(ARM_HWCAP_ARM_ASIMDBF16)] = "asimdbf16", + [__builtin_ctz(ARM_HWCAP_ARM_I8MM )] = "i8mm", }; return bit < ARRAY_SIZE(hwcap_str) ? hwcap_str[bit] : NULL; @@ -553,6 +579,8 @@ const char *elf_hwcap2_str(uint32_t bit) [__builtin_ctz(ARM_HWCAP2_ARM_SHA1 )] = "sha1", [__builtin_ctz(ARM_HWCAP2_ARM_SHA2 )] = "sha2", [__builtin_ctz(ARM_HWCAP2_ARM_CRC32)] = "crc32", + [__builtin_ctz(ARM_HWCAP2_ARM_SB )] = "sb", + [__builtin_ctz(ARM_HWCAP2_ARM_SSBS )] = "ssbs", }; return bit < ARRAY_SIZE(hwcap_str) ? hwcap_str[bit] : NULL; @@ -696,6 +724,20 @@ enum { ARM_HWCAP2_A64_SME_B16F32 = 1 << 28, ARM_HWCAP2_A64_SME_F32F32 = 1 << 29, ARM_HWCAP2_A64_SME_FA64 = 1 << 30, + ARM_HWCAP2_A64_WFXT = 1ULL << 31, + ARM_HWCAP2_A64_EBF16 = 1ULL << 32, + ARM_HWCAP2_A64_SVE_EBF16 = 1ULL << 33, + ARM_HWCAP2_A64_CSSC = 1ULL << 34, + ARM_HWCAP2_A64_RPRFM = 1ULL << 35, + ARM_HWCAP2_A64_SVE2P1 = 1ULL << 36, + ARM_HWCAP2_A64_SME2 = 1ULL << 37, + ARM_HWCAP2_A64_SME2P1 = 1ULL << 38, + ARM_HWCAP2_A64_SME_I16I32 = 1ULL << 39, + ARM_HWCAP2_A64_SME_BI32I32 = 1ULL << 40, + ARM_HWCAP2_A64_SME_B16B16 = 1ULL << 41, + ARM_HWCAP2_A64_SME_F16F16 = 1ULL << 42, + ARM_HWCAP2_A64_MOPS = 1ULL << 43, + ARM_HWCAP2_A64_HBC = 1ULL << 44, }; #define ELF_HWCAP get_elf_hwcap() @@ -773,6 +815,8 @@ uint32_t get_elf_hwcap2(void) GET_FEATURE_ID(aa64_sme_f64f64, ARM_HWCAP2_A64_SME_F64F64); GET_FEATURE_ID(aa64_sme_i16i64, ARM_HWCAP2_A64_SME_I16I64); GET_FEATURE_ID(aa64_sme_fa64, ARM_HWCAP2_A64_SME_FA64); + GET_FEATURE_ID(aa64_hbc, ARM_HWCAP2_A64_HBC); + GET_FEATURE_ID(aa64_mops, ARM_HWCAP2_A64_MOPS); return hwcaps; } @@ -844,13 +888,27 @@ const char *elf_hwcap2_str(uint32_t bit) [__builtin_ctz(ARM_HWCAP2_A64_RPRES )] = "rpres", [__builtin_ctz(ARM_HWCAP2_A64_MTE3 )] = "mte3", [__builtin_ctz(ARM_HWCAP2_A64_SME )] = "sme", - [__builtin_ctz(ARM_HWCAP2_A64_SME_I16I64 )] = "sme_i16i64", - [__builtin_ctz(ARM_HWCAP2_A64_SME_F64F64 )] = "sme_f64f64", - [__builtin_ctz(ARM_HWCAP2_A64_SME_I8I32 )] = "sme_i8i32", - [__builtin_ctz(ARM_HWCAP2_A64_SME_F16F32 )] = "sme_f16f32", - [__builtin_ctz(ARM_HWCAP2_A64_SME_B16F32 )] = "sme_b16f32", - [__builtin_ctz(ARM_HWCAP2_A64_SME_F32F32 )] = "sme_f32f32", - [__builtin_ctz(ARM_HWCAP2_A64_SME_FA64 )] = "sme_fa64", + [__builtin_ctz(ARM_HWCAP2_A64_SME_I16I64 )] = "smei16i64", + [__builtin_ctz(ARM_HWCAP2_A64_SME_F64F64 )] = "smef64f64", + [__builtin_ctz(ARM_HWCAP2_A64_SME_I8I32 )] = "smei8i32", + [__builtin_ctz(ARM_HWCAP2_A64_SME_F16F32 )] = "smef16f32", + [__builtin_ctz(ARM_HWCAP2_A64_SME_B16F32 )] = "smeb16f32", + [__builtin_ctz(ARM_HWCAP2_A64_SME_F32F32 )] = "smef32f32", + [__builtin_ctz(ARM_HWCAP2_A64_SME_FA64 )] = "smefa64", + [__builtin_ctz(ARM_HWCAP2_A64_WFXT )] = "wfxt", + [__builtin_ctzll(ARM_HWCAP2_A64_EBF16 )] = "ebf16", + [__builtin_ctzll(ARM_HWCAP2_A64_SVE_EBF16 )] = "sveebf16", + [__builtin_ctzll(ARM_HWCAP2_A64_CSSC )] = "cssc", + [__builtin_ctzll(ARM_HWCAP2_A64_RPRFM )] = "rprfm", + [__builtin_ctzll(ARM_HWCAP2_A64_SVE2P1 )] = "sve2p1", + [__builtin_ctzll(ARM_HWCAP2_A64_SME2 )] = "sme2", + [__builtin_ctzll(ARM_HWCAP2_A64_SME2P1 )] = "sme2p1", + [__builtin_ctzll(ARM_HWCAP2_A64_SME_I16I32 )] = "smei16i32", + [__builtin_ctzll(ARM_HWCAP2_A64_SME_BI32I32)] = "smebi32i32", + [__builtin_ctzll(ARM_HWCAP2_A64_SME_B16B16 )] = "smeb16b16", + [__builtin_ctzll(ARM_HWCAP2_A64_SME_F16F16 )] = "smef16f16", + [__builtin_ctzll(ARM_HWCAP2_A64_MOPS )] = "mops", + [__builtin_ctzll(ARM_HWCAP2_A64_HBC )] = "hbc", }; return bit < ARRAY_SIZE(hwcap_str) ? hwcap_str[bit] : NULL; diff --git a/linux-user/hppa/signal.c b/linux-user/hppa/signal.c index f253a15864..bda6e54655 100644 --- a/linux-user/hppa/signal.c +++ b/linux-user/hppa/signal.c @@ -149,16 +149,18 @@ void setup_rt_frame(int sig, struct target_sigaction *ka, target_ulong *fdesc, dest; haddr &= -4; - if (!lock_user_struct(VERIFY_READ, fdesc, haddr, 1)) { + fdesc = lock_user(VERIFY_READ, haddr, 2 * sizeof(target_ulong), 1); + if (!fdesc) { goto give_sigsegv; } __get_user(dest, fdesc); __get_user(env->gr[19], fdesc + 1); - unlock_user_struct(fdesc, haddr, 1); + unlock_user(fdesc, haddr, 0); haddr = dest; } env->iaoq_f = haddr; env->iaoq_b = haddr + 4; + env->psw_n = 0; return; give_sigsegv: diff --git a/linux-user/loongarch64/signal.c b/linux-user/loongarch64/signal.c index bb8efb1172..39572c1190 100644 --- a/linux-user/loongarch64/signal.c +++ b/linux-user/loongarch64/signal.c @@ -12,6 +12,7 @@ #include "linux-user/trace.h" #include "target/loongarch/internals.h" +#include "target/loongarch/vec.h" /* FP context was used */ #define SC_USED_FP (1 << 0) diff --git a/meson.build b/meson.build index 5150a74831..f426861d90 100644 --- a/meson.build +++ b/meson.build @@ -1873,6 +1873,13 @@ if libbpf.found() and not cc.links(''' endif endif +# libxdp +libxdp = not_found +if not get_option('af_xdp').auto() or have_system + libxdp = dependency('libxdp', required: get_option('af_xdp'), + version: '>=1.4.0', method: 'pkg-config') +endif + # libdw libdw = not_found if not get_option('libdw').auto() or \ @@ -2099,6 +2106,7 @@ config_host_data.set('CONFIG_HEXAGON_IDEF_PARSER', get_option('hexagon_idef_pars config_host_data.set('CONFIG_LIBATTR', have_old_libattr) config_host_data.set('CONFIG_LIBCAP_NG', libcap_ng.found()) config_host_data.set('CONFIG_EBPF', libbpf.found()) +config_host_data.set('CONFIG_AF_XDP', libxdp.found()) config_host_data.set('CONFIG_LIBDAXCTL', libdaxctl.found()) config_host_data.set('CONFIG_LIBISCSI', libiscsi.found()) config_host_data.set('CONFIG_LIBNFS', libnfs.found()) @@ -4270,6 +4278,7 @@ summary_info = {} if targetos == 'darwin' summary_info += {'vmnet.framework support': vmnet} endif +summary_info += {'AF_XDP support': libxdp} summary_info += {'slirp support': slirp} summary_info += {'vde support': vde} summary_info += {'netmap support': have_netmap} diff --git a/meson_options.txt b/meson_options.txt index f82d88b7c6..2ca40f22e9 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -122,6 +122,8 @@ option('avx512bw', type: 'feature', value: 'auto', option('keyring', type: 'feature', value: 'auto', description: 'Linux keyring support') +option('af_xdp', type : 'feature', value : 'auto', + description: 'AF_XDP network backend support') option('attr', type : 'feature', value : 'auto', description: 'attr/xattr support') option('auth_pam', type : 'feature', value : 'auto', diff --git a/net/af-xdp.c b/net/af-xdp.c new file mode 100644 index 0000000000..6c65028fb0 --- /dev/null +++ b/net/af-xdp.c @@ -0,0 +1,526 @@ +/* + * AF_XDP network backend. + * + * Copyright (c) 2023 Red Hat, Inc. + * + * Authors: + * Ilya Maximets <i.maximets@ovn.org> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + + +#include "qemu/osdep.h" +#include <bpf/bpf.h> +#include <inttypes.h> +#include <linux/if_link.h> +#include <linux/if_xdp.h> +#include <net/if.h> +#include <xdp/xsk.h> + +#include "clients.h" +#include "monitor/monitor.h" +#include "net/net.h" +#include "qapi/error.h" +#include "qemu/cutils.h" +#include "qemu/error-report.h" +#include "qemu/iov.h" +#include "qemu/main-loop.h" +#include "qemu/memalign.h" + + +typedef struct AFXDPState { + NetClientState nc; + + struct xsk_socket *xsk; + struct xsk_ring_cons rx; + struct xsk_ring_prod tx; + struct xsk_ring_cons cq; + struct xsk_ring_prod fq; + + char ifname[IFNAMSIZ]; + int ifindex; + bool read_poll; + bool write_poll; + uint32_t outstanding_tx; + + uint64_t *pool; + uint32_t n_pool; + char *buffer; + struct xsk_umem *umem; + + uint32_t n_queues; + uint32_t xdp_flags; + bool inhibit; +} AFXDPState; + +#define AF_XDP_BATCH_SIZE 64 + +static void af_xdp_send(void *opaque); +static void af_xdp_writable(void *opaque); + +/* Set the event-loop handlers for the af-xdp backend. */ +static void af_xdp_update_fd_handler(AFXDPState *s) +{ + qemu_set_fd_handler(xsk_socket__fd(s->xsk), + s->read_poll ? af_xdp_send : NULL, + s->write_poll ? af_xdp_writable : NULL, + s); +} + +/* Update the read handler. */ +static void af_xdp_read_poll(AFXDPState *s, bool enable) +{ + if (s->read_poll != enable) { + s->read_poll = enable; + af_xdp_update_fd_handler(s); + } +} + +/* Update the write handler. */ +static void af_xdp_write_poll(AFXDPState *s, bool enable) +{ + if (s->write_poll != enable) { + s->write_poll = enable; + af_xdp_update_fd_handler(s); + } +} + +static void af_xdp_poll(NetClientState *nc, bool enable) +{ + AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc); + + if (s->read_poll != enable || s->write_poll != enable) { + s->write_poll = enable; + s->read_poll = enable; + af_xdp_update_fd_handler(s); + } +} + +static void af_xdp_complete_tx(AFXDPState *s) +{ + uint32_t idx = 0; + uint32_t done, i; + uint64_t *addr; + + done = xsk_ring_cons__peek(&s->cq, XSK_RING_CONS__DEFAULT_NUM_DESCS, &idx); + + for (i = 0; i < done; i++) { + addr = (void *) xsk_ring_cons__comp_addr(&s->cq, idx++); + s->pool[s->n_pool++] = *addr; + s->outstanding_tx--; + } + + if (done) { + xsk_ring_cons__release(&s->cq, done); + } +} + +/* + * The fd_write() callback, invoked if the fd is marked as writable + * after a poll. + */ +static void af_xdp_writable(void *opaque) +{ + AFXDPState *s = opaque; + + /* Try to recover buffers that are already sent. */ + af_xdp_complete_tx(s); + + /* + * Unregister the handler, unless we still have packets to transmit + * and kernel needs a wake up. + */ + if (!s->outstanding_tx || !xsk_ring_prod__needs_wakeup(&s->tx)) { + af_xdp_write_poll(s, false); + } + + /* Flush any buffered packets. */ + qemu_flush_queued_packets(&s->nc); +} + +static ssize_t af_xdp_receive(NetClientState *nc, + const uint8_t *buf, size_t size) +{ + AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc); + struct xdp_desc *desc; + uint32_t idx; + void *data; + + /* Try to recover buffers that are already sent. */ + af_xdp_complete_tx(s); + + if (size > XSK_UMEM__DEFAULT_FRAME_SIZE) { + /* We can't transmit packet this size... */ + return size; + } + + if (!s->n_pool || !xsk_ring_prod__reserve(&s->tx, 1, &idx)) { + /* + * Out of buffers or space in tx ring. Poll until we can write. + * This will also kick the Tx, if it was waiting on CQ. + */ + af_xdp_write_poll(s, true); + return 0; + } + + desc = xsk_ring_prod__tx_desc(&s->tx, idx); + desc->addr = s->pool[--s->n_pool]; + desc->len = size; + + data = xsk_umem__get_data(s->buffer, desc->addr); + memcpy(data, buf, size); + + xsk_ring_prod__submit(&s->tx, 1); + s->outstanding_tx++; + + if (xsk_ring_prod__needs_wakeup(&s->tx)) { + af_xdp_write_poll(s, true); + } + + return size; +} + +/* + * Complete a previous send (backend --> guest) and enable the + * fd_read callback. + */ +static void af_xdp_send_completed(NetClientState *nc, ssize_t len) +{ + AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc); + + af_xdp_read_poll(s, true); +} + +static void af_xdp_fq_refill(AFXDPState *s, uint32_t n) +{ + uint32_t i, idx = 0; + + /* Leave one packet for Tx, just in case. */ + if (s->n_pool < n + 1) { + n = s->n_pool; + } + + if (!n || !xsk_ring_prod__reserve(&s->fq, n, &idx)) { + return; + } + + for (i = 0; i < n; i++) { + *xsk_ring_prod__fill_addr(&s->fq, idx++) = s->pool[--s->n_pool]; + } + xsk_ring_prod__submit(&s->fq, n); + + if (xsk_ring_prod__needs_wakeup(&s->fq)) { + /* Receive was blocked by not having enough buffers. Wake it up. */ + af_xdp_read_poll(s, true); + } +} + +static void af_xdp_send(void *opaque) +{ + uint32_t i, n_rx, idx = 0; + AFXDPState *s = opaque; + + n_rx = xsk_ring_cons__peek(&s->rx, AF_XDP_BATCH_SIZE, &idx); + if (!n_rx) { + return; + } + + for (i = 0; i < n_rx; i++) { + const struct xdp_desc *desc; + struct iovec iov; + + desc = xsk_ring_cons__rx_desc(&s->rx, idx++); + + iov.iov_base = xsk_umem__get_data(s->buffer, desc->addr); + iov.iov_len = desc->len; + + s->pool[s->n_pool++] = desc->addr; + + if (!qemu_sendv_packet_async(&s->nc, &iov, 1, + af_xdp_send_completed)) { + /* + * The peer does not receive anymore. Packet is queued, stop + * reading from the backend until af_xdp_send_completed(). + */ + af_xdp_read_poll(s, false); + + /* Return unused descriptors to not break the ring cache. */ + xsk_ring_cons__cancel(&s->rx, n_rx - i - 1); + n_rx = i + 1; + break; + } + } + + /* Release actually sent descriptors and try to re-fill. */ + xsk_ring_cons__release(&s->rx, n_rx); + af_xdp_fq_refill(s, AF_XDP_BATCH_SIZE); +} + +/* Flush and close. */ +static void af_xdp_cleanup(NetClientState *nc) +{ + AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc); + + qemu_purge_queued_packets(nc); + + af_xdp_poll(nc, false); + + xsk_socket__delete(s->xsk); + s->xsk = NULL; + g_free(s->pool); + s->pool = NULL; + xsk_umem__delete(s->umem); + s->umem = NULL; + qemu_vfree(s->buffer); + s->buffer = NULL; + + /* Remove the program if it's the last open queue. */ + if (!s->inhibit && nc->queue_index == s->n_queues - 1 && s->xdp_flags + && bpf_xdp_detach(s->ifindex, s->xdp_flags, NULL) != 0) { + fprintf(stderr, + "af-xdp: unable to remove XDP program from '%s', ifindex: %d\n", + s->ifname, s->ifindex); + } +} + +static int af_xdp_umem_create(AFXDPState *s, int sock_fd, Error **errp) +{ + struct xsk_umem_config config = { + .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, + .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, + .frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE, + .frame_headroom = 0, + }; + uint64_t n_descs; + uint64_t size; + int64_t i; + int ret; + + /* Number of descriptors if all 4 queues (rx, tx, cq, fq) are full. */ + n_descs = (XSK_RING_PROD__DEFAULT_NUM_DESCS + + XSK_RING_CONS__DEFAULT_NUM_DESCS) * 2; + size = n_descs * XSK_UMEM__DEFAULT_FRAME_SIZE; + + s->buffer = qemu_memalign(qemu_real_host_page_size(), size); + memset(s->buffer, 0, size); + + if (sock_fd < 0) { + ret = xsk_umem__create(&s->umem, s->buffer, size, + &s->fq, &s->cq, &config); + } else { + ret = xsk_umem__create_with_fd(&s->umem, sock_fd, s->buffer, size, + &s->fq, &s->cq, &config); + } + + if (ret) { + qemu_vfree(s->buffer); + error_setg_errno(errp, errno, + "failed to create umem for %s queue_index: %d", + s->ifname, s->nc.queue_index); + return -1; + } + + s->pool = g_new(uint64_t, n_descs); + /* Fill the pool in the opposite order, because it's a LIFO queue. */ + for (i = n_descs; i >= 0; i--) { + s->pool[i] = i * XSK_UMEM__DEFAULT_FRAME_SIZE; + } + s->n_pool = n_descs; + + af_xdp_fq_refill(s, XSK_RING_PROD__DEFAULT_NUM_DESCS); + + return 0; +} + +static int af_xdp_socket_create(AFXDPState *s, + const NetdevAFXDPOptions *opts, Error **errp) +{ + struct xsk_socket_config cfg = { + .rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, + .tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, + .libxdp_flags = 0, + .bind_flags = XDP_USE_NEED_WAKEUP, + .xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST, + }; + int queue_id, error = 0; + + s->inhibit = opts->has_inhibit && opts->inhibit; + if (s->inhibit) { + cfg.libxdp_flags |= XSK_LIBXDP_FLAGS__INHIBIT_PROG_LOAD; + } + + if (opts->has_force_copy && opts->force_copy) { + cfg.bind_flags |= XDP_COPY; + } + + queue_id = s->nc.queue_index; + if (opts->has_start_queue && opts->start_queue > 0) { + queue_id += opts->start_queue; + } + + if (opts->has_mode) { + /* Specific mode requested. */ + cfg.xdp_flags |= (opts->mode == AFXDP_MODE_NATIVE) + ? XDP_FLAGS_DRV_MODE : XDP_FLAGS_SKB_MODE; + if (xsk_socket__create(&s->xsk, s->ifname, queue_id, + s->umem, &s->rx, &s->tx, &cfg)) { + error = errno; + } + } else { + /* No mode requested, try native first. */ + cfg.xdp_flags |= XDP_FLAGS_DRV_MODE; + + if (xsk_socket__create(&s->xsk, s->ifname, queue_id, + s->umem, &s->rx, &s->tx, &cfg)) { + /* Can't use native mode, try skb. */ + cfg.xdp_flags &= ~XDP_FLAGS_DRV_MODE; + cfg.xdp_flags |= XDP_FLAGS_SKB_MODE; + + if (xsk_socket__create(&s->xsk, s->ifname, queue_id, + s->umem, &s->rx, &s->tx, &cfg)) { + error = errno; + } + } + } + + if (error) { + error_setg_errno(errp, error, + "failed to create AF_XDP socket for %s queue_id: %d", + s->ifname, queue_id); + return -1; + } + + s->xdp_flags = cfg.xdp_flags; + + return 0; +} + +/* NetClientInfo methods. */ +static NetClientInfo net_af_xdp_info = { + .type = NET_CLIENT_DRIVER_AF_XDP, + .size = sizeof(AFXDPState), + .receive = af_xdp_receive, + .poll = af_xdp_poll, + .cleanup = af_xdp_cleanup, +}; + +static int *parse_socket_fds(const char *sock_fds_str, + int64_t n_expected, Error **errp) +{ + gchar **substrings = g_strsplit(sock_fds_str, ":", -1); + int64_t i, n_sock_fds = g_strv_length(substrings); + int *sock_fds = NULL; + + if (n_sock_fds != n_expected) { + error_setg(errp, "expected %"PRIi64" socket fds, got %"PRIi64, + n_expected, n_sock_fds); + goto exit; + } + + sock_fds = g_new(int, n_sock_fds); + + for (i = 0; i < n_sock_fds; i++) { + sock_fds[i] = monitor_fd_param(monitor_cur(), substrings[i], errp); + if (sock_fds[i] < 0) { + g_free(sock_fds); + sock_fds = NULL; + goto exit; + } + } + +exit: + g_strfreev(substrings); + return sock_fds; +} + +/* + * The exported init function. + * + * ... -netdev af-xdp,ifname="..." + */ +int net_init_af_xdp(const Netdev *netdev, + const char *name, NetClientState *peer, Error **errp) +{ + const NetdevAFXDPOptions *opts = &netdev->u.af_xdp; + NetClientState *nc, *nc0 = NULL; + unsigned int ifindex; + uint32_t prog_id = 0; + int *sock_fds = NULL; + int64_t i, queues; + Error *err = NULL; + AFXDPState *s; + + ifindex = if_nametoindex(opts->ifname); + if (!ifindex) { + error_setg_errno(errp, errno, "failed to get ifindex for '%s'", + opts->ifname); + return -1; + } + + queues = opts->has_queues ? opts->queues : 1; + if (queues < 1) { + error_setg(errp, "invalid number of queues (%" PRIi64 ") for '%s'", + queues, opts->ifname); + return -1; + } + + if ((opts->has_inhibit && opts->inhibit) != !!opts->sock_fds) { + error_setg(errp, "'inhibit=on' requires 'sock-fds' and vice versa"); + return -1; + } + + if (opts->sock_fds) { + sock_fds = parse_socket_fds(opts->sock_fds, queues, errp); + if (!sock_fds) { + return -1; + } + } + + for (i = 0; i < queues; i++) { + nc = qemu_new_net_client(&net_af_xdp_info, peer, "af-xdp", name); + qemu_set_info_str(nc, "af-xdp%"PRIi64" to %s", i, opts->ifname); + nc->queue_index = i; + + if (!nc0) { + nc0 = nc; + } + + s = DO_UPCAST(AFXDPState, nc, nc); + + pstrcpy(s->ifname, sizeof(s->ifname), opts->ifname); + s->ifindex = ifindex; + s->n_queues = queues; + + if (af_xdp_umem_create(s, sock_fds ? sock_fds[i] : -1, errp) + || af_xdp_socket_create(s, opts, errp)) { + /* Make sure the XDP program will be removed. */ + s->n_queues = i; + error_propagate(errp, err); + goto err; + } + } + + if (nc0) { + s = DO_UPCAST(AFXDPState, nc, nc0); + if (bpf_xdp_query_id(s->ifindex, s->xdp_flags, &prog_id) || !prog_id) { + error_setg_errno(errp, errno, + "no XDP program loaded on '%s', ifindex: %d", + s->ifname, s->ifindex); + goto err; + } + } + + af_xdp_read_poll(s, true); /* Initially only poll for reads. */ + + return 0; + +err: + g_free(sock_fds); + if (nc0) { + qemu_del_net_client(nc0); + } + + return -1; +} diff --git a/net/clients.h b/net/clients.h index ed8bdfff1e..be53794582 100644 --- a/net/clients.h +++ b/net/clients.h @@ -64,6 +64,11 @@ int net_init_netmap(const Netdev *netdev, const char *name, NetClientState *peer, Error **errp); #endif +#ifdef CONFIG_AF_XDP +int net_init_af_xdp(const Netdev *netdev, const char *name, + NetClientState *peer, Error **errp); +#endif + int net_init_vhost_user(const Netdev *netdev, const char *name, NetClientState *peer, Error **errp); diff --git a/net/dump.c b/net/dump.c index 7d05f16ca7..16073f2458 100644 --- a/net/dump.c +++ b/net/dump.c @@ -68,7 +68,7 @@ static ssize_t dump_receive_iov(DumpState *s, const struct iovec *iov, int cnt, int64_t ts; int caplen; size_t size = iov_size(iov, cnt) - offset; - struct iovec dumpiov[cnt + 1]; + g_autofree struct iovec *dumpiov = g_new(struct iovec, cnt + 1); /* Early return in case of previous error. */ if (s->fd < 0) { diff --git a/net/meson.build b/net/meson.build index 51caa42c9d..ce99bd4447 100644 --- a/net/meson.build +++ b/net/meson.build @@ -36,6 +36,9 @@ system_ss.add(when: vde, if_true: files('vde.c')) if have_netmap system_ss.add(files('netmap.c')) endif + +system_ss.add(when: libxdp, if_true: files('af-xdp.c')) + if have_vhost_net_user system_ss.add(when: 'CONFIG_VIRTIO_NET', if_true: files('vhost-user.c'), if_false: files('vhost-user-stub.c')) system_ss.add(when: 'CONFIG_ALL', if_true: files('vhost-user-stub.c')) diff --git a/net/net.c b/net/net.c index 6492ad530e..1c0bfdaa6c 100644 --- a/net/net.c +++ b/net/net.c @@ -495,6 +495,15 @@ bool qemu_has_ufo(NetClientState *nc) return nc->info->has_ufo(nc); } +bool qemu_has_uso(NetClientState *nc) +{ + if (!nc || !nc->info->has_uso) { + return false; + } + + return nc->info->has_uso(nc); +} + bool qemu_has_vnet_hdr(NetClientState *nc) { if (!nc || !nc->info->has_vnet_hdr) { @@ -532,13 +541,13 @@ void qemu_using_vnet_hdr(NetClientState *nc, bool enable) } void qemu_set_offload(NetClientState *nc, int csum, int tso4, int tso6, - int ecn, int ufo) + int ecn, int ufo, int uso4, int uso6) { if (!nc || !nc->info->set_offload) { return; } - nc->info->set_offload(nc, csum, tso4, tso6, ecn, ufo); + nc->info->set_offload(nc, csum, tso4, tso6, ecn, ufo, uso4, uso6); } int qemu_get_vnet_hdr_len(NetClientState *nc) @@ -1082,6 +1091,9 @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])( #ifdef CONFIG_NETMAP [NET_CLIENT_DRIVER_NETMAP] = net_init_netmap, #endif +#ifdef CONFIG_AF_XDP + [NET_CLIENT_DRIVER_AF_XDP] = net_init_af_xdp, +#endif #ifdef CONFIG_NET_BRIDGE [NET_CLIENT_DRIVER_BRIDGE] = net_init_bridge, #endif @@ -1186,6 +1198,9 @@ void show_netdevs(void) #ifdef CONFIG_NETMAP "netmap", #endif +#ifdef CONFIG_AF_XDP + "af-xdp", +#endif #ifdef CONFIG_POSIX "vhost-user", #endif diff --git a/net/netmap.c b/net/netmap.c index 9e0cec58d3..241b27c8e9 100644 --- a/net/netmap.c +++ b/net/netmap.c @@ -371,7 +371,7 @@ static void netmap_set_vnet_hdr_len(NetClientState *nc, int len) } static void netmap_set_offload(NetClientState *nc, int csum, int tso4, int tso6, - int ecn, int ufo) + int ecn, int ufo, int uso4, int uso6) { NetmapState *s = DO_UPCAST(NetmapState, nc, nc); diff --git a/net/tap-bsd.c b/net/tap-bsd.c index 4c98fdd337..274ea7bd2c 100644 --- a/net/tap-bsd.c +++ b/net/tap-bsd.c @@ -212,6 +212,11 @@ int tap_probe_has_ufo(int fd) return 0; } +int tap_probe_has_uso(int fd) +{ + return 0; +} + int tap_probe_vnet_hdr_len(int fd, int len) { return 0; @@ -232,7 +237,7 @@ int tap_fd_set_vnet_be(int fd, int is_be) } void tap_fd_set_offload(int fd, int csum, int tso4, - int tso6, int ecn, int ufo) + int tso6, int ecn, int ufo, int uso4, int uso6) { } diff --git a/net/tap-linux.c b/net/tap-linux.c index f54f308d35..c7e514ecb0 100644 --- a/net/tap-linux.c +++ b/net/tap-linux.c @@ -173,6 +173,18 @@ int tap_probe_has_ufo(int fd) return 1; } +int tap_probe_has_uso(int fd) +{ + unsigned offload; + + offload = TUN_F_CSUM | TUN_F_USO4 | TUN_F_USO6; + + if (ioctl(fd, TUNSETOFFLOAD, offload) < 0) { + return 0; + } + return 1; +} + /* Verify that we can assign given length */ int tap_probe_vnet_hdr_len(int fd, int len) { @@ -237,7 +249,7 @@ int tap_fd_set_vnet_be(int fd, int is_be) } void tap_fd_set_offload(int fd, int csum, int tso4, - int tso6, int ecn, int ufo) + int tso6, int ecn, int ufo, int uso4, int uso6) { unsigned int offload = 0; @@ -256,13 +268,22 @@ void tap_fd_set_offload(int fd, int csum, int tso4, offload |= TUN_F_TSO_ECN; if (ufo) offload |= TUN_F_UFO; + if (uso4) { + offload |= TUN_F_USO4; + } + if (uso6) { + offload |= TUN_F_USO6; + } } if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) { - offload &= ~TUN_F_UFO; + offload &= ~(TUN_F_USO4 | TUN_F_USO6); if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) { - fprintf(stderr, "TUNSETOFFLOAD ioctl() failed: %s\n", + offload &= ~TUN_F_UFO; + if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) { + fprintf(stderr, "TUNSETOFFLOAD ioctl() failed: %s\n", strerror(errno)); + } } } } diff --git a/net/tap-linux.h b/net/tap-linux.h index bbbb62c2a7..9a58cecb7f 100644 --- a/net/tap-linux.h +++ b/net/tap-linux.h @@ -50,5 +50,7 @@ #define TUN_F_TSO6 0x04 /* I can handle TSO for IPv6 packets */ #define TUN_F_TSO_ECN 0x08 /* I can handle TSO with ECN bits. */ #define TUN_F_UFO 0x10 /* I can handle UFO packets */ +#define TUN_F_USO4 0x20 /* I can handle USO for IPv4 packets */ +#define TUN_F_USO6 0x40 /* I can handle USO for IPv6 packets */ #endif /* QEMU_TAP_LINUX_H */ diff --git a/net/tap-solaris.c b/net/tap-solaris.c index 38e15028bf..08b13af512 100644 --- a/net/tap-solaris.c +++ b/net/tap-solaris.c @@ -216,6 +216,11 @@ int tap_probe_has_ufo(int fd) return 0; } +int tap_probe_has_uso(int fd) +{ + return 0; +} + int tap_probe_vnet_hdr_len(int fd, int len) { return 0; @@ -236,7 +241,7 @@ int tap_fd_set_vnet_be(int fd, int is_be) } void tap_fd_set_offload(int fd, int csum, int tso4, - int tso6, int ecn, int ufo) + int tso6, int ecn, int ufo, int uso4, int uso6) { } diff --git a/net/tap-stub.c b/net/tap-stub.c index a0fa25804b..4b24f61e3a 100644 --- a/net/tap-stub.c +++ b/net/tap-stub.c @@ -47,6 +47,11 @@ int tap_probe_has_ufo(int fd) return 0; } +int tap_probe_has_uso(int fd) +{ + return 0; +} + int tap_probe_vnet_hdr_len(int fd, int len) { return 0; @@ -67,7 +72,7 @@ int tap_fd_set_vnet_be(int fd, int is_be) } void tap_fd_set_offload(int fd, int csum, int tso4, - int tso6, int ecn, int ufo) + int tso6, int ecn, int ufo, int uso4, int uso6) { } diff --git a/net/tap-win32.c b/net/tap-win32.c index f327d62ab0..7b8b4be02c 100644 --- a/net/tap-win32.c +++ b/net/tap-win32.c @@ -741,7 +741,7 @@ static void tap_using_vnet_hdr(NetClientState *nc, bool using_vnet_hdr) } static void tap_set_offload(NetClientState *nc, int csum, int tso4, - int tso6, int ecn, int ufo) + int tso6, int ecn, int ufo, int uso4, int uso6) { } diff --git a/net/tap.c b/net/tap.c index 1bf085d422..c23d0323c2 100644 --- a/net/tap.c +++ b/net/tap.c @@ -57,6 +57,7 @@ typedef struct TAPState { bool write_poll; bool using_vnet_hdr; bool has_ufo; + bool has_uso; bool enabled; VHostNetState *vhost_net; unsigned host_vnet_hdr_len; @@ -117,10 +118,11 @@ static ssize_t tap_receive_iov(NetClientState *nc, const struct iovec *iov, { TAPState *s = DO_UPCAST(TAPState, nc, nc); const struct iovec *iovp = iov; - struct iovec iov_copy[iovcnt + 1]; + g_autofree struct iovec *iov_copy = NULL; struct virtio_net_hdr_mrg_rxbuf hdr = { }; if (s->host_vnet_hdr_len && !s->using_vnet_hdr) { + iov_copy = g_new(struct iovec, iovcnt + 1); iov_copy[0].iov_base = &hdr; iov_copy[0].iov_len = s->host_vnet_hdr_len; memcpy(&iov_copy[1], iov, iovcnt * sizeof(*iov)); @@ -237,6 +239,15 @@ static bool tap_has_ufo(NetClientState *nc) return s->has_ufo; } +static bool tap_has_uso(NetClientState *nc) +{ + TAPState *s = DO_UPCAST(TAPState, nc, nc); + + assert(nc->info->type == NET_CLIENT_DRIVER_TAP); + + return s->has_uso; +} + static bool tap_has_vnet_hdr(NetClientState *nc) { TAPState *s = DO_UPCAST(TAPState, nc, nc); @@ -307,14 +318,14 @@ static int tap_set_vnet_be(NetClientState *nc, bool is_be) } static void tap_set_offload(NetClientState *nc, int csum, int tso4, - int tso6, int ecn, int ufo) + int tso6, int ecn, int ufo, int uso4, int uso6) { TAPState *s = DO_UPCAST(TAPState, nc, nc); if (s->fd < 0) { return; } - tap_fd_set_offload(s->fd, csum, tso4, tso6, ecn, ufo); + tap_fd_set_offload(s->fd, csum, tso4, tso6, ecn, ufo, uso4, uso6); } static void tap_exit_notify(Notifier *notifier, void *data) @@ -384,6 +395,7 @@ static NetClientInfo net_tap_info = { .poll = tap_poll, .cleanup = tap_cleanup, .has_ufo = tap_has_ufo, + .has_uso = tap_has_uso, .has_vnet_hdr = tap_has_vnet_hdr, .has_vnet_hdr_len = tap_has_vnet_hdr_len, .get_using_vnet_hdr = tap_get_using_vnet_hdr, @@ -413,8 +425,9 @@ static TAPState *net_tap_fd_init(NetClientState *peer, s->host_vnet_hdr_len = vnet_hdr ? sizeof(struct virtio_net_hdr) : 0; s->using_vnet_hdr = false; s->has_ufo = tap_probe_has_ufo(s->fd); + s->has_uso = tap_probe_has_uso(s->fd); s->enabled = true; - tap_set_offload(&s->nc, 0, 0, 0, 0, 0); + tap_set_offload(&s->nc, 0, 0, 0, 0, 0, 0, 0); /* * Make sure host header length is set correctly in tap: * it might have been modified by another instance of qemu. diff --git a/net/tap_int.h b/net/tap_int.h index 547f8a5a28..9a2175655b 100644 --- a/net/tap_int.h +++ b/net/tap_int.h @@ -37,7 +37,9 @@ void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp); int tap_probe_vnet_hdr(int fd, Error **errp); int tap_probe_vnet_hdr_len(int fd, int len); int tap_probe_has_ufo(int fd); -void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo); +int tap_probe_has_uso(int fd); +void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo, + int uso4, int uso6); void tap_fd_set_vnet_hdr_len(int fd, int len); int tap_fd_set_vnet_le(int fd, int vnet_is_le); int tap_fd_set_vnet_be(int fd, int vnet_is_be); diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c index 34202ca009..4e94c50bc7 100644 --- a/net/vhost-vdpa.c +++ b/net/vhost-vdpa.c @@ -75,11 +75,14 @@ const int vdpa_feature_bits[] = { VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, VIRTIO_NET_F_GUEST_UFO, + VIRTIO_NET_F_GUEST_USO4, + VIRTIO_NET_F_GUEST_USO6, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_TSO6, VIRTIO_NET_F_HOST_UFO, + VIRTIO_NET_F_HOST_USO, VIRTIO_NET_F_MQ, VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_MTU, diff --git a/pc-bios/edk2-aarch64-code.fd.bz2 b/pc-bios/edk2-aarch64-code.fd.bz2 index 4bc824e48d..985e69a66a 100644 --- a/pc-bios/edk2-aarch64-code.fd.bz2 +++ b/pc-bios/edk2-aarch64-code.fd.bz2 Binary files differdiff --git a/pc-bios/edk2-arm-code.fd.bz2 b/pc-bios/edk2-arm-code.fd.bz2 index 7899fca426..ae797a8c8e 100644 --- a/pc-bios/edk2-arm-code.fd.bz2 +++ b/pc-bios/edk2-arm-code.fd.bz2 Binary files differdiff --git a/pc-bios/edk2-i386-code.fd.bz2 b/pc-bios/edk2-i386-code.fd.bz2 index a68ae4fa15..e703c2f954 100644 --- a/pc-bios/edk2-i386-code.fd.bz2 +++ b/pc-bios/edk2-i386-code.fd.bz2 Binary files differdiff --git a/pc-bios/edk2-i386-secure-code.fd.bz2 b/pc-bios/edk2-i386-secure-code.fd.bz2 index 91936ebbc9..7230d44615 100644 --- a/pc-bios/edk2-i386-secure-code.fd.bz2 +++ b/pc-bios/edk2-i386-secure-code.fd.bz2 Binary files differdiff --git a/pc-bios/edk2-riscv-code.fd.bz2 b/pc-bios/edk2-riscv-code.fd.bz2 new file mode 100644 index 0000000000..c1cc08561d --- /dev/null +++ b/pc-bios/edk2-riscv-code.fd.bz2 Binary files differdiff --git a/pc-bios/edk2-riscv-vars.fd.bz2 b/pc-bios/edk2-riscv-vars.fd.bz2 new file mode 100644 index 0000000000..40da6591ad --- /dev/null +++ b/pc-bios/edk2-riscv-vars.fd.bz2 Binary files differdiff --git a/pc-bios/edk2-riscv.fd.bz2 b/pc-bios/edk2-riscv.fd.bz2 deleted file mode 100644 index ef566b374a..0000000000 --- a/pc-bios/edk2-riscv.fd.bz2 +++ /dev/null Binary files differdiff --git a/pc-bios/edk2-x86_64-code.fd.bz2 b/pc-bios/edk2-x86_64-code.fd.bz2 index 35c2340e46..9b7767a3ac 100644 --- a/pc-bios/edk2-x86_64-code.fd.bz2 +++ b/pc-bios/edk2-x86_64-code.fd.bz2 Binary files differdiff --git a/pc-bios/edk2-x86_64-microvm.fd.bz2 b/pc-bios/edk2-x86_64-microvm.fd.bz2 index 742abf06c5..17460dd380 100644 --- a/pc-bios/edk2-x86_64-microvm.fd.bz2 +++ b/pc-bios/edk2-x86_64-microvm.fd.bz2 Binary files differdiff --git a/pc-bios/edk2-x86_64-secure-code.fd.bz2 b/pc-bios/edk2-x86_64-secure-code.fd.bz2 index 311b7b91d7..fd0efeacbf 100644 --- a/pc-bios/edk2-x86_64-secure-code.fd.bz2 +++ b/pc-bios/edk2-x86_64-secure-code.fd.bz2 Binary files differdiff --git a/pc-bios/hppa-firmware.img b/pc-bios/hppa-firmware.img index 0fa3808f16..c7196143b1 100644 --- a/pc-bios/hppa-firmware.img +++ b/pc-bios/hppa-firmware.img Binary files differdiff --git a/plugins/api.c b/plugins/api.c index 2078b16edb..5521b0ad36 100644 --- a/plugins/api.c +++ b/plugins/api.c @@ -316,22 +316,7 @@ uint64_t qemu_plugin_hwaddr_phys_addr(const struct qemu_plugin_hwaddr *haddr) { #ifdef CONFIG_SOFTMMU if (haddr) { - if (!haddr->is_io) { - RAMBlock *block; - ram_addr_t offset; - void *hostaddr = haddr->v.ram.hostaddr; - - block = qemu_ram_block_from_host(hostaddr, false, &offset); - if (!block) { - error_report("Bad host ram pointer %p", haddr->v.ram.hostaddr); - abort(); - } - - return block->offset + offset + block->mr->addr; - } else { - MemoryRegionSection *mrs = haddr->v.io.section; - return mrs->offset_within_address_space + haddr->v.io.offset; - } + return haddr->phys_addr; } #endif return 0; @@ -341,13 +326,13 @@ const char *qemu_plugin_hwaddr_device_name(const struct qemu_plugin_hwaddr *h) { #ifdef CONFIG_SOFTMMU if (h && h->is_io) { - MemoryRegionSection *mrs = h->v.io.section; - if (!mrs->mr->name) { - unsigned long maddr = 0xffffffff & (uintptr_t) mrs->mr; - g_autofree char *temp = g_strdup_printf("anon%08lx", maddr); + MemoryRegion *mr = h->mr; + if (!mr->name) { + unsigned maddr = (uintptr_t)mr; + g_autofree char *temp = g_strdup_printf("anon%08x", maddr); return g_intern_string(temp); } else { - return g_intern_string(mrs->mr->name); + return g_intern_string(mr->name); } } else { return g_intern_static_string("RAM"); diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py index c16a0b6fed..35d5a672db 100644 --- a/python/qemu/machine/machine.py +++ b/python/qemu/machine/machine.py @@ -191,6 +191,7 @@ class QEMUMachine: self.sock_dir, f"{self._name}.con" ) self._console_socket: Optional[socket.socket] = None + self._console_file: Optional[socket.SocketIO] = None self._remove_files: List[str] = [] self._user_killed = False self._quit_issued = False @@ -509,6 +510,11 @@ class QEMUMachine: # If we keep the console socket open, we may deadlock waiting # for QEMU to exit, while QEMU is waiting for the socket to # become writable. + if self._console_file is not None: + LOG.debug("Closing console file") + self._console_file.close() + self._console_file = None + if self._console_socket is not None: LOG.debug("Closing console socket") self._console_socket.close() @@ -874,6 +880,7 @@ class QEMUMachine: Returns a socket connected to the console """ if self._console_socket is None: + LOG.debug("Opening console socket") self._console_socket = console_socket.ConsoleSocket( self._console_address, file=self._console_log_path, @@ -881,6 +888,18 @@ class QEMUMachine: return self._console_socket @property + def console_file(self) -> socket.SocketIO: + """ + Returns a file associated with the console socket + """ + if self._console_file is None: + LOG.debug("Opening console file") + self._console_file = self.console_socket.makefile(mode='rb', + buffering=0, + encoding='utf-8') + return self._console_file + + @property def temp_dir(self) -> str: """ Returns a temporary directory to be used for this machine diff --git a/qapi/block-core.json b/qapi/block-core.json index 2b1d493d6e..89751d81f2 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -409,6 +409,8 @@ # # @zero: whether the virtual blocks read as zeroes # +# @compressed: true if the data is stored compressed (since 8.2) +# # @depth: number of layers (0 = top image, 1 = top image's backing # file, ..., n - 1 = bottom image (where n is the number of images # in the chain)) before reaching one for which the range is @@ -426,8 +428,8 @@ ## { 'struct': 'MapEntry', 'data': {'start': 'int', 'length': 'int', 'data': 'bool', - 'zero': 'bool', 'depth': 'int', 'present': 'bool', - '*offset': 'int', '*filename': 'str' } } + 'zero': 'bool', 'compressed': 'bool', 'depth': 'int', + 'present': 'bool', '*offset': 'int', '*filename': 'str' } } ## # @BlockdevCacheInfo: diff --git a/qapi/net.json b/qapi/net.json index 313c8a606e..8095b68fa8 100644 --- a/qapi/net.json +++ b/qapi/net.json @@ -409,6 +409,60 @@ '*devname': 'str' } } ## +# @AFXDPMode: +# +# Attach mode for a default XDP program +# +# @skb: generic mode, no driver support necessary +# +# @native: DRV mode, program is attached to a driver, packets are passed to +# the socket without allocation of skb. +# +# Since: 8.2 +## +{ 'enum': 'AFXDPMode', + 'data': [ 'native', 'skb' ], + 'if': 'CONFIG_AF_XDP' } + +## +# @NetdevAFXDPOptions: +# +# AF_XDP network backend +# +# @ifname: The name of an existing network interface. +# +# @mode: Attach mode for a default XDP program. If not specified, then +# 'native' will be tried first, then 'skb'. +# +# @force-copy: Force XDP copy mode even if device supports zero-copy. +# (default: false) +# +# @queues: number of queues to be used for multiqueue interfaces (default: 1). +# +# @start-queue: Use @queues starting from this queue number (default: 0). +# +# @inhibit: Don't load a default XDP program, use one already loaded to +# the interface (default: false). Requires @sock-fds. +# +# @sock-fds: A colon (:) separated list of file descriptors for already open +# but not bound AF_XDP sockets in the queue order. One fd per queue. +# These descriptors should already be added into XDP socket map for +# corresponding queues. Requires @inhibit. +# +# Since: 8.2 +## +{ 'struct': 'NetdevAFXDPOptions', + 'data': { + 'ifname': 'str', + '*mode': 'AFXDPMode', + '*force-copy': 'bool', + '*queues': 'int', + '*start-queue': 'int', + '*inhibit': 'bool', + '*sock-fds': 'str' }, + 'if': 'CONFIG_AF_XDP' } + +## # @NetdevVhostUserOptions: # # Vhost-user network backend @@ -642,6 +696,7 @@ # @vmnet-bridged: since 7.1 # @stream: since 7.2 # @dgram: since 7.2 +# @af-xdp: since 8.2 # # Since: 2.7 ## @@ -649,6 +704,7 @@ 'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'stream', 'dgram', 'vde', 'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa', + { 'name': 'af-xdp', 'if': 'CONFIG_AF_XDP' }, { 'name': 'vmnet-host', 'if': 'CONFIG_VMNET' }, { 'name': 'vmnet-shared', 'if': 'CONFIG_VMNET' }, { 'name': 'vmnet-bridged', 'if': 'CONFIG_VMNET' }] } @@ -679,6 +735,8 @@ 'bridge': 'NetdevBridgeOptions', 'hubport': 'NetdevHubPortOptions', 'netmap': 'NetdevNetmapOptions', + 'af-xdp': { 'type': 'NetdevAFXDPOptions', + 'if': 'CONFIG_AF_XDP' }, 'vhost-user': 'NetdevVhostUserOptions', 'vhost-vdpa': 'NetdevVhostVDPAOptions', 'vmnet-host': { 'type': 'NetdevVmnetHostOptions', diff --git a/qapi/qom.json b/qapi/qom.json index fa3e88c8e6..c53ef978ff 100644 --- a/qapi/qom.json +++ b/qapi/qom.json @@ -668,6 +668,20 @@ # @readonly: if true, the backing file is opened read-only; if false, # it is opened read-write. (default: false) # +# @rom: whether to create Read Only Memory (ROM) that cannot be modified +# by the VM. Any write attempts to such ROM will be denied. Most +# use cases want writable RAM instead of ROM. However, selected use +# cases, like R/O NVDIMMs, can benefit from ROM. If set to 'on', +# create ROM; if set to 'off', create writable RAM; if set to +# 'auto', the value of the @readonly property is used. This +# property is primarily helpful when we want to have proper RAM in +# configurations that would traditionally create ROM before this +# property was introduced: VM templating, where we want to open a +# file readonly (@readonly set to true) and mark the memory to be +# private for QEMU (@share set to false). For this use case, we need +# writable RAM instead of ROM, and want to set this property to 'off'. +# (default: auto, since 8.2) +# # Since: 2.1 ## { 'struct': 'MemoryBackendFileProperties', @@ -677,7 +691,8 @@ '*discard-data': 'bool', 'mem-path': 'str', '*pmem': { 'type': 'bool', 'if': 'CONFIG_LIBPMEM' }, - '*readonly': 'bool' } } + '*readonly': 'bool', + '*rom': 'OnOffAuto' } } ## # @MemoryBackendMemfdProperties: diff --git a/qemu-img.c b/qemu-img.c index 0756dbb835..a48edb7101 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -3108,10 +3108,12 @@ static int dump_map_entry(OutputFormat output_format, MapEntry *e, case OFORMAT_JSON: printf("{ \"start\": %"PRId64", \"length\": %"PRId64"," " \"depth\": %"PRId64", \"present\": %s, \"zero\": %s," - " \"data\": %s", e->start, e->length, e->depth, + " \"data\": %s, \"compressed\": %s", + e->start, e->length, e->depth, e->present ? "true" : "false", e->zero ? "true" : "false", - e->data ? "true" : "false"); + e->data ? "true" : "false", + e->compressed ? "true" : "false"); if (e->has_offset) { printf(", \"offset\": %"PRId64"", e->offset); } @@ -3172,6 +3174,7 @@ static int get_block_status(BlockDriverState *bs, int64_t offset, .length = bytes, .data = !!(ret & BDRV_BLOCK_DATA), .zero = !!(ret & BDRV_BLOCK_ZERO), + .compressed = !!(ret & BDRV_BLOCK_COMPRESSED), .offset = map, .has_offset = has_offset, .depth = depth, @@ -3189,6 +3192,7 @@ static inline bool entry_mergeable(const MapEntry *curr, const MapEntry *next) } if (curr->zero != next->zero || curr->data != next->data || + curr->compressed != next->compressed || curr->depth != next->depth || curr->present != next->present || !curr->filename != !next->filename || diff --git a/qemu-options.hx b/qemu-options.hx index 6be621c232..bcd77255cb 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -2882,6 +2882,19 @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev, " VALE port (created on the fly) called 'name' ('nmname' is name of the \n" " netmap device, defaults to '/dev/netmap')\n" #endif +#ifdef CONFIG_AF_XDP + "-netdev af-xdp,id=str,ifname=name[,mode=native|skb][,force-copy=on|off]\n" + " [,queues=n][,start-queue=m][,inhibit=on|off][,sock-fds=x:y:...:z]\n" + " attach to the existing network interface 'name' with AF_XDP socket\n" + " use 'mode=MODE' to specify an XDP program attach mode\n" + " use 'force-copy=on|off' to force XDP copy mode even if device supports zero-copy (default: off)\n" + " use 'inhibit=on|off' to inhibit loading of a default XDP program (default: off)\n" + " with inhibit=on,\n" + " use 'sock-fds' to provide file descriptors for already open AF_XDP sockets\n" + " added to a socket map in XDP program. One socket per queue.\n" + " use 'queues=n' to specify how many queues of a multiqueue interface should be used\n" + " use 'start-queue=m' to specify the first queue that should be used\n" +#endif #ifdef CONFIG_POSIX "-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n" " configure a vhost-user network, backed by a chardev 'dev'\n" @@ -2927,6 +2940,9 @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic, #ifdef CONFIG_NETMAP "netmap|" #endif +#ifdef CONFIG_AF_XDP + "af-xdp|" +#endif #ifdef CONFIG_POSIX "vhost-user|" #endif @@ -2955,6 +2971,9 @@ DEF("net", HAS_ARG, QEMU_OPTION_net, #ifdef CONFIG_NETMAP "netmap|" #endif +#ifdef CONFIG_AF_XDP + "af-xdp|" +#endif #ifdef CONFIG_VMNET "vmnet-host|vmnet-shared|vmnet-bridged|" #endif @@ -2962,7 +2981,7 @@ DEF("net", HAS_ARG, QEMU_OPTION_net, " old way to initialize a host network interface\n" " (use the -netdev option if possible instead)\n", QEMU_ARCH_ALL) SRST -``-nic [tap|bridge|user|l2tpv3|vde|netmap|vhost-user|socket][,...][,mac=macaddr][,model=mn]`` +``-nic [tap|bridge|user|l2tpv3|vde|netmap|af-xdp|vhost-user|socket][,...][,mac=macaddr][,model=mn]`` This option is a shortcut for configuring both the on-board (default) guest NIC hardware and the host network backend in one go. The host backend options are the same as with the corresponding @@ -3376,6 +3395,55 @@ SRST # launch QEMU instance |qemu_system| linux.img -nic vde,sock=/tmp/myswitch +``-netdev af-xdp,id=str,ifname=name[,mode=native|skb][,force-copy=on|off][,queues=n][,start-queue=m][,inhibit=on|off][,sock-fds=x:y:...:z]`` + Configure AF_XDP backend to connect to a network interface 'name' + using AF_XDP socket. A specific program attach mode for a default + XDP program can be forced with 'mode', defaults to best-effort, + where the likely most performant mode will be in use. Number of queues + 'n' should generally match the number or queues in the interface, + defaults to 1. Traffic arriving on non-configured device queues will + not be delivered to the network backend. + + .. parsed-literal:: + + # set number of queues to 4 + ethtool -L eth0 combined 4 + # launch QEMU instance + |qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\ + -netdev af-xdp,id=n1,ifname=eth0,queues=4 + + 'start-queue' option can be specified if a particular range of queues + [m, m + n] should be in use. For example, this is may be necessary in + order to use certain NICs in native mode. Kernel allows the driver to + create a separate set of XDP queues on top of regular ones, and only + these queues can be used for AF_XDP sockets. NICs that work this way + may also require an additional traffic redirection with ethtool to these + special queues. + + .. parsed-literal:: + + # set number of queues to 1 + ethtool -L eth0 combined 1 + # redirect all the traffic to the second queue (id: 1) + # note: drivers may require non-empty key/mask pair. + ethtool -N eth0 flow-type ether \\ + dst 00:00:00:00:00:00 m FF:FF:FF:FF:FF:FE action 1 + ethtool -N eth0 flow-type ether \\ + dst 00:00:00:00:00:01 m FF:FF:FF:FF:FF:FE action 1 + # launch QEMU instance + |qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\ + -netdev af-xdp,id=n1,ifname=eth0,queues=1,start-queue=1 + + XDP program can also be loaded externally. In this case 'inhibit' option + should be set to 'on' and 'sock-fds' provided with file descriptors for + already open but not bound XDP sockets already added to a socket map for + corresponding queues. One socket per queue. + + .. parsed-literal:: + + |qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\ + -netdev af-xdp,id=n1,ifname=eth0,queues=3,inhibit=on,sock-fds=15:16:17 + ``-netdev vhost-user,chardev=id[,vhostforce=on|off][,queues=n]`` Establish a vhost-user netdev, backed by a chardev id. The chardev should be a unix domain socket backed one. The vhost-user uses a @@ -4995,7 +5063,7 @@ SRST they are specified. Note that the 'id' property must be set. These objects are placed in the '/objects' path. - ``-object memory-backend-file,id=id,size=size,mem-path=dir,share=on|off,discard-data=on|off,merge=on|off,dump=on|off,prealloc=on|off,host-nodes=host-nodes,policy=default|preferred|bind|interleave,align=align,offset=offset,readonly=on|off`` + ``-object memory-backend-file,id=id,size=size,mem-path=dir,share=on|off,discard-data=on|off,merge=on|off,dump=on|off,prealloc=on|off,host-nodes=host-nodes,policy=default|preferred|bind|interleave,align=align,offset=offset,readonly=on|off,rom=on|off|auto`` Creates a memory file backend object, which can be used to back the guest RAM with huge pages. @@ -5085,6 +5153,20 @@ SRST The ``readonly`` option specifies whether the backing file is opened read-only or read-write (default). + The ``rom`` option specifies whether to create Read Only Memory + (ROM) that cannot be modified by the VM. Any write attempts to such + ROM will be denied. Most use cases want proper RAM instead of ROM. + However, selected use cases, like R/O NVDIMMs, can benefit from + ROM. If set to ``on``, create ROM; if set to ``off``, create + writable RAM; if set to ``auto`` (default), the value of the + ``readonly`` option is used. This option is primarily helpful when + we want to have writable RAM in configurations that would + traditionally create ROM before the ``rom`` option was introduced: + VM templating, where we want to open a file readonly + (``readonly=on``) and mark the memory to be private for QEMU + (``share=off``). For this use case, we need writable RAM instead + of ROM, and want to also set ``rom=off``. + ``-object memory-backend-ram,id=id,merge=on|off,dump=on|off,share=on|off,prealloc=on|off,size=size,host-nodes=host-nodes,policy=default|preferred|bind|interleave`` Creates a memory backend object, which can be used to back the guest RAM. Memory backend objects offer more control than the diff --git a/roms/edk2 b/roms/edk2 -Subproject f80f052277c88a67c55e107b550f504eeea947d +Subproject 819cfc6b42a68790a23509e4fcc58ceb70e1965 diff --git a/roms/edk2-build.config b/roms/edk2-build.config index 66ef9ffcb9..bab6a9caeb 100644 --- a/roms/edk2-build.config +++ b/roms/edk2-build.config @@ -26,6 +26,9 @@ DEBUG_PRINT_ERROR_LEVEL = 0x80000000 # grub.efi uses EfiLoaderData for code PcdDxeNxMemoryProtectionPolicy = 0xC000000000007FD1 +[pcds.workaround.202308] +PcdFirstTimeWakeUpAPsBySipi = FALSE + #################################################################################### # i386 @@ -57,6 +60,7 @@ desc = ovmf build (64-bit) conf = OvmfPkg/OvmfPkgX64.dsc arch = X64 opts = common +pcds = workaround.202308 plat = OvmfX64 dest = ../pc-bios cpy1 = FV/OVMF_CODE.fd edk2-x86_64-code.fd @@ -67,6 +71,7 @@ conf = OvmfPkg/OvmfPkgIa32X64.dsc arch = IA32 X64 opts = common ovmf.sb.smm +pcds = workaround.202308 plat = Ovmf3264 dest = ../pc-bios cpy1 = FV/OVMF_CODE.fd edk2-x86_64-secure-code.fd @@ -76,6 +81,7 @@ desc = ovmf build for microvm conf = OvmfPkg/Microvm/MicrovmX64.dsc arch = X64 opts = common +pcds = workaround.202308 plat = MicrovmX64 dest = ../pc-bios cpy1 = FV/MICROVM.fd edk2-x86_64-microvm.fd @@ -120,5 +126,7 @@ conf = OvmfPkg/RiscVVirt/RiscVVirtQemu.dsc arch = RISCV64 plat = RiscVVirtQemu dest = ../pc-bios -cpy1 = FV/RISCV_VIRT.fd edk2-riscv.fd -pad1 = edk2-riscv.fd 32m +cpy1 = FV/RISCV_VIRT_CODE.fd edk2-riscv-code.fd +cpy2 = FV/RISCV_VIRT_VARS.fd edk2-riscv-vars.fd +pad1 = edk2-riscv-code.fd 32m +pad2 = edk2-riscv-vars.fd 32m diff --git a/roms/edk2-build.py b/roms/edk2-build.py index 870893f7c8..e564765aaa 100755 --- a/roms/edk2-build.py +++ b/roms/edk2-build.py @@ -6,6 +6,7 @@ https://gitlab.com/kraxel/edk2-build-config """ import os import sys +import time import shutil import argparse import subprocess @@ -45,19 +46,28 @@ def get_coredir(cfg): return os.path.abspath(cfg['global']['core']) return os.getcwd() -def get_version(cfg): +def get_toolchain(cfg, build): + if cfg.has_option(build, 'tool'): + return cfg[build]['tool'] + if cfg.has_option('global', 'tool'): + return cfg['global']['tool'] + return 'GCC5' + +def get_version(cfg, silent = False): coredir = get_coredir(cfg) if version_override: version = version_override - print('') - print(f'### version [override]: {version}') + if not silent: + print('') + print(f'### version [override]: {version}') return version if os.environ.get('RPM_PACKAGE_NAME'): version = os.environ.get('RPM_PACKAGE_NAME') version += '-' + os.environ.get('RPM_PACKAGE_VERSION') version += '-' + os.environ.get('RPM_PACKAGE_RELEASE') - print('') - print(f'### version [rpmbuild]: {version}') + if not silent: + print('') + print(f'### version [rpmbuild]: {version}') return version if os.path.exists(coredir + '/.git'): cmdline = [ 'git', 'describe', '--tags', '--abbrev=8', @@ -66,16 +76,17 @@ def get_version(cfg): stdout = subprocess.PIPE, check = True) version = result.stdout.decode().strip() - print('') - print(f'### version [git]: {version}') + if not silent: + print('') + print(f'### version [git]: {version}') return version return None def pcd_string(name, value): return f'{name}=L{value}\\0' -def pcd_version(cfg): - version = get_version(cfg) +def pcd_version(cfg, silent = False): + version = get_version(cfg, silent) if version is None: return [] return [ '--pcd', pcd_string('PcdFirmwareVersionString', version) ] @@ -85,49 +96,58 @@ def pcd_release_date(): return [] return [ '--pcd', pcd_string('PcdFirmwareReleaseDateString', release_date) ] -def build_message(line, line2 = None): +def build_message(line, line2 = None, silent = False): if os.environ.get('TERM') in [ 'xterm', 'xterm-256color' ]: # setxterm title start = '\x1b]2;' end = '\x07' print(f'{start}{rebase_prefix}{line}{end}', end = '') - print('') - print('###') - print(f'### {rebase_prefix}{line}') - if line2: - print(f'### {line2}') - print('###', flush = True) + if silent: + print(f'### {rebase_prefix}{line}', flush = True) + else: + print('') + print('###') + print(f'### {rebase_prefix}{line}') + if line2: + print(f'### {line2}') + print('###', flush = True) -def build_run(cmdline, name, section, silent = False): - print(cmdline, flush = True) +def build_run(cmdline, name, section, silent = False, nologs = False): if silent: - print('### building in silent mode ...', flush = True) + logfile = f'{section}.log' + if nologs: + print(f'### building in silent mode [no log] ...', flush = True) + else: + print(f'### building in silent mode [{logfile}] ...', flush = True) + start = time.time() result = subprocess.run(cmdline, check = False, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) - - logfile = f'{section}.log' - print(f'### writing log to {logfile} ...') - with open(logfile, 'wb') as f: - f.write(result.stdout) + if not nologs: + with open(logfile, 'wb') as f: + f.write(result.stdout) if result.returncode: print('### BUILD FAILURE') + print('### cmdline') + print(cmdline) print('### output') print(result.stdout.decode()) print(f'### exit code: {result.returncode}') else: - print('### OK') + secs = int(time.time() - start) + print(f'### OK ({int(secs/60)}:{secs%60:02d})') else: + print(cmdline, flush = True) result = subprocess.run(cmdline, check = False) if result.returncode: print(f'ERROR: {cmdline[0]} exited with {result.returncode}' f' while building {name}') sys.exit(result.returncode) -def build_copy(plat, tgt, dstdir, copy): - srcdir = f'Build/{plat}/{tgt}_GCC5' +def build_copy(plat, tgt, toolchain, dstdir, copy): + srcdir = f'Build/{plat}/{tgt}_{toolchain}' names = copy.split() srcfile = names[0] if len(names) > 1: @@ -156,66 +176,68 @@ def pad_file(dstdir, pad): subprocess.run(cmdline, check = True) # pylint: disable=too-many-branches -def build_one(cfg, build, jobs = None, silent = False): +def build_one(cfg, build, jobs = None, silent = False, nologs = False): + b = cfg[build] + cmdline = [ 'build' ] - cmdline += [ '-t', 'GCC5' ] - cmdline += [ '-p', cfg[build]['conf'] ] + cmdline += [ '-t', get_toolchain(cfg, build) ] + cmdline += [ '-p', b['conf'] ] - if (cfg[build]['conf'].startswith('OvmfPkg/') or - cfg[build]['conf'].startswith('ArmVirtPkg/')): - cmdline += pcd_version(cfg) + if (b['conf'].startswith('OvmfPkg/') or + b['conf'].startswith('ArmVirtPkg/')): + cmdline += pcd_version(cfg, silent) cmdline += pcd_release_date() if jobs: cmdline += [ '-n', jobs ] - for arch in cfg[build]['arch'].split(): + for arch in b['arch'].split(): cmdline += [ '-a', arch ] - if 'opts' in cfg[build]: - for name in cfg[build]['opts'].split(): + if 'opts' in b: + for name in b['opts'].split(): section = 'opts.' + name for opt in cfg[section]: cmdline += [ '-D', opt + '=' + cfg[section][opt] ] - if 'pcds' in cfg[build]: - for name in cfg[build]['pcds'].split(): + if 'pcds' in b: + for name in b['pcds'].split(): section = 'pcds.' + name for pcd in cfg[section]: cmdline += [ '--pcd', pcd + '=' + cfg[section][pcd] ] - if 'tgts' in cfg[build]: - tgts = cfg[build]['tgts'].split() + if 'tgts' in b: + tgts = b['tgts'].split() else: tgts = [ 'DEBUG' ] for tgt in tgts: desc = None - if 'desc' in cfg[build]: - desc = cfg[build]['desc'] - build_message(f'building: {cfg[build]["conf"]} ({cfg[build]["arch"]}, {tgt})', - f'description: {desc}') + if 'desc' in b: + desc = b['desc'] + build_message(f'building: {b["conf"]} ({b["arch"]}, {tgt})', + f'description: {desc}', + silent = silent) build_run(cmdline + [ '-b', tgt ], - cfg[build]['conf'], + b['conf'], build + '.' + tgt, - silent) + silent, + nologs) - if 'plat' in cfg[build]: + if 'plat' in b: # copy files - for cpy in cfg[build]: + for cpy in b: if not cpy.startswith('cpy'): continue - build_copy(cfg[build]['plat'], - tgt, - cfg[build]['dest'], - cfg[build][cpy]) + build_copy(b['plat'], tgt, + get_toolchain(cfg, build), + b['dest'], b[cpy]) # pad builds - for pad in cfg[build]: + for pad in b: if not pad.startswith('pad'): continue - pad_file(cfg[build]['dest'], - cfg[build][pad]) + pad_file(b['dest'], b[pad]) -def build_basetools(silent = False): - build_message('building: BaseTools') +def build_basetools(silent = False, nologs = False): + build_message('building: BaseTools', silent = silent) basedir = os.environ['EDK_TOOLS_PATH'] cmdline = [ 'make', '-C', basedir ] - build_run(cmdline, 'BaseTools', 'build.basetools', silent) + build_run(cmdline, 'BaseTools', 'build.basetools', silent, nologs) def binary_exists(name): for pdir in os.environ['PATH'].split(':'): @@ -223,7 +245,7 @@ def binary_exists(name): return True return False -def prepare_env(cfg): +def prepare_env(cfg, silent = False): """ mimic Conf/BuildEnv.sh """ workspace = os.getcwd() packages = [ workspace, ] @@ -253,7 +275,7 @@ def prepare_env(cfg): toolsdef = coredir + '/Conf/tools_def.txt' if not os.path.exists(toolsdef): os.makedirs(os.path.dirname(toolsdef), exist_ok = True) - build_message('running BaseTools/BuildEnv') + build_message('running BaseTools/BuildEnv', silent = silent) cmdline = [ 'bash', 'BaseTools/BuildEnv' ] subprocess.run(cmdline, cwd = coredir, check = True) @@ -267,20 +289,32 @@ def prepare_env(cfg): os.environ['PYTHONHASHSEED'] = '1' # for cross builds - if binary_exists('arm-linux-gnu-gcc'): + if binary_exists('arm-linux-gnueabi-gcc'): + # ubuntu + os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnueabi-' + os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnueabi-' + elif binary_exists('arm-linux-gnu-gcc'): + # fedora os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnu-' + os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnu-' if binary_exists('loongarch64-linux-gnu-gcc'): os.environ['GCC5_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-' + os.environ['GCC_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-' hostarch = os.uname().machine if binary_exists('aarch64-linux-gnu-gcc') and hostarch != 'aarch64': os.environ['GCC5_AARCH64_PREFIX'] = 'aarch64-linux-gnu-' + os.environ['GCC_AARCH64_PREFIX'] = 'aarch64-linux-gnu-' if binary_exists('riscv64-linux-gnu-gcc') and hostarch != 'riscv64': os.environ['GCC5_RISCV64_PREFIX'] = 'riscv64-linux-gnu-' + os.environ['GCC_RISCV64_PREFIX'] = 'riscv64-linux-gnu-' if binary_exists('x86_64-linux-gnu-gcc') and hostarch != 'x86_64': os.environ['GCC5_IA32_PREFIX'] = 'x86_64-linux-gnu-' os.environ['GCC5_X64_PREFIX'] = 'x86_64-linux-gnu-' os.environ['GCC5_BIN'] = 'x86_64-linux-gnu-' + os.environ['GCC_IA32_PREFIX'] = 'x86_64-linux-gnu-' + os.environ['GCC_X64_PREFIX'] = 'x86_64-linux-gnu-' + os.environ['GCC_BIN'] = 'x86_64-linux-gnu-' def build_list(cfg): for build in cfg.sections(): @@ -303,10 +337,12 @@ def main(): parser.add_argument('-j', '--jobs', dest = 'jobs', type = str, help = 'allow up to JOBS parallel build jobs', metavar = 'JOBS') - parser.add_argument('-m', '--match', dest = 'match', type = str, + parser.add_argument('-m', '--match', dest = 'match', + type = str, action = 'append', help = 'only run builds matching INCLUDE (substring)', metavar = 'INCLUDE') - parser.add_argument('-x', '--exclude', dest = 'exclude', type = str, + parser.add_argument('-x', '--exclude', dest = 'exclude', + type = str, action = 'append', help = 'skip builds matching EXCLUDE (substring)', metavar = 'EXCLUDE') parser.add_argument('-l', '--list', dest = 'list', @@ -316,6 +352,9 @@ def main(): action = 'store_true', default = False, help = 'write build output to logfiles, ' 'write to console only on errors') + parser.add_argument('--no-logs', dest = 'nologs', + action = 'store_true', default = False, + help = 'do not write build log files (with --silent)') parser.add_argument('--core', dest = 'core', type = str, metavar = 'DIR', help = 'location of the core edk2 repository ' '(i.e. where BuildTools are located)') @@ -323,6 +362,9 @@ def main(): type = str, action = 'append', metavar = 'DIR', help = 'location(s) of additional packages ' '(can be specified multiple times)') + parser.add_argument('-t', '--toolchain', dest = 'toolchain', + type = str, metavar = 'NAME', + help = 'tool chain to be used to build edk2') parser.add_argument('--version-override', dest = 'version_override', type = str, metavar = 'VERSION', help = 'set firmware build version') @@ -335,7 +377,7 @@ def main(): os.chdir(options.directory) if not os.path.exists(options.configfile): - print('config file "{options.configfile}" not found') + print(f'config file "{options.configfile}" not found') return 1 cfg = configparser.ConfigParser() @@ -344,7 +386,7 @@ def main(): if options.list: build_list(cfg) - return + return 0 if not cfg.has_section('global'): cfg.add_section('global') @@ -352,6 +394,8 @@ def main(): cfg.set('global', 'core', options.core) if options.pkgs: cfg.set('global', 'pkgs', ' '.join(options.pkgs)) + if options.toolchain: + cfg.set('global', 'tool', options.toolchain) global version_override global release_date @@ -361,18 +405,28 @@ def main(): if options.release_date: release_date = options.release_date - prepare_env(cfg) - build_basetools(options.silent) + prepare_env(cfg, options.silent) + build_basetools(options.silent, options.nologs) for build in cfg.sections(): if not build.startswith('build.'): continue - if options.match and options.match not in build: - print(f'# skipping "{build}" (not matching "{options.match}")') - continue - if options.exclude and options.exclude in build: - print(f'# skipping "{build}" (matching "{options.exclude}")') - continue - build_one(cfg, build, options.jobs, options.silent) + if options.match: + matching = False + for item in options.match: + if item in build: + matching = True + if not matching: + print(f'# skipping "{build}" (not matching "{"|".join(options.match)}")') + continue + if options.exclude: + exclude = False + for item in options.exclude: + if item in build: + print(f'# skipping "{build}" (matching "{item}")') + exclude = True + if exclude: + continue + build_one(cfg, build, options.jobs, options.silent, options.nologs) return 0 diff --git a/roms/seabios-hppa b/roms/seabios-hppa -Subproject 673d2595d4f773cc266cbf8dbaf2f475a6adb94 +Subproject 763e3b73499db5fef94087bd310bfc8ccbcf785 diff --git a/scripts/block-coroutine-wrapper.py b/scripts/block-coroutine-wrapper.py index d4a183db61..685d0b4ed4 100644 --- a/scripts/block-coroutine-wrapper.py +++ b/scripts/block-coroutine-wrapper.py @@ -71,10 +71,13 @@ class FuncDecl: self.args = [ParamDecl(arg.strip()) for arg in args.split(',')] self.create_only_co = 'mixed' not in variant self.graph_rdlock = 'bdrv_rdlock' in variant + self.graph_wrlock = 'bdrv_wrlock' in variant self.wrapper_type = wrapper_type if wrapper_type == 'co': + if self.graph_wrlock: + raise ValueError(f"co function can't be wrlock: {self.name}") subsystem, subname = self.name.split('_', 1) self.target_name = f'{subsystem}_co_{subname}' else: @@ -102,12 +105,13 @@ class FuncDecl: def gen_ctx(self, prefix: str = '') -> str: t = self.args[0].type + name = self.args[0].name if t == 'BlockDriverState *': - return f'bdrv_get_aio_context({prefix}bs)' + return f'bdrv_get_aio_context({prefix}{name})' elif t == 'BdrvChild *': - return f'bdrv_get_aio_context({prefix}child->bs)' + return f'bdrv_get_aio_context({prefix}{name}->bs)' elif t == 'BlockBackend *': - return f'blk_get_aio_context({prefix}blk)' + return f'blk_get_aio_context({prefix}{name})' else: return 'qemu_get_aio_context()' @@ -250,6 +254,12 @@ def gen_no_co_wrapper(func: FuncDecl) -> str: name = func.target_name struct_name = func.struct_name + graph_lock='' + graph_unlock='' + if func.graph_wrlock: + graph_lock=' bdrv_graph_wrlock(NULL);' + graph_unlock=' bdrv_graph_wrunlock();' + return f"""\ /* * Wrappers for {name} @@ -266,9 +276,11 @@ static void {name}_bh(void *opaque) {struct_name} *s = opaque; AioContext *ctx = {func.gen_ctx('s->')}; +{graph_lock} aio_context_acquire(ctx); {func.get_result}{name}({ func.gen_list('s->{name}') }); aio_context_release(ctx); +{graph_unlock} aio_co_wake(s->co); }} diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure index 131f8ee5f3..76781f17f4 100755 --- a/scripts/ci/org.centos/stream/8/x86_64/configure +++ b/scripts/ci/org.centos/stream/8/x86_64/configure @@ -35,6 +35,7 @@ --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \ --with-coroutine=ucontext \ --tls-priority=@QEMU,SYSTEM \ +--disable-af-xdp \ --disable-attr \ --disable-auth-pam \ --disable-avx2 \ diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index e1d178370c..230119346a 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -76,6 +76,7 @@ meson_options_help() { printf "%s\n" 'disabled with --disable-FEATURE, default is enabled if available' printf "%s\n" '(unless built with --without-default-features):' printf "%s\n" '' + printf "%s\n" ' af-xdp AF_XDP network backend support' printf "%s\n" ' alsa ALSA sound support' printf "%s\n" ' attr attr/xattr support' printf "%s\n" ' auth-pam PAM access control' @@ -208,6 +209,8 @@ meson_options_help() { } _meson_option_parse() { case $1 in + --enable-af-xdp) printf "%s" -Daf_xdp=enabled ;; + --disable-af-xdp) printf "%s" -Daf_xdp=disabled ;; --enable-alsa) printf "%s" -Dalsa=enabled ;; --disable-alsa) printf "%s" -Dalsa=disabled ;; --enable-attr) printf "%s" -Dattr=enabled ;; diff --git a/softmmu/async-teardown.c b/softmmu/async-teardown.c index 62cdeb0f20..396963c091 100644 --- a/softmmu/async-teardown.c +++ b/softmmu/async-teardown.c @@ -121,10 +121,7 @@ static void *new_stack_for_clone(void) /* Allocate a new stack and get a pointer to its top. */ stack_ptr = qemu_alloc_stack(&stack_size); -#if !defined(HOST_HPPA) - /* The top is at the end of the area, except on HPPA. */ stack_ptr += stack_size; -#endif return stack_ptr; } diff --git a/softmmu/dma-helpers.c b/softmmu/dma-helpers.c index 2463964805..36211acc7e 100644 --- a/softmmu/dma-helpers.c +++ b/softmmu/dma-helpers.c @@ -206,17 +206,9 @@ static void dma_aio_cancel(BlockAIOCB *acb) } } -static AioContext *dma_get_aio_context(BlockAIOCB *acb) -{ - DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common); - - return dbs->ctx; -} - static const AIOCBInfo dma_aiocb_info = { .aiocb_size = sizeof(DMAAIOCB), .cancel_async = dma_aio_cancel, - .get_aio_context = dma_get_aio_context, }; BlockAIOCB *dma_blk_io(AioContext *ctx, diff --git a/softmmu/memory.c b/softmmu/memory.c index 7d9494ce70..c0383a163d 100644 --- a/softmmu/memory.c +++ b/softmmu/memory.c @@ -842,6 +842,10 @@ static void address_space_update_ioeventfds(AddressSpace *as) AddrRange tmp; unsigned i; + if (!as->ioeventfd_notifiers) { + return; + } + /* * It is likely that the number of ioeventfds hasn't changed much, so use * the previous size as the starting value, with some headroom to avoid @@ -1620,18 +1624,17 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, uint32_t ram_flags, const char *path, ram_addr_t offset, - bool readonly, Error **errp) { Error *err = NULL; memory_region_init(mr, owner, name, size); mr->ram = true; - mr->readonly = readonly; + mr->readonly = !!(ram_flags & RAM_READONLY); mr->terminates = true; mr->destructor = memory_region_destructor_ram; mr->align = align; mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path, - offset, readonly, &err); + offset, &err); if (err) { mr->size = int128_zero(); object_unparent(OBJECT(mr)); @@ -1651,10 +1654,11 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr, Error *err = NULL; memory_region_init(mr, owner, name, size); mr->ram = true; + mr->readonly = !!(ram_flags & RAM_READONLY); mr->terminates = true; mr->destructor = memory_region_destructor_ram; mr->ram_block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, - false, &err); + &err); if (err) { mr->size = int128_zero(); object_unparent(OBJECT(mr)); @@ -3075,6 +3079,10 @@ void memory_listener_register(MemoryListener *listener, AddressSpace *as) } listener_add_address_space(listener, as); + + if (listener->eventfd_add || listener->eventfd_del) { + as->ioeventfd_notifiers++; + } } void memory_listener_unregister(MemoryListener *listener) @@ -3083,6 +3091,10 @@ void memory_listener_unregister(MemoryListener *listener) return; } + if (listener->eventfd_add || listener->eventfd_del) { + listener->address_space->ioeventfd_notifiers--; + } + listener_del_address_space(listener, listener->address_space); QTAILQ_REMOVE(&memory_listeners, listener, link); QTAILQ_REMOVE(&listener->address_space->listeners, listener, link_as); diff --git a/softmmu/physmem.c b/softmmu/physmem.c index 18277ddd67..4f6ca653b3 100644 --- a/softmmu/physmem.c +++ b/softmmu/physmem.c @@ -1288,8 +1288,7 @@ static int64_t get_file_align(int fd) static int file_ram_open(const char *path, const char *region_name, bool readonly, - bool *created, - Error **errp) + bool *created) { char *filename; char *sanitized_name; @@ -1300,10 +1299,33 @@ static int file_ram_open(const char *path, for (;;) { fd = open(path, readonly ? O_RDONLY : O_RDWR); if (fd >= 0) { + /* + * open(O_RDONLY) won't fail with EISDIR. Check manually if we + * opened a directory and fail similarly to how we fail ENOENT + * in readonly mode. Note that mkstemp() would imply O_RDWR. + */ + if (readonly) { + struct stat file_stat; + + if (fstat(fd, &file_stat)) { + close(fd); + if (errno == EINTR) { + continue; + } + return -errno; + } else if (S_ISDIR(file_stat.st_mode)) { + close(fd); + return -EISDIR; + } + } /* @path names an existing file, use it */ break; } if (errno == ENOENT) { + if (readonly) { + /* Refuse to create new, readonly files. */ + return -ENOENT; + } /* @path names a file that doesn't exist, create it */ fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644); if (fd >= 0) { @@ -1333,10 +1355,7 @@ static int file_ram_open(const char *path, g_free(filename); } if (errno != EEXIST && errno != EINTR) { - error_setg_errno(errp, errno, - "can't open backing store %s for guest RAM", - path); - return -1; + return -errno; } /* * Try again on EINTR and EEXIST. The latter happens when @@ -1350,7 +1369,6 @@ static int file_ram_open(const char *path, static void *file_ram_alloc(RAMBlock *block, ram_addr_t memory, int fd, - bool readonly, bool truncate, off_t offset, Error **errp) @@ -1408,7 +1426,7 @@ static void *file_ram_alloc(RAMBlock *block, perror("ftruncate"); } - qemu_map_flags = readonly ? QEMU_MAP_READONLY : 0; + qemu_map_flags = (block->flags & RAM_READONLY) ? QEMU_MAP_READONLY : 0; qemu_map_flags |= (block->flags & RAM_SHARED) ? QEMU_MAP_SHARED : 0; qemu_map_flags |= (block->flags & RAM_PMEM) ? QEMU_MAP_SYNC : 0; qemu_map_flags |= (block->flags & RAM_NORESERVE) ? QEMU_MAP_NORESERVE : 0; @@ -1876,7 +1894,7 @@ static void ram_block_add(RAMBlock *new_block, Error **errp) #ifdef CONFIG_POSIX RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, int fd, off_t offset, - bool readonly, Error **errp) + Error **errp) { RAMBlock *new_block; Error *local_err = NULL; @@ -1884,7 +1902,8 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, /* Just support these ram flags by now. */ assert((ram_flags & ~(RAM_SHARED | RAM_PMEM | RAM_NORESERVE | - RAM_PROTECTED | RAM_NAMED_FILE)) == 0); + RAM_PROTECTED | RAM_NAMED_FILE | RAM_READONLY | + RAM_READONLY_FD)) == 0); if (xen_enabled()) { error_setg(errp, "-mem-path not supported with Xen"); @@ -1919,8 +1938,8 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, new_block->used_length = size; new_block->max_length = size; new_block->flags = ram_flags; - new_block->host = file_ram_alloc(new_block, size, fd, readonly, - !file_size, offset, errp); + new_block->host = file_ram_alloc(new_block, size, fd, !file_size, offset, + errp); if (!new_block->host) { g_free(new_block); return NULL; @@ -1939,20 +1958,40 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, const char *mem_path, - off_t offset, bool readonly, Error **errp) + off_t offset, Error **errp) { int fd; bool created; RAMBlock *block; - fd = file_ram_open(mem_path, memory_region_name(mr), readonly, &created, - errp); + fd = file_ram_open(mem_path, memory_region_name(mr), + !!(ram_flags & RAM_READONLY_FD), &created); if (fd < 0) { + error_setg_errno(errp, -fd, "can't open backing store %s for guest RAM", + mem_path); + if (!(ram_flags & RAM_READONLY_FD) && !(ram_flags & RAM_SHARED) && + fd == -EACCES) { + /* + * If we can open the file R/O (note: will never create a new file) + * and we are dealing with a private mapping, there are still ways + * to consume such files and get RAM instead of ROM. + */ + fd = file_ram_open(mem_path, memory_region_name(mr), true, + &created); + if (fd < 0) { + return NULL; + } + assert(!created); + close(fd); + error_append_hint(errp, "Consider opening the backing store" + " read-only but still creating writable RAM using" + " '-object memory-backend-file,readonly=on,rom=off...'" + " (see \"VM templating\" documentation)\n"); + } return NULL; } - block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, readonly, - errp); + block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, errp); if (!block) { if (created) { unlink(mem_path); @@ -2070,6 +2109,7 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length) ram_addr_t offset; int flags; void *area, *vaddr; + int prot; RAMBLOCK_FOREACH(block) { offset = addr - block->offset; @@ -2084,13 +2124,14 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length) flags |= block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE; flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0; + prot = PROT_READ; + prot |= block->flags & RAM_READONLY ? 0 : PROT_WRITE; if (block->fd >= 0) { - area = mmap(vaddr, length, PROT_READ | PROT_WRITE, - flags, block->fd, offset + block->fd_offset); + area = mmap(vaddr, length, prot, flags, block->fd, + offset + block->fd_offset); } else { flags |= MAP_ANONYMOUS; - area = mmap(vaddr, length, PROT_READ | PROT_WRITE, - flags, -1, 0); + area = mmap(vaddr, length, prot, flags, -1, 0); } if (area != vaddr) { error_report("Could not remap addr: " @@ -3481,6 +3522,16 @@ int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length) */ #ifdef CONFIG_FALLOCATE_PUNCH_HOLE /* + * fallocate() will fail with readonly files. Let's print a + * proper error message. + */ + if (rb->flags & RAM_READONLY_FD) { + error_report("ram_block_discard_range: Discarding RAM" + " with readonly files is not supported"); + goto err; + + } + /* * We'll discard data from the actual file, even though we only * have a MAP_PRIVATE mapping, possibly messing with other * MAP_PRIVATE/MAP_SHARED mappings. There is no easy way to diff --git a/subprojects/berkeley-softfloat-3.wrap b/subprojects/berkeley-softfloat-3.wrap index a8fd87740b..c3e356d42f 100644 --- a/subprojects/berkeley-softfloat-3.wrap +++ b/subprojects/berkeley-softfloat-3.wrap @@ -1,5 +1,5 @@ [wrap-git] -url = https://gitlab.com/qemu-project/berkeley-softfloat-3 +url = https://gitlab.com/qemu-project/berkeley-softfloat-3.git revision = b64af41c3276f97f0e181920400ee056b9c88037 patch_directory = berkeley-softfloat-3 depth = 1 diff --git a/subprojects/berkeley-testfloat-3.wrap b/subprojects/berkeley-testfloat-3.wrap index c86dc078a8..b8b12e7629 100644 --- a/subprojects/berkeley-testfloat-3.wrap +++ b/subprojects/berkeley-testfloat-3.wrap @@ -1,5 +1,5 @@ [wrap-git] -url = https://gitlab.com/qemu-project/berkeley-testfloat-3 +url = https://gitlab.com/qemu-project/berkeley-testfloat-3.git revision = e7af9751d9f9fd3b47911f51a5cfd08af256a9ab patch_directory = berkeley-testfloat-3 depth = 1 diff --git a/subprojects/slirp.wrap b/subprojects/slirp.wrap index 08291a4cf9..a93b048962 100644 --- a/subprojects/slirp.wrap +++ b/subprojects/slirp.wrap @@ -1,5 +1,5 @@ [wrap-git] -url = https://gitlab.freedesktop.org/slirp/libslirp +url = https://gitlab.freedesktop.org/slirp/libslirp.git revision = 26be815b86e8d49add8c9a8b320239b9594ff03d [provide] diff --git a/target/arm/cpu.h b/target/arm/cpu.h index f2e3dc49a6..bd55c5dabf 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -1315,6 +1315,7 @@ void pmu_init(ARMCPU *cpu); #define SCTLR_EnIB (1U << 30) /* v8.3, AArch64 only */ #define SCTLR_EnIA (1U << 31) /* v8.3, AArch64 only */ #define SCTLR_DSSBS_32 (1U << 31) /* v8.5, AArch32 only */ +#define SCTLR_MSCEN (1ULL << 33) /* FEAT_MOPS */ #define SCTLR_BT0 (1ULL << 35) /* v8.5-BTI */ #define SCTLR_BT1 (1ULL << 36) /* v8.5-BTI */ #define SCTLR_ITFSB (1ULL << 37) /* v8.5-MemTag */ @@ -2166,6 +2167,7 @@ FIELD(ID_AA64ISAR0, SHA1, 8, 4) FIELD(ID_AA64ISAR0, SHA2, 12, 4) FIELD(ID_AA64ISAR0, CRC32, 16, 4) FIELD(ID_AA64ISAR0, ATOMIC, 20, 4) +FIELD(ID_AA64ISAR0, TME, 24, 4) FIELD(ID_AA64ISAR0, RDM, 28, 4) FIELD(ID_AA64ISAR0, SHA3, 32, 4) FIELD(ID_AA64ISAR0, SM3, 36, 4) @@ -2200,6 +2202,13 @@ FIELD(ID_AA64ISAR2, APA3, 12, 4) FIELD(ID_AA64ISAR2, MOPS, 16, 4) FIELD(ID_AA64ISAR2, BC, 20, 4) FIELD(ID_AA64ISAR2, PAC_FRAC, 24, 4) +FIELD(ID_AA64ISAR2, CLRBHB, 28, 4) +FIELD(ID_AA64ISAR2, SYSREG_128, 32, 4) +FIELD(ID_AA64ISAR2, SYSINSTR_128, 36, 4) +FIELD(ID_AA64ISAR2, PRFMSLC, 40, 4) +FIELD(ID_AA64ISAR2, RPRFM, 48, 4) +FIELD(ID_AA64ISAR2, CSSC, 52, 4) +FIELD(ID_AA64ISAR2, ATS1A, 60, 4) FIELD(ID_AA64PFR0, EL0, 0, 4) FIELD(ID_AA64PFR0, EL1, 4, 4) @@ -2227,6 +2236,12 @@ FIELD(ID_AA64PFR1, SME, 24, 4) FIELD(ID_AA64PFR1, RNDR_TRAP, 28, 4) FIELD(ID_AA64PFR1, CSV2_FRAC, 32, 4) FIELD(ID_AA64PFR1, NMI, 36, 4) +FIELD(ID_AA64PFR1, MTE_FRAC, 40, 4) +FIELD(ID_AA64PFR1, GCS, 44, 4) +FIELD(ID_AA64PFR1, THE, 48, 4) +FIELD(ID_AA64PFR1, MTEX, 52, 4) +FIELD(ID_AA64PFR1, DF2, 56, 4) +FIELD(ID_AA64PFR1, PFAR, 60, 4) FIELD(ID_AA64MMFR0, PARANGE, 0, 4) FIELD(ID_AA64MMFR0, ASIDBITS, 4, 4) @@ -2258,6 +2273,7 @@ FIELD(ID_AA64MMFR1, AFP, 44, 4) FIELD(ID_AA64MMFR1, NTLBPA, 48, 4) FIELD(ID_AA64MMFR1, TIDCP1, 52, 4) FIELD(ID_AA64MMFR1, CMOW, 56, 4) +FIELD(ID_AA64MMFR1, ECBHB, 60, 4) FIELD(ID_AA64MMFR2, CNP, 0, 4) FIELD(ID_AA64MMFR2, UAO, 4, 4) @@ -2279,7 +2295,9 @@ FIELD(ID_AA64DFR0, DEBUGVER, 0, 4) FIELD(ID_AA64DFR0, TRACEVER, 4, 4) FIELD(ID_AA64DFR0, PMUVER, 8, 4) FIELD(ID_AA64DFR0, BRPS, 12, 4) +FIELD(ID_AA64DFR0, PMSS, 16, 4) FIELD(ID_AA64DFR0, WRPS, 20, 4) +FIELD(ID_AA64DFR0, SEBEP, 24, 4) FIELD(ID_AA64DFR0, CTX_CMPS, 28, 4) FIELD(ID_AA64DFR0, PMSVER, 32, 4) FIELD(ID_AA64DFR0, DOUBLELOCK, 36, 4) @@ -2287,12 +2305,14 @@ FIELD(ID_AA64DFR0, TRACEFILT, 40, 4) FIELD(ID_AA64DFR0, TRACEBUFFER, 44, 4) FIELD(ID_AA64DFR0, MTPMU, 48, 4) FIELD(ID_AA64DFR0, BRBE, 52, 4) +FIELD(ID_AA64DFR0, EXTTRCBUFF, 56, 4) FIELD(ID_AA64DFR0, HPMN0, 60, 4) FIELD(ID_AA64ZFR0, SVEVER, 0, 4) FIELD(ID_AA64ZFR0, AES, 4, 4) FIELD(ID_AA64ZFR0, BITPERM, 16, 4) FIELD(ID_AA64ZFR0, BFLOAT16, 20, 4) +FIELD(ID_AA64ZFR0, B16B16, 24, 4) FIELD(ID_AA64ZFR0, SHA3, 32, 4) FIELD(ID_AA64ZFR0, SM4, 40, 4) FIELD(ID_AA64ZFR0, I8MM, 44, 4) @@ -2300,9 +2320,13 @@ FIELD(ID_AA64ZFR0, F32MM, 52, 4) FIELD(ID_AA64ZFR0, F64MM, 56, 4) FIELD(ID_AA64SMFR0, F32F32, 32, 1) +FIELD(ID_AA64SMFR0, BI32I32, 33, 1) FIELD(ID_AA64SMFR0, B16F32, 34, 1) FIELD(ID_AA64SMFR0, F16F32, 35, 1) FIELD(ID_AA64SMFR0, I8I32, 36, 4) +FIELD(ID_AA64SMFR0, F16F16, 42, 1) +FIELD(ID_AA64SMFR0, B16B16, 43, 1) +FIELD(ID_AA64SMFR0, I16I32, 44, 4) FIELD(ID_AA64SMFR0, F64F64, 48, 1) FIELD(ID_AA64SMFR0, I16I64, 52, 4) FIELD(ID_AA64SMFR0, SMEVER, 56, 4) @@ -3147,6 +3171,7 @@ FIELD(TBFLAG_A64, SVL, 24, 4) FIELD(TBFLAG_A64, SME_TRAP_NONSTREAMING, 28, 1) FIELD(TBFLAG_A64, FGT_ERET, 29, 1) FIELD(TBFLAG_A64, NAA, 30, 1) +FIELD(TBFLAG_A64, ATA0, 31, 1) /* * Helpers for using the above. @@ -4065,6 +4090,11 @@ static inline bool isar_feature_aa64_i8mm(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, I8MM) != 0; } +static inline bool isar_feature_aa64_hbc(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64isar2, ID_AA64ISAR2, BC) != 0; +} + static inline bool isar_feature_aa64_tgran4_lpa2(const ARMISARegisters *id) { return FIELD_SEX64(id->id_aa64mmfr0, ID_AA64MMFR0, TGRAN4) >= 1; @@ -4253,6 +4283,11 @@ static inline bool isar_feature_aa64_doublelock(const ARMISARegisters *id) return FIELD_SEX64(id->id_aa64dfr0, ID_AA64DFR0, DOUBLELOCK) >= 0; } +static inline bool isar_feature_aa64_mops(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64isar2, ID_AA64ISAR2, MOPS); +} + /* * Feature tests for "does this exist in either 32-bit or 64-bit?" */ diff --git a/target/arm/helper.c b/target/arm/helper.c index 3b22596eab..83620787b4 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -5980,7 +5980,10 @@ static void hcrx_write(CPUARMState *env, const ARMCPRegInfo *ri, { uint64_t valid_mask = 0; - /* No features adding bits to HCRX are implemented. */ + /* FEAT_MOPS adds MSCEn and MCE2 */ + if (cpu_isar_feature(aa64_mops, env_archcpu(env))) { + valid_mask |= HCRX_MSCEN | HCRX_MCE2; + } /* Clear RES0 bits. */ env->cp15.hcrx_el2 = value & valid_mask; @@ -6009,13 +6012,24 @@ uint64_t arm_hcrx_el2_eff(CPUARMState *env) { /* * The bits in this register behave as 0 for all purposes other than - * direct reads of the register if: - * - EL2 is not enabled in the current security state, - * - SCR_EL3.HXEn is 0. - */ - if (!arm_is_el2_enabled(env) - || (arm_feature(env, ARM_FEATURE_EL3) - && !(env->cp15.scr_el3 & SCR_HXEN))) { + * direct reads of the register if SCR_EL3.HXEn is 0. + * If EL2 is not enabled in the current security state, then the + * bit may behave as if 0, or as if 1, depending on the bit. + * For the moment, we treat the EL2-disabled case as taking + * priority over the HXEn-disabled case. This is true for the only + * bit for a feature which we implement where the answer is different + * for the two cases (MSCEn for FEAT_MOPS). + * This may need to be revisited for future bits. + */ + if (!arm_is_el2_enabled(env)) { + uint64_t hcrx = 0; + if (cpu_isar_feature(aa64_mops, env_archcpu(env))) { + /* MSCEn behaves as 1 if EL2 is not enabled */ + hcrx |= HCRX_MSCEN; + } + return hcrx; + } + if (arm_feature(env, ARM_FEATURE_EL3) && !(env->cp15.scr_el3 & SCR_HXEN)) { return 0; } return env->cp15.hcrx_el2; @@ -8621,11 +8635,16 @@ void register_cp_regs_for_features(ARMCPU *cpu) R_ID_AA64ZFR0_F64MM_MASK }, { .name = "ID_AA64SMFR0_EL1", .exported_bits = R_ID_AA64SMFR0_F32F32_MASK | + R_ID_AA64SMFR0_BI32I32_MASK | R_ID_AA64SMFR0_B16F32_MASK | R_ID_AA64SMFR0_F16F32_MASK | R_ID_AA64SMFR0_I8I32_MASK | + R_ID_AA64SMFR0_F16F16_MASK | + R_ID_AA64SMFR0_B16B16_MASK | + R_ID_AA64SMFR0_I16I32_MASK | R_ID_AA64SMFR0_F64F64_MASK | R_ID_AA64SMFR0_I16I64_MASK | + R_ID_AA64SMFR0_SMEVER_MASK | R_ID_AA64SMFR0_FA64_MASK }, { .name = "ID_AA64MMFR0_EL1", .exported_bits = R_ID_AA64MMFR0_ECV_MASK, @@ -8676,7 +8695,11 @@ void register_cp_regs_for_features(ARMCPU *cpu) .exported_bits = R_ID_AA64ISAR2_WFXT_MASK | R_ID_AA64ISAR2_RPRES_MASK | R_ID_AA64ISAR2_GPA3_MASK | - R_ID_AA64ISAR2_APA3_MASK }, + R_ID_AA64ISAR2_APA3_MASK | + R_ID_AA64ISAR2_MOPS_MASK | + R_ID_AA64ISAR2_BC_MASK | + R_ID_AA64ISAR2_RPRFM_MASK | + R_ID_AA64ISAR2_CSSC_MASK }, { .name = "ID_AA64ISAR*_EL1_RESERVED", .is_glob = true }, }; diff --git a/target/arm/internals.h b/target/arm/internals.h index 5f5393b25c..1dd9182a54 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -1272,6 +1272,61 @@ FIELD(MTEDESC, SIZEM1, 12, SIMD_DATA_BITS - 12) /* size - 1 */ bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr); uint64_t mte_check(CPUARMState *env, uint32_t desc, uint64_t ptr, uintptr_t ra); +/** + * mte_mops_probe: Check where the next MTE failure is for a FEAT_MOPS operation + * @env: CPU env + * @ptr: start address of memory region (dirty pointer) + * @size: length of region (guaranteed not to cross a page boundary) + * @desc: MTEDESC descriptor word (0 means no MTE checks) + * Returns: the size of the region that can be copied without hitting + * an MTE tag failure + * + * Note that we assume that the caller has already checked the TBI + * and TCMA bits with mte_checks_needed() and an MTE check is definitely + * required. + */ +uint64_t mte_mops_probe(CPUARMState *env, uint64_t ptr, uint64_t size, + uint32_t desc); + +/** + * mte_mops_probe_rev: Check where the next MTE failure is for a FEAT_MOPS + * operation going in the reverse direction + * @env: CPU env + * @ptr: *end* address of memory region (dirty pointer) + * @size: length of region (guaranteed not to cross a page boundary) + * @desc: MTEDESC descriptor word (0 means no MTE checks) + * Returns: the size of the region that can be copied without hitting + * an MTE tag failure + * + * Note that we assume that the caller has already checked the TBI + * and TCMA bits with mte_checks_needed() and an MTE check is definitely + * required. + */ +uint64_t mte_mops_probe_rev(CPUARMState *env, uint64_t ptr, uint64_t size, + uint32_t desc); + +/** + * mte_check_fail: Record an MTE tag check failure + * @env: CPU env + * @desc: MTEDESC descriptor word + * @dirty_ptr: Failing dirty address + * @ra: TCG retaddr + * + * This may never return (if the MTE tag checks are configured to fault). + */ +void mte_check_fail(CPUARMState *env, uint32_t desc, + uint64_t dirty_ptr, uintptr_t ra); + +/** + * mte_mops_set_tags: Set MTE tags for a portion of a FEAT_MOPS operation + * @env: CPU env + * @dirty_ptr: Start address of memory region (dirty pointer) + * @size: length of region (guaranteed not to cross page boundary) + * @desc: MTEDESC descriptor word + */ +void mte_mops_set_tags(CPUARMState *env, uint64_t dirty_ptr, uint64_t size, + uint32_t desc); + static inline int allocation_tag_from_addr(uint64_t ptr) { return extract64(ptr, 56, 4); diff --git a/target/arm/syndrome.h b/target/arm/syndrome.h index 8a6b8f8162..5d34755508 100644 --- a/target/arm/syndrome.h +++ b/target/arm/syndrome.h @@ -58,6 +58,7 @@ enum arm_exception_class { EC_DATAABORT = 0x24, EC_DATAABORT_SAME_EL = 0x25, EC_SPALIGNMENT = 0x26, + EC_MOP = 0x27, EC_AA32_FPTRAP = 0x28, EC_AA64_FPTRAP = 0x2c, EC_SERROR = 0x2f, @@ -334,4 +335,15 @@ static inline uint32_t syn_serror(uint32_t extra) return (EC_SERROR << ARM_EL_EC_SHIFT) | ARM_EL_IL | extra; } +static inline uint32_t syn_mop(bool is_set, bool is_setg, int options, + bool epilogue, bool wrong_option, bool option_a, + int destreg, int srcreg, int sizereg) +{ + return (EC_MOP << ARM_EL_EC_SHIFT) | ARM_EL_IL | + (is_set << 24) | (is_setg << 23) | (options << 19) | + (epilogue << 18) | (wrong_option << 17) | (option_a << 16) | + (destreg << 10) | (srcreg << 5) | sizereg; +} + + #endif /* TARGET_ARM_SYNDROME_H */ diff --git a/target/arm/tcg/a64.decode b/target/arm/tcg/a64.decode index ef64a3f9cb..0cf1147074 100644 --- a/target/arm/tcg/a64.decode +++ b/target/arm/tcg/a64.decode @@ -126,7 +126,8 @@ CBZ sf:1 011010 nz:1 ................... rt:5 &cbz imm=%imm19 TBZ . 011011 nz:1 ..... .............. rt:5 &tbz imm=%imm14 bitpos=%imm31_19 -B_cond 0101010 0 ................... 0 cond:4 imm=%imm19 +# B.cond and BC.cond +B_cond 0101010 0 ................... c:1 cond:4 imm=%imm19 BR 1101011 0000 11111 000000 rn:5 00000 &r BLR 1101011 0001 11111 000000 rn:5 00000 &r @@ -553,3 +554,38 @@ LDGM 11011001 11 1 ......... 00 ..... ..... @ldst_tag_mult p=0 w=0 STZ2G 11011001 11 1 ......... 01 ..... ..... @ldst_tag p=1 w=1 STZ2G 11011001 11 1 ......... 10 ..... ..... @ldst_tag p=0 w=0 STZ2G 11011001 11 1 ......... 11 ..... ..... @ldst_tag p=0 w=1 + +# Memory operations (memset, memcpy, memmove) +# Each of these comes in a set of three, eg SETP (prologue), SETM (main), +# SETE (epilogue), and each of those has different flavours to +# indicate whether memory accesses should be unpriv or non-temporal. +# We don't distinguish temporal and non-temporal accesses, but we +# do need to report it in syndrome register values. + +# Memset +&set rs rn rd unpriv nontemp +# op2 bit 1 is nontemporal bit +@set .. ......... rs:5 .. nontemp:1 unpriv:1 .. rn:5 rd:5 &set + +SETP 00 011001110 ..... 00 . . 01 ..... ..... @set +SETM 00 011001110 ..... 01 . . 01 ..... ..... @set +SETE 00 011001110 ..... 10 . . 01 ..... ..... @set + +# Like SET, but also setting MTE tags +SETGP 00 011101110 ..... 00 . . 01 ..... ..... @set +SETGM 00 011101110 ..... 01 . . 01 ..... ..... @set +SETGE 00 011101110 ..... 10 . . 01 ..... ..... @set + +# Memmove/Memcopy: the CPY insns allow overlapping src/dest and +# copy in the correct direction; the CPYF insns always copy forwards. +# +# options has the nontemporal and unpriv bits for src and dest +&cpy rs rn rd options +@cpy .. ... . ..... rs:5 options:4 .. rn:5 rd:5 &cpy + +CPYFP 00 011 0 01000 ..... .... 01 ..... ..... @cpy +CPYFM 00 011 0 01010 ..... .... 01 ..... ..... @cpy +CPYFE 00 011 0 01100 ..... .... 01 ..... ..... @cpy +CPYP 00 011 1 01000 ..... .... 01 ..... ..... @cpy +CPYM 00 011 1 01010 ..... .... 01 ..... ..... @cpy +CPYE 00 011 1 01100 ..... .... 01 ..... ..... @cpy diff --git a/target/arm/tcg/cpu64.c b/target/arm/tcg/cpu64.c index 7264ab5ead..68928e5127 100644 --- a/target/arm/tcg/cpu64.c +++ b/target/arm/tcg/cpu64.c @@ -1027,6 +1027,11 @@ void aarch64_max_tcg_initfn(Object *obj) t = FIELD_DP64(t, ID_AA64ISAR1, I8MM, 1); /* FEAT_I8MM */ cpu->isar.id_aa64isar1 = t; + t = cpu->isar.id_aa64isar2; + t = FIELD_DP64(t, ID_AA64ISAR2, MOPS, 1); /* FEAT_MOPS */ + t = FIELD_DP64(t, ID_AA64ISAR2, BC, 1); /* FEAT_HBC */ + cpu->isar.id_aa64isar2 = t; + t = cpu->isar.id_aa64pfr0; t = FIELD_DP64(t, ID_AA64PFR0, FP, 1); /* FEAT_FP16 */ t = FIELD_DP64(t, ID_AA64PFR0, ADVSIMD, 1); /* FEAT_FP16 */ diff --git a/target/arm/tcg/helper-a64.c b/target/arm/tcg/helper-a64.c index 0cf56f6dc4..84f54750fc 100644 --- a/target/arm/tcg/helper-a64.c +++ b/target/arm/tcg/helper-a64.c @@ -968,3 +968,881 @@ void HELPER(unaligned_access)(CPUARMState *env, uint64_t addr, arm_cpu_do_unaligned_access(env_cpu(env), addr, access_type, mmu_idx, GETPC()); } + +/* Memory operations (memset, memmove, memcpy) */ + +/* + * Return true if the CPY* and SET* insns can execute; compare + * pseudocode CheckMOPSEnabled(), though we refactor it a little. + */ +static bool mops_enabled(CPUARMState *env) +{ + int el = arm_current_el(env); + + if (el < 2 && + (arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE) && + !(arm_hcrx_el2_eff(env) & HCRX_MSCEN)) { + return false; + } + + if (el == 0) { + if (!el_is_in_host(env, 0)) { + return env->cp15.sctlr_el[1] & SCTLR_MSCEN; + } else { + return env->cp15.sctlr_el[2] & SCTLR_MSCEN; + } + } + return true; +} + +static void check_mops_enabled(CPUARMState *env, uintptr_t ra) +{ + if (!mops_enabled(env)) { + raise_exception_ra(env, EXCP_UDEF, syn_uncategorized(), + exception_target_el(env), ra); + } +} + +/* + * Return the target exception level for an exception due + * to mismatched arguments in a FEAT_MOPS copy or set. + * Compare pseudocode MismatchedCpySetTargetEL() + */ +static int mops_mismatch_exception_target_el(CPUARMState *env) +{ + int el = arm_current_el(env); + + if (el > 1) { + return el; + } + if (el == 0 && (arm_hcr_el2_eff(env) & HCR_TGE)) { + return 2; + } + if (el == 1 && (arm_hcrx_el2_eff(env) & HCRX_MCE2)) { + return 2; + } + return 1; +} + +/* + * Check whether an M or E instruction was executed with a CF value + * indicating the wrong option for this implementation. + * Assumes we are always Option A. + */ +static void check_mops_wrong_option(CPUARMState *env, uint32_t syndrome, + uintptr_t ra) +{ + if (env->CF != 0) { + syndrome |= 1 << 17; /* Set the wrong-option bit */ + raise_exception_ra(env, EXCP_UDEF, syndrome, + mops_mismatch_exception_target_el(env), ra); + } +} + +/* + * Return the maximum number of bytes we can transfer starting at addr + * without crossing a page boundary. + */ +static uint64_t page_limit(uint64_t addr) +{ + return TARGET_PAGE_ALIGN(addr + 1) - addr; +} + +/* + * Return the number of bytes we can copy starting from addr and working + * backwards without crossing a page boundary. + */ +static uint64_t page_limit_rev(uint64_t addr) +{ + return (addr & ~TARGET_PAGE_MASK) + 1; +} + +/* + * Perform part of a memory set on an area of guest memory starting at + * toaddr (a dirty address) and extending for setsize bytes. + * + * Returns the number of bytes actually set, which might be less than + * setsize; the caller should loop until the whole set has been done. + * The caller should ensure that the guest registers are correct + * for the possibility that the first byte of the set encounters + * an exception or watchpoint. We guarantee not to take any faults + * for bytes other than the first. + */ +static uint64_t set_step(CPUARMState *env, uint64_t toaddr, + uint64_t setsize, uint32_t data, int memidx, + uint32_t *mtedesc, uintptr_t ra) +{ + void *mem; + + setsize = MIN(setsize, page_limit(toaddr)); + if (*mtedesc) { + uint64_t mtesize = mte_mops_probe(env, toaddr, setsize, *mtedesc); + if (mtesize == 0) { + /* Trap, or not. All CPU state is up to date */ + mte_check_fail(env, *mtedesc, toaddr, ra); + /* Continue, with no further MTE checks required */ + *mtedesc = 0; + } else { + /* Advance to the end, or to the tag mismatch */ + setsize = MIN(setsize, mtesize); + } + } + + toaddr = useronly_clean_ptr(toaddr); + /* + * Trapless lookup: returns NULL for invalid page, I/O, + * watchpoints, clean pages, etc. + */ + mem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, memidx); + +#ifndef CONFIG_USER_ONLY + if (unlikely(!mem)) { + /* + * Slow-path: just do one byte write. This will handle the + * watchpoint, invalid page, etc handling correctly. + * For clean code pages, the next iteration will see + * the page dirty and will use the fast path. + */ + cpu_stb_mmuidx_ra(env, toaddr, data, memidx, ra); + return 1; + } +#endif + /* Easy case: just memset the host memory */ + memset(mem, data, setsize); + return setsize; +} + +/* + * Similar, but setting tags. The architecture requires us to do this + * in 16-byte chunks. SETP accesses are not tag checked; they set + * the tags. + */ +static uint64_t set_step_tags(CPUARMState *env, uint64_t toaddr, + uint64_t setsize, uint32_t data, int memidx, + uint32_t *mtedesc, uintptr_t ra) +{ + void *mem; + uint64_t cleanaddr; + + setsize = MIN(setsize, page_limit(toaddr)); + + cleanaddr = useronly_clean_ptr(toaddr); + /* + * Trapless lookup: returns NULL for invalid page, I/O, + * watchpoints, clean pages, etc. + */ + mem = tlb_vaddr_to_host(env, cleanaddr, MMU_DATA_STORE, memidx); + +#ifndef CONFIG_USER_ONLY + if (unlikely(!mem)) { + /* + * Slow-path: just do one write. This will handle the + * watchpoint, invalid page, etc handling correctly. + * The architecture requires that we do 16 bytes at a time, + * and we know both ptr and size are 16 byte aligned. + * For clean code pages, the next iteration will see + * the page dirty and will use the fast path. + */ + uint64_t repldata = data * 0x0101010101010101ULL; + MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, memidx); + cpu_st16_mmu(env, toaddr, int128_make128(repldata, repldata), oi16, ra); + mte_mops_set_tags(env, toaddr, 16, *mtedesc); + return 16; + } +#endif + /* Easy case: just memset the host memory */ + memset(mem, data, setsize); + mte_mops_set_tags(env, toaddr, setsize, *mtedesc); + return setsize; +} + +typedef uint64_t StepFn(CPUARMState *env, uint64_t toaddr, + uint64_t setsize, uint32_t data, + int memidx, uint32_t *mtedesc, uintptr_t ra); + +/* Extract register numbers from a MOPS exception syndrome value */ +static int mops_destreg(uint32_t syndrome) +{ + return extract32(syndrome, 10, 5); +} + +static int mops_srcreg(uint32_t syndrome) +{ + return extract32(syndrome, 5, 5); +} + +static int mops_sizereg(uint32_t syndrome) +{ + return extract32(syndrome, 0, 5); +} + +/* + * Return true if TCMA and TBI bits mean we need to do MTE checks. + * We only need to do this once per MOPS insn, not for every page. + */ +static bool mte_checks_needed(uint64_t ptr, uint32_t desc) +{ + int bit55 = extract64(ptr, 55, 1); + + /* + * Note that tbi_check() returns true for "access checked" but + * tcma_check() returns true for "access unchecked". + */ + if (!tbi_check(desc, bit55)) { + return false; + } + return !tcma_check(desc, bit55, allocation_tag_from_addr(ptr)); +} + +/* Take an exception if the SETG addr/size are not granule aligned */ +static void check_setg_alignment(CPUARMState *env, uint64_t ptr, uint64_t size, + uint32_t memidx, uintptr_t ra) +{ + if ((size != 0 && !QEMU_IS_ALIGNED(ptr, TAG_GRANULE)) || + !QEMU_IS_ALIGNED(size, TAG_GRANULE)) { + arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE, + memidx, ra); + + } +} + +/* + * For the Memory Set operation, our implementation chooses + * always to use "option A", where we update Xd to the final + * address in the SETP insn, and set Xn to be -(bytes remaining). + * On SETM and SETE insns we only need update Xn. + * + * @env: CPU + * @syndrome: syndrome value for mismatch exceptions + * (also contains the register numbers we need to use) + * @mtedesc: MTE descriptor word + * @stepfn: function which does a single part of the set operation + * @is_setg: true if this is the tag-setting SETG variant + */ +static void do_setp(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, + StepFn *stepfn, bool is_setg, uintptr_t ra) +{ + /* Prologue: we choose to do up to the next page boundary */ + int rd = mops_destreg(syndrome); + int rs = mops_srcreg(syndrome); + int rn = mops_sizereg(syndrome); + uint8_t data = env->xregs[rs]; + uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); + uint64_t toaddr = env->xregs[rd]; + uint64_t setsize = env->xregs[rn]; + uint64_t stagesetsize, step; + + check_mops_enabled(env, ra); + + if (setsize > INT64_MAX) { + setsize = INT64_MAX; + if (is_setg) { + setsize &= ~0xf; + } + } + + if (unlikely(is_setg)) { + check_setg_alignment(env, toaddr, setsize, memidx, ra); + } else if (!mte_checks_needed(toaddr, mtedesc)) { + mtedesc = 0; + } + + stagesetsize = MIN(setsize, page_limit(toaddr)); + while (stagesetsize) { + env->xregs[rd] = toaddr; + env->xregs[rn] = setsize; + step = stepfn(env, toaddr, stagesetsize, data, memidx, &mtedesc, ra); + toaddr += step; + setsize -= step; + stagesetsize -= step; + } + /* Insn completed, so update registers to the Option A format */ + env->xregs[rd] = toaddr + setsize; + env->xregs[rn] = -setsize; + + /* Set NZCV = 0000 to indicate we are an Option A implementation */ + env->NF = 0; + env->ZF = 1; /* our env->ZF encoding is inverted */ + env->CF = 0; + env->VF = 0; + return; +} + +void HELPER(setp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) +{ + do_setp(env, syndrome, mtedesc, set_step, false, GETPC()); +} + +void HELPER(setgp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) +{ + do_setp(env, syndrome, mtedesc, set_step_tags, true, GETPC()); +} + +static void do_setm(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, + StepFn *stepfn, bool is_setg, uintptr_t ra) +{ + /* Main: we choose to do all the full-page chunks */ + CPUState *cs = env_cpu(env); + int rd = mops_destreg(syndrome); + int rs = mops_srcreg(syndrome); + int rn = mops_sizereg(syndrome); + uint8_t data = env->xregs[rs]; + uint64_t toaddr = env->xregs[rd] + env->xregs[rn]; + uint64_t setsize = -env->xregs[rn]; + uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); + uint64_t step, stagesetsize; + + check_mops_enabled(env, ra); + + /* + * We're allowed to NOP out "no data to copy" before the consistency + * checks; we choose to do so. + */ + if (env->xregs[rn] == 0) { + return; + } + + check_mops_wrong_option(env, syndrome, ra); + + /* + * Our implementation will work fine even if we have an unaligned + * destination address, and because we update Xn every time around + * the loop below and the return value from stepfn() may be less + * than requested, we might find toaddr is unaligned. So we don't + * have an IMPDEF check for alignment here. + */ + + if (unlikely(is_setg)) { + check_setg_alignment(env, toaddr, setsize, memidx, ra); + } else if (!mte_checks_needed(toaddr, mtedesc)) { + mtedesc = 0; + } + + /* Do the actual memset: we leave the last partial page to SETE */ + stagesetsize = setsize & TARGET_PAGE_MASK; + while (stagesetsize > 0) { + step = stepfn(env, toaddr, setsize, data, memidx, &mtedesc, ra); + toaddr += step; + setsize -= step; + stagesetsize -= step; + env->xregs[rn] = -setsize; + if (stagesetsize > 0 && unlikely(cpu_loop_exit_requested(cs))) { + cpu_loop_exit_restore(cs, ra); + } + } +} + +void HELPER(setm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) +{ + do_setm(env, syndrome, mtedesc, set_step, false, GETPC()); +} + +void HELPER(setgm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) +{ + do_setm(env, syndrome, mtedesc, set_step_tags, true, GETPC()); +} + +static void do_sete(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, + StepFn *stepfn, bool is_setg, uintptr_t ra) +{ + /* Epilogue: do the last partial page */ + int rd = mops_destreg(syndrome); + int rs = mops_srcreg(syndrome); + int rn = mops_sizereg(syndrome); + uint8_t data = env->xregs[rs]; + uint64_t toaddr = env->xregs[rd] + env->xregs[rn]; + uint64_t setsize = -env->xregs[rn]; + uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); + uint64_t step; + + check_mops_enabled(env, ra); + + /* + * We're allowed to NOP out "no data to copy" before the consistency + * checks; we choose to do so. + */ + if (setsize == 0) { + return; + } + + check_mops_wrong_option(env, syndrome, ra); + + /* + * Our implementation has no address alignment requirements, but + * we do want to enforce the "less than a page" size requirement, + * so we don't need to have the "check for interrupts" here. + */ + if (setsize >= TARGET_PAGE_SIZE) { + raise_exception_ra(env, EXCP_UDEF, syndrome, + mops_mismatch_exception_target_el(env), ra); + } + + if (unlikely(is_setg)) { + check_setg_alignment(env, toaddr, setsize, memidx, ra); + } else if (!mte_checks_needed(toaddr, mtedesc)) { + mtedesc = 0; + } + + /* Do the actual memset */ + while (setsize > 0) { + step = stepfn(env, toaddr, setsize, data, memidx, &mtedesc, ra); + toaddr += step; + setsize -= step; + env->xregs[rn] = -setsize; + } +} + +void HELPER(sete)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) +{ + do_sete(env, syndrome, mtedesc, set_step, false, GETPC()); +} + +void HELPER(setge)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) +{ + do_sete(env, syndrome, mtedesc, set_step_tags, true, GETPC()); +} + +/* + * Perform part of a memory copy from the guest memory at fromaddr + * and extending for copysize bytes, to the guest memory at + * toaddr. Both addreses are dirty. + * + * Returns the number of bytes actually set, which might be less than + * copysize; the caller should loop until the whole copy has been done. + * The caller should ensure that the guest registers are correct + * for the possibility that the first byte of the copy encounters + * an exception or watchpoint. We guarantee not to take any faults + * for bytes other than the first. + */ +static uint64_t copy_step(CPUARMState *env, uint64_t toaddr, uint64_t fromaddr, + uint64_t copysize, int wmemidx, int rmemidx, + uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra) +{ + void *rmem; + void *wmem; + + /* Don't cross a page boundary on either source or destination */ + copysize = MIN(copysize, page_limit(toaddr)); + copysize = MIN(copysize, page_limit(fromaddr)); + /* + * Handle MTE tag checks: either handle the tag mismatch for byte 0, + * or else copy up to but not including the byte with the mismatch. + */ + if (*rdesc) { + uint64_t mtesize = mte_mops_probe(env, fromaddr, copysize, *rdesc); + if (mtesize == 0) { + mte_check_fail(env, *rdesc, fromaddr, ra); + *rdesc = 0; + } else { + copysize = MIN(copysize, mtesize); + } + } + if (*wdesc) { + uint64_t mtesize = mte_mops_probe(env, toaddr, copysize, *wdesc); + if (mtesize == 0) { + mte_check_fail(env, *wdesc, toaddr, ra); + *wdesc = 0; + } else { + copysize = MIN(copysize, mtesize); + } + } + + toaddr = useronly_clean_ptr(toaddr); + fromaddr = useronly_clean_ptr(fromaddr); + /* Trapless lookup of whether we can get a host memory pointer */ + wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx); + rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx); + +#ifndef CONFIG_USER_ONLY + /* + * If we don't have host memory for both source and dest then just + * do a single byte copy. This will handle watchpoints, invalid pages, + * etc correctly. For clean code pages, the next iteration will see + * the page dirty and will use the fast path. + */ + if (unlikely(!rmem || !wmem)) { + uint8_t byte; + if (rmem) { + byte = *(uint8_t *)rmem; + } else { + byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra); + } + if (wmem) { + *(uint8_t *)wmem = byte; + } else { + cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra); + } + return 1; + } +#endif + /* Easy case: just memmove the host memory */ + memmove(wmem, rmem, copysize); + return copysize; +} + +/* + * Do part of a backwards memory copy. Here toaddr and fromaddr point + * to the *last* byte to be copied. + */ +static uint64_t copy_step_rev(CPUARMState *env, uint64_t toaddr, + uint64_t fromaddr, + uint64_t copysize, int wmemidx, int rmemidx, + uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra) +{ + void *rmem; + void *wmem; + + /* Don't cross a page boundary on either source or destination */ + copysize = MIN(copysize, page_limit_rev(toaddr)); + copysize = MIN(copysize, page_limit_rev(fromaddr)); + + /* + * Handle MTE tag checks: either handle the tag mismatch for byte 0, + * or else copy up to but not including the byte with the mismatch. + */ + if (*rdesc) { + uint64_t mtesize = mte_mops_probe_rev(env, fromaddr, copysize, *rdesc); + if (mtesize == 0) { + mte_check_fail(env, *rdesc, fromaddr, ra); + *rdesc = 0; + } else { + copysize = MIN(copysize, mtesize); + } + } + if (*wdesc) { + uint64_t mtesize = mte_mops_probe_rev(env, toaddr, copysize, *wdesc); + if (mtesize == 0) { + mte_check_fail(env, *wdesc, toaddr, ra); + *wdesc = 0; + } else { + copysize = MIN(copysize, mtesize); + } + } + + toaddr = useronly_clean_ptr(toaddr); + fromaddr = useronly_clean_ptr(fromaddr); + /* Trapless lookup of whether we can get a host memory pointer */ + wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx); + rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx); + +#ifndef CONFIG_USER_ONLY + /* + * If we don't have host memory for both source and dest then just + * do a single byte copy. This will handle watchpoints, invalid pages, + * etc correctly. For clean code pages, the next iteration will see + * the page dirty and will use the fast path. + */ + if (unlikely(!rmem || !wmem)) { + uint8_t byte; + if (rmem) { + byte = *(uint8_t *)rmem; + } else { + byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra); + } + if (wmem) { + *(uint8_t *)wmem = byte; + } else { + cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra); + } + return 1; + } +#endif + /* + * Easy case: just memmove the host memory. Note that wmem and + * rmem here point to the *last* byte to copy. + */ + memmove(wmem - (copysize - 1), rmem - (copysize - 1), copysize); + return copysize; +} + +/* + * for the Memory Copy operation, our implementation chooses always + * to use "option A", where we update Xd and Xs to the final addresses + * in the CPYP insn, and then in CPYM and CPYE only need to update Xn. + * + * @env: CPU + * @syndrome: syndrome value for mismatch exceptions + * (also contains the register numbers we need to use) + * @wdesc: MTE descriptor for the writes (destination) + * @rdesc: MTE descriptor for the reads (source) + * @move: true if this is CPY (memmove), false for CPYF (memcpy forwards) + */ +static void do_cpyp(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, + uint32_t rdesc, uint32_t move, uintptr_t ra) +{ + int rd = mops_destreg(syndrome); + int rs = mops_srcreg(syndrome); + int rn = mops_sizereg(syndrome); + uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); + uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); + bool forwards = true; + uint64_t toaddr = env->xregs[rd]; + uint64_t fromaddr = env->xregs[rs]; + uint64_t copysize = env->xregs[rn]; + uint64_t stagecopysize, step; + + check_mops_enabled(env, ra); + + + if (move) { + /* + * Copy backwards if necessary. The direction for a non-overlapping + * copy is IMPDEF; we choose forwards. + */ + if (copysize > 0x007FFFFFFFFFFFFFULL) { + copysize = 0x007FFFFFFFFFFFFFULL; + } + uint64_t fs = extract64(fromaddr, 0, 56); + uint64_t ts = extract64(toaddr, 0, 56); + uint64_t fe = extract64(fromaddr + copysize, 0, 56); + + if (fs < ts && fe > ts) { + forwards = false; + } + } else { + if (copysize > INT64_MAX) { + copysize = INT64_MAX; + } + } + + if (!mte_checks_needed(fromaddr, rdesc)) { + rdesc = 0; + } + if (!mte_checks_needed(toaddr, wdesc)) { + wdesc = 0; + } + + if (forwards) { + stagecopysize = MIN(copysize, page_limit(toaddr)); + stagecopysize = MIN(stagecopysize, page_limit(fromaddr)); + while (stagecopysize) { + env->xregs[rd] = toaddr; + env->xregs[rs] = fromaddr; + env->xregs[rn] = copysize; + step = copy_step(env, toaddr, fromaddr, stagecopysize, + wmemidx, rmemidx, &wdesc, &rdesc, ra); + toaddr += step; + fromaddr += step; + copysize -= step; + stagecopysize -= step; + } + /* Insn completed, so update registers to the Option A format */ + env->xregs[rd] = toaddr + copysize; + env->xregs[rs] = fromaddr + copysize; + env->xregs[rn] = -copysize; + } else { + /* + * In a reverse copy the to and from addrs in Xs and Xd are the start + * of the range, but it's more convenient for us to work with pointers + * to the last byte being copied. + */ + toaddr += copysize - 1; + fromaddr += copysize - 1; + stagecopysize = MIN(copysize, page_limit_rev(toaddr)); + stagecopysize = MIN(stagecopysize, page_limit_rev(fromaddr)); + while (stagecopysize) { + env->xregs[rn] = copysize; + step = copy_step_rev(env, toaddr, fromaddr, stagecopysize, + wmemidx, rmemidx, &wdesc, &rdesc, ra); + copysize -= step; + stagecopysize -= step; + toaddr -= step; + fromaddr -= step; + } + /* + * Insn completed, so update registers to the Option A format. + * For a reverse copy this is no different to the CPYP input format. + */ + env->xregs[rn] = copysize; + } + + /* Set NZCV = 0000 to indicate we are an Option A implementation */ + env->NF = 0; + env->ZF = 1; /* our env->ZF encoding is inverted */ + env->CF = 0; + env->VF = 0; + return; +} + +void HELPER(cpyp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, + uint32_t rdesc) +{ + do_cpyp(env, syndrome, wdesc, rdesc, true, GETPC()); +} + +void HELPER(cpyfp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, + uint32_t rdesc) +{ + do_cpyp(env, syndrome, wdesc, rdesc, false, GETPC()); +} + +static void do_cpym(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, + uint32_t rdesc, uint32_t move, uintptr_t ra) +{ + /* Main: we choose to copy until less than a page remaining */ + CPUState *cs = env_cpu(env); + int rd = mops_destreg(syndrome); + int rs = mops_srcreg(syndrome); + int rn = mops_sizereg(syndrome); + uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); + uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); + bool forwards = true; + uint64_t toaddr, fromaddr, copysize, step; + + check_mops_enabled(env, ra); + + /* We choose to NOP out "no data to copy" before consistency checks */ + if (env->xregs[rn] == 0) { + return; + } + + check_mops_wrong_option(env, syndrome, ra); + + if (move) { + forwards = (int64_t)env->xregs[rn] < 0; + } + + if (forwards) { + toaddr = env->xregs[rd] + env->xregs[rn]; + fromaddr = env->xregs[rs] + env->xregs[rn]; + copysize = -env->xregs[rn]; + } else { + copysize = env->xregs[rn]; + /* This toaddr and fromaddr point to the *last* byte to copy */ + toaddr = env->xregs[rd] + copysize - 1; + fromaddr = env->xregs[rs] + copysize - 1; + } + + if (!mte_checks_needed(fromaddr, rdesc)) { + rdesc = 0; + } + if (!mte_checks_needed(toaddr, wdesc)) { + wdesc = 0; + } + + /* Our implementation has no particular parameter requirements for CPYM */ + + /* Do the actual memmove */ + if (forwards) { + while (copysize >= TARGET_PAGE_SIZE) { + step = copy_step(env, toaddr, fromaddr, copysize, + wmemidx, rmemidx, &wdesc, &rdesc, ra); + toaddr += step; + fromaddr += step; + copysize -= step; + env->xregs[rn] = -copysize; + if (copysize >= TARGET_PAGE_SIZE && + unlikely(cpu_loop_exit_requested(cs))) { + cpu_loop_exit_restore(cs, ra); + } + } + } else { + while (copysize >= TARGET_PAGE_SIZE) { + step = copy_step_rev(env, toaddr, fromaddr, copysize, + wmemidx, rmemidx, &wdesc, &rdesc, ra); + toaddr -= step; + fromaddr -= step; + copysize -= step; + env->xregs[rn] = copysize; + if (copysize >= TARGET_PAGE_SIZE && + unlikely(cpu_loop_exit_requested(cs))) { + cpu_loop_exit_restore(cs, ra); + } + } + } +} + +void HELPER(cpym)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, + uint32_t rdesc) +{ + do_cpym(env, syndrome, wdesc, rdesc, true, GETPC()); +} + +void HELPER(cpyfm)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, + uint32_t rdesc) +{ + do_cpym(env, syndrome, wdesc, rdesc, false, GETPC()); +} + +static void do_cpye(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, + uint32_t rdesc, uint32_t move, uintptr_t ra) +{ + /* Epilogue: do the last partial page */ + int rd = mops_destreg(syndrome); + int rs = mops_srcreg(syndrome); + int rn = mops_sizereg(syndrome); + uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); + uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); + bool forwards = true; + uint64_t toaddr, fromaddr, copysize, step; + + check_mops_enabled(env, ra); + + /* We choose to NOP out "no data to copy" before consistency checks */ + if (env->xregs[rn] == 0) { + return; + } + + check_mops_wrong_option(env, syndrome, ra); + + if (move) { + forwards = (int64_t)env->xregs[rn] < 0; + } + + if (forwards) { + toaddr = env->xregs[rd] + env->xregs[rn]; + fromaddr = env->xregs[rs] + env->xregs[rn]; + copysize = -env->xregs[rn]; + } else { + copysize = env->xregs[rn]; + /* This toaddr and fromaddr point to the *last* byte to copy */ + toaddr = env->xregs[rd] + copysize - 1; + fromaddr = env->xregs[rs] + copysize - 1; + } + + if (!mte_checks_needed(fromaddr, rdesc)) { + rdesc = 0; + } + if (!mte_checks_needed(toaddr, wdesc)) { + wdesc = 0; + } + + /* Check the size; we don't want to have do a check-for-interrupts */ + if (copysize >= TARGET_PAGE_SIZE) { + raise_exception_ra(env, EXCP_UDEF, syndrome, + mops_mismatch_exception_target_el(env), ra); + } + + /* Do the actual memmove */ + if (forwards) { + while (copysize > 0) { + step = copy_step(env, toaddr, fromaddr, copysize, + wmemidx, rmemidx, &wdesc, &rdesc, ra); + toaddr += step; + fromaddr += step; + copysize -= step; + env->xregs[rn] = -copysize; + } + } else { + while (copysize > 0) { + step = copy_step_rev(env, toaddr, fromaddr, copysize, + wmemidx, rmemidx, &wdesc, &rdesc, ra); + toaddr -= step; + fromaddr -= step; + copysize -= step; + env->xregs[rn] = copysize; + } + } +} + +void HELPER(cpye)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, + uint32_t rdesc) +{ + do_cpye(env, syndrome, wdesc, rdesc, true, GETPC()); +} + +void HELPER(cpyfe)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, + uint32_t rdesc) +{ + do_cpye(env, syndrome, wdesc, rdesc, false, GETPC()); +} diff --git a/target/arm/tcg/helper-a64.h b/target/arm/tcg/helper-a64.h index 57cfd68569..575a5dab7d 100644 --- a/target/arm/tcg/helper-a64.h +++ b/target/arm/tcg/helper-a64.h @@ -117,3 +117,17 @@ DEF_HELPER_FLAGS_3(stzgm_tags, TCG_CALL_NO_WG, void, env, i64, i64) DEF_HELPER_FLAGS_4(unaligned_access, TCG_CALL_NO_WG, noreturn, env, i64, i32, i32) + +DEF_HELPER_3(setp, void, env, i32, i32) +DEF_HELPER_3(setm, void, env, i32, i32) +DEF_HELPER_3(sete, void, env, i32, i32) +DEF_HELPER_3(setgp, void, env, i32, i32) +DEF_HELPER_3(setgm, void, env, i32, i32) +DEF_HELPER_3(setge, void, env, i32, i32) + +DEF_HELPER_4(cpyp, void, env, i32, i32, i32) +DEF_HELPER_4(cpym, void, env, i32, i32, i32) +DEF_HELPER_4(cpye, void, env, i32, i32, i32) +DEF_HELPER_4(cpyfp, void, env, i32, i32, i32) +DEF_HELPER_4(cpyfm, void, env, i32, i32, i32) +DEF_HELPER_4(cpyfe, void, env, i32, i32, i32) diff --git a/target/arm/tcg/hflags.c b/target/arm/tcg/hflags.c index 616c5fa723..cea1adb7b6 100644 --- a/target/arm/tcg/hflags.c +++ b/target/arm/tcg/hflags.c @@ -306,6 +306,15 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, && !(env->pstate & PSTATE_TCO) && (sctlr & (el == 0 ? SCTLR_TCF0 : SCTLR_TCF))) { DP_TBFLAG_A64(flags, MTE_ACTIVE, 1); + if (!EX_TBFLAG_A64(flags, UNPRIV)) { + /* + * In non-unpriv contexts (eg EL0), unpriv load/stores + * act like normal ones; duplicate the MTE info to + * avoid translate-a64.c having to check UNPRIV to see + * whether it is OK to index into MTE_ACTIVE[]. + */ + DP_TBFLAG_A64(flags, MTE0_ACTIVE, 1); + } } } /* And again for unprivileged accesses, if required. */ @@ -316,6 +325,18 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, && allocation_tag_access_enabled(env, 0, sctlr)) { DP_TBFLAG_A64(flags, MTE0_ACTIVE, 1); } + /* + * For unpriv tag-setting accesses we alse need ATA0. Again, in + * contexts where unpriv and normal insns are the same we + * duplicate the ATA bit to save effort for translate-a64.c. + */ + if (EX_TBFLAG_A64(flags, UNPRIV)) { + if (allocation_tag_access_enabled(env, 0, sctlr)) { + DP_TBFLAG_A64(flags, ATA0, 1); + } + } else { + DP_TBFLAG_A64(flags, ATA0, EX_TBFLAG_A64(flags, ATA)); + } /* Cache TCMA as well as TBI. */ DP_TBFLAG_A64(flags, TCMA, aa64_va_parameter_tcma(tcr, mmu_idx)); } diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c index b23d11563a..2dd7eb3edb 100644 --- a/target/arm/tcg/mte_helper.c +++ b/target/arm/tcg/mte_helper.c @@ -50,14 +50,14 @@ static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude) } /** - * allocation_tag_mem: + * allocation_tag_mem_probe: * @env: the cpu environment * @ptr_mmu_idx: the addressing regime to use for the virtual address * @ptr: the virtual address for which to look up tag memory * @ptr_access: the access to use for the virtual address * @ptr_size: the number of bytes in the normal memory access * @tag_access: the access to use for the tag memory - * @tag_size: the number of bytes in the tag memory access + * @probe: true to merely probe, never taking an exception * @ra: the return address for exception handling * * Our tag memory is formatted as a sequence of little-endian nibbles. @@ -66,18 +66,25 @@ static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude) * for the higher addr. * * Here, resolve the physical address from the virtual address, and return - * a pointer to the corresponding tag byte. Exit with exception if the - * virtual address is not accessible for @ptr_access. - * - * The @ptr_size and @tag_size values may not have an obvious relation - * due to the alignment of @ptr, and the number of tag checks required. + * a pointer to the corresponding tag byte. * * If there is no tag storage corresponding to @ptr, return NULL. + * + * If the page is inaccessible for @ptr_access, or has a watchpoint, there are + * three options: + * (1) probe = true, ra = 0 : pure probe -- we return NULL if the page is not + * accessible, and do not take watchpoint traps. The calling code must + * handle those cases in the right priority compared to MTE traps. + * (2) probe = false, ra = 0 : probe, no fault expected -- the caller guarantees + * that the page is going to be accessible. We will take watchpoint traps. + * (3) probe = false, ra != 0 : non-probe -- we will take both memory access + * traps and watchpoint traps. + * (probe = true, ra != 0 is invalid and will assert.) */ -static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, - uint64_t ptr, MMUAccessType ptr_access, - int ptr_size, MMUAccessType tag_access, - int tag_size, uintptr_t ra) +static uint8_t *allocation_tag_mem_probe(CPUARMState *env, int ptr_mmu_idx, + uint64_t ptr, MMUAccessType ptr_access, + int ptr_size, MMUAccessType tag_access, + bool probe, uintptr_t ra) { #ifdef CONFIG_USER_ONLY uint64_t clean_ptr = useronly_clean_ptr(ptr); @@ -85,6 +92,8 @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, uint8_t *tags; uintptr_t index; + assert(!(probe && ra)); + if (!(flags & (ptr_access == MMU_DATA_STORE ? PAGE_WRITE_ORG : PAGE_READ))) { cpu_loop_exit_sigsegv(env_cpu(env), ptr, ptr_access, !(flags & PAGE_VALID), ra); @@ -115,12 +124,16 @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, * exception for inaccessible pages, and resolves the virtual address * into the softmmu tlb. * - * When RA == 0, this is for mte_probe. The page is expected to be - * valid. Indicate to probe_access_flags no-fault, then assert that - * we received a valid page. + * When RA == 0, this is either a pure probe or a no-fault-expected probe. + * Indicate to probe_access_flags no-fault, then either return NULL + * for the pure probe, or assert that we received a valid page for the + * no-fault-expected probe. */ flags = probe_access_full(env, ptr, 0, ptr_access, ptr_mmu_idx, ra == 0, &host, &full, ra); + if (probe && (flags & TLB_INVALID_MASK)) { + return NULL; + } assert(!(flags & TLB_INVALID_MASK)); /* If the virtual page MemAttr != Tagged, access unchecked. */ @@ -161,7 +174,7 @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, } /* Any debug exception has priority over a tag check exception. */ - if (unlikely(flags & TLB_WATCHPOINT)) { + if (!probe && unlikely(flags & TLB_WATCHPOINT)) { int wp = ptr_access == MMU_DATA_LOAD ? BP_MEM_READ : BP_MEM_WRITE; assert(ra != 0); cpu_check_watchpoint(env_cpu(env), ptr, ptr_size, attrs, wp, ra); @@ -203,6 +216,15 @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, #endif } +static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, + uint64_t ptr, MMUAccessType ptr_access, + int ptr_size, MMUAccessType tag_access, + uintptr_t ra) +{ + return allocation_tag_mem_probe(env, ptr_mmu_idx, ptr, ptr_access, + ptr_size, tag_access, false, ra); +} + uint64_t HELPER(irg)(CPUARMState *env, uint64_t rn, uint64_t rm) { uint16_t exclude = extract32(rm | env->cp15.gcr_el1, 0, 16); @@ -275,7 +297,7 @@ uint64_t HELPER(ldg)(CPUARMState *env, uint64_t ptr, uint64_t xt) /* Trap if accessing an invalid page. */ mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD, 1, - MMU_DATA_LOAD, 1, GETPC()); + MMU_DATA_LOAD, GETPC()); /* Load if page supports tags. */ if (mem) { @@ -329,7 +351,7 @@ static inline void do_stg(CPUARMState *env, uint64_t ptr, uint64_t xt, /* Trap if accessing an invalid page. */ mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, TAG_GRANULE, - MMU_DATA_STORE, 1, ra); + MMU_DATA_STORE, ra); /* Store if page supports tags. */ if (mem) { @@ -372,10 +394,10 @@ static inline void do_st2g(CPUARMState *env, uint64_t ptr, uint64_t xt, if (ptr & TAG_GRANULE) { /* Two stores unaligned mod TAG_GRANULE*2 -- modify two bytes. */ mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, - TAG_GRANULE, MMU_DATA_STORE, 1, ra); + TAG_GRANULE, MMU_DATA_STORE, ra); mem2 = allocation_tag_mem(env, mmu_idx, ptr + TAG_GRANULE, MMU_DATA_STORE, TAG_GRANULE, - MMU_DATA_STORE, 1, ra); + MMU_DATA_STORE, ra); /* Store if page(s) support tags. */ if (mem1) { @@ -387,7 +409,7 @@ static inline void do_st2g(CPUARMState *env, uint64_t ptr, uint64_t xt, } else { /* Two stores aligned mod TAG_GRANULE*2 -- modify one byte. */ mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, - 2 * TAG_GRANULE, MMU_DATA_STORE, 1, ra); + 2 * TAG_GRANULE, MMU_DATA_STORE, ra); if (mem1) { tag |= tag << 4; qatomic_set(mem1, tag); @@ -435,8 +457,7 @@ uint64_t HELPER(ldgm)(CPUARMState *env, uint64_t ptr) /* Trap if accessing an invalid page. */ tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD, - gm_bs_bytes, MMU_DATA_LOAD, - gm_bs_bytes / (2 * TAG_GRANULE), ra); + gm_bs_bytes, MMU_DATA_LOAD, ra); /* The tag is squashed to zero if the page does not support tags. */ if (!tag_mem) { @@ -495,8 +516,7 @@ void HELPER(stgm)(CPUARMState *env, uint64_t ptr, uint64_t val) /* Trap if accessing an invalid page. */ tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, - gm_bs_bytes, MMU_DATA_LOAD, - gm_bs_bytes / (2 * TAG_GRANULE), ra); + gm_bs_bytes, MMU_DATA_LOAD, ra); /* * Tag store only happens if the page support tags, @@ -552,7 +572,7 @@ void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val) ptr &= -dcz_bytes; mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, dcz_bytes, - MMU_DATA_STORE, tag_bytes, ra); + MMU_DATA_STORE, ra); if (mem) { int tag_pair = (val & 0xf) * 0x11; memset(mem, tag_pair, tag_bytes); @@ -597,8 +617,8 @@ static void mte_async_check_fail(CPUARMState *env, uint64_t dirty_ptr, } /* Record a tag check failure. */ -static void mte_check_fail(CPUARMState *env, uint32_t desc, - uint64_t dirty_ptr, uintptr_t ra) +void mte_check_fail(CPUARMState *env, uint32_t desc, + uint64_t dirty_ptr, uintptr_t ra) { int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); ARMMMUIdx arm_mmu_idx = core_to_aa64_mmu_idx(mmu_idx); @@ -715,6 +735,55 @@ static int checkN(uint8_t *mem, int odd, int cmp, int count) } /** + * checkNrev: + * @tag: tag memory to test + * @odd: true to begin testing at tags at odd nibble + * @cmp: the tag to compare against + * @count: number of tags to test + * + * Return the number of successful tests. + * Thus a return value < @count indicates a failure. + * + * This is like checkN, but it runs backwards, checking the + * tags starting with @tag and then the tags preceding it. + * This is needed by the backwards-memory-copying operations. + */ +static int checkNrev(uint8_t *mem, int odd, int cmp, int count) +{ + int n = 0, diff; + + /* Replicate the test tag and compare. */ + cmp *= 0x11; + diff = *mem-- ^ cmp; + + if (!odd) { + goto start_even; + } + + while (1) { + /* Test odd tag. */ + if (unlikely((diff) & 0xf0)) { + break; + } + if (++n == count) { + break; + } + + start_even: + /* Test even tag. */ + if (unlikely((diff) & 0x0f)) { + break; + } + if (++n == count) { + break; + } + + diff = *mem-- ^ cmp; + } + return n; +} + +/** * mte_probe_int() - helper for mte_probe and mte_check * @env: CPU environment * @desc: MTEDESC descriptor @@ -732,8 +801,7 @@ static int mte_probe_int(CPUARMState *env, uint32_t desc, uint64_t ptr, int mmu_idx, ptr_tag, bit55; uint64_t ptr_last, prev_page, next_page; uint64_t tag_first, tag_last; - uint64_t tag_byte_first, tag_byte_last; - uint32_t sizem1, tag_count, tag_size, n, c; + uint32_t sizem1, tag_count, n, c; uint8_t *mem1, *mem2; MMUAccessType type; @@ -763,19 +831,14 @@ static int mte_probe_int(CPUARMState *env, uint32_t desc, uint64_t ptr, tag_last = QEMU_ALIGN_DOWN(ptr_last, TAG_GRANULE); tag_count = ((tag_last - tag_first) / TAG_GRANULE) + 1; - /* Round the bounds to twice the tag granule, and compute the bytes. */ - tag_byte_first = QEMU_ALIGN_DOWN(ptr, 2 * TAG_GRANULE); - tag_byte_last = QEMU_ALIGN_DOWN(ptr_last, 2 * TAG_GRANULE); - /* Locate the page boundaries. */ prev_page = ptr & TARGET_PAGE_MASK; next_page = prev_page + TARGET_PAGE_SIZE; if (likely(tag_last - prev_page < TARGET_PAGE_SIZE)) { /* Memory access stays on one page. */ - tag_size = ((tag_byte_last - tag_byte_first) / (2 * TAG_GRANULE)) + 1; mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, sizem1 + 1, - MMU_DATA_LOAD, tag_size, ra); + MMU_DATA_LOAD, ra); if (!mem1) { return 1; } @@ -783,14 +846,12 @@ static int mte_probe_int(CPUARMState *env, uint32_t desc, uint64_t ptr, n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, tag_count); } else { /* Memory access crosses to next page. */ - tag_size = (next_page - tag_byte_first) / (2 * TAG_GRANULE); mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, next_page - ptr, - MMU_DATA_LOAD, tag_size, ra); + MMU_DATA_LOAD, ra); - tag_size = ((tag_byte_last - next_page) / (2 * TAG_GRANULE)) + 1; mem2 = allocation_tag_mem(env, mmu_idx, next_page, type, ptr_last - next_page + 1, - MMU_DATA_LOAD, tag_size, ra); + MMU_DATA_LOAD, ra); /* * Perform all of the comparisons. @@ -918,7 +979,7 @@ uint64_t HELPER(mte_check_zva)(CPUARMState *env, uint32_t desc, uint64_t ptr) mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); (void) probe_write(env, ptr, 1, mmu_idx, ra); mem = allocation_tag_mem(env, mmu_idx, align_ptr, MMU_DATA_STORE, - dcz_bytes, MMU_DATA_LOAD, tag_bytes, ra); + dcz_bytes, MMU_DATA_LOAD, ra); if (!mem) { goto done; } @@ -979,3 +1040,143 @@ uint64_t HELPER(mte_check_zva)(CPUARMState *env, uint32_t desc, uint64_t ptr) done: return useronly_clean_ptr(ptr); } + +uint64_t mte_mops_probe(CPUARMState *env, uint64_t ptr, uint64_t size, + uint32_t desc) +{ + int mmu_idx, tag_count; + uint64_t ptr_tag, tag_first, tag_last; + void *mem; + bool w = FIELD_EX32(desc, MTEDESC, WRITE); + uint32_t n; + + mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); + /* True probe; this will never fault */ + mem = allocation_tag_mem_probe(env, mmu_idx, ptr, + w ? MMU_DATA_STORE : MMU_DATA_LOAD, + size, MMU_DATA_LOAD, true, 0); + if (!mem) { + return size; + } + + /* + * TODO: checkN() is not designed for checks of the size we expect + * for FEAT_MOPS operations, so we should implement this differently. + * Maybe we should do something like + * if (region start and size are aligned nicely) { + * do direct loads of 64 tag bits at a time; + * } else { + * call checkN() + * } + */ + /* Round the bounds to the tag granule, and compute the number of tags. */ + ptr_tag = allocation_tag_from_addr(ptr); + tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE); + tag_last = QEMU_ALIGN_DOWN(ptr + size - 1, TAG_GRANULE); + tag_count = ((tag_last - tag_first) / TAG_GRANULE) + 1; + n = checkN(mem, ptr & TAG_GRANULE, ptr_tag, tag_count); + if (likely(n == tag_count)) { + return size; + } + + /* + * Failure; for the first granule, it's at @ptr. Otherwise + * it's at the first byte of the nth granule. Calculate how + * many bytes we can access without hitting that failure. + */ + if (n == 0) { + return 0; + } else { + return n * TAG_GRANULE - (ptr - tag_first); + } +} + +uint64_t mte_mops_probe_rev(CPUARMState *env, uint64_t ptr, uint64_t size, + uint32_t desc) +{ + int mmu_idx, tag_count; + uint64_t ptr_tag, tag_first, tag_last; + void *mem; + bool w = FIELD_EX32(desc, MTEDESC, WRITE); + uint32_t n; + + mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); + /* True probe; this will never fault */ + mem = allocation_tag_mem_probe(env, mmu_idx, ptr, + w ? MMU_DATA_STORE : MMU_DATA_LOAD, + size, MMU_DATA_LOAD, true, 0); + if (!mem) { + return size; + } + + /* + * TODO: checkNrev() is not designed for checks of the size we expect + * for FEAT_MOPS operations, so we should implement this differently. + * Maybe we should do something like + * if (region start and size are aligned nicely) { + * do direct loads of 64 tag bits at a time; + * } else { + * call checkN() + * } + */ + /* Round the bounds to the tag granule, and compute the number of tags. */ + ptr_tag = allocation_tag_from_addr(ptr); + tag_first = QEMU_ALIGN_DOWN(ptr - (size - 1), TAG_GRANULE); + tag_last = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE); + tag_count = ((tag_last - tag_first) / TAG_GRANULE) + 1; + n = checkNrev(mem, ptr & TAG_GRANULE, ptr_tag, tag_count); + if (likely(n == tag_count)) { + return size; + } + + /* + * Failure; for the first granule, it's at @ptr. Otherwise + * it's at the last byte of the nth granule. Calculate how + * many bytes we can access without hitting that failure. + */ + if (n == 0) { + return 0; + } else { + return (n - 1) * TAG_GRANULE + ((ptr + 1) - tag_last); + } +} + +void mte_mops_set_tags(CPUARMState *env, uint64_t ptr, uint64_t size, + uint32_t desc) +{ + int mmu_idx, tag_count; + uint64_t ptr_tag; + void *mem; + + if (!desc) { + /* Tags not actually enabled */ + return; + } + + mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); + /* True probe: this will never fault */ + mem = allocation_tag_mem_probe(env, mmu_idx, ptr, MMU_DATA_STORE, size, + MMU_DATA_STORE, true, 0); + if (!mem) { + return; + } + + /* + * We know that ptr and size are both TAG_GRANULE aligned; store + * the tag from the pointer value into the tag memory. + */ + ptr_tag = allocation_tag_from_addr(ptr); + tag_count = size / TAG_GRANULE; + if (ptr & TAG_GRANULE) { + /* Not 2*TAG_GRANULE-aligned: store tag to first nibble */ + store_tag1_parallel(TAG_GRANULE, mem, ptr_tag); + mem++; + tag_count--; + } + memset(mem, ptr_tag | (ptr_tag << 4), tag_count / 2); + if (tag_count & 1) { + /* Final trailing unaligned nibble */ + mem += tag_count / 2; + store_tag1_parallel(0, mem, ptr_tag); + } +} diff --git a/target/arm/tcg/mve_helper.c b/target/arm/tcg/mve_helper.c index 403b345ea3..c666a96ba1 100644 --- a/target/arm/tcg/mve_helper.c +++ b/target/arm/tcg/mve_helper.c @@ -26,6 +26,7 @@ #include "exec/exec-all.h" #include "tcg/tcg.h" #include "fpu/softfloat.h" +#include "crypto/clmul.h" static uint16_t mve_eci_mask(CPUARMState *env) { @@ -984,17 +985,10 @@ DO_2OP_L(vmulltuw, 1, 4, uint32_t, 8, uint64_t, DO_MUL) * Polynomial multiply. We can always do this generating 64 bits * of the result at a time, so we don't need to use DO_2OP_L. */ -#define VMULLPH_MASK 0x00ff00ff00ff00ffULL -#define VMULLPW_MASK 0x0000ffff0000ffffULL -#define DO_VMULLPBH(N, M) pmull_h((N) & VMULLPH_MASK, (M) & VMULLPH_MASK) -#define DO_VMULLPTH(N, M) DO_VMULLPBH((N) >> 8, (M) >> 8) -#define DO_VMULLPBW(N, M) pmull_w((N) & VMULLPW_MASK, (M) & VMULLPW_MASK) -#define DO_VMULLPTW(N, M) DO_VMULLPBW((N) >> 16, (M) >> 16) - -DO_2OP(vmullpbh, 8, uint64_t, DO_VMULLPBH) -DO_2OP(vmullpth, 8, uint64_t, DO_VMULLPTH) -DO_2OP(vmullpbw, 8, uint64_t, DO_VMULLPBW) -DO_2OP(vmullptw, 8, uint64_t, DO_VMULLPTW) +DO_2OP(vmullpbh, 8, uint64_t, clmul_8x4_even) +DO_2OP(vmullpth, 8, uint64_t, clmul_8x4_odd) +DO_2OP(vmullpbw, 8, uint64_t, clmul_16x2_even) +DO_2OP(vmullptw, 8, uint64_t, clmul_16x2_odd) /* * Because the computation type is at least twice as large as required, diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index 1b6fbb61e2..97f25b4451 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -105,9 +105,17 @@ void a64_translate_init(void) } /* - * Return the core mmu_idx to use for A64 "unprivileged load/store" insns + * Return the core mmu_idx to use for A64 load/store insns which + * have a "unprivileged load/store" variant. Those insns access + * EL0 if executed from an EL which has control over EL0 (usually + * EL1) but behave like normal loads and stores if executed from + * elsewhere (eg EL3). + * + * @unpriv : true for the unprivileged encoding; false for the + * normal encoding (in which case we will return the same + * thing as get_mem_index(). */ -static int get_a64_user_mem_index(DisasContext *s) +static int get_a64_user_mem_index(DisasContext *s, bool unpriv) { /* * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL, @@ -115,7 +123,7 @@ static int get_a64_user_mem_index(DisasContext *s) */ ARMMMUIdx useridx = s->mmu_idx; - if (s->unpriv) { + if (unpriv && s->unpriv) { /* * We have pre-computed the condition for AccType_UNPRIV. * Therefore we should never get here with a mmu_idx for @@ -1453,6 +1461,10 @@ static bool trans_TBZ(DisasContext *s, arg_tbz *a) static bool trans_B_cond(DisasContext *s, arg_B_cond *a) { + /* BC.cond is only present with FEAT_HBC */ + if (a->c && !dc_isar_feature(aa64_hbc, s)) { + return false; + } reset_btype(s); if (a->cond < 0x0e) { /* genuinely conditional branches */ @@ -2260,7 +2272,7 @@ static void handle_sys(DisasContext *s, bool isread, clean_addr = clean_data_tbi(s, tcg_rt); gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8); - if (s->ata) { + if (s->ata[0]) { /* Extract the tag from the register to match STZGM. */ tag = tcg_temp_new_i64(); tcg_gen_shri_i64(tag, tcg_rt, 56); @@ -2277,7 +2289,7 @@ static void handle_sys(DisasContext *s, bool isread, clean_addr = clean_data_tbi(s, tcg_rt); gen_helper_dc_zva(cpu_env, clean_addr); - if (s->ata) { + if (s->ata[0]) { /* Extract the tag from the register to match STZGM. */ tag = tcg_temp_new_i64(); tcg_gen_shri_i64(tag, tcg_rt, 56); @@ -3058,7 +3070,7 @@ static bool trans_STGP(DisasContext *s, arg_ldstpair *a) tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop); /* Perform the tag store, if tag access enabled. */ - if (s->ata) { + if (s->ata[0]) { if (tb_cflags(s->base.tb) & CF_PARALLEL) { gen_helper_stg_parallel(cpu_env, dirty_addr, dirty_addr); } else { @@ -3084,7 +3096,7 @@ static void op_addr_ldst_imm_pre(DisasContext *s, arg_ldst_imm *a, if (!a->p) { tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset); } - memidx = a->unpriv ? get_a64_user_mem_index(s) : get_mem_index(s); + memidx = get_a64_user_mem_index(s, a->unpriv); *clean_addr = gen_mte_check1_mmuidx(s, *dirty_addr, is_store, a->w || a->rn != 31, mop, a->unpriv, memidx); @@ -3105,7 +3117,7 @@ static bool trans_STR_i(DisasContext *s, arg_ldst_imm *a) { bool iss_sf, iss_valid = !a->w; TCGv_i64 clean_addr, dirty_addr, tcg_rt; - int memidx = a->unpriv ? get_a64_user_mem_index(s) : get_mem_index(s); + int memidx = get_a64_user_mem_index(s, a->unpriv); MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); @@ -3123,7 +3135,7 @@ static bool trans_LDR_i(DisasContext *s, arg_ldst_imm *a) { bool iss_sf, iss_valid = !a->w; TCGv_i64 clean_addr, dirty_addr, tcg_rt; - int memidx = a->unpriv ? get_a64_user_mem_index(s) : get_mem_index(s); + int memidx = get_a64_user_mem_index(s, a->unpriv); MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); @@ -3756,7 +3768,7 @@ static bool trans_STZGM(DisasContext *s, arg_ldst_tag *a) tcg_gen_addi_i64(addr, addr, a->imm); tcg_rt = cpu_reg(s, a->rt); - if (s->ata) { + if (s->ata[0]) { gen_helper_stzgm_tags(cpu_env, addr, tcg_rt); } /* @@ -3788,7 +3800,7 @@ static bool trans_STGM(DisasContext *s, arg_ldst_tag *a) tcg_gen_addi_i64(addr, addr, a->imm); tcg_rt = cpu_reg(s, a->rt); - if (s->ata) { + if (s->ata[0]) { gen_helper_stgm(cpu_env, addr, tcg_rt); } else { MMUAccessType acc = MMU_DATA_STORE; @@ -3820,7 +3832,7 @@ static bool trans_LDGM(DisasContext *s, arg_ldst_tag *a) tcg_gen_addi_i64(addr, addr, a->imm); tcg_rt = cpu_reg(s, a->rt); - if (s->ata) { + if (s->ata[0]) { gen_helper_ldgm(tcg_rt, cpu_env, addr); } else { MMUAccessType acc = MMU_DATA_LOAD; @@ -3855,7 +3867,7 @@ static bool trans_LDG(DisasContext *s, arg_ldst_tag *a) tcg_gen_andi_i64(addr, addr, -TAG_GRANULE); tcg_rt = cpu_reg(s, a->rt); - if (s->ata) { + if (s->ata[0]) { gen_helper_ldg(tcg_rt, cpu_env, addr, tcg_rt); } else { /* @@ -3892,7 +3904,7 @@ static bool do_STG(DisasContext *s, arg_ldst_tag *a, bool is_zero, bool is_pair) tcg_gen_addi_i64(addr, addr, a->imm); } tcg_rt = cpu_reg_sp(s, a->rt); - if (!s->ata) { + if (!s->ata[0]) { /* * For STG and ST2G, we need to check alignment and probe memory. * TODO: For STZG and STZ2G, we could rely on the stores below, @@ -3950,6 +3962,123 @@ TRANS_FEAT(STZG, aa64_mte_insn_reg, do_STG, a, true, false) TRANS_FEAT(ST2G, aa64_mte_insn_reg, do_STG, a, false, true) TRANS_FEAT(STZ2G, aa64_mte_insn_reg, do_STG, a, true, true) +typedef void SetFn(TCGv_env, TCGv_i32, TCGv_i32); + +static bool do_SET(DisasContext *s, arg_set *a, bool is_epilogue, + bool is_setg, SetFn fn) +{ + int memidx; + uint32_t syndrome, desc = 0; + + if (is_setg && !dc_isar_feature(aa64_mte, s)) { + return false; + } + + /* + * UNPREDICTABLE cases: we choose to UNDEF, which allows + * us to pull this check before the CheckMOPSEnabled() test + * (which we do in the helper function) + */ + if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || + a->rd == 31 || a->rn == 31) { + return false; + } + + memidx = get_a64_user_mem_index(s, a->unpriv); + + /* + * We pass option_a == true, matching our implementation; + * we pass wrong_option == false: helper function may set that bit. + */ + syndrome = syn_mop(true, is_setg, (a->nontemp << 1) | a->unpriv, + is_epilogue, false, true, a->rd, a->rs, a->rn); + + if (is_setg ? s->ata[a->unpriv] : s->mte_active[a->unpriv]) { + /* We may need to do MTE tag checking, so assemble the descriptor */ + desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); + desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); + desc = FIELD_DP32(desc, MTEDESC, WRITE, true); + /* SIZEM1 and ALIGN we leave 0 (byte write) */ + } + /* The helper function always needs the memidx even with MTE disabled */ + desc = FIELD_DP32(desc, MTEDESC, MIDX, memidx); + + /* + * The helper needs the register numbers, but since they're in + * the syndrome anyway, we let it extract them from there rather + * than passing in an extra three integer arguments. + */ + fn(cpu_env, tcg_constant_i32(syndrome), tcg_constant_i32(desc)); + return true; +} + +TRANS_FEAT(SETP, aa64_mops, do_SET, a, false, false, gen_helper_setp) +TRANS_FEAT(SETM, aa64_mops, do_SET, a, false, false, gen_helper_setm) +TRANS_FEAT(SETE, aa64_mops, do_SET, a, true, false, gen_helper_sete) +TRANS_FEAT(SETGP, aa64_mops, do_SET, a, false, true, gen_helper_setgp) +TRANS_FEAT(SETGM, aa64_mops, do_SET, a, false, true, gen_helper_setgm) +TRANS_FEAT(SETGE, aa64_mops, do_SET, a, true, true, gen_helper_setge) + +typedef void CpyFn(TCGv_env, TCGv_i32, TCGv_i32, TCGv_i32); + +static bool do_CPY(DisasContext *s, arg_cpy *a, bool is_epilogue, CpyFn fn) +{ + int rmemidx, wmemidx; + uint32_t syndrome, rdesc = 0, wdesc = 0; + bool wunpriv = extract32(a->options, 0, 1); + bool runpriv = extract32(a->options, 1, 1); + + /* + * UNPREDICTABLE cases: we choose to UNDEF, which allows + * us to pull this check before the CheckMOPSEnabled() test + * (which we do in the helper function) + */ + if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || + a->rd == 31 || a->rs == 31 || a->rn == 31) { + return false; + } + + rmemidx = get_a64_user_mem_index(s, runpriv); + wmemidx = get_a64_user_mem_index(s, wunpriv); + + /* + * We pass option_a == true, matching our implementation; + * we pass wrong_option == false: helper function may set that bit. + */ + syndrome = syn_mop(false, false, a->options, is_epilogue, + false, true, a->rd, a->rs, a->rn); + + /* If we need to do MTE tag checking, assemble the descriptors */ + if (s->mte_active[runpriv]) { + rdesc = FIELD_DP32(rdesc, MTEDESC, TBI, s->tbid); + rdesc = FIELD_DP32(rdesc, MTEDESC, TCMA, s->tcma); + } + if (s->mte_active[wunpriv]) { + wdesc = FIELD_DP32(wdesc, MTEDESC, TBI, s->tbid); + wdesc = FIELD_DP32(wdesc, MTEDESC, TCMA, s->tcma); + wdesc = FIELD_DP32(wdesc, MTEDESC, WRITE, true); + } + /* The helper function needs these parts of the descriptor regardless */ + rdesc = FIELD_DP32(rdesc, MTEDESC, MIDX, rmemidx); + wdesc = FIELD_DP32(wdesc, MTEDESC, MIDX, wmemidx); + + /* + * The helper needs the register numbers, but since they're in + * the syndrome anyway, we let it extract them from there rather + * than passing in an extra three integer arguments. + */ + fn(cpu_env, tcg_constant_i32(syndrome), tcg_constant_i32(wdesc), + tcg_constant_i32(rdesc)); + return true; +} + +TRANS_FEAT(CPYP, aa64_mops, do_CPY, a, false, gen_helper_cpyp) +TRANS_FEAT(CPYM, aa64_mops, do_CPY, a, false, gen_helper_cpym) +TRANS_FEAT(CPYE, aa64_mops, do_CPY, a, true, gen_helper_cpye) +TRANS_FEAT(CPYFP, aa64_mops, do_CPY, a, false, gen_helper_cpyfp) +TRANS_FEAT(CPYFM, aa64_mops, do_CPY, a, false, gen_helper_cpyfm) +TRANS_FEAT(CPYFE, aa64_mops, do_CPY, a, true, gen_helper_cpyfe) + typedef void ArithTwoOp(TCGv_i64, TCGv_i64, TCGv_i64); static bool gen_rri(DisasContext *s, arg_rri_sf *a, @@ -4012,7 +4141,7 @@ static bool gen_add_sub_imm_with_tags(DisasContext *s, arg_rri_tag *a, tcg_rn = cpu_reg_sp(s, a->rn); tcg_rd = cpu_reg_sp(s, a->rd); - if (s->ata) { + if (s->ata[0]) { gen_helper_addsubg(tcg_rd, cpu_env, tcg_rn, tcg_constant_i32(imm), tcg_constant_i32(a->uimm4)); @@ -5399,7 +5528,7 @@ static void disas_data_proc_2src(DisasContext *s, uint32_t insn) if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) { goto do_unallocated; } - if (s->ata) { + if (s->ata[0]) { gen_helper_irg(cpu_reg_sp(s, rd), cpu_env, cpu_reg_sp(s, rn), cpu_reg(s, rm)); } else { @@ -13890,7 +14019,8 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, dc->bt = EX_TBFLAG_A64(tb_flags, BT); dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV); - dc->ata = EX_TBFLAG_A64(tb_flags, ATA); + dc->ata[0] = EX_TBFLAG_A64(tb_flags, ATA); + dc->ata[1] = EX_TBFLAG_A64(tb_flags, ATA0); dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE); dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE); dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM); diff --git a/target/arm/tcg/translate.c b/target/arm/tcg/translate.c index 976b704200..d83a0e772c 100644 --- a/target/arm/tcg/translate.c +++ b/target/arm/tcg/translate.c @@ -2943,54 +2943,16 @@ void gen_gvec_sqrdmlsh_qc(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, gen_gvec_fn3_qc(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, fns[vece - 1]); } -#define GEN_CMP0(NAME, COND) \ - static void gen_##NAME##0_i32(TCGv_i32 d, TCGv_i32 a) \ - { \ - tcg_gen_negsetcond_i32(COND, d, a, tcg_constant_i32(0)); \ - } \ - static void gen_##NAME##0_i64(TCGv_i64 d, TCGv_i64 a) \ - { \ - tcg_gen_negsetcond_i64(COND, d, a, tcg_constant_i64(0)); \ - } \ - static void gen_##NAME##0_vec(unsigned vece, TCGv_vec d, TCGv_vec a) \ - { \ - TCGv_vec zero = tcg_constant_vec_matching(d, vece, 0); \ - tcg_gen_cmp_vec(COND, vece, d, a, zero); \ - } \ - void gen_gvec_##NAME##0(unsigned vece, uint32_t d, uint32_t m, \ - uint32_t opr_sz, uint32_t max_sz) \ - { \ - const GVecGen2 op[4] = { \ - { .fno = gen_helper_gvec_##NAME##0_b, \ - .fniv = gen_##NAME##0_vec, \ - .opt_opc = vecop_list_cmp, \ - .vece = MO_8 }, \ - { .fno = gen_helper_gvec_##NAME##0_h, \ - .fniv = gen_##NAME##0_vec, \ - .opt_opc = vecop_list_cmp, \ - .vece = MO_16 }, \ - { .fni4 = gen_##NAME##0_i32, \ - .fniv = gen_##NAME##0_vec, \ - .opt_opc = vecop_list_cmp, \ - .vece = MO_32 }, \ - { .fni8 = gen_##NAME##0_i64, \ - .fniv = gen_##NAME##0_vec, \ - .opt_opc = vecop_list_cmp, \ - .prefer_i64 = TCG_TARGET_REG_BITS == 64, \ - .vece = MO_64 }, \ - }; \ - tcg_gen_gvec_2(d, m, opr_sz, max_sz, &op[vece]); \ - } - -static const TCGOpcode vecop_list_cmp[] = { - INDEX_op_cmp_vec, 0 -}; - -GEN_CMP0(ceq, TCG_COND_EQ) -GEN_CMP0(cle, TCG_COND_LE) -GEN_CMP0(cge, TCG_COND_GE) -GEN_CMP0(clt, TCG_COND_LT) -GEN_CMP0(cgt, TCG_COND_GT) +#define GEN_CMP0(NAME, COND) \ + void NAME(unsigned vece, uint32_t d, uint32_t m, \ + uint32_t opr_sz, uint32_t max_sz) \ + { tcg_gen_gvec_cmpi(COND, vece, d, m, 0, opr_sz, max_sz); } + +GEN_CMP0(gen_gvec_ceq0, TCG_COND_EQ) +GEN_CMP0(gen_gvec_cle0, TCG_COND_LE) +GEN_CMP0(gen_gvec_cge0, TCG_COND_GE) +GEN_CMP0(gen_gvec_clt0, TCG_COND_LT) +GEN_CMP0(gen_gvec_cgt0, TCG_COND_GT) #undef GEN_CMP0 diff --git a/target/arm/tcg/translate.h b/target/arm/tcg/translate.h index f748ba6f39..63922f8bad 100644 --- a/target/arm/tcg/translate.h +++ b/target/arm/tcg/translate.h @@ -114,8 +114,8 @@ typedef struct DisasContext { bool unpriv; /* True if v8.3-PAuth is active. */ bool pauth_active; - /* True if v8.5-MTE access to tags is enabled. */ - bool ata; + /* True if v8.5-MTE access to tags is enabled; index with is_unpriv. */ + bool ata[2]; /* True if v8.5-MTE tag checks affect the PE; index with is_unpriv. */ bool mte_active[2]; /* True with v8.5-BTI and SCTLR_ELx.BT* set. */ diff --git a/target/arm/tcg/vec_helper.c b/target/arm/tcg/vec_helper.c index 6712a2c790..1f93510b85 100644 --- a/target/arm/tcg/vec_helper.c +++ b/target/arm/tcg/vec_helper.c @@ -23,6 +23,7 @@ #include "tcg/tcg-gvec-desc.h" #include "fpu/softfloat.h" #include "qemu/int128.h" +#include "crypto/clmul.h" #include "vec_internal.h" /* @@ -1986,21 +1987,11 @@ void HELPER(gvec_ushl_h)(void *vd, void *vn, void *vm, uint32_t desc) */ void HELPER(gvec_pmul_b)(void *vd, void *vn, void *vm, uint32_t desc) { - intptr_t i, j, opr_sz = simd_oprsz(desc); + intptr_t i, opr_sz = simd_oprsz(desc); uint64_t *d = vd, *n = vn, *m = vm; for (i = 0; i < opr_sz / 8; ++i) { - uint64_t nn = n[i]; - uint64_t mm = m[i]; - uint64_t rr = 0; - - for (j = 0; j < 8; ++j) { - uint64_t mask = (nn & 0x0101010101010101ull) * 0xff; - rr ^= mm & mask; - mm = (mm << 1) & 0xfefefefefefefefeull; - nn >>= 1; - } - d[i] = rr; + d[i] = clmul_8x8_low(n[i], m[i]); } clear_tail(d, opr_sz, simd_maxsz(desc)); } @@ -2012,84 +2003,28 @@ void HELPER(gvec_pmul_b)(void *vd, void *vn, void *vm, uint32_t desc) */ void HELPER(gvec_pmull_q)(void *vd, void *vn, void *vm, uint32_t desc) { - intptr_t i, j, opr_sz = simd_oprsz(desc); + intptr_t i, opr_sz = simd_oprsz(desc); intptr_t hi = simd_data(desc); uint64_t *d = vd, *n = vn, *m = vm; for (i = 0; i < opr_sz / 8; i += 2) { - uint64_t nn = n[i + hi]; - uint64_t mm = m[i + hi]; - uint64_t rhi = 0; - uint64_t rlo = 0; - - /* Bit 0 can only influence the low 64-bit result. */ - if (nn & 1) { - rlo = mm; - } - - for (j = 1; j < 64; ++j) { - uint64_t mask = -((nn >> j) & 1); - rlo ^= (mm << j) & mask; - rhi ^= (mm >> (64 - j)) & mask; - } - d[i] = rlo; - d[i + 1] = rhi; + Int128 r = clmul_64(n[i + hi], m[i + hi]); + d[i] = int128_getlo(r); + d[i + 1] = int128_gethi(r); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -/* - * 8x8->16 polynomial multiply. - * - * The byte inputs are expanded to (or extracted from) half-words. - * Note that neon and sve2 get the inputs from different positions. - * This allows 4 bytes to be processed in parallel with uint64_t. - */ - -static uint64_t expand_byte_to_half(uint64_t x) -{ - return (x & 0x000000ff) - | ((x & 0x0000ff00) << 8) - | ((x & 0x00ff0000) << 16) - | ((x & 0xff000000) << 24); -} - -uint64_t pmull_w(uint64_t op1, uint64_t op2) -{ - uint64_t result = 0; - int i; - for (i = 0; i < 16; ++i) { - uint64_t mask = (op1 & 0x0000000100000001ull) * 0xffffffff; - result ^= op2 & mask; - op1 >>= 1; - op2 <<= 1; - } - return result; -} - -uint64_t pmull_h(uint64_t op1, uint64_t op2) -{ - uint64_t result = 0; - int i; - for (i = 0; i < 8; ++i) { - uint64_t mask = (op1 & 0x0001000100010001ull) * 0xffff; - result ^= op2 & mask; - op1 >>= 1; - op2 <<= 1; - } - return result; -} - void HELPER(neon_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc) { int hi = simd_data(desc); uint64_t *d = vd, *n = vn, *m = vm; uint64_t nn = n[hi], mm = m[hi]; - d[0] = pmull_h(expand_byte_to_half(nn), expand_byte_to_half(mm)); + d[0] = clmul_8x4_packed(nn, mm); nn >>= 32; mm >>= 32; - d[1] = pmull_h(expand_byte_to_half(nn), expand_byte_to_half(mm)); + d[1] = clmul_8x4_packed(nn, mm); clear_tail(d, 16, simd_maxsz(desc)); } @@ -2102,23 +2037,8 @@ void HELPER(sve2_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc) uint64_t *d = vd, *n = vn, *m = vm; for (i = 0; i < opr_sz / 8; ++i) { - uint64_t nn = (n[i] >> shift) & 0x00ff00ff00ff00ffull; - uint64_t mm = (m[i] >> shift) & 0x00ff00ff00ff00ffull; - - d[i] = pmull_h(nn, mm); - } -} - -static uint64_t pmull_d(uint64_t op1, uint64_t op2) -{ - uint64_t result = 0; - int i; - - for (i = 0; i < 32; ++i) { - uint64_t mask = -((op1 >> i) & 1); - result ^= (op2 << i) & mask; + d[i] = clmul_8x4_even(n[i] >> shift, m[i] >> shift); } - return result; } void HELPER(sve2_pmull_d)(void *vd, void *vn, void *vm, uint32_t desc) @@ -2129,7 +2049,7 @@ void HELPER(sve2_pmull_d)(void *vd, void *vn, void *vm, uint32_t desc) uint64_t *d = vd; for (i = 0; i < opr_sz / 8; ++i) { - d[i] = pmull_d(n[2 * i + sel], m[2 * i + sel]); + d[i] = clmul_32(n[2 * i + sel], m[2 * i + sel]); } } #endif diff --git a/target/arm/tcg/vec_internal.h b/target/arm/tcg/vec_internal.h index 1f4ed80ff7..3ca1b94ccf 100644 --- a/target/arm/tcg/vec_internal.h +++ b/target/arm/tcg/vec_internal.h @@ -219,17 +219,6 @@ int16_t do_sqrdmlah_h(int16_t, int16_t, int16_t, bool, bool, uint32_t *); int32_t do_sqrdmlah_s(int32_t, int32_t, int32_t, bool, bool, uint32_t *); int64_t do_sqrdmlah_d(int64_t, int64_t, int64_t, bool, bool); -/* - * 8 x 8 -> 16 vector polynomial multiply where the inputs are - * in the low 8 bits of each 16-bit element -*/ -uint64_t pmull_h(uint64_t op1, uint64_t op2); -/* - * 16 x 16 -> 32 vector polynomial multiply where the inputs are - * in the low 16 bits of each 32-bit element - */ -uint64_t pmull_w(uint64_t op1, uint64_t op2); - /** * bfdotadd: * @sum: addend diff --git a/target/hppa/cpu.h b/target/hppa/cpu.h index fa13694dab..730f35231a 100644 --- a/target/hppa/cpu.h +++ b/target/hppa/cpu.h @@ -211,8 +211,14 @@ typedef struct CPUArchState { target_ureg shadow[7]; /* shadow registers */ /* ??? The number of entries isn't specified by the architecture. */ +#ifdef TARGET_HPPA64 +#define HPPA_BTLB_FIXED 0 /* BTLBs are not supported in 64-bit machines */ +#else +#define HPPA_BTLB_FIXED 16 +#endif +#define HPPA_BTLB_VARIABLE 0 #define HPPA_TLB_ENTRIES 256 -#define HPPA_BTLB_ENTRIES 0 +#define HPPA_BTLB_ENTRIES (HPPA_BTLB_FIXED + HPPA_BTLB_VARIABLE) /* ??? Implement a unified itlb/dtlb for the moment. */ /* ??? We should use a more intelligent data structure. */ @@ -344,7 +350,8 @@ bool hppa_cpu_tlb_fill(CPUState *cs, vaddr address, int size, void hppa_cpu_do_interrupt(CPUState *cpu); bool hppa_cpu_exec_interrupt(CPUState *cpu, int int_req); int hppa_get_physical_address(CPUHPPAState *env, vaddr addr, int mmu_idx, - int type, hwaddr *pphys, int *pprot); + int type, hwaddr *pphys, int *pprot, + hppa_tlb_entry **tlb_entry); extern const MemoryRegionOps hppa_io_eir_ops; extern const VMStateDescription vmstate_hppa_cpu; void hppa_cpu_alarm_timer(void *); diff --git a/target/hppa/helper.h b/target/hppa/helper.h index c7e35ce8c7..647f043c85 100644 --- a/target/hppa/helper.h +++ b/target/hppa/helper.h @@ -95,4 +95,5 @@ DEF_HELPER_FLAGS_2(ptlb, TCG_CALL_NO_RWG, void, env, tl) DEF_HELPER_FLAGS_1(ptlbe, TCG_CALL_NO_RWG, void, env) DEF_HELPER_FLAGS_2(lpa, TCG_CALL_NO_WG, tr, env, tl) DEF_HELPER_FLAGS_1(change_prot_id, TCG_CALL_NO_RWG, void, env) +DEF_HELPER_1(diag_btlb, void, env) #endif diff --git a/target/hppa/insns.decode b/target/hppa/insns.decode index 27341d27b2..aebe03ccfd 100644 --- a/target/hppa/insns.decode +++ b/target/hppa/insns.decode @@ -528,4 +528,4 @@ fdiv_d 001110 ..... ..... 011 ..... ... ..... @f0e_d_3 xmpyu 001110 ..... ..... 010 .0111 .00 t:5 r1=%ra64 r2=%rb64 # diag -diag 000101 ----- ----- ---- ---- ---- ---- +diag 000101 i:26 diff --git a/target/hppa/int_helper.c b/target/hppa/int_helper.c index bebc732c97..3ab9934a1d 100644 --- a/target/hppa/int_helper.c +++ b/target/hppa/int_helper.c @@ -154,7 +154,7 @@ void hppa_cpu_do_interrupt(CPUState *cs) vaddr = hppa_form_gva_psw(old_psw, iasq_f, vaddr); t = hppa_get_physical_address(env, vaddr, MMU_KERNEL_IDX, - 0, &paddr, &prot); + 0, &paddr, &prot, NULL); if (t >= 0) { /* We can't re-load the instruction. */ env->cr[CR_IIR] = 0; diff --git a/target/hppa/mem_helper.c b/target/hppa/mem_helper.c index 46c3dcaf15..520fd311f8 100644 --- a/target/hppa/mem_helper.c +++ b/target/hppa/mem_helper.c @@ -41,16 +41,24 @@ static hppa_tlb_entry *hppa_find_tlb(CPUHPPAState *env, vaddr addr) return NULL; } -static void hppa_flush_tlb_ent(CPUHPPAState *env, hppa_tlb_entry *ent) +static void hppa_flush_tlb_ent(CPUHPPAState *env, hppa_tlb_entry *ent, + bool force_flush_btlb) { CPUState *cs = env_cpu(env); - unsigned i, n = 1 << (2 * ent->page_size); - uint64_t addr = ent->va_b; + + if (!ent->entry_valid) { + return; + } trace_hppa_tlb_flush_ent(env, ent, ent->va_b, ent->va_e, ent->pa); - for (i = 0; i < n; ++i, addr += TARGET_PAGE_SIZE) { - tlb_flush_page_by_mmuidx(cs, addr, HPPA_MMU_FLUSH_MASK); + tlb_flush_range_by_mmuidx(cs, ent->va_b, + ent->va_e - ent->va_b + 1, + HPPA_MMU_FLUSH_MASK, TARGET_LONG_BITS); + + /* never clear BTLBs, unless forced to do so. */ + if (ent < &env->tlb[HPPA_BTLB_ENTRIES] && !force_flush_btlb) { + return; } memset(ent, 0, sizeof(*ent)); @@ -60,23 +68,35 @@ static void hppa_flush_tlb_ent(CPUHPPAState *env, hppa_tlb_entry *ent) static hppa_tlb_entry *hppa_alloc_tlb_ent(CPUHPPAState *env) { hppa_tlb_entry *ent; - uint32_t i = env->tlb_last; + uint32_t i; + + if (env->tlb_last < HPPA_BTLB_ENTRIES || env->tlb_last >= ARRAY_SIZE(env->tlb)) { + i = HPPA_BTLB_ENTRIES; + env->tlb_last = HPPA_BTLB_ENTRIES + 1; + } else { + i = env->tlb_last; + env->tlb_last++; + } - env->tlb_last = (i == ARRAY_SIZE(env->tlb) - 1 ? 0 : i + 1); ent = &env->tlb[i]; - hppa_flush_tlb_ent(env, ent); + hppa_flush_tlb_ent(env, ent, false); return ent; } int hppa_get_physical_address(CPUHPPAState *env, vaddr addr, int mmu_idx, - int type, hwaddr *pphys, int *pprot) + int type, hwaddr *pphys, int *pprot, + hppa_tlb_entry **tlb_entry) { hwaddr phys; int prot, r_prot, w_prot, x_prot, priv; hppa_tlb_entry *ent; int ret = -1; + if (tlb_entry) { + *tlb_entry = NULL; + } + /* Virtual translation disabled. Direct map virtual to physical. */ if (mmu_idx == MMU_PHYS_IDX) { phys = addr; @@ -93,8 +113,12 @@ int hppa_get_physical_address(CPUHPPAState *env, vaddr addr, int mmu_idx, goto egress; } + if (tlb_entry) { + *tlb_entry = ent; + } + /* We now know the physical address. */ - phys = ent->pa + (addr & ~TARGET_PAGE_MASK); + phys = ent->pa + (addr - ent->va_b); /* Map TLB access_rights field to QEMU protection. */ priv = MMU_IDX_TO_PRIV(mmu_idx); @@ -193,7 +217,7 @@ hwaddr hppa_cpu_get_phys_page_debug(CPUState *cs, vaddr addr) } excp = hppa_get_physical_address(&cpu->env, addr, MMU_KERNEL_IDX, 0, - &phys, &prot); + &phys, &prot, NULL); /* Since we're translating for debugging, the only error that is a hard error is no translation at all. Otherwise, while a real cpu @@ -207,6 +231,7 @@ bool hppa_cpu_tlb_fill(CPUState *cs, vaddr addr, int size, { HPPACPU *cpu = HPPA_CPU(cs); CPUHPPAState *env = &cpu->env; + hppa_tlb_entry *ent; int prot, excp, a_prot; hwaddr phys; @@ -223,7 +248,7 @@ bool hppa_cpu_tlb_fill(CPUState *cs, vaddr addr, int size, } excp = hppa_get_physical_address(env, addr, mmu_idx, - a_prot, &phys, &prot); + a_prot, &phys, &prot, &ent); if (unlikely(excp >= 0)) { if (probe) { return false; @@ -243,7 +268,7 @@ bool hppa_cpu_tlb_fill(CPUState *cs, vaddr addr, int size, phys & TARGET_PAGE_MASK, size, type, mmu_idx); /* Success! Store the translation into the QEMU TLB. */ tlb_set_page(cs, addr & TARGET_PAGE_MASK, phys & TARGET_PAGE_MASK, - prot, mmu_idx, TARGET_PAGE_SIZE); + prot, mmu_idx, TARGET_PAGE_SIZE << (ent ? 2 * ent->page_size : 0)); return true; } @@ -254,11 +279,11 @@ void HELPER(itlba)(CPUHPPAState *env, target_ulong addr, target_ureg reg) int i; /* Zap any old entries covering ADDR; notice empty entries on the way. */ - for (i = 0; i < ARRAY_SIZE(env->tlb); ++i) { + for (i = HPPA_BTLB_ENTRIES; i < ARRAY_SIZE(env->tlb); ++i) { hppa_tlb_entry *ent = &env->tlb[i]; if (ent->va_b <= addr && addr <= ent->va_e) { if (ent->entry_valid) { - hppa_flush_tlb_ent(env, ent); + hppa_flush_tlb_ent(env, ent, false); } if (!empty) { empty = ent; @@ -278,16 +303,8 @@ void HELPER(itlba)(CPUHPPAState *env, target_ulong addr, target_ureg reg) trace_hppa_tlb_itlba(env, empty, empty->va_b, empty->va_e, empty->pa); } -/* Insert (Insn/Data) TLB Protection. Note this is PA 1.1 only. */ -void HELPER(itlbp)(CPUHPPAState *env, target_ulong addr, target_ureg reg) +static void set_access_bits(CPUHPPAState *env, hppa_tlb_entry *ent, target_ureg reg) { - hppa_tlb_entry *ent = hppa_find_tlb(env, addr); - - if (unlikely(ent == NULL)) { - qemu_log_mask(LOG_GUEST_ERROR, "ITLBP not following ITLBA\n"); - return; - } - ent->access_id = extract32(reg, 1, 18); ent->u = extract32(reg, 19, 1); ent->ar_pl2 = extract32(reg, 20, 2); @@ -301,6 +318,19 @@ void HELPER(itlbp)(CPUHPPAState *env, target_ulong addr, target_ureg reg) ent->ar_pl1, ent->ar_type, ent->b, ent->d, ent->t); } +/* Insert (Insn/Data) TLB Protection. Note this is PA 1.1 only. */ +void HELPER(itlbp)(CPUHPPAState *env, target_ulong addr, target_ureg reg) +{ + hppa_tlb_entry *ent = hppa_find_tlb(env, addr); + + if (unlikely(ent == NULL)) { + qemu_log_mask(LOG_GUEST_ERROR, "ITLBP not following ITLBA\n"); + return; + } + + set_access_bits(env, ent, reg); +} + /* Purge (Insn/Data) TLB. This is explicitly page-based, and is synchronous across all processors. */ static void ptlb_work(CPUState *cpu, run_on_cpu_data data) @@ -310,7 +340,7 @@ static void ptlb_work(CPUState *cpu, run_on_cpu_data data) hppa_tlb_entry *ent = hppa_find_tlb(env, addr); if (ent && ent->entry_valid) { - hppa_flush_tlb_ent(env, ent); + hppa_flush_tlb_ent(env, ent, false); } } @@ -334,7 +364,10 @@ void HELPER(ptlb)(CPUHPPAState *env, target_ulong addr) void HELPER(ptlbe)(CPUHPPAState *env) { trace_hppa_tlb_ptlbe(env); - memset(env->tlb, 0, sizeof(env->tlb)); + qemu_log_mask(CPU_LOG_MMU, "FLUSH ALL TLB ENTRIES\n"); + memset(&env->tlb[HPPA_BTLB_ENTRIES], 0, + sizeof(env->tlb) - HPPA_BTLB_ENTRIES * sizeof(env->tlb[0])); + env->tlb_last = HPPA_BTLB_ENTRIES; tlb_flush_by_mmuidx(env_cpu(env), HPPA_MMU_FLUSH_MASK); } @@ -356,7 +389,7 @@ target_ureg HELPER(lpa)(CPUHPPAState *env, target_ulong addr) int prot, excp; excp = hppa_get_physical_address(env, addr, MMU_KERNEL_IDX, 0, - &phys, &prot); + &phys, &prot, NULL); if (excp >= 0) { if (env->psw & PSW_Q) { /* ??? Needs tweaking for hppa64. */ @@ -379,3 +412,95 @@ int hppa_artype_for_page(CPUHPPAState *env, target_ulong vaddr) hppa_tlb_entry *ent = hppa_find_tlb(env, vaddr); return ent ? ent->ar_type : -1; } + +/* + * diag_btlb() emulates the PDC PDC_BLOCK_TLB firmware call to + * allow operating systems to modify the Block TLB (BTLB) entries. + * For implementation details see page 1-13 in + * https://parisc.wiki.kernel.org/images-parisc/e/ef/Pdc11-v0.96-Ch1-procs.pdf + */ +void HELPER(diag_btlb)(CPUHPPAState *env) +{ + unsigned int phys_page, len, slot; + int mmu_idx = cpu_mmu_index(env, 0); + uintptr_t ra = GETPC(); + hppa_tlb_entry *btlb; + uint64_t virt_page; + uint32_t *vaddr; + +#ifdef TARGET_HPPA64 + /* BTLBs are not supported on 64-bit CPUs */ + env->gr[28] = -1; /* nonexistent procedure */ + return; +#endif + env->gr[28] = 0; /* PDC_OK */ + + switch (env->gr[25]) { + case 0: + /* return BTLB parameters */ + qemu_log_mask(CPU_LOG_MMU, "PDC_BLOCK_TLB: PDC_BTLB_INFO\n"); + vaddr = probe_access(env, env->gr[24], 4 * sizeof(target_ulong), + MMU_DATA_STORE, mmu_idx, ra); + if (vaddr == NULL) { + env->gr[28] = -10; /* invalid argument */ + } else { + vaddr[0] = cpu_to_be32(1); + vaddr[1] = cpu_to_be32(16 * 1024); + vaddr[2] = cpu_to_be32(HPPA_BTLB_FIXED); + vaddr[3] = cpu_to_be32(HPPA_BTLB_VARIABLE); + } + break; + case 1: + /* insert BTLB entry */ + virt_page = env->gr[24]; /* upper 32 bits */ + virt_page <<= 32; + virt_page |= env->gr[23]; /* lower 32 bits */ + phys_page = env->gr[22]; + len = env->gr[21]; + slot = env->gr[19]; + qemu_log_mask(CPU_LOG_MMU, "PDC_BLOCK_TLB: PDC_BTLB_INSERT " + "0x%08llx-0x%08llx: vpage 0x%llx for phys page 0x%04x len %d " + "into slot %d\n", + (long long) virt_page << TARGET_PAGE_BITS, + (long long) (virt_page + len) << TARGET_PAGE_BITS, + (long long) virt_page, phys_page, len, slot); + if (slot < HPPA_BTLB_ENTRIES) { + btlb = &env->tlb[slot]; + /* force flush of possibly existing BTLB entry */ + hppa_flush_tlb_ent(env, btlb, true); + /* create new BTLB entry */ + btlb->va_b = virt_page << TARGET_PAGE_BITS; + btlb->va_e = btlb->va_b + len * TARGET_PAGE_SIZE - 1; + btlb->pa = phys_page << TARGET_PAGE_BITS; + set_access_bits(env, btlb, env->gr[20]); + btlb->t = 0; + btlb->d = 1; + } else { + env->gr[28] = -10; /* invalid argument */ + } + break; + case 2: + /* Purge BTLB entry */ + slot = env->gr[22]; + qemu_log_mask(CPU_LOG_MMU, "PDC_BLOCK_TLB: PDC_BTLB_PURGE slot %d\n", + slot); + if (slot < HPPA_BTLB_ENTRIES) { + btlb = &env->tlb[slot]; + hppa_flush_tlb_ent(env, btlb, true); + } else { + env->gr[28] = -10; /* invalid argument */ + } + break; + case 3: + /* Purge all BTLB entries */ + qemu_log_mask(CPU_LOG_MMU, "PDC_BLOCK_TLB: PDC_BTLB_PURGE_ALL\n"); + for (slot = 0; slot < HPPA_BTLB_ENTRIES; slot++) { + btlb = &env->tlb[slot]; + hppa_flush_tlb_ent(env, btlb, true); + } + break; + default: + env->gr[28] = -2; /* nonexistent option */ + break; + } +} diff --git a/target/hppa/op_helper.c b/target/hppa/op_helper.c index f25a5a72aa..837e2b3117 100644 --- a/target/hppa/op_helper.c +++ b/target/hppa/op_helper.c @@ -179,7 +179,8 @@ target_ureg HELPER(probe)(CPUHPPAState *env, target_ulong addr, return 0; } - excp = hppa_get_physical_address(env, addr, level, 0, &phys, &prot); + excp = hppa_get_physical_address(env, addr, level, 0, &phys, + &prot, NULL); if (excp >= 0) { if (env->psw & PSW_Q) { /* ??? Needs tweaking for hppa64. */ diff --git a/target/hppa/translate.c b/target/hppa/translate.c index c04dc15228..650bbcfe95 100644 --- a/target/hppa/translate.c +++ b/target/hppa/translate.c @@ -4042,9 +4042,18 @@ static bool trans_fmpyfadd_d(DisasContext *ctx, arg_fmpyfadd_d *a) static bool trans_diag(DisasContext *ctx, arg_diag *a) { - qemu_log_mask(LOG_UNIMP, "DIAG opcode ignored\n"); - cond_free(&ctx->null_cond); - return true; + nullify_over(ctx); + CHECK_MOST_PRIVILEGED(EXCP_PRIV_OPR); +#ifndef CONFIG_USER_ONLY + if (a->i == 0x100) { + /* emulate PDC BTLB, called by SeaBIOS-hppa */ + gen_helper_diag_btlb(cpu_env); + } else +#endif + { + qemu_log_mask(LOG_UNIMP, "DIAG opcode 0x%04x ignored\n", a->i); + } + return nullify_end(ctx); } static void hppa_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index f9e51a9d87..7836aa6692 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -5341,7 +5341,7 @@ static const char *x86_cpu_feature_name(FeatureWord w, int bitnr) return name; } -/* Compatibily hack to maintain legacy +-feat semantic, +/* Compatibility hack to maintain legacy +-feat semantic, * where +-feat overwrites any feature set by * feat=on|feat even if the later is parsed after +-feat * (i.e. "-x2apic,x2apic=on" will result in x2apic disabled) @@ -6304,7 +6304,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count, * The initial value of xcr0 and ebx == 0, On host without kvm * commit 412a3c41(e.g., CentOS 6), the ebx's value always == 0 * even through guest update xcr0, this will crash some legacy guest - * (e.g., CentOS 6), So set ebx == ecx to workaroud it. + * (e.g., CentOS 6), So set ebx == ecx to workaround it. */ *ebx = kvm_enabled() ? *ecx : xsave_area_size(env->xcr0, false); } else if (count == 1) { diff --git a/target/i386/cpu.h b/target/i386/cpu.h index d1ffadd78b..d3f377d48a 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -728,7 +728,7 @@ uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, #define CPUID_EXT2_3DNOWEXT (1U << 30) #define CPUID_EXT2_3DNOW (1U << 31) -/* CPUID[8000_0001].EDX bits that are aliase of CPUID[1].EDX bits on AMD CPUs */ +/* CPUID[8000_0001].EDX bits that are aliases of CPUID[1].EDX bits on AMD CPUs */ #define CPUID_EXT2_AMD_ALIASES (CPUID_EXT2_FPU | CPUID_EXT2_VME | \ CPUID_EXT2_DE | CPUID_EXT2_PSE | \ CPUID_EXT2_TSC | CPUID_EXT2_MSR | \ @@ -2072,7 +2072,7 @@ hwaddr x86_cpu_get_phys_page_attrs_debug(CPUState *cpu, vaddr addr, MemTxAttrs *attrs); int cpu_get_pic_interrupt(CPUX86State *s); -/* MSDOS compatibility mode FPU exception support */ +/* MS-DOS compatibility mode FPU exception support */ void x86_register_ferr_irq(qemu_irq irq); void fpu_check_raise_ferr_irq(CPUX86State *s); void cpu_set_ignne(void); diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index e5cd7cc806..af101fcdf6 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -4729,7 +4729,7 @@ int kvm_arch_put_registers(CPUState *cpu, int level) /* * Put MSR_IA32_FEATURE_CONTROL first, this ensures the VM gets out of VMX * root operation upon vCPU reset. kvm_put_msr_feature_control() should also - * preceed kvm_put_nested_state() when 'real' nested state is set. + * precede kvm_put_nested_state() when 'real' nested state is set. */ if (level >= KVM_PUT_RESET_STATE) { ret = kvm_put_msr_feature_control(x86_cpu); @@ -5653,7 +5653,7 @@ int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route, } /* - * Handled untranslated compatibilty format interrupt with + * Handled untranslated compatibility format interrupt with * extended destination ID in the low bits 11-5. */ dst.address = kvm_swizzle_msi_ext_dest_id(dst.address); diff --git a/target/i386/kvm/xen-emu.c b/target/i386/kvm/xen-emu.c index a8146115f0..76348f9d5d 100644 --- a/target/i386/kvm/xen-emu.c +++ b/target/i386/kvm/xen-emu.c @@ -1033,7 +1033,7 @@ static int do_set_periodic_timer(CPUState *target, uint64_t period_ns) #define MILLISECS(_ms) ((int64_t)((_ms) * 1000000ULL)) #define MICROSECS(_us) ((int64_t)((_us) * 1000ULL)) #define STIME_MAX ((time_t)((int64_t)~0ull >> 1)) -/* Chosen so (NOW() + delta) wont overflow without an uptime of 200 years */ +/* Chosen so (NOW() + delta) won't overflow without an uptime of 200 years */ #define STIME_DELTA_MAX ((int64_t)((uint64_t)~0ull >> 2)) static int vcpuop_set_periodic_timer(CPUState *cs, CPUState *target, diff --git a/target/i386/machine.c b/target/i386/machine.c index c7ac8084b2..a1041ef828 100644 --- a/target/i386/machine.c +++ b/target/i386/machine.c @@ -282,12 +282,12 @@ static int cpu_pre_save(void *opaque) * hypervisor, its exception payload (CR2/DR6 on #PF/#DB) * should not be set yet in the respective vCPU register. * Thus, in case an exception is pending, it is - * important to save the exception payload seperately. + * important to save the exception payload separately. * * Therefore, if an exception is not in a pending state * or vCPU is not in guest-mode, it is not important to * distinguish between a pending and injected exception - * and we don't need to store seperately the exception payload. + * and we don't need to store separately the exception payload. * * In order to preserve better backwards-compatible migration, * convert a pending exception to an injected exception in diff --git a/target/i386/ops_sse.h b/target/i386/ops_sse.h index a0e425733f..33908c0691 100644 --- a/target/i386/ops_sse.h +++ b/target/i386/ops_sse.h @@ -20,6 +20,7 @@ #include "crypto/aes.h" #include "crypto/aes-round.h" +#include "crypto/clmul.h" #if SHIFT == 0 #define Reg MMXReg @@ -2122,41 +2123,18 @@ target_ulong helper_crc32(uint32_t crc1, target_ulong msg, uint32_t len) #endif -#if SHIFT == 1 -static void clmulq(uint64_t *dest_l, uint64_t *dest_h, - uint64_t a, uint64_t b) -{ - uint64_t al, ah, resh, resl; - - ah = 0; - al = a; - resh = resl = 0; - - while (b) { - if (b & 1) { - resl ^= al; - resh ^= ah; - } - ah = (ah << 1) | (al >> 63); - al <<= 1; - b >>= 1; - } - - *dest_l = resl; - *dest_h = resh; -} -#endif - void glue(helper_pclmulqdq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, uint32_t ctrl) { - uint64_t a, b; - int i; + int a_idx = (ctrl & 1) != 0; + int b_idx = (ctrl & 16) != 0; - for (i = 0; i < 1 << SHIFT; i += 2) { - a = v->Q(((ctrl & 1) != 0) + i); - b = s->Q(((ctrl & 16) != 0) + i); - clmulq(&d->Q(i), &d->Q(i + 1), a, b); + for (int i = 0; i < SHIFT; i++) { + uint64_t a = v->Q(2 * i + a_idx); + uint64_t b = s->Q(2 * i + b_idx); + Int128 *r = (Int128 *)&d->ZMM_X(i); + + *r = clmul_64(a, b); } } diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index e0a622941c..c98e42f17a 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -1069,7 +1069,7 @@ static CCPrepare gen_prepare_eflags_z(DisasContext *s, TCGv reg) } /* perform a conditional store into register 'reg' according to jump opcode - value 'b'. In the fast case, T0 is guaranted not to be used. */ + value 'b'. In the fast case, T0 is guaranteed not to be used. */ static CCPrepare gen_prepare_cc(DisasContext *s, int b, TCGv reg) { int inv, jcc_op, cond; @@ -1202,7 +1202,7 @@ static inline void gen_compute_eflags_c(DisasContext *s, TCGv reg) } /* generate a conditional jump to label 'l1' according to jump opcode - value 'b'. In the fast case, T0 is guaranted not to be used. */ + value 'b'. In the fast case, T0 is guaranteed not to be used. */ static inline void gen_jcc1_noeob(DisasContext *s, int b, TCGLabel *l1) { CCPrepare cc = gen_prepare_cc(s, b, s->T0); @@ -1219,7 +1219,7 @@ static inline void gen_jcc1_noeob(DisasContext *s, int b, TCGLabel *l1) } /* Generate a conditional jump to label 'l1' according to jump opcode - value 'b'. In the fast case, T0 is guaranted not to be used. + value 'b'. In the fast case, T0 is guaranteed not to be used. A translation block must end soon. */ static inline void gen_jcc1(DisasContext *s, int b, TCGLabel *l1) { @@ -5355,7 +5355,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (s->prefix & PREFIX_LOCK) { switch (op) { case 0: /* bt */ - /* Needs no atomic ops; we surpressed the normal + /* Needs no atomic ops; we suppressed the normal memory load for LOCK above so do it now. */ gen_op_ld_v(s, ot, s->T0, s->A0); break; diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c index 65f9320e34..fc7f70fbe5 100644 --- a/target/loongarch/cpu.c +++ b/target/loongarch/cpu.c @@ -19,6 +19,7 @@ #include "cpu-csr.h" #include "sysemu/reset.h" #include "tcg/tcg.h" +#include "vec.h" const char * const regnames[32] = { "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", @@ -54,6 +55,7 @@ static const char * const excp_names[] = { [EXCCODE_DBP] = "Debug breakpoint", [EXCCODE_BCE] = "Bound Check Exception", [EXCCODE_SXD] = "128 bit vector instructions Disable exception", + [EXCCODE_ASXD] = "256 bit vector instructions Disable exception", }; const char *loongarch_exception_name(int32_t exception) @@ -189,6 +191,7 @@ static void loongarch_cpu_do_interrupt(CPUState *cs) case EXCCODE_FPD: case EXCCODE_FPE: case EXCCODE_SXD: + case EXCCODE_ASXD: env->CSR_BADV = env->pc; QEMU_FALLTHROUGH; case EXCCODE_BCE: @@ -390,6 +393,7 @@ static void loongarch_la464_initfn(Object *obj) data = FIELD_DP32(data, CPUCFG2, FP_DP, 1); data = FIELD_DP32(data, CPUCFG2, FP_VER, 1); data = FIELD_DP32(data, CPUCFG2, LSX, 1), + data = FIELD_DP32(data, CPUCFG2, LASX, 1), data = FIELD_DP32(data, CPUCFG2, LLFTP, 1); data = FIELD_DP32(data, CPUCFG2, LLFTP_VER, 1); data = FIELD_DP32(data, CPUCFG2, LSPW, 1); diff --git a/target/loongarch/cpu.h b/target/loongarch/cpu.h index 4d7201995a..f125a8e49b 100644 --- a/target/loongarch/cpu.h +++ b/target/loongarch/cpu.h @@ -251,18 +251,20 @@ FIELD(TLB_MISC, ASID, 1, 10) FIELD(TLB_MISC, VPPN, 13, 35) FIELD(TLB_MISC, PS, 48, 6) -#define LSX_LEN (128) +#define LSX_LEN (128) +#define LASX_LEN (256) + typedef union VReg { - int8_t B[LSX_LEN / 8]; - int16_t H[LSX_LEN / 16]; - int32_t W[LSX_LEN / 32]; - int64_t D[LSX_LEN / 64]; - uint8_t UB[LSX_LEN / 8]; - uint16_t UH[LSX_LEN / 16]; - uint32_t UW[LSX_LEN / 32]; - uint64_t UD[LSX_LEN / 64]; - Int128 Q[LSX_LEN / 128]; -}VReg; + int8_t B[LASX_LEN / 8]; + int16_t H[LASX_LEN / 16]; + int32_t W[LASX_LEN / 32]; + int64_t D[LASX_LEN / 64]; + uint8_t UB[LASX_LEN / 8]; + uint16_t UH[LASX_LEN / 16]; + uint32_t UW[LASX_LEN / 32]; + uint64_t UD[LASX_LEN / 64]; + Int128 Q[LASX_LEN / 128]; +} VReg; typedef union fpr_t fpr_t; union fpr_t { @@ -460,6 +462,7 @@ static inline void set_pc(CPULoongArchState *env, uint64_t value) #define HW_FLAGS_CRMD_PG R_CSR_CRMD_PG_MASK /* 0x10 */ #define HW_FLAGS_EUEN_FPE 0x04 #define HW_FLAGS_EUEN_SXE 0x08 +#define HW_FLAGS_EUEN_ASXE 0x10 #define HW_FLAGS_VA32 0x20 static inline void cpu_get_tb_cpu_state(CPULoongArchState *env, vaddr *pc, @@ -470,6 +473,7 @@ static inline void cpu_get_tb_cpu_state(CPULoongArchState *env, vaddr *pc, *flags = env->CSR_CRMD & (R_CSR_CRMD_PLV_MASK | R_CSR_CRMD_PG_MASK); *flags |= FIELD_EX64(env->CSR_EUEN, CSR_EUEN, FPE) * HW_FLAGS_EUEN_FPE; *flags |= FIELD_EX64(env->CSR_EUEN, CSR_EUEN, SXE) * HW_FLAGS_EUEN_SXE; + *flags |= FIELD_EX64(env->CSR_EUEN, CSR_EUEN, ASXE) * HW_FLAGS_EUEN_ASXE; *flags |= is_va32(env) * HW_FLAGS_VA32; } diff --git a/target/loongarch/disas.c b/target/loongarch/disas.c index 5c402d944d..c8a29eac2b 100644 --- a/target/loongarch/disas.c +++ b/target/loongarch/disas.c @@ -1695,3 +1695,927 @@ INSN_LSX(vstelm_d, vr_ii) INSN_LSX(vstelm_w, vr_ii) INSN_LSX(vstelm_h, vr_ii) INSN_LSX(vstelm_b, vr_ii) + +#define INSN_LASX(insn, type) \ +static bool trans_##insn(DisasContext *ctx, arg_##type * a) \ +{ \ + output_##type ## _x(ctx, a, #insn); \ + return true; \ +} + +static void output_cv_x(DisasContext *ctx, arg_cv *a, const char *mnemonic) +{ + output(ctx, mnemonic, "fcc%d, x%d", a->cd, a->vj); +} + +static void output_v_i_x(DisasContext *ctx, arg_v_i *a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, 0x%x", a->vd, a->imm); +} + +static void output_vvvv_x(DisasContext *ctx, arg_vvvv *a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, x%d, x%d, x%d", a->vd, a->vj, a->vk, a->va); +} + +static void output_vvv_x(DisasContext *ctx, arg_vvv * a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, x%d, x%d", a->vd, a->vj, a->vk); +} + +static void output_vr_x(DisasContext *ctx, arg_vr *a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, r%d", a->vd, a->rj); +} + +static void output_vv_i_x(DisasContext *ctx, arg_vv_i *a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, x%d, 0x%x", a->vd, a->vj, a->imm); +} + +static void output_vv_x(DisasContext *ctx, arg_vv *a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, x%d", a->vd, a->vj); +} + +static void output_vr_i_x(DisasContext *ctx, arg_vr_i *a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, r%d, 0x%x", a->vd, a->rj, a->imm); +} + +static void output_rv_i_x(DisasContext *ctx, arg_rv_i *a, const char *mnemonic) +{ + output(ctx, mnemonic, "r%d, x%d, 0x%x", a->rd, a->vj, a->imm); +} + +static void output_vvr_x(DisasContext *ctx, arg_vvr *a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, x%d, r%d", a->vd, a->vj, a->rk); +} + +static void output_vrr_x(DisasContext *ctx, arg_vrr *a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, r%d, r%d", a->vd, a->rj, a->rk); +} + +static void output_vr_ii_x(DisasContext *ctx, arg_vr_ii *a, const char *mnemonic) +{ + output(ctx, mnemonic, "x%d, r%d, 0x%x, 0x%x", a->vd, a->rj, a->imm, a->imm2); +} + +INSN_LASX(xvadd_b, vvv) +INSN_LASX(xvadd_h, vvv) +INSN_LASX(xvadd_w, vvv) +INSN_LASX(xvadd_d, vvv) +INSN_LASX(xvadd_q, vvv) +INSN_LASX(xvsub_b, vvv) +INSN_LASX(xvsub_h, vvv) +INSN_LASX(xvsub_w, vvv) +INSN_LASX(xvsub_d, vvv) +INSN_LASX(xvsub_q, vvv) + +INSN_LASX(xvaddi_bu, vv_i) +INSN_LASX(xvaddi_hu, vv_i) +INSN_LASX(xvaddi_wu, vv_i) +INSN_LASX(xvaddi_du, vv_i) +INSN_LASX(xvsubi_bu, vv_i) +INSN_LASX(xvsubi_hu, vv_i) +INSN_LASX(xvsubi_wu, vv_i) +INSN_LASX(xvsubi_du, vv_i) + +INSN_LASX(xvneg_b, vv) +INSN_LASX(xvneg_h, vv) +INSN_LASX(xvneg_w, vv) +INSN_LASX(xvneg_d, vv) + +INSN_LASX(xvsadd_b, vvv) +INSN_LASX(xvsadd_h, vvv) +INSN_LASX(xvsadd_w, vvv) +INSN_LASX(xvsadd_d, vvv) +INSN_LASX(xvsadd_bu, vvv) +INSN_LASX(xvsadd_hu, vvv) +INSN_LASX(xvsadd_wu, vvv) +INSN_LASX(xvsadd_du, vvv) +INSN_LASX(xvssub_b, vvv) +INSN_LASX(xvssub_h, vvv) +INSN_LASX(xvssub_w, vvv) +INSN_LASX(xvssub_d, vvv) +INSN_LASX(xvssub_bu, vvv) +INSN_LASX(xvssub_hu, vvv) +INSN_LASX(xvssub_wu, vvv) +INSN_LASX(xvssub_du, vvv) + +INSN_LASX(xvhaddw_h_b, vvv) +INSN_LASX(xvhaddw_w_h, vvv) +INSN_LASX(xvhaddw_d_w, vvv) +INSN_LASX(xvhaddw_q_d, vvv) +INSN_LASX(xvhaddw_hu_bu, vvv) +INSN_LASX(xvhaddw_wu_hu, vvv) +INSN_LASX(xvhaddw_du_wu, vvv) +INSN_LASX(xvhaddw_qu_du, vvv) +INSN_LASX(xvhsubw_h_b, vvv) +INSN_LASX(xvhsubw_w_h, vvv) +INSN_LASX(xvhsubw_d_w, vvv) +INSN_LASX(xvhsubw_q_d, vvv) +INSN_LASX(xvhsubw_hu_bu, vvv) +INSN_LASX(xvhsubw_wu_hu, vvv) +INSN_LASX(xvhsubw_du_wu, vvv) +INSN_LASX(xvhsubw_qu_du, vvv) + +INSN_LASX(xvaddwev_h_b, vvv) +INSN_LASX(xvaddwev_w_h, vvv) +INSN_LASX(xvaddwev_d_w, vvv) +INSN_LASX(xvaddwev_q_d, vvv) +INSN_LASX(xvaddwod_h_b, vvv) +INSN_LASX(xvaddwod_w_h, vvv) +INSN_LASX(xvaddwod_d_w, vvv) +INSN_LASX(xvaddwod_q_d, vvv) +INSN_LASX(xvsubwev_h_b, vvv) +INSN_LASX(xvsubwev_w_h, vvv) +INSN_LASX(xvsubwev_d_w, vvv) +INSN_LASX(xvsubwev_q_d, vvv) +INSN_LASX(xvsubwod_h_b, vvv) +INSN_LASX(xvsubwod_w_h, vvv) +INSN_LASX(xvsubwod_d_w, vvv) +INSN_LASX(xvsubwod_q_d, vvv) + +INSN_LASX(xvaddwev_h_bu, vvv) +INSN_LASX(xvaddwev_w_hu, vvv) +INSN_LASX(xvaddwev_d_wu, vvv) +INSN_LASX(xvaddwev_q_du, vvv) +INSN_LASX(xvaddwod_h_bu, vvv) +INSN_LASX(xvaddwod_w_hu, vvv) +INSN_LASX(xvaddwod_d_wu, vvv) +INSN_LASX(xvaddwod_q_du, vvv) +INSN_LASX(xvsubwev_h_bu, vvv) +INSN_LASX(xvsubwev_w_hu, vvv) +INSN_LASX(xvsubwev_d_wu, vvv) +INSN_LASX(xvsubwev_q_du, vvv) +INSN_LASX(xvsubwod_h_bu, vvv) +INSN_LASX(xvsubwod_w_hu, vvv) +INSN_LASX(xvsubwod_d_wu, vvv) +INSN_LASX(xvsubwod_q_du, vvv) + +INSN_LASX(xvaddwev_h_bu_b, vvv) +INSN_LASX(xvaddwev_w_hu_h, vvv) +INSN_LASX(xvaddwev_d_wu_w, vvv) +INSN_LASX(xvaddwev_q_du_d, vvv) +INSN_LASX(xvaddwod_h_bu_b, vvv) +INSN_LASX(xvaddwod_w_hu_h, vvv) +INSN_LASX(xvaddwod_d_wu_w, vvv) +INSN_LASX(xvaddwod_q_du_d, vvv) + +INSN_LASX(xvavg_b, vvv) +INSN_LASX(xvavg_h, vvv) +INSN_LASX(xvavg_w, vvv) +INSN_LASX(xvavg_d, vvv) +INSN_LASX(xvavg_bu, vvv) +INSN_LASX(xvavg_hu, vvv) +INSN_LASX(xvavg_wu, vvv) +INSN_LASX(xvavg_du, vvv) +INSN_LASX(xvavgr_b, vvv) +INSN_LASX(xvavgr_h, vvv) +INSN_LASX(xvavgr_w, vvv) +INSN_LASX(xvavgr_d, vvv) +INSN_LASX(xvavgr_bu, vvv) +INSN_LASX(xvavgr_hu, vvv) +INSN_LASX(xvavgr_wu, vvv) +INSN_LASX(xvavgr_du, vvv) + +INSN_LASX(xvabsd_b, vvv) +INSN_LASX(xvabsd_h, vvv) +INSN_LASX(xvabsd_w, vvv) +INSN_LASX(xvabsd_d, vvv) +INSN_LASX(xvabsd_bu, vvv) +INSN_LASX(xvabsd_hu, vvv) +INSN_LASX(xvabsd_wu, vvv) +INSN_LASX(xvabsd_du, vvv) + +INSN_LASX(xvadda_b, vvv) +INSN_LASX(xvadda_h, vvv) +INSN_LASX(xvadda_w, vvv) +INSN_LASX(xvadda_d, vvv) + +INSN_LASX(xvmax_b, vvv) +INSN_LASX(xvmax_h, vvv) +INSN_LASX(xvmax_w, vvv) +INSN_LASX(xvmax_d, vvv) +INSN_LASX(xvmin_b, vvv) +INSN_LASX(xvmin_h, vvv) +INSN_LASX(xvmin_w, vvv) +INSN_LASX(xvmin_d, vvv) +INSN_LASX(xvmax_bu, vvv) +INSN_LASX(xvmax_hu, vvv) +INSN_LASX(xvmax_wu, vvv) +INSN_LASX(xvmax_du, vvv) +INSN_LASX(xvmin_bu, vvv) +INSN_LASX(xvmin_hu, vvv) +INSN_LASX(xvmin_wu, vvv) +INSN_LASX(xvmin_du, vvv) + +INSN_LASX(xvmaxi_b, vv_i) +INSN_LASX(xvmaxi_h, vv_i) +INSN_LASX(xvmaxi_w, vv_i) +INSN_LASX(xvmaxi_d, vv_i) +INSN_LASX(xvmini_b, vv_i) +INSN_LASX(xvmini_h, vv_i) +INSN_LASX(xvmini_w, vv_i) +INSN_LASX(xvmini_d, vv_i) +INSN_LASX(xvmaxi_bu, vv_i) +INSN_LASX(xvmaxi_hu, vv_i) +INSN_LASX(xvmaxi_wu, vv_i) +INSN_LASX(xvmaxi_du, vv_i) +INSN_LASX(xvmini_bu, vv_i) +INSN_LASX(xvmini_hu, vv_i) +INSN_LASX(xvmini_wu, vv_i) +INSN_LASX(xvmini_du, vv_i) + +INSN_LASX(xvmul_b, vvv) +INSN_LASX(xvmul_h, vvv) +INSN_LASX(xvmul_w, vvv) +INSN_LASX(xvmul_d, vvv) +INSN_LASX(xvmuh_b, vvv) +INSN_LASX(xvmuh_h, vvv) +INSN_LASX(xvmuh_w, vvv) +INSN_LASX(xvmuh_d, vvv) +INSN_LASX(xvmuh_bu, vvv) +INSN_LASX(xvmuh_hu, vvv) +INSN_LASX(xvmuh_wu, vvv) +INSN_LASX(xvmuh_du, vvv) + +INSN_LASX(xvmulwev_h_b, vvv) +INSN_LASX(xvmulwev_w_h, vvv) +INSN_LASX(xvmulwev_d_w, vvv) +INSN_LASX(xvmulwev_q_d, vvv) +INSN_LASX(xvmulwod_h_b, vvv) +INSN_LASX(xvmulwod_w_h, vvv) +INSN_LASX(xvmulwod_d_w, vvv) +INSN_LASX(xvmulwod_q_d, vvv) +INSN_LASX(xvmulwev_h_bu, vvv) +INSN_LASX(xvmulwev_w_hu, vvv) +INSN_LASX(xvmulwev_d_wu, vvv) +INSN_LASX(xvmulwev_q_du, vvv) +INSN_LASX(xvmulwod_h_bu, vvv) +INSN_LASX(xvmulwod_w_hu, vvv) +INSN_LASX(xvmulwod_d_wu, vvv) +INSN_LASX(xvmulwod_q_du, vvv) +INSN_LASX(xvmulwev_h_bu_b, vvv) +INSN_LASX(xvmulwev_w_hu_h, vvv) +INSN_LASX(xvmulwev_d_wu_w, vvv) +INSN_LASX(xvmulwev_q_du_d, vvv) +INSN_LASX(xvmulwod_h_bu_b, vvv) +INSN_LASX(xvmulwod_w_hu_h, vvv) +INSN_LASX(xvmulwod_d_wu_w, vvv) +INSN_LASX(xvmulwod_q_du_d, vvv) + +INSN_LASX(xvmadd_b, vvv) +INSN_LASX(xvmadd_h, vvv) +INSN_LASX(xvmadd_w, vvv) +INSN_LASX(xvmadd_d, vvv) +INSN_LASX(xvmsub_b, vvv) +INSN_LASX(xvmsub_h, vvv) +INSN_LASX(xvmsub_w, vvv) +INSN_LASX(xvmsub_d, vvv) + +INSN_LASX(xvmaddwev_h_b, vvv) +INSN_LASX(xvmaddwev_w_h, vvv) +INSN_LASX(xvmaddwev_d_w, vvv) +INSN_LASX(xvmaddwev_q_d, vvv) +INSN_LASX(xvmaddwod_h_b, vvv) +INSN_LASX(xvmaddwod_w_h, vvv) +INSN_LASX(xvmaddwod_d_w, vvv) +INSN_LASX(xvmaddwod_q_d, vvv) +INSN_LASX(xvmaddwev_h_bu, vvv) +INSN_LASX(xvmaddwev_w_hu, vvv) +INSN_LASX(xvmaddwev_d_wu, vvv) +INSN_LASX(xvmaddwev_q_du, vvv) +INSN_LASX(xvmaddwod_h_bu, vvv) +INSN_LASX(xvmaddwod_w_hu, vvv) +INSN_LASX(xvmaddwod_d_wu, vvv) +INSN_LASX(xvmaddwod_q_du, vvv) +INSN_LASX(xvmaddwev_h_bu_b, vvv) +INSN_LASX(xvmaddwev_w_hu_h, vvv) +INSN_LASX(xvmaddwev_d_wu_w, vvv) +INSN_LASX(xvmaddwev_q_du_d, vvv) +INSN_LASX(xvmaddwod_h_bu_b, vvv) +INSN_LASX(xvmaddwod_w_hu_h, vvv) +INSN_LASX(xvmaddwod_d_wu_w, vvv) +INSN_LASX(xvmaddwod_q_du_d, vvv) + +INSN_LASX(xvdiv_b, vvv) +INSN_LASX(xvdiv_h, vvv) +INSN_LASX(xvdiv_w, vvv) +INSN_LASX(xvdiv_d, vvv) +INSN_LASX(xvdiv_bu, vvv) +INSN_LASX(xvdiv_hu, vvv) +INSN_LASX(xvdiv_wu, vvv) +INSN_LASX(xvdiv_du, vvv) +INSN_LASX(xvmod_b, vvv) +INSN_LASX(xvmod_h, vvv) +INSN_LASX(xvmod_w, vvv) +INSN_LASX(xvmod_d, vvv) +INSN_LASX(xvmod_bu, vvv) +INSN_LASX(xvmod_hu, vvv) +INSN_LASX(xvmod_wu, vvv) +INSN_LASX(xvmod_du, vvv) + +INSN_LASX(xvsat_b, vv_i) +INSN_LASX(xvsat_h, vv_i) +INSN_LASX(xvsat_w, vv_i) +INSN_LASX(xvsat_d, vv_i) +INSN_LASX(xvsat_bu, vv_i) +INSN_LASX(xvsat_hu, vv_i) +INSN_LASX(xvsat_wu, vv_i) +INSN_LASX(xvsat_du, vv_i) + +INSN_LASX(xvexth_h_b, vv) +INSN_LASX(xvexth_w_h, vv) +INSN_LASX(xvexth_d_w, vv) +INSN_LASX(xvexth_q_d, vv) +INSN_LASX(xvexth_hu_bu, vv) +INSN_LASX(xvexth_wu_hu, vv) +INSN_LASX(xvexth_du_wu, vv) +INSN_LASX(xvexth_qu_du, vv) + +INSN_LASX(vext2xv_h_b, vv) +INSN_LASX(vext2xv_w_b, vv) +INSN_LASX(vext2xv_d_b, vv) +INSN_LASX(vext2xv_w_h, vv) +INSN_LASX(vext2xv_d_h, vv) +INSN_LASX(vext2xv_d_w, vv) +INSN_LASX(vext2xv_hu_bu, vv) +INSN_LASX(vext2xv_wu_bu, vv) +INSN_LASX(vext2xv_du_bu, vv) +INSN_LASX(vext2xv_wu_hu, vv) +INSN_LASX(vext2xv_du_hu, vv) +INSN_LASX(vext2xv_du_wu, vv) + +INSN_LASX(xvsigncov_b, vvv) +INSN_LASX(xvsigncov_h, vvv) +INSN_LASX(xvsigncov_w, vvv) +INSN_LASX(xvsigncov_d, vvv) + +INSN_LASX(xvmskltz_b, vv) +INSN_LASX(xvmskltz_h, vv) +INSN_LASX(xvmskltz_w, vv) +INSN_LASX(xvmskltz_d, vv) +INSN_LASX(xvmskgez_b, vv) +INSN_LASX(xvmsknz_b, vv) + +INSN_LASX(xvldi, v_i) + +INSN_LASX(xvand_v, vvv) +INSN_LASX(xvor_v, vvv) +INSN_LASX(xvxor_v, vvv) +INSN_LASX(xvnor_v, vvv) +INSN_LASX(xvandn_v, vvv) +INSN_LASX(xvorn_v, vvv) + +INSN_LASX(xvandi_b, vv_i) +INSN_LASX(xvori_b, vv_i) +INSN_LASX(xvxori_b, vv_i) +INSN_LASX(xvnori_b, vv_i) + +INSN_LASX(xvsll_b, vvv) +INSN_LASX(xvsll_h, vvv) +INSN_LASX(xvsll_w, vvv) +INSN_LASX(xvsll_d, vvv) +INSN_LASX(xvslli_b, vv_i) +INSN_LASX(xvslli_h, vv_i) +INSN_LASX(xvslli_w, vv_i) +INSN_LASX(xvslli_d, vv_i) + +INSN_LASX(xvsrl_b, vvv) +INSN_LASX(xvsrl_h, vvv) +INSN_LASX(xvsrl_w, vvv) +INSN_LASX(xvsrl_d, vvv) +INSN_LASX(xvsrli_b, vv_i) +INSN_LASX(xvsrli_h, vv_i) +INSN_LASX(xvsrli_w, vv_i) +INSN_LASX(xvsrli_d, vv_i) + +INSN_LASX(xvsra_b, vvv) +INSN_LASX(xvsra_h, vvv) +INSN_LASX(xvsra_w, vvv) +INSN_LASX(xvsra_d, vvv) +INSN_LASX(xvsrai_b, vv_i) +INSN_LASX(xvsrai_h, vv_i) +INSN_LASX(xvsrai_w, vv_i) +INSN_LASX(xvsrai_d, vv_i) + +INSN_LASX(xvrotr_b, vvv) +INSN_LASX(xvrotr_h, vvv) +INSN_LASX(xvrotr_w, vvv) +INSN_LASX(xvrotr_d, vvv) +INSN_LASX(xvrotri_b, vv_i) +INSN_LASX(xvrotri_h, vv_i) +INSN_LASX(xvrotri_w, vv_i) +INSN_LASX(xvrotri_d, vv_i) + +INSN_LASX(xvsllwil_h_b, vv_i) +INSN_LASX(xvsllwil_w_h, vv_i) +INSN_LASX(xvsllwil_d_w, vv_i) +INSN_LASX(xvextl_q_d, vv) +INSN_LASX(xvsllwil_hu_bu, vv_i) +INSN_LASX(xvsllwil_wu_hu, vv_i) +INSN_LASX(xvsllwil_du_wu, vv_i) +INSN_LASX(xvextl_qu_du, vv) + +INSN_LASX(xvsrlr_b, vvv) +INSN_LASX(xvsrlr_h, vvv) +INSN_LASX(xvsrlr_w, vvv) +INSN_LASX(xvsrlr_d, vvv) +INSN_LASX(xvsrlri_b, vv_i) +INSN_LASX(xvsrlri_h, vv_i) +INSN_LASX(xvsrlri_w, vv_i) +INSN_LASX(xvsrlri_d, vv_i) + +INSN_LASX(xvsrar_b, vvv) +INSN_LASX(xvsrar_h, vvv) +INSN_LASX(xvsrar_w, vvv) +INSN_LASX(xvsrar_d, vvv) +INSN_LASX(xvsrari_b, vv_i) +INSN_LASX(xvsrari_h, vv_i) +INSN_LASX(xvsrari_w, vv_i) +INSN_LASX(xvsrari_d, vv_i) + +INSN_LASX(xvsrln_b_h, vvv) +INSN_LASX(xvsrln_h_w, vvv) +INSN_LASX(xvsrln_w_d, vvv) +INSN_LASX(xvsran_b_h, vvv) +INSN_LASX(xvsran_h_w, vvv) +INSN_LASX(xvsran_w_d, vvv) + +INSN_LASX(xvsrlni_b_h, vv_i) +INSN_LASX(xvsrlni_h_w, vv_i) +INSN_LASX(xvsrlni_w_d, vv_i) +INSN_LASX(xvsrlni_d_q, vv_i) +INSN_LASX(xvsrani_b_h, vv_i) +INSN_LASX(xvsrani_h_w, vv_i) +INSN_LASX(xvsrani_w_d, vv_i) +INSN_LASX(xvsrani_d_q, vv_i) + +INSN_LASX(xvsrlrn_b_h, vvv) +INSN_LASX(xvsrlrn_h_w, vvv) +INSN_LASX(xvsrlrn_w_d, vvv) +INSN_LASX(xvsrarn_b_h, vvv) +INSN_LASX(xvsrarn_h_w, vvv) +INSN_LASX(xvsrarn_w_d, vvv) + +INSN_LASX(xvsrlrni_b_h, vv_i) +INSN_LASX(xvsrlrni_h_w, vv_i) +INSN_LASX(xvsrlrni_w_d, vv_i) +INSN_LASX(xvsrlrni_d_q, vv_i) +INSN_LASX(xvsrarni_b_h, vv_i) +INSN_LASX(xvsrarni_h_w, vv_i) +INSN_LASX(xvsrarni_w_d, vv_i) +INSN_LASX(xvsrarni_d_q, vv_i) + +INSN_LASX(xvssrln_b_h, vvv) +INSN_LASX(xvssrln_h_w, vvv) +INSN_LASX(xvssrln_w_d, vvv) +INSN_LASX(xvssran_b_h, vvv) +INSN_LASX(xvssran_h_w, vvv) +INSN_LASX(xvssran_w_d, vvv) +INSN_LASX(xvssrln_bu_h, vvv) +INSN_LASX(xvssrln_hu_w, vvv) +INSN_LASX(xvssrln_wu_d, vvv) +INSN_LASX(xvssran_bu_h, vvv) +INSN_LASX(xvssran_hu_w, vvv) +INSN_LASX(xvssran_wu_d, vvv) + +INSN_LASX(xvssrlni_b_h, vv_i) +INSN_LASX(xvssrlni_h_w, vv_i) +INSN_LASX(xvssrlni_w_d, vv_i) +INSN_LASX(xvssrlni_d_q, vv_i) +INSN_LASX(xvssrani_b_h, vv_i) +INSN_LASX(xvssrani_h_w, vv_i) +INSN_LASX(xvssrani_w_d, vv_i) +INSN_LASX(xvssrani_d_q, vv_i) +INSN_LASX(xvssrlni_bu_h, vv_i) +INSN_LASX(xvssrlni_hu_w, vv_i) +INSN_LASX(xvssrlni_wu_d, vv_i) +INSN_LASX(xvssrlni_du_q, vv_i) +INSN_LASX(xvssrani_bu_h, vv_i) +INSN_LASX(xvssrani_hu_w, vv_i) +INSN_LASX(xvssrani_wu_d, vv_i) +INSN_LASX(xvssrani_du_q, vv_i) + +INSN_LASX(xvssrlrn_b_h, vvv) +INSN_LASX(xvssrlrn_h_w, vvv) +INSN_LASX(xvssrlrn_w_d, vvv) +INSN_LASX(xvssrarn_b_h, vvv) +INSN_LASX(xvssrarn_h_w, vvv) +INSN_LASX(xvssrarn_w_d, vvv) +INSN_LASX(xvssrlrn_bu_h, vvv) +INSN_LASX(xvssrlrn_hu_w, vvv) +INSN_LASX(xvssrlrn_wu_d, vvv) +INSN_LASX(xvssrarn_bu_h, vvv) +INSN_LASX(xvssrarn_hu_w, vvv) +INSN_LASX(xvssrarn_wu_d, vvv) + +INSN_LASX(xvssrlrni_b_h, vv_i) +INSN_LASX(xvssrlrni_h_w, vv_i) +INSN_LASX(xvssrlrni_w_d, vv_i) +INSN_LASX(xvssrlrni_d_q, vv_i) +INSN_LASX(xvssrlrni_bu_h, vv_i) +INSN_LASX(xvssrlrni_hu_w, vv_i) +INSN_LASX(xvssrlrni_wu_d, vv_i) +INSN_LASX(xvssrlrni_du_q, vv_i) +INSN_LASX(xvssrarni_b_h, vv_i) +INSN_LASX(xvssrarni_h_w, vv_i) +INSN_LASX(xvssrarni_w_d, vv_i) +INSN_LASX(xvssrarni_d_q, vv_i) +INSN_LASX(xvssrarni_bu_h, vv_i) +INSN_LASX(xvssrarni_hu_w, vv_i) +INSN_LASX(xvssrarni_wu_d, vv_i) +INSN_LASX(xvssrarni_du_q, vv_i) + +INSN_LASX(xvclo_b, vv) +INSN_LASX(xvclo_h, vv) +INSN_LASX(xvclo_w, vv) +INSN_LASX(xvclo_d, vv) +INSN_LASX(xvclz_b, vv) +INSN_LASX(xvclz_h, vv) +INSN_LASX(xvclz_w, vv) +INSN_LASX(xvclz_d, vv) + +INSN_LASX(xvpcnt_b, vv) +INSN_LASX(xvpcnt_h, vv) +INSN_LASX(xvpcnt_w, vv) +INSN_LASX(xvpcnt_d, vv) + +INSN_LASX(xvbitclr_b, vvv) +INSN_LASX(xvbitclr_h, vvv) +INSN_LASX(xvbitclr_w, vvv) +INSN_LASX(xvbitclr_d, vvv) +INSN_LASX(xvbitclri_b, vv_i) +INSN_LASX(xvbitclri_h, vv_i) +INSN_LASX(xvbitclri_w, vv_i) +INSN_LASX(xvbitclri_d, vv_i) +INSN_LASX(xvbitset_b, vvv) +INSN_LASX(xvbitset_h, vvv) +INSN_LASX(xvbitset_w, vvv) +INSN_LASX(xvbitset_d, vvv) +INSN_LASX(xvbitseti_b, vv_i) +INSN_LASX(xvbitseti_h, vv_i) +INSN_LASX(xvbitseti_w, vv_i) +INSN_LASX(xvbitseti_d, vv_i) +INSN_LASX(xvbitrev_b, vvv) +INSN_LASX(xvbitrev_h, vvv) +INSN_LASX(xvbitrev_w, vvv) +INSN_LASX(xvbitrev_d, vvv) +INSN_LASX(xvbitrevi_b, vv_i) +INSN_LASX(xvbitrevi_h, vv_i) +INSN_LASX(xvbitrevi_w, vv_i) +INSN_LASX(xvbitrevi_d, vv_i) + +INSN_LASX(xvfrstp_b, vvv) +INSN_LASX(xvfrstp_h, vvv) +INSN_LASX(xvfrstpi_b, vv_i) +INSN_LASX(xvfrstpi_h, vv_i) + +INSN_LASX(xvfadd_s, vvv) +INSN_LASX(xvfadd_d, vvv) +INSN_LASX(xvfsub_s, vvv) +INSN_LASX(xvfsub_d, vvv) +INSN_LASX(xvfmul_s, vvv) +INSN_LASX(xvfmul_d, vvv) +INSN_LASX(xvfdiv_s, vvv) +INSN_LASX(xvfdiv_d, vvv) + +INSN_LASX(xvfmadd_s, vvvv) +INSN_LASX(xvfmadd_d, vvvv) +INSN_LASX(xvfmsub_s, vvvv) +INSN_LASX(xvfmsub_d, vvvv) +INSN_LASX(xvfnmadd_s, vvvv) +INSN_LASX(xvfnmadd_d, vvvv) +INSN_LASX(xvfnmsub_s, vvvv) +INSN_LASX(xvfnmsub_d, vvvv) + +INSN_LASX(xvfmax_s, vvv) +INSN_LASX(xvfmax_d, vvv) +INSN_LASX(xvfmin_s, vvv) +INSN_LASX(xvfmin_d, vvv) + +INSN_LASX(xvfmaxa_s, vvv) +INSN_LASX(xvfmaxa_d, vvv) +INSN_LASX(xvfmina_s, vvv) +INSN_LASX(xvfmina_d, vvv) + +INSN_LASX(xvflogb_s, vv) +INSN_LASX(xvflogb_d, vv) + +INSN_LASX(xvfclass_s, vv) +INSN_LASX(xvfclass_d, vv) + +INSN_LASX(xvfsqrt_s, vv) +INSN_LASX(xvfsqrt_d, vv) +INSN_LASX(xvfrecip_s, vv) +INSN_LASX(xvfrecip_d, vv) +INSN_LASX(xvfrsqrt_s, vv) +INSN_LASX(xvfrsqrt_d, vv) + +INSN_LASX(xvfcvtl_s_h, vv) +INSN_LASX(xvfcvth_s_h, vv) +INSN_LASX(xvfcvtl_d_s, vv) +INSN_LASX(xvfcvth_d_s, vv) +INSN_LASX(xvfcvt_h_s, vvv) +INSN_LASX(xvfcvt_s_d, vvv) + +INSN_LASX(xvfrint_s, vv) +INSN_LASX(xvfrint_d, vv) +INSN_LASX(xvfrintrm_s, vv) +INSN_LASX(xvfrintrm_d, vv) +INSN_LASX(xvfrintrp_s, vv) +INSN_LASX(xvfrintrp_d, vv) +INSN_LASX(xvfrintrz_s, vv) +INSN_LASX(xvfrintrz_d, vv) +INSN_LASX(xvfrintrne_s, vv) +INSN_LASX(xvfrintrne_d, vv) + +INSN_LASX(xvftint_w_s, vv) +INSN_LASX(xvftint_l_d, vv) +INSN_LASX(xvftintrm_w_s, vv) +INSN_LASX(xvftintrm_l_d, vv) +INSN_LASX(xvftintrp_w_s, vv) +INSN_LASX(xvftintrp_l_d, vv) +INSN_LASX(xvftintrz_w_s, vv) +INSN_LASX(xvftintrz_l_d, vv) +INSN_LASX(xvftintrne_w_s, vv) +INSN_LASX(xvftintrne_l_d, vv) +INSN_LASX(xvftint_wu_s, vv) +INSN_LASX(xvftint_lu_d, vv) +INSN_LASX(xvftintrz_wu_s, vv) +INSN_LASX(xvftintrz_lu_d, vv) +INSN_LASX(xvftint_w_d, vvv) +INSN_LASX(xvftintrm_w_d, vvv) +INSN_LASX(xvftintrp_w_d, vvv) +INSN_LASX(xvftintrz_w_d, vvv) +INSN_LASX(xvftintrne_w_d, vvv) +INSN_LASX(xvftintl_l_s, vv) +INSN_LASX(xvftinth_l_s, vv) +INSN_LASX(xvftintrml_l_s, vv) +INSN_LASX(xvftintrmh_l_s, vv) +INSN_LASX(xvftintrpl_l_s, vv) +INSN_LASX(xvftintrph_l_s, vv) +INSN_LASX(xvftintrzl_l_s, vv) +INSN_LASX(xvftintrzh_l_s, vv) +INSN_LASX(xvftintrnel_l_s, vv) +INSN_LASX(xvftintrneh_l_s, vv) + +INSN_LASX(xvffint_s_w, vv) +INSN_LASX(xvffint_s_wu, vv) +INSN_LASX(xvffint_d_l, vv) +INSN_LASX(xvffint_d_lu, vv) +INSN_LASX(xvffintl_d_w, vv) +INSN_LASX(xvffinth_d_w, vv) +INSN_LASX(xvffint_s_l, vvv) + +INSN_LASX(xvseq_b, vvv) +INSN_LASX(xvseq_h, vvv) +INSN_LASX(xvseq_w, vvv) +INSN_LASX(xvseq_d, vvv) +INSN_LASX(xvseqi_b, vv_i) +INSN_LASX(xvseqi_h, vv_i) +INSN_LASX(xvseqi_w, vv_i) +INSN_LASX(xvseqi_d, vv_i) + +INSN_LASX(xvsle_b, vvv) +INSN_LASX(xvsle_h, vvv) +INSN_LASX(xvsle_w, vvv) +INSN_LASX(xvsle_d, vvv) +INSN_LASX(xvslei_b, vv_i) +INSN_LASX(xvslei_h, vv_i) +INSN_LASX(xvslei_w, vv_i) +INSN_LASX(xvslei_d, vv_i) +INSN_LASX(xvsle_bu, vvv) +INSN_LASX(xvsle_hu, vvv) +INSN_LASX(xvsle_wu, vvv) +INSN_LASX(xvsle_du, vvv) +INSN_LASX(xvslei_bu, vv_i) +INSN_LASX(xvslei_hu, vv_i) +INSN_LASX(xvslei_wu, vv_i) +INSN_LASX(xvslei_du, vv_i) + +INSN_LASX(xvslt_b, vvv) +INSN_LASX(xvslt_h, vvv) +INSN_LASX(xvslt_w, vvv) +INSN_LASX(xvslt_d, vvv) +INSN_LASX(xvslti_b, vv_i) +INSN_LASX(xvslti_h, vv_i) +INSN_LASX(xvslti_w, vv_i) +INSN_LASX(xvslti_d, vv_i) +INSN_LASX(xvslt_bu, vvv) +INSN_LASX(xvslt_hu, vvv) +INSN_LASX(xvslt_wu, vvv) +INSN_LASX(xvslt_du, vvv) +INSN_LASX(xvslti_bu, vv_i) +INSN_LASX(xvslti_hu, vv_i) +INSN_LASX(xvslti_wu, vv_i) +INSN_LASX(xvslti_du, vv_i) + +#define output_xvfcmp(C, PREFIX, SUFFIX) \ +{ \ + (C)->info->fprintf_func((C)->info->stream, "%08x %s%s\tx%d, x%d, x%d", \ + (C)->insn, PREFIX, SUFFIX, a->vd, \ + a->vj, a->vk); \ +} +static bool output_xxx_fcond(DisasContext *ctx, arg_vvv_fcond * a, + const char *suffix) +{ + bool ret = true; + switch (a->fcond) { + case 0x0: + output_xvfcmp(ctx, "xvfcmp_caf_", suffix); + break; + case 0x1: + output_xvfcmp(ctx, "xvfcmp_saf_", suffix); + break; + case 0x2: + output_xvfcmp(ctx, "xvfcmp_clt_", suffix); + break; + case 0x3: + output_xvfcmp(ctx, "xvfcmp_slt_", suffix); + break; + case 0x4: + output_xvfcmp(ctx, "xvfcmp_ceq_", suffix); + break; + case 0x5: + output_xvfcmp(ctx, "xvfcmp_seq_", suffix); + break; + case 0x6: + output_xvfcmp(ctx, "xvfcmp_cle_", suffix); + break; + case 0x7: + output_xvfcmp(ctx, "xvfcmp_sle_", suffix); + break; + case 0x8: + output_xvfcmp(ctx, "xvfcmp_cun_", suffix); + break; + case 0x9: + output_xvfcmp(ctx, "xvfcmp_sun_", suffix); + break; + case 0xA: + output_xvfcmp(ctx, "xvfcmp_cult_", suffix); + break; + case 0xB: + output_xvfcmp(ctx, "xvfcmp_sult_", suffix); + break; + case 0xC: + output_xvfcmp(ctx, "xvfcmp_cueq_", suffix); + break; + case 0xD: + output_xvfcmp(ctx, "xvfcmp_sueq_", suffix); + break; + case 0xE: + output_xvfcmp(ctx, "xvfcmp_cule_", suffix); + break; + case 0xF: + output_xvfcmp(ctx, "xvfcmp_sule_", suffix); + break; + case 0x10: + output_xvfcmp(ctx, "xvfcmp_cne_", suffix); + break; + case 0x11: + output_xvfcmp(ctx, "xvfcmp_sne_", suffix); + break; + case 0x14: + output_xvfcmp(ctx, "xvfcmp_cor_", suffix); + break; + case 0x15: + output_xvfcmp(ctx, "xvfcmp_sor_", suffix); + break; + case 0x18: + output_xvfcmp(ctx, "xvfcmp_cune_", suffix); + break; + case 0x19: + output_xvfcmp(ctx, "xvfcmp_sune_", suffix); + break; + default: + ret = false; + } + return ret; +} + +#define LASX_FCMP_INSN(suffix) \ +static bool trans_xvfcmp_cond_##suffix(DisasContext *ctx, \ + arg_vvv_fcond * a) \ +{ \ + return output_xxx_fcond(ctx, a, #suffix); \ +} + +LASX_FCMP_INSN(s) +LASX_FCMP_INSN(d) + +INSN_LASX(xvbitsel_v, vvvv) +INSN_LASX(xvbitseli_b, vv_i) + +INSN_LASX(xvseteqz_v, cv) +INSN_LASX(xvsetnez_v, cv) +INSN_LASX(xvsetanyeqz_b, cv) +INSN_LASX(xvsetanyeqz_h, cv) +INSN_LASX(xvsetanyeqz_w, cv) +INSN_LASX(xvsetanyeqz_d, cv) +INSN_LASX(xvsetallnez_b, cv) +INSN_LASX(xvsetallnez_h, cv) +INSN_LASX(xvsetallnez_w, cv) +INSN_LASX(xvsetallnez_d, cv) + +INSN_LASX(xvinsgr2vr_w, vr_i) +INSN_LASX(xvinsgr2vr_d, vr_i) +INSN_LASX(xvpickve2gr_w, rv_i) +INSN_LASX(xvpickve2gr_d, rv_i) +INSN_LASX(xvpickve2gr_wu, rv_i) +INSN_LASX(xvpickve2gr_du, rv_i) + +INSN_LASX(xvreplgr2vr_b, vr) +INSN_LASX(xvreplgr2vr_h, vr) +INSN_LASX(xvreplgr2vr_w, vr) +INSN_LASX(xvreplgr2vr_d, vr) + +INSN_LASX(xvreplve_b, vvr) +INSN_LASX(xvreplve_h, vvr) +INSN_LASX(xvreplve_w, vvr) +INSN_LASX(xvreplve_d, vvr) +INSN_LASX(xvrepl128vei_b, vv_i) +INSN_LASX(xvrepl128vei_h, vv_i) +INSN_LASX(xvrepl128vei_w, vv_i) +INSN_LASX(xvrepl128vei_d, vv_i) + +INSN_LASX(xvreplve0_b, vv) +INSN_LASX(xvreplve0_h, vv) +INSN_LASX(xvreplve0_w, vv) +INSN_LASX(xvreplve0_d, vv) +INSN_LASX(xvreplve0_q, vv) + +INSN_LASX(xvinsve0_w, vv_i) +INSN_LASX(xvinsve0_d, vv_i) + +INSN_LASX(xvpickve_w, vv_i) +INSN_LASX(xvpickve_d, vv_i) + +INSN_LASX(xvbsll_v, vv_i) +INSN_LASX(xvbsrl_v, vv_i) + +INSN_LASX(xvpackev_b, vvv) +INSN_LASX(xvpackev_h, vvv) +INSN_LASX(xvpackev_w, vvv) +INSN_LASX(xvpackev_d, vvv) +INSN_LASX(xvpackod_b, vvv) +INSN_LASX(xvpackod_h, vvv) +INSN_LASX(xvpackod_w, vvv) +INSN_LASX(xvpackod_d, vvv) + +INSN_LASX(xvpickev_b, vvv) +INSN_LASX(xvpickev_h, vvv) +INSN_LASX(xvpickev_w, vvv) +INSN_LASX(xvpickev_d, vvv) +INSN_LASX(xvpickod_b, vvv) +INSN_LASX(xvpickod_h, vvv) +INSN_LASX(xvpickod_w, vvv) +INSN_LASX(xvpickod_d, vvv) + +INSN_LASX(xvilvl_b, vvv) +INSN_LASX(xvilvl_h, vvv) +INSN_LASX(xvilvl_w, vvv) +INSN_LASX(xvilvl_d, vvv) +INSN_LASX(xvilvh_b, vvv) +INSN_LASX(xvilvh_h, vvv) +INSN_LASX(xvilvh_w, vvv) +INSN_LASX(xvilvh_d, vvv) + +INSN_LASX(xvshuf_b, vvvv) +INSN_LASX(xvshuf_h, vvv) +INSN_LASX(xvshuf_w, vvv) +INSN_LASX(xvshuf_d, vvv) + +INSN_LASX(xvperm_w, vvv) + +INSN_LASX(xvshuf4i_b, vv_i) +INSN_LASX(xvshuf4i_h, vv_i) +INSN_LASX(xvshuf4i_w, vv_i) +INSN_LASX(xvshuf4i_d, vv_i) + +INSN_LASX(xvpermi_w, vv_i) +INSN_LASX(xvpermi_d, vv_i) +INSN_LASX(xvpermi_q, vv_i) + +INSN_LASX(xvextrins_d, vv_i) +INSN_LASX(xvextrins_w, vv_i) +INSN_LASX(xvextrins_h, vv_i) +INSN_LASX(xvextrins_b, vv_i) + +INSN_LASX(xvld, vr_i) +INSN_LASX(xvst, vr_i) +INSN_LASX(xvldx, vrr) +INSN_LASX(xvstx, vrr) + +INSN_LASX(xvldrepl_d, vr_i) +INSN_LASX(xvldrepl_w, vr_i) +INSN_LASX(xvldrepl_h, vr_i) +INSN_LASX(xvldrepl_b, vr_i) +INSN_LASX(xvstelm_d, vr_ii) +INSN_LASX(xvstelm_w, vr_ii) +INSN_LASX(xvstelm_h, vr_ii) +INSN_LASX(xvstelm_b, vr_ii) diff --git a/target/loongarch/gdbstub.c b/target/loongarch/gdbstub.c index b09804b62f..5fc2f19e96 100644 --- a/target/loongarch/gdbstub.c +++ b/target/loongarch/gdbstub.c @@ -11,6 +11,7 @@ #include "internals.h" #include "exec/gdbstub.h" #include "gdbstub/helpers.h" +#include "vec.h" uint64_t read_fcc(CPULoongArchState *env) { diff --git a/target/loongarch/helper.h b/target/loongarch/helper.h index ffb1e0b0bf..b3b64a0215 100644 --- a/target/loongarch/helper.h +++ b/target/loongarch/helper.h @@ -133,22 +133,22 @@ DEF_HELPER_1(idle, void, env) #endif /* LoongArch LSX */ -DEF_HELPER_4(vhaddw_h_b, void, env, i32, i32, i32) -DEF_HELPER_4(vhaddw_w_h, void, env, i32, i32, i32) -DEF_HELPER_4(vhaddw_d_w, void, env, i32, i32, i32) -DEF_HELPER_4(vhaddw_q_d, void, env, i32, i32, i32) -DEF_HELPER_4(vhaddw_hu_bu, void, env, i32, i32, i32) -DEF_HELPER_4(vhaddw_wu_hu, void, env, i32, i32, i32) -DEF_HELPER_4(vhaddw_du_wu, void, env, i32, i32, i32) -DEF_HELPER_4(vhaddw_qu_du, void, env, i32, i32, i32) -DEF_HELPER_4(vhsubw_h_b, void, env, i32, i32, i32) -DEF_HELPER_4(vhsubw_w_h, void, env, i32, i32, i32) -DEF_HELPER_4(vhsubw_d_w, void, env, i32, i32, i32) -DEF_HELPER_4(vhsubw_q_d, void, env, i32, i32, i32) -DEF_HELPER_4(vhsubw_hu_bu, void, env, i32, i32, i32) -DEF_HELPER_4(vhsubw_wu_hu, void, env, i32, i32, i32) -DEF_HELPER_4(vhsubw_du_wu, void, env, i32, i32, i32) -DEF_HELPER_4(vhsubw_qu_du, void, env, i32, i32, i32) +DEF_HELPER_FLAGS_4(vhaddw_h_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhaddw_w_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhaddw_d_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhaddw_q_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhaddw_hu_bu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhaddw_wu_hu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhaddw_du_wu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhaddw_qu_du, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhsubw_h_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhsubw_w_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhsubw_d_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhsubw_q_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhsubw_hu_bu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhsubw_wu_hu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhsubw_du_wu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vhsubw_qu_du, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vaddwev_h_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vaddwev_w_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) @@ -305,22 +305,22 @@ DEF_HELPER_FLAGS_4(vmaddwod_h_bu_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vmaddwod_w_hu_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vmaddwod_d_wu_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) -DEF_HELPER_4(vdiv_b, void, env, i32, i32, i32) -DEF_HELPER_4(vdiv_h, void, env, i32, i32, i32) -DEF_HELPER_4(vdiv_w, void, env, i32, i32, i32) -DEF_HELPER_4(vdiv_d, void, env, i32, i32, i32) -DEF_HELPER_4(vdiv_bu, void, env, i32, i32, i32) -DEF_HELPER_4(vdiv_hu, void, env, i32, i32, i32) -DEF_HELPER_4(vdiv_wu, void, env, i32, i32, i32) -DEF_HELPER_4(vdiv_du, void, env, i32, i32, i32) -DEF_HELPER_4(vmod_b, void, env, i32, i32, i32) -DEF_HELPER_4(vmod_h, void, env, i32, i32, i32) -DEF_HELPER_4(vmod_w, void, env, i32, i32, i32) -DEF_HELPER_4(vmod_d, void, env, i32, i32, i32) -DEF_HELPER_4(vmod_bu, void, env, i32, i32, i32) -DEF_HELPER_4(vmod_hu, void, env, i32, i32, i32) -DEF_HELPER_4(vmod_wu, void, env, i32, i32, i32) -DEF_HELPER_4(vmod_du, void, env, i32, i32, i32) +DEF_HELPER_FLAGS_4(vdiv_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vdiv_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vdiv_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vdiv_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vdiv_bu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vdiv_hu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vdiv_wu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vdiv_du, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vmod_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vmod_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vmod_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vmod_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vmod_bu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vmod_hu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vmod_wu, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vmod_du, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vsat_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) DEF_HELPER_FLAGS_4(vsat_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) @@ -331,161 +331,174 @@ DEF_HELPER_FLAGS_4(vsat_hu, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) DEF_HELPER_FLAGS_4(vsat_wu, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) DEF_HELPER_FLAGS_4(vsat_du, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) -DEF_HELPER_3(vexth_h_b, void, env, i32, i32) -DEF_HELPER_3(vexth_w_h, void, env, i32, i32) -DEF_HELPER_3(vexth_d_w, void, env, i32, i32) -DEF_HELPER_3(vexth_q_d, void, env, i32, i32) -DEF_HELPER_3(vexth_hu_bu, void, env, i32, i32) -DEF_HELPER_3(vexth_wu_hu, void, env, i32, i32) -DEF_HELPER_3(vexth_du_wu, void, env, i32, i32) -DEF_HELPER_3(vexth_qu_du, void, env, i32, i32) +DEF_HELPER_FLAGS_3(vexth_h_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vexth_w_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vexth_d_w, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vexth_q_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vexth_hu_bu, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vexth_wu_hu, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vexth_du_wu, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vexth_qu_du, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(vext2xv_h_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_w_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_d_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_w_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_d_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_d_w, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_hu_bu, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_wu_bu, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_du_bu, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_wu_hu, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_du_hu, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vext2xv_du_wu, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vsigncov_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vsigncov_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vsigncov_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vsigncov_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) -DEF_HELPER_3(vmskltz_b, void, env, i32, i32) -DEF_HELPER_3(vmskltz_h, void, env, i32, i32) -DEF_HELPER_3(vmskltz_w, void, env, i32, i32) -DEF_HELPER_3(vmskltz_d, void, env, i32, i32) -DEF_HELPER_3(vmskgez_b, void, env, i32, i32) -DEF_HELPER_3(vmsknz_b, void, env, i32,i32) +DEF_HELPER_FLAGS_3(vmskltz_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vmskltz_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vmskltz_w, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vmskltz_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vmskgez_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vmsknz_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vnori_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) -DEF_HELPER_4(vsllwil_h_b, void, env, i32, i32, i32) -DEF_HELPER_4(vsllwil_w_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsllwil_d_w, void, env, i32, i32, i32) -DEF_HELPER_3(vextl_q_d, void, env, i32, i32) -DEF_HELPER_4(vsllwil_hu_bu, void, env, i32, i32, i32) -DEF_HELPER_4(vsllwil_wu_hu, void, env, i32, i32, i32) -DEF_HELPER_4(vsllwil_du_wu, void, env, i32, i32, i32) -DEF_HELPER_3(vextl_qu_du, void, env, i32, i32) - -DEF_HELPER_4(vsrlr_b, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlr_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlr_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlr_d, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlri_b, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlri_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlri_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlri_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vsrar_b, void, env, i32, i32, i32) -DEF_HELPER_4(vsrar_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrar_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrar_d, void, env, i32, i32, i32) -DEF_HELPER_4(vsrari_b, void, env, i32, i32, i32) -DEF_HELPER_4(vsrari_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrari_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrari_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vsrln_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrln_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrln_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vsran_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsran_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsran_w_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vsrlni_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlni_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlni_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlni_d_q, void, env, i32, i32, i32) -DEF_HELPER_4(vsrani_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrani_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrani_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vsrani_d_q, void, env, i32, i32, i32) - -DEF_HELPER_4(vsrlrn_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlrn_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlrn_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vsrarn_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrarn_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrarn_w_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vsrlrni_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlrni_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlrni_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vsrlrni_d_q, void, env, i32, i32, i32) -DEF_HELPER_4(vsrarni_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vsrarni_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vsrarni_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vsrarni_d_q, void, env, i32, i32, i32) - -DEF_HELPER_4(vssrln_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrln_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrln_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssran_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssran_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssran_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrln_bu_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrln_hu_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrln_wu_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssran_bu_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssran_hu_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssran_wu_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vssrlni_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlni_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlni_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlni_d_q, void, env, i32, i32, i32) -DEF_HELPER_4(vssrani_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrani_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrani_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrani_d_q, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlni_bu_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlni_hu_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlni_wu_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlni_du_q, void, env, i32, i32, i32) -DEF_HELPER_4(vssrani_bu_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrani_hu_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrani_wu_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrani_du_q, void, env, i32, i32, i32) - -DEF_HELPER_4(vssrlrn_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrn_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrn_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarn_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarn_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarn_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrn_bu_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrn_hu_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrn_wu_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarn_bu_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarn_hu_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarn_wu_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vssrlrni_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrni_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrni_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrni_d_q, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarni_b_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarni_h_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarni_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarni_d_q, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrni_bu_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrni_hu_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrni_wu_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrlrni_du_q, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarni_bu_h, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarni_hu_w, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarni_wu_d, void, env, i32, i32, i32) -DEF_HELPER_4(vssrarni_du_q, void, env, i32, i32, i32) - -DEF_HELPER_3(vclo_b, void, env, i32, i32) -DEF_HELPER_3(vclo_h, void, env, i32, i32) -DEF_HELPER_3(vclo_w, void, env, i32, i32) -DEF_HELPER_3(vclo_d, void, env, i32, i32) -DEF_HELPER_3(vclz_b, void, env, i32, i32) -DEF_HELPER_3(vclz_h, void, env, i32, i32) -DEF_HELPER_3(vclz_w, void, env, i32, i32) -DEF_HELPER_3(vclz_d, void, env, i32, i32) - -DEF_HELPER_3(vpcnt_b, void, env, i32, i32) -DEF_HELPER_3(vpcnt_h, void, env, i32, i32) -DEF_HELPER_3(vpcnt_w, void, env, i32, i32) -DEF_HELPER_3(vpcnt_d, void, env, i32, i32) +DEF_HELPER_FLAGS_4(vsllwil_h_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsllwil_w_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsllwil_d_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_3(vextl_q_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsllwil_hu_bu, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsllwil_wu_hu, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsllwil_du_wu, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_3(vextl_qu_du, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(vsrlr_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrlr_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrlr_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrlr_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrlri_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrlri_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrlri_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrlri_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(vsrar_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrar_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrar_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrari_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrari_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrari_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrari_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(vsrln_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrln_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrln_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsran_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsran_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsran_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(vsrlni_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrlni_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrlni_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrlni_d_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrani_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrani_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrani_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrani_d_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(vsrlrn_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrlrn_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrlrn_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrarn_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrarn_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vsrarn_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(vsrlrni_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrlrni_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrlrni_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrlrni_d_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrarni_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrarni_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrarni_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vsrarni_d_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(vssrln_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrln_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrln_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssran_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssran_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssran_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrln_bu_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrln_hu_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrln_wu_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssran_bu_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssran_hu_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssran_wu_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(vssrlni_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlni_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlni_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlni_d_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrani_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrani_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrani_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrani_d_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlni_bu_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlni_hu_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlni_wu_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlni_du_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrani_bu_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrani_hu_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrani_wu_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrani_du_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(vssrlrn_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrlrn_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrlrn_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrarn_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrarn_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrarn_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrlrn_bu_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrlrn_hu_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrlrn_wu_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrarn_bu_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrarn_hu_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vssrarn_wu_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(vssrlrni_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlrni_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlrni_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlrni_d_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrarni_b_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrarni_h_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrarni_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrarni_d_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlrni_bu_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlrni_hu_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlrni_wu_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrlrni_du_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrarni_bu_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrarni_hu_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrarni_wu_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vssrarni_du_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_3(vclo_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vclo_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vclo_w, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vclo_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vclz_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vclz_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vclz_w, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vclz_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) + +DEF_HELPER_FLAGS_3(vpcnt_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vpcnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vpcnt_w, TCG_CALL_NO_RWG, void, ptr, ptr, i32) +DEF_HELPER_FLAGS_3(vpcnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vbitclr_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(vbitclr_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) @@ -514,107 +527,107 @@ DEF_HELPER_FLAGS_4(vbitrevi_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) DEF_HELPER_FLAGS_4(vbitrevi_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) DEF_HELPER_FLAGS_4(vbitrevi_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) -DEF_HELPER_4(vfrstp_b, void, env, i32, i32, i32) -DEF_HELPER_4(vfrstp_h, void, env, i32, i32, i32) -DEF_HELPER_4(vfrstpi_b, void, env, i32, i32, i32) -DEF_HELPER_4(vfrstpi_h, void, env, i32, i32, i32) - -DEF_HELPER_4(vfadd_s, void, env, i32, i32, i32) -DEF_HELPER_4(vfadd_d, void, env, i32, i32, i32) -DEF_HELPER_4(vfsub_s, void, env, i32, i32, i32) -DEF_HELPER_4(vfsub_d, void, env, i32, i32, i32) -DEF_HELPER_4(vfmul_s, void, env, i32, i32, i32) -DEF_HELPER_4(vfmul_d, void, env, i32, i32, i32) -DEF_HELPER_4(vfdiv_s, void, env, i32, i32, i32) -DEF_HELPER_4(vfdiv_d, void, env, i32, i32, i32) - -DEF_HELPER_5(vfmadd_s, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfmadd_d, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfmsub_s, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfmsub_d, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfnmadd_s, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfnmadd_d, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfnmsub_s, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfnmsub_d, void, env, i32, i32, i32, i32) - -DEF_HELPER_4(vfmax_s, void, env, i32, i32, i32) -DEF_HELPER_4(vfmax_d, void, env, i32, i32, i32) -DEF_HELPER_4(vfmin_s, void, env, i32, i32, i32) -DEF_HELPER_4(vfmin_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vfmaxa_s, void, env, i32, i32, i32) -DEF_HELPER_4(vfmaxa_d, void, env, i32, i32, i32) -DEF_HELPER_4(vfmina_s, void, env, i32, i32, i32) -DEF_HELPER_4(vfmina_d, void, env, i32, i32, i32) - -DEF_HELPER_3(vflogb_s, void, env, i32, i32) -DEF_HELPER_3(vflogb_d, void, env, i32, i32) - -DEF_HELPER_3(vfclass_s, void, env, i32, i32) -DEF_HELPER_3(vfclass_d, void, env, i32, i32) - -DEF_HELPER_3(vfsqrt_s, void, env, i32, i32) -DEF_HELPER_3(vfsqrt_d, void, env, i32, i32) -DEF_HELPER_3(vfrecip_s, void, env, i32, i32) -DEF_HELPER_3(vfrecip_d, void, env, i32, i32) -DEF_HELPER_3(vfrsqrt_s, void, env, i32, i32) -DEF_HELPER_3(vfrsqrt_d, void, env, i32, i32) - -DEF_HELPER_3(vfcvtl_s_h, void, env, i32, i32) -DEF_HELPER_3(vfcvth_s_h, void, env, i32, i32) -DEF_HELPER_3(vfcvtl_d_s, void, env, i32, i32) -DEF_HELPER_3(vfcvth_d_s, void, env, i32, i32) -DEF_HELPER_4(vfcvt_h_s, void, env, i32, i32, i32) -DEF_HELPER_4(vfcvt_s_d, void, env, i32, i32, i32) - -DEF_HELPER_3(vfrintrne_s, void, env, i32, i32) -DEF_HELPER_3(vfrintrne_d, void, env, i32, i32) -DEF_HELPER_3(vfrintrz_s, void, env, i32, i32) -DEF_HELPER_3(vfrintrz_d, void, env, i32, i32) -DEF_HELPER_3(vfrintrp_s, void, env, i32, i32) -DEF_HELPER_3(vfrintrp_d, void, env, i32, i32) -DEF_HELPER_3(vfrintrm_s, void, env, i32, i32) -DEF_HELPER_3(vfrintrm_d, void, env, i32, i32) -DEF_HELPER_3(vfrint_s, void, env, i32, i32) -DEF_HELPER_3(vfrint_d, void, env, i32, i32) - -DEF_HELPER_3(vftintrne_w_s, void, env, i32, i32) -DEF_HELPER_3(vftintrne_l_d, void, env, i32, i32) -DEF_HELPER_3(vftintrz_w_s, void, env, i32, i32) -DEF_HELPER_3(vftintrz_l_d, void, env, i32, i32) -DEF_HELPER_3(vftintrp_w_s, void, env, i32, i32) -DEF_HELPER_3(vftintrp_l_d, void, env, i32, i32) -DEF_HELPER_3(vftintrm_w_s, void, env, i32, i32) -DEF_HELPER_3(vftintrm_l_d, void, env, i32, i32) -DEF_HELPER_3(vftint_w_s, void, env, i32, i32) -DEF_HELPER_3(vftint_l_d, void, env, i32, i32) -DEF_HELPER_3(vftintrz_wu_s, void, env, i32, i32) -DEF_HELPER_3(vftintrz_lu_d, void, env, i32, i32) -DEF_HELPER_3(vftint_wu_s, void, env, i32, i32) -DEF_HELPER_3(vftint_lu_d, void, env, i32, i32) -DEF_HELPER_4(vftintrne_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vftintrz_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vftintrp_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vftintrm_w_d, void, env, i32, i32, i32) -DEF_HELPER_4(vftint_w_d, void, env, i32, i32, i32) -DEF_HELPER_3(vftintrnel_l_s, void, env, i32, i32) -DEF_HELPER_3(vftintrneh_l_s, void, env, i32, i32) -DEF_HELPER_3(vftintrzl_l_s, void, env, i32, i32) -DEF_HELPER_3(vftintrzh_l_s, void, env, i32, i32) -DEF_HELPER_3(vftintrpl_l_s, void, env, i32, i32) -DEF_HELPER_3(vftintrph_l_s, void, env, i32, i32) -DEF_HELPER_3(vftintrml_l_s, void, env, i32, i32) -DEF_HELPER_3(vftintrmh_l_s, void, env, i32, i32) -DEF_HELPER_3(vftintl_l_s, void, env, i32, i32) -DEF_HELPER_3(vftinth_l_s, void, env, i32, i32) - -DEF_HELPER_3(vffint_s_w, void, env, i32, i32) -DEF_HELPER_3(vffint_d_l, void, env, i32, i32) -DEF_HELPER_3(vffint_s_wu, void, env, i32, i32) -DEF_HELPER_3(vffint_d_lu, void, env, i32, i32) -DEF_HELPER_3(vffintl_d_w, void, env, i32, i32) -DEF_HELPER_3(vffinth_d_w, void, env, i32, i32) -DEF_HELPER_4(vffint_s_l, void, env, i32, i32, i32) +DEF_HELPER_FLAGS_4(vfrstp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vfrstp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vfrstpi_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vfrstpi_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_5(vfadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfsub_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfsub_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfmul_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfmul_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfdiv_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfdiv_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_6(vfmadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_6(vfmadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_6(vfmsub_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_6(vfmsub_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_6(vfnmadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_6(vfnmadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_6(vfnmsub_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_6(vfnmsub_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_5(vfmax_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfmax_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfmin_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfmin_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_5(vfmaxa_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfmaxa_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfmina_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfmina_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_4(vflogb_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vflogb_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_4(vfclass_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfclass_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_4(vfsqrt_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfsqrt_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrecip_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrecip_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrsqrt_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrsqrt_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_4(vfcvtl_s_h, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfcvth_s_h, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfcvtl_d_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfcvth_d_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfcvt_h_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vfcvt_s_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_4(vfrintrne_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrintrne_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrintrz_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrintrz_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrintrp_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrintrp_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrintrm_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrintrm_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrint_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vfrint_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_4(vftintrne_w_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrne_l_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrz_w_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrz_l_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrp_w_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrp_l_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrm_w_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrm_l_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftint_w_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftint_l_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrz_wu_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrz_lu_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftint_wu_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftint_lu_d, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vftintrne_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vftintrz_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vftintrp_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vftintrm_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vftint_w_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrnel_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrneh_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrzl_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrzh_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrpl_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrph_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrml_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintrmh_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftintl_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vftinth_l_s, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) + +DEF_HELPER_FLAGS_4(vffint_s_w, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vffint_d_l, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vffint_s_wu, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vffint_d_lu, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vffintl_d_w, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_4(vffinth_d_w, TCG_CALL_NO_RWG, void, ptr, ptr, env, i32) +DEF_HELPER_FLAGS_5(vffint_s_l, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env, i32) DEF_HELPER_FLAGS_4(vseqi_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) DEF_HELPER_FLAGS_4(vseqi_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) @@ -639,61 +652,69 @@ DEF_HELPER_FLAGS_4(vslti_hu, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) DEF_HELPER_FLAGS_4(vslti_wu, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) DEF_HELPER_FLAGS_4(vslti_du, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) -DEF_HELPER_5(vfcmp_c_s, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfcmp_s_s, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfcmp_c_d, void, env, i32, i32, i32, i32) -DEF_HELPER_5(vfcmp_s_d, void, env, i32, i32, i32, i32) +DEF_HELPER_6(vfcmp_c_s, void, env, i32, i32, i32, i32, i32) +DEF_HELPER_6(vfcmp_s_s, void, env, i32, i32, i32, i32, i32) +DEF_HELPER_6(vfcmp_c_d, void, env, i32, i32, i32, i32, i32) +DEF_HELPER_6(vfcmp_s_d, void, env, i32, i32, i32, i32, i32) DEF_HELPER_FLAGS_4(vbitseli_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) -DEF_HELPER_3(vsetanyeqz_b, void, env, i32, i32) -DEF_HELPER_3(vsetanyeqz_h, void, env, i32, i32) -DEF_HELPER_3(vsetanyeqz_w, void, env, i32, i32) -DEF_HELPER_3(vsetanyeqz_d, void, env, i32, i32) -DEF_HELPER_3(vsetallnez_b, void, env, i32, i32) -DEF_HELPER_3(vsetallnez_h, void, env, i32, i32) -DEF_HELPER_3(vsetallnez_w, void, env, i32, i32) -DEF_HELPER_3(vsetallnez_d, void, env, i32, i32) - -DEF_HELPER_4(vpackev_b, void, env, i32, i32, i32) -DEF_HELPER_4(vpackev_h, void, env, i32, i32, i32) -DEF_HELPER_4(vpackev_w, void, env, i32, i32, i32) -DEF_HELPER_4(vpackev_d, void, env, i32, i32, i32) -DEF_HELPER_4(vpackod_b, void, env, i32, i32, i32) -DEF_HELPER_4(vpackod_h, void, env, i32, i32, i32) -DEF_HELPER_4(vpackod_w, void, env, i32, i32, i32) -DEF_HELPER_4(vpackod_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vpickev_b, void, env, i32, i32, i32) -DEF_HELPER_4(vpickev_h, void, env, i32, i32, i32) -DEF_HELPER_4(vpickev_w, void, env, i32, i32, i32) -DEF_HELPER_4(vpickev_d, void, env, i32, i32, i32) -DEF_HELPER_4(vpickod_b, void, env, i32, i32, i32) -DEF_HELPER_4(vpickod_h, void, env, i32, i32, i32) -DEF_HELPER_4(vpickod_w, void, env, i32, i32, i32) -DEF_HELPER_4(vpickod_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vilvl_b, void, env, i32, i32, i32) -DEF_HELPER_4(vilvl_h, void, env, i32, i32, i32) -DEF_HELPER_4(vilvl_w, void, env, i32, i32, i32) -DEF_HELPER_4(vilvl_d, void, env, i32, i32, i32) -DEF_HELPER_4(vilvh_b, void, env, i32, i32, i32) -DEF_HELPER_4(vilvh_h, void, env, i32, i32, i32) -DEF_HELPER_4(vilvh_w, void, env, i32, i32, i32) -DEF_HELPER_4(vilvh_d, void, env, i32, i32, i32) - -DEF_HELPER_5(vshuf_b, void, env, i32, i32, i32, i32) -DEF_HELPER_4(vshuf_h, void, env, i32, i32, i32) -DEF_HELPER_4(vshuf_w, void, env, i32, i32, i32) -DEF_HELPER_4(vshuf_d, void, env, i32, i32, i32) -DEF_HELPER_4(vshuf4i_b, void, env, i32, i32, i32) -DEF_HELPER_4(vshuf4i_h, void, env, i32, i32, i32) -DEF_HELPER_4(vshuf4i_w, void, env, i32, i32, i32) -DEF_HELPER_4(vshuf4i_d, void, env, i32, i32, i32) - -DEF_HELPER_4(vpermi_w, void, env, i32, i32, i32) - -DEF_HELPER_4(vextrins_b, void, env, i32, i32, i32) -DEF_HELPER_4(vextrins_h, void, env, i32, i32, i32) -DEF_HELPER_4(vextrins_w, void, env, i32, i32, i32) -DEF_HELPER_4(vextrins_d, void, env, i32, i32, i32) +DEF_HELPER_4(vsetanyeqz_b, void, env, i32, i32, i32) +DEF_HELPER_4(vsetanyeqz_h, void, env, i32, i32, i32) +DEF_HELPER_4(vsetanyeqz_w, void, env, i32, i32, i32) +DEF_HELPER_4(vsetanyeqz_d, void, env, i32, i32, i32) +DEF_HELPER_4(vsetallnez_b, void, env, i32, i32, i32) +DEF_HELPER_4(vsetallnez_h, void, env, i32, i32, i32) +DEF_HELPER_4(vsetallnez_w, void, env, i32, i32, i32) +DEF_HELPER_4(vsetallnez_d, void, env, i32, i32, i32) + +DEF_HELPER_FLAGS_4(xvinsve0_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(xvinsve0_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(xvpickve_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(xvpickve_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(vpackev_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpackev_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpackev_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpackev_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpackod_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpackod_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpackod_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpackod_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(vpickev_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpickev_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpickev_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpickev_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpickod_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpickod_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpickod_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpickod_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(vilvl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vilvl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vilvl_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vilvl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vilvh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vilvh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vilvh_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vilvh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(vshuf_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vshuf_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vshuf_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vshuf_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vshuf4i_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vshuf4i_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vshuf4i_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vshuf4i_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(vperm_w, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(vpermi_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vpermi_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vpermi_q, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) + +DEF_HELPER_FLAGS_4(vextrins_b, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vextrins_h, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vextrins_w, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) +DEF_HELPER_FLAGS_4(vextrins_d, TCG_CALL_NO_RWG, void, ptr, ptr, i64, i32) diff --git a/target/loongarch/insn_trans/trans_lsx.c.inc b/target/loongarch/insn_trans/trans_vec.c.inc index 5fbf2718f7..c647137372 100644 --- a/target/loongarch/insn_trans/trans_lsx.c.inc +++ b/target/loongarch/insn_trans/trans_vec.c.inc @@ -1,78 +1,254 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * LSX translate functions + * LoongArch vector translate functions * Copyright (c) 2022-2023 Loongson Technology Corporation Limited */ #ifndef CONFIG_USER_ONLY -#define CHECK_SXE do { \ - if ((ctx->base.tb->flags & HW_FLAGS_EUEN_SXE) == 0) { \ - generate_exception(ctx, EXCCODE_SXD); \ - return true; \ - } \ -} while (0) + +static bool check_vec(DisasContext *ctx, uint32_t oprsz) +{ + if ((oprsz == 16) && ((ctx->base.tb->flags & HW_FLAGS_EUEN_SXE) == 0)) { + generate_exception(ctx, EXCCODE_SXD); + return false; + } + + if ((oprsz == 32) && ((ctx->base.tb->flags & HW_FLAGS_EUEN_ASXE) == 0)) { + generate_exception(ctx, EXCCODE_ASXD); + return false; + } + + return true; +} + #else -#define CHECK_SXE + +static bool check_vec(DisasContext *ctx, uint32_t oprsz) +{ + return true; +} + #endif +static bool gen_vvvv_ptr_vl(DisasContext *ctx, arg_vvvv *a, uint32_t oprsz, + gen_helper_gvec_4_ptr *fn) +{ + if (!check_vec(ctx, oprsz)) { + return true; + } + + tcg_gen_gvec_4_ptr(vec_full_offset(a->vd), + vec_full_offset(a->vj), + vec_full_offset(a->vk), + vec_full_offset(a->va), + cpu_env, + oprsz, ctx->vl / 8, 0, fn); + return true; +} + +static bool gen_vvvv_ptr(DisasContext *ctx, arg_vvvv *a, + gen_helper_gvec_4_ptr *fn) +{ + return gen_vvvv_ptr_vl(ctx, a, 16, fn); +} + +static bool gen_xxxx_ptr(DisasContext *ctx, arg_vvvv *a, + gen_helper_gvec_4_ptr *fn) +{ + return gen_vvvv_ptr_vl(ctx, a, 32, fn); +} + +static bool gen_vvvv_vl(DisasContext *ctx, arg_vvvv *a, uint32_t oprsz, + gen_helper_gvec_4 *fn) +{ + if (!check_vec(ctx, oprsz)) { + return true; + } + + tcg_gen_gvec_4_ool(vec_full_offset(a->vd), + vec_full_offset(a->vj), + vec_full_offset(a->vk), + vec_full_offset(a->va), + oprsz, ctx->vl / 8, 0, fn); + return true; +} + static bool gen_vvvv(DisasContext *ctx, arg_vvvv *a, - void (*func)(TCGv_ptr, TCGv_i32, TCGv_i32, - TCGv_i32, TCGv_i32)) + gen_helper_gvec_4 *fn) { - TCGv_i32 vd = tcg_constant_i32(a->vd); - TCGv_i32 vj = tcg_constant_i32(a->vj); - TCGv_i32 vk = tcg_constant_i32(a->vk); - TCGv_i32 va = tcg_constant_i32(a->va); + return gen_vvvv_vl(ctx, a, 16, fn); +} + +static bool gen_xxxx(DisasContext *ctx, arg_vvvv *a, + gen_helper_gvec_4 *fn) +{ + return gen_vvvv_vl(ctx, a, 32, fn); +} - CHECK_SXE; - func(cpu_env, vd, vj, vk, va); +static bool gen_vvv_ptr_vl(DisasContext *ctx, arg_vvv *a, uint32_t oprsz, + gen_helper_gvec_3_ptr *fn) +{ + if (!check_vec(ctx, oprsz)) { + return true; + } + tcg_gen_gvec_3_ptr(vec_full_offset(a->vd), + vec_full_offset(a->vj), + vec_full_offset(a->vk), + cpu_env, + oprsz, ctx->vl / 8, 0, fn); return true; } -static bool gen_vvv(DisasContext *ctx, arg_vvv *a, - void (*func)(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32)) +static bool gen_vvv_ptr(DisasContext *ctx, arg_vvv *a, + gen_helper_gvec_3_ptr *fn) { - TCGv_i32 vd = tcg_constant_i32(a->vd); - TCGv_i32 vj = tcg_constant_i32(a->vj); - TCGv_i32 vk = tcg_constant_i32(a->vk); + return gen_vvv_ptr_vl(ctx, a, 16, fn); +} - CHECK_SXE; +static bool gen_xxx_ptr(DisasContext *ctx, arg_vvv *a, + gen_helper_gvec_3_ptr *fn) +{ + return gen_vvv_ptr_vl(ctx, a, 32, fn); +} + +static bool gen_vvv_vl(DisasContext *ctx, arg_vvv *a, uint32_t oprsz, + gen_helper_gvec_3 *fn) +{ + if (!check_vec(ctx, oprsz)) { + return true; + } - func(cpu_env, vd, vj, vk); + tcg_gen_gvec_3_ool(vec_full_offset(a->vd), + vec_full_offset(a->vj), + vec_full_offset(a->vk), + oprsz, ctx->vl / 8, 0, fn); return true; } -static bool gen_vv(DisasContext *ctx, arg_vv *a, - void (*func)(TCGv_ptr, TCGv_i32, TCGv_i32)) +static bool gen_vvv(DisasContext *ctx, arg_vvv *a, gen_helper_gvec_3 *fn) { - TCGv_i32 vd = tcg_constant_i32(a->vd); - TCGv_i32 vj = tcg_constant_i32(a->vj); + return gen_vvv_vl(ctx, a, 16, fn); +} - CHECK_SXE; - func(cpu_env, vd, vj); +static bool gen_xxx(DisasContext *ctx, arg_vvv *a, gen_helper_gvec_3 *fn) +{ + return gen_vvv_vl(ctx, a, 32, fn); +} + +static bool gen_vv_ptr_vl(DisasContext *ctx, arg_vv *a, uint32_t oprsz, + gen_helper_gvec_2_ptr *fn) +{ + if (!check_vec(ctx, oprsz)) { + return true; + } + + tcg_gen_gvec_2_ptr(vec_full_offset(a->vd), + vec_full_offset(a->vj), + cpu_env, + oprsz, ctx->vl / 8, 0, fn); return true; } -static bool gen_vv_i(DisasContext *ctx, arg_vv_i *a, - void (*func)(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32)) +static bool gen_vv_ptr(DisasContext *ctx, arg_vv *a, + gen_helper_gvec_2_ptr *fn) { - TCGv_i32 vd = tcg_constant_i32(a->vd); - TCGv_i32 vj = tcg_constant_i32(a->vj); - TCGv_i32 imm = tcg_constant_i32(a->imm); + return gen_vv_ptr_vl(ctx, a, 16, fn); +} + +static bool gen_xx_ptr(DisasContext *ctx, arg_vv *a, + gen_helper_gvec_2_ptr *fn) +{ + return gen_vv_ptr_vl(ctx, a, 32, fn); +} + +static bool gen_vv_vl(DisasContext *ctx, arg_vv *a, uint32_t oprsz, + gen_helper_gvec_2 *fn) +{ + if (!check_vec(ctx, oprsz)) { + return true; + } - CHECK_SXE; - func(cpu_env, vd, vj, imm); + tcg_gen_gvec_2_ool(vec_full_offset(a->vd), + vec_full_offset(a->vj), + oprsz, ctx->vl / 8, 0, fn); return true; } -static bool gen_cv(DisasContext *ctx, arg_cv *a, - void (*func)(TCGv_ptr, TCGv_i32, TCGv_i32)) +static bool gen_vv(DisasContext *ctx, arg_vv *a, gen_helper_gvec_2 *fn) +{ + return gen_vv_vl(ctx, a, 16, fn); +} + +static bool gen_xx(DisasContext *ctx, arg_vv *a, gen_helper_gvec_2 *fn) { + return gen_vv_vl(ctx, a, 32, fn); +} + +static bool gen_vv_i_vl(DisasContext *ctx, arg_vv_i *a, uint32_t oprsz, + gen_helper_gvec_2i *fn) +{ + if (!check_vec(ctx, oprsz)) { + return true; + } + + tcg_gen_gvec_2i_ool(vec_full_offset(a->vd), + vec_full_offset(a->vj), + tcg_constant_i64(a->imm), + oprsz, ctx->vl / 8, 0, fn); + return true; +} + +static bool gen_vv_i(DisasContext *ctx, arg_vv_i *a, gen_helper_gvec_2i *fn) +{ + return gen_vv_i_vl(ctx, a, 16, fn); +} + +static bool gen_xx_i(DisasContext *ctx, arg_vv_i *a, gen_helper_gvec_2i *fn) +{ + return gen_vv_i_vl(ctx, a, 32, fn); +} + +static bool gen_cv_vl(DisasContext *ctx, arg_cv *a, uint32_t sz, + void (*func)(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32)) +{ + if (!check_vec(ctx, sz)) { + return true; + } + TCGv_i32 vj = tcg_constant_i32(a->vj); TCGv_i32 cd = tcg_constant_i32(a->cd); + TCGv_i32 oprsz = tcg_constant_i32(sz); + + func(cpu_env, oprsz, cd, vj); + return true; +} + +static bool gen_cv(DisasContext *ctx, arg_cv *a, + void (*func)(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32)) +{ + return gen_cv_vl(ctx, a, 16, func); +} + +static bool gen_cx(DisasContext *ctx, arg_cv *a, + void (*func)(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32)) +{ + return gen_cv_vl(ctx, a, 32, func); +} - CHECK_SXE; - func(cpu_env, cd, vj); +static bool gvec_vvv_vl(DisasContext *ctx, arg_vvv *a, + uint32_t oprsz, MemOp mop, + void (*func)(unsigned, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t)) +{ + uint32_t vd_ofs = vec_full_offset(a->vd); + uint32_t vj_ofs = vec_full_offset(a->vj); + uint32_t vk_ofs = vec_full_offset(a->vk); + + if (!check_vec(ctx, oprsz)) { + return true; + } + + func(mop, vd_ofs, vj_ofs, vk_ofs, oprsz, ctx->vl / 8); return true; } @@ -80,30 +256,60 @@ static bool gvec_vvv(DisasContext *ctx, arg_vvv *a, MemOp mop, void (*func)(unsigned, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t)) { - uint32_t vd_ofs, vj_ofs, vk_ofs; + return gvec_vvv_vl(ctx, a, 16, mop, func); +} - CHECK_SXE; +static bool gvec_xxx(DisasContext *ctx, arg_vvv *a, MemOp mop, + void (*func)(unsigned, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t)) +{ + return gvec_vvv_vl(ctx, a, 32, mop, func); +} - vd_ofs = vec_full_offset(a->vd); - vj_ofs = vec_full_offset(a->vj); - vk_ofs = vec_full_offset(a->vk); +static bool gvec_vv_vl(DisasContext *ctx, arg_vv *a, + uint32_t oprsz, MemOp mop, + void (*func)(unsigned, uint32_t, uint32_t, + uint32_t, uint32_t)) +{ + uint32_t vd_ofs = vec_full_offset(a->vd); + uint32_t vj_ofs = vec_full_offset(a->vj); + + if (!check_vec(ctx, oprsz)) { + return true; + } - func(mop, vd_ofs, vj_ofs, vk_ofs, 16, ctx->vl/8); + func(mop, vd_ofs, vj_ofs, oprsz, ctx->vl / 8); return true; } + static bool gvec_vv(DisasContext *ctx, arg_vv *a, MemOp mop, void (*func)(unsigned, uint32_t, uint32_t, uint32_t, uint32_t)) { - uint32_t vd_ofs, vj_ofs; + return gvec_vv_vl(ctx, a, 16, mop, func); +} + +static bool gvec_xx(DisasContext *ctx, arg_vv *a, MemOp mop, + void (*func)(unsigned, uint32_t, uint32_t, + uint32_t, uint32_t)) +{ + return gvec_vv_vl(ctx, a, 32, mop, func); +} - CHECK_SXE; +static bool gvec_vv_i_vl(DisasContext *ctx, arg_vv_i *a, + uint32_t oprsz, MemOp mop, + void (*func)(unsigned, uint32_t, uint32_t, + int64_t, uint32_t, uint32_t)) +{ + uint32_t vd_ofs = vec_full_offset(a->vd); + uint32_t vj_ofs = vec_full_offset(a->vj); - vd_ofs = vec_full_offset(a->vd); - vj_ofs = vec_full_offset(a->vj); + if (!check_vec(ctx, oprsz)) { + return true; + } - func(mop, vd_ofs, vj_ofs, 16, ctx->vl/8); + func(mop, vd_ofs, vj_ofs, a->imm, oprsz, ctx->vl / 8); return true; } @@ -111,73 +317,108 @@ static bool gvec_vv_i(DisasContext *ctx, arg_vv_i *a, MemOp mop, void (*func)(unsigned, uint32_t, uint32_t, int64_t, uint32_t, uint32_t)) { - uint32_t vd_ofs, vj_ofs; + return gvec_vv_i_vl(ctx, a, 16, mop, func); +} + +static bool gvec_xx_i(DisasContext *ctx, arg_vv_i *a, MemOp mop, + void (*func)(unsigned, uint32_t, uint32_t, + int64_t, uint32_t, uint32_t)) +{ + return gvec_vv_i_vl(ctx,a, 32, mop, func); +} - CHECK_SXE; +static bool gvec_subi_vl(DisasContext *ctx, arg_vv_i *a, + uint32_t oprsz, MemOp mop) +{ + uint32_t vd_ofs = vec_full_offset(a->vd); + uint32_t vj_ofs = vec_full_offset(a->vj); - vd_ofs = vec_full_offset(a->vd); - vj_ofs = vec_full_offset(a->vj); + if (!check_vec(ctx, oprsz)) { + return true; + } - func(mop, vd_ofs, vj_ofs, a->imm , 16, ctx->vl/8); + tcg_gen_gvec_addi(mop, vd_ofs, vj_ofs, -a->imm, oprsz, ctx->vl / 8); return true; } static bool gvec_subi(DisasContext *ctx, arg_vv_i *a, MemOp mop) { - uint32_t vd_ofs, vj_ofs; - - CHECK_SXE; - - vd_ofs = vec_full_offset(a->vd); - vj_ofs = vec_full_offset(a->vj); + return gvec_subi_vl(ctx, a, 16, mop); +} - tcg_gen_gvec_addi(mop, vd_ofs, vj_ofs, -a->imm, 16, ctx->vl/8); - return true; +static bool gvec_xsubi(DisasContext *ctx, arg_vv_i *a, MemOp mop) +{ + return gvec_subi_vl(ctx, a, 32, mop); } TRANS(vadd_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_add) TRANS(vadd_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_add) TRANS(vadd_w, LSX, gvec_vvv, MO_32, tcg_gen_gvec_add) TRANS(vadd_d, LSX, gvec_vvv, MO_64, tcg_gen_gvec_add) +TRANS(xvadd_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_add) +TRANS(xvadd_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_add) +TRANS(xvadd_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_add) +TRANS(xvadd_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_add) + +static bool gen_vaddsub_q_vl(DisasContext *ctx, arg_vvv *a, uint32_t oprsz, + void (*func)(TCGv_i64, TCGv_i64, TCGv_i64, + TCGv_i64, TCGv_i64, TCGv_i64)) +{ + int i; + TCGv_i64 rh, rl, ah, al, bh, bl; + + if (!check_vec(ctx, oprsz)) { + return true; + } + + rh = tcg_temp_new_i64(); + rl = tcg_temp_new_i64(); + ah = tcg_temp_new_i64(); + al = tcg_temp_new_i64(); + bh = tcg_temp_new_i64(); + bl = tcg_temp_new_i64(); + + for (i = 0; i < oprsz / 16; i++) { + get_vreg64(ah, a->vj, 1 + i * 2); + get_vreg64(al, a->vj, i * 2); + get_vreg64(bh, a->vk, 1 + i * 2); + get_vreg64(bl, a->vk, i * 2); + + func(rl, rh, al, ah, bl, bh); + + set_vreg64(rh, a->vd, 1 + i * 2); + set_vreg64(rl, a->vd, i * 2); + } + return true; +} -#define VADDSUB_Q(NAME) \ -static bool trans_v## NAME ##_q(DisasContext *ctx, arg_vvv *a) \ -{ \ - TCGv_i64 rh, rl, ah, al, bh, bl; \ - \ - if (!avail_LSX(ctx)) { \ - return false; \ - } \ - \ - CHECK_SXE; \ - \ - rh = tcg_temp_new_i64(); \ - rl = tcg_temp_new_i64(); \ - ah = tcg_temp_new_i64(); \ - al = tcg_temp_new_i64(); \ - bh = tcg_temp_new_i64(); \ - bl = tcg_temp_new_i64(); \ - \ - get_vreg64(ah, a->vj, 1); \ - get_vreg64(al, a->vj, 0); \ - get_vreg64(bh, a->vk, 1); \ - get_vreg64(bl, a->vk, 0); \ - \ - tcg_gen_## NAME ##2_i64(rl, rh, al, ah, bl, bh); \ - \ - set_vreg64(rh, a->vd, 1); \ - set_vreg64(rl, a->vd, 0); \ - \ - return true; \ -} - -VADDSUB_Q(add) -VADDSUB_Q(sub) +static bool gen_vaddsub_q(DisasContext *ctx, arg_vvv *a, + void (*func)(TCGv_i64, TCGv_i64, TCGv_i64, + TCGv_i64, TCGv_i64, TCGv_i64)) +{ + return gen_vaddsub_q_vl(ctx, a, 16, func); +} + +static bool gen_xvaddsub_q(DisasContext *ctx, arg_vvv *a, + void (*func)(TCGv_i64, TCGv_i64, TCGv_i64, + TCGv_i64, TCGv_i64, TCGv_i64)) +{ + return gen_vaddsub_q_vl(ctx, a, 32, func); +} TRANS(vsub_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_sub) TRANS(vsub_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_sub) TRANS(vsub_w, LSX, gvec_vvv, MO_32, tcg_gen_gvec_sub) TRANS(vsub_d, LSX, gvec_vvv, MO_64, tcg_gen_gvec_sub) +TRANS(xvsub_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_sub) +TRANS(xvsub_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_sub) +TRANS(xvsub_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_sub) +TRANS(xvsub_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_sub) + +TRANS(vadd_q, LSX, gen_vaddsub_q, tcg_gen_add2_i64) +TRANS(vsub_q, LSX, gen_vaddsub_q, tcg_gen_sub2_i64) +TRANS(xvadd_q, LASX, gen_xvaddsub_q, tcg_gen_add2_i64) +TRANS(xvsub_q, LASX, gen_xvaddsub_q, tcg_gen_sub2_i64) TRANS(vaddi_bu, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_addi) TRANS(vaddi_hu, LSX, gvec_vv_i, MO_16, tcg_gen_gvec_addi) @@ -187,11 +428,23 @@ TRANS(vsubi_bu, LSX, gvec_subi, MO_8) TRANS(vsubi_hu, LSX, gvec_subi, MO_16) TRANS(vsubi_wu, LSX, gvec_subi, MO_32) TRANS(vsubi_du, LSX, gvec_subi, MO_64) +TRANS(xvaddi_bu, LASX, gvec_xx_i, MO_8, tcg_gen_gvec_addi) +TRANS(xvaddi_hu, LASX, gvec_xx_i, MO_16, tcg_gen_gvec_addi) +TRANS(xvaddi_wu, LASX, gvec_xx_i, MO_32, tcg_gen_gvec_addi) +TRANS(xvaddi_du, LASX, gvec_xx_i, MO_64, tcg_gen_gvec_addi) +TRANS(xvsubi_bu, LASX, gvec_xsubi, MO_8) +TRANS(xvsubi_hu, LASX, gvec_xsubi, MO_16) +TRANS(xvsubi_wu, LASX, gvec_xsubi, MO_32) +TRANS(xvsubi_du, LASX, gvec_xsubi, MO_64) TRANS(vneg_b, LSX, gvec_vv, MO_8, tcg_gen_gvec_neg) TRANS(vneg_h, LSX, gvec_vv, MO_16, tcg_gen_gvec_neg) TRANS(vneg_w, LSX, gvec_vv, MO_32, tcg_gen_gvec_neg) TRANS(vneg_d, LSX, gvec_vv, MO_64, tcg_gen_gvec_neg) +TRANS(xvneg_b, LASX, gvec_xx, MO_8, tcg_gen_gvec_neg) +TRANS(xvneg_h, LASX, gvec_xx, MO_16, tcg_gen_gvec_neg) +TRANS(xvneg_w, LASX, gvec_xx, MO_32, tcg_gen_gvec_neg) +TRANS(xvneg_d, LASX, gvec_xx, MO_64, tcg_gen_gvec_neg) TRANS(vsadd_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_ssadd) TRANS(vsadd_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_ssadd) @@ -210,6 +463,23 @@ TRANS(vssub_hu, LSX, gvec_vvv, MO_16, tcg_gen_gvec_ussub) TRANS(vssub_wu, LSX, gvec_vvv, MO_32, tcg_gen_gvec_ussub) TRANS(vssub_du, LSX, gvec_vvv, MO_64, tcg_gen_gvec_ussub) +TRANS(xvsadd_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_ssadd) +TRANS(xvsadd_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_ssadd) +TRANS(xvsadd_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_ssadd) +TRANS(xvsadd_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_ssadd) +TRANS(xvsadd_bu, LASX, gvec_xxx, MO_8, tcg_gen_gvec_usadd) +TRANS(xvsadd_hu, LASX, gvec_xxx, MO_16, tcg_gen_gvec_usadd) +TRANS(xvsadd_wu, LASX, gvec_xxx, MO_32, tcg_gen_gvec_usadd) +TRANS(xvsadd_du, LASX, gvec_xxx, MO_64, tcg_gen_gvec_usadd) +TRANS(xvssub_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_sssub) +TRANS(xvssub_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_sssub) +TRANS(xvssub_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_sssub) +TRANS(xvssub_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_sssub) +TRANS(xvssub_bu, LASX, gvec_xxx, MO_8, tcg_gen_gvec_ussub) +TRANS(xvssub_hu, LASX, gvec_xxx, MO_16, tcg_gen_gvec_ussub) +TRANS(xvssub_wu, LASX, gvec_xxx, MO_32, tcg_gen_gvec_ussub) +TRANS(xvssub_du, LASX, gvec_xxx, MO_64, tcg_gen_gvec_ussub) + TRANS(vhaddw_h_b, LSX, gen_vvv, gen_helper_vhaddw_h_b) TRANS(vhaddw_w_h, LSX, gen_vvv, gen_helper_vhaddw_w_h) TRANS(vhaddw_d_w, LSX, gen_vvv, gen_helper_vhaddw_d_w) @@ -227,6 +497,23 @@ TRANS(vhsubw_wu_hu, LSX, gen_vvv, gen_helper_vhsubw_wu_hu) TRANS(vhsubw_du_wu, LSX, gen_vvv, gen_helper_vhsubw_du_wu) TRANS(vhsubw_qu_du, LSX, gen_vvv, gen_helper_vhsubw_qu_du) +TRANS(xvhaddw_h_b, LASX, gen_xxx, gen_helper_vhaddw_h_b) +TRANS(xvhaddw_w_h, LASX, gen_xxx, gen_helper_vhaddw_w_h) +TRANS(xvhaddw_d_w, LASX, gen_xxx, gen_helper_vhaddw_d_w) +TRANS(xvhaddw_q_d, LASX, gen_xxx, gen_helper_vhaddw_q_d) +TRANS(xvhaddw_hu_bu, LASX, gen_xxx, gen_helper_vhaddw_hu_bu) +TRANS(xvhaddw_wu_hu, LASX, gen_xxx, gen_helper_vhaddw_wu_hu) +TRANS(xvhaddw_du_wu, LASX, gen_xxx, gen_helper_vhaddw_du_wu) +TRANS(xvhaddw_qu_du, LASX, gen_xxx, gen_helper_vhaddw_qu_du) +TRANS(xvhsubw_h_b, LASX, gen_xxx, gen_helper_vhsubw_h_b) +TRANS(xvhsubw_w_h, LASX, gen_xxx, gen_helper_vhsubw_w_h) +TRANS(xvhsubw_d_w, LASX, gen_xxx, gen_helper_vhsubw_d_w) +TRANS(xvhsubw_q_d, LASX, gen_xxx, gen_helper_vhsubw_q_d) +TRANS(xvhsubw_hu_bu, LASX, gen_xxx, gen_helper_vhsubw_hu_bu) +TRANS(xvhsubw_wu_hu, LASX, gen_xxx, gen_helper_vhsubw_wu_hu) +TRANS(xvhsubw_du_wu, LASX, gen_xxx, gen_helper_vhsubw_du_wu) +TRANS(xvhsubw_qu_du, LASX, gen_xxx, gen_helper_vhsubw_qu_du) + static void gen_vaddwev_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { TCGv_vec t1, t2; @@ -309,6 +596,10 @@ TRANS(vaddwev_h_b, LSX, gvec_vvv, MO_8, do_vaddwev_s) TRANS(vaddwev_w_h, LSX, gvec_vvv, MO_16, do_vaddwev_s) TRANS(vaddwev_d_w, LSX, gvec_vvv, MO_32, do_vaddwev_s) TRANS(vaddwev_q_d, LSX, gvec_vvv, MO_64, do_vaddwev_s) +TRANS(xvaddwev_h_b, LASX, gvec_xxx, MO_8, do_vaddwev_s) +TRANS(xvaddwev_w_h, LASX, gvec_xxx, MO_16, do_vaddwev_s) +TRANS(xvaddwev_d_w, LASX, gvec_xxx, MO_32, do_vaddwev_s) +TRANS(xvaddwev_q_d, LASX, gvec_xxx, MO_64, do_vaddwev_s) static void gen_vaddwod_w_h(TCGv_i32 t, TCGv_i32 a, TCGv_i32 b) { @@ -388,6 +679,11 @@ TRANS(vaddwod_h_b, LSX, gvec_vvv, MO_8, do_vaddwod_s) TRANS(vaddwod_w_h, LSX, gvec_vvv, MO_16, do_vaddwod_s) TRANS(vaddwod_d_w, LSX, gvec_vvv, MO_32, do_vaddwod_s) TRANS(vaddwod_q_d, LSX, gvec_vvv, MO_64, do_vaddwod_s) +TRANS(xvaddwod_h_b, LASX, gvec_xxx, MO_8, do_vaddwod_s) +TRANS(xvaddwod_w_h, LASX, gvec_xxx, MO_16, do_vaddwod_s) +TRANS(xvaddwod_d_w, LASX, gvec_xxx, MO_32, do_vaddwod_s) +TRANS(xvaddwod_q_d, LASX, gvec_xxx, MO_64, do_vaddwod_s) + static void gen_vsubwev_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -471,6 +767,10 @@ TRANS(vsubwev_h_b, LSX, gvec_vvv, MO_8, do_vsubwev_s) TRANS(vsubwev_w_h, LSX, gvec_vvv, MO_16, do_vsubwev_s) TRANS(vsubwev_d_w, LSX, gvec_vvv, MO_32, do_vsubwev_s) TRANS(vsubwev_q_d, LSX, gvec_vvv, MO_64, do_vsubwev_s) +TRANS(xvsubwev_h_b, LASX, gvec_xxx, MO_8, do_vsubwev_s) +TRANS(xvsubwev_w_h, LASX, gvec_xxx, MO_16, do_vsubwev_s) +TRANS(xvsubwev_d_w, LASX, gvec_xxx, MO_32, do_vsubwev_s) +TRANS(xvsubwev_q_d, LASX, gvec_xxx, MO_64, do_vsubwev_s) static void gen_vsubwod_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -550,6 +850,10 @@ TRANS(vsubwod_h_b, LSX, gvec_vvv, MO_8, do_vsubwod_s) TRANS(vsubwod_w_h, LSX, gvec_vvv, MO_16, do_vsubwod_s) TRANS(vsubwod_d_w, LSX, gvec_vvv, MO_32, do_vsubwod_s) TRANS(vsubwod_q_d, LSX, gvec_vvv, MO_64, do_vsubwod_s) +TRANS(xvsubwod_h_b, LASX, gvec_xxx, MO_8, do_vsubwod_s) +TRANS(xvsubwod_w_h, LASX, gvec_xxx, MO_16, do_vsubwod_s) +TRANS(xvsubwod_d_w, LASX, gvec_xxx, MO_32, do_vsubwod_s) +TRANS(xvsubwod_q_d, LASX, gvec_xxx, MO_64, do_vsubwod_s) static void gen_vaddwev_u(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -625,6 +929,10 @@ TRANS(vaddwev_h_bu, LSX, gvec_vvv, MO_8, do_vaddwev_u) TRANS(vaddwev_w_hu, LSX, gvec_vvv, MO_16, do_vaddwev_u) TRANS(vaddwev_d_wu, LSX, gvec_vvv, MO_32, do_vaddwev_u) TRANS(vaddwev_q_du, LSX, gvec_vvv, MO_64, do_vaddwev_u) +TRANS(xvaddwev_h_bu, LASX, gvec_xxx, MO_8, do_vaddwev_u) +TRANS(xvaddwev_w_hu, LASX, gvec_xxx, MO_16, do_vaddwev_u) +TRANS(xvaddwev_d_wu, LASX, gvec_xxx, MO_32, do_vaddwev_u) +TRANS(xvaddwev_q_du, LASX, gvec_xxx, MO_64, do_vaddwev_u) static void gen_vaddwod_u(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -704,6 +1012,10 @@ TRANS(vaddwod_h_bu, LSX, gvec_vvv, MO_8, do_vaddwod_u) TRANS(vaddwod_w_hu, LSX, gvec_vvv, MO_16, do_vaddwod_u) TRANS(vaddwod_d_wu, LSX, gvec_vvv, MO_32, do_vaddwod_u) TRANS(vaddwod_q_du, LSX, gvec_vvv, MO_64, do_vaddwod_u) +TRANS(xvaddwod_h_bu, LASX, gvec_xxx, MO_8, do_vaddwod_u) +TRANS(xvaddwod_w_hu, LASX, gvec_xxx, MO_16, do_vaddwod_u) +TRANS(xvaddwod_d_wu, LASX, gvec_xxx, MO_32, do_vaddwod_u) +TRANS(xvaddwod_q_du, LASX, gvec_xxx, MO_64, do_vaddwod_u) static void gen_vsubwev_u(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -779,6 +1091,10 @@ TRANS(vsubwev_h_bu, LSX, gvec_vvv, MO_8, do_vsubwev_u) TRANS(vsubwev_w_hu, LSX, gvec_vvv, MO_16, do_vsubwev_u) TRANS(vsubwev_d_wu, LSX, gvec_vvv, MO_32, do_vsubwev_u) TRANS(vsubwev_q_du, LSX, gvec_vvv, MO_64, do_vsubwev_u) +TRANS(xvsubwev_h_bu, LASX, gvec_xxx, MO_8, do_vsubwev_u) +TRANS(xvsubwev_w_hu, LASX, gvec_xxx, MO_16, do_vsubwev_u) +TRANS(xvsubwev_d_wu, LASX, gvec_xxx, MO_32, do_vsubwev_u) +TRANS(xvsubwev_q_du, LASX, gvec_xxx, MO_64, do_vsubwev_u) static void gen_vsubwod_u(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -858,6 +1174,10 @@ TRANS(vsubwod_h_bu, LSX, gvec_vvv, MO_8, do_vsubwod_u) TRANS(vsubwod_w_hu, LSX, gvec_vvv, MO_16, do_vsubwod_u) TRANS(vsubwod_d_wu, LSX, gvec_vvv, MO_32, do_vsubwod_u) TRANS(vsubwod_q_du, LSX, gvec_vvv, MO_64, do_vsubwod_u) +TRANS(xvsubwod_h_bu, LASX, gvec_xxx, MO_8, do_vsubwod_u) +TRANS(xvsubwod_w_hu, LASX, gvec_xxx, MO_16, do_vsubwod_u) +TRANS(xvsubwod_d_wu, LASX, gvec_xxx, MO_32, do_vsubwod_u) +TRANS(xvsubwod_q_du, LASX, gvec_xxx, MO_64, do_vsubwod_u) static void gen_vaddwev_u_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -941,6 +1261,10 @@ TRANS(vaddwev_h_bu_b, LSX, gvec_vvv, MO_8, do_vaddwev_u_s) TRANS(vaddwev_w_hu_h, LSX, gvec_vvv, MO_16, do_vaddwev_u_s) TRANS(vaddwev_d_wu_w, LSX, gvec_vvv, MO_32, do_vaddwev_u_s) TRANS(vaddwev_q_du_d, LSX, gvec_vvv, MO_64, do_vaddwev_u_s) +TRANS(xvaddwev_h_bu_b, LASX, gvec_xxx, MO_8, do_vaddwev_u_s) +TRANS(xvaddwev_w_hu_h, LASX, gvec_xxx, MO_16, do_vaddwev_u_s) +TRANS(xvaddwev_d_wu_w, LASX, gvec_xxx, MO_32, do_vaddwev_u_s) +TRANS(xvaddwev_q_du_d, LASX, gvec_xxx, MO_64, do_vaddwev_u_s) static void gen_vaddwod_u_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -1021,6 +1345,10 @@ TRANS(vaddwod_h_bu_b, LSX, gvec_vvv, MO_8, do_vaddwod_u_s) TRANS(vaddwod_w_hu_h, LSX, gvec_vvv, MO_16, do_vaddwod_u_s) TRANS(vaddwod_d_wu_w, LSX, gvec_vvv, MO_32, do_vaddwod_u_s) TRANS(vaddwod_q_du_d, LSX, gvec_vvv, MO_64, do_vaddwod_u_s) +TRANS(xvaddwod_h_bu_b, LSX, gvec_xxx, MO_8, do_vaddwod_u_s) +TRANS(xvaddwod_w_hu_h, LSX, gvec_xxx, MO_16, do_vaddwod_u_s) +TRANS(xvaddwod_d_wu_w, LSX, gvec_xxx, MO_32, do_vaddwod_u_s) +TRANS(xvaddwod_q_du_d, LSX, gvec_xxx, MO_64, do_vaddwod_u_s) static void do_vavg(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b, void (*gen_shr_vec)(unsigned, TCGv_vec, @@ -1137,6 +1465,14 @@ TRANS(vavg_bu, LSX, gvec_vvv, MO_8, do_vavg_u) TRANS(vavg_hu, LSX, gvec_vvv, MO_16, do_vavg_u) TRANS(vavg_wu, LSX, gvec_vvv, MO_32, do_vavg_u) TRANS(vavg_du, LSX, gvec_vvv, MO_64, do_vavg_u) +TRANS(xvavg_b, LASX, gvec_xxx, MO_8, do_vavg_s) +TRANS(xvavg_h, LASX, gvec_xxx, MO_16, do_vavg_s) +TRANS(xvavg_w, LASX, gvec_xxx, MO_32, do_vavg_s) +TRANS(xvavg_d, LASX, gvec_xxx, MO_64, do_vavg_s) +TRANS(xvavg_bu, LASX, gvec_xxx, MO_8, do_vavg_u) +TRANS(xvavg_hu, LASX, gvec_xxx, MO_16, do_vavg_u) +TRANS(xvavg_wu, LASX, gvec_xxx, MO_32, do_vavg_u) +TRANS(xvavg_du, LASX, gvec_xxx, MO_64, do_vavg_u) static void do_vavgr_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, uint32_t vk_ofs, uint32_t oprsz, uint32_t maxsz) @@ -1218,6 +1554,14 @@ TRANS(vavgr_bu, LSX, gvec_vvv, MO_8, do_vavgr_u) TRANS(vavgr_hu, LSX, gvec_vvv, MO_16, do_vavgr_u) TRANS(vavgr_wu, LSX, gvec_vvv, MO_32, do_vavgr_u) TRANS(vavgr_du, LSX, gvec_vvv, MO_64, do_vavgr_u) +TRANS(xvavgr_b, LASX, gvec_xxx, MO_8, do_vavgr_s) +TRANS(xvavgr_h, LASX, gvec_xxx, MO_16, do_vavgr_s) +TRANS(xvavgr_w, LASX, gvec_xxx, MO_32, do_vavgr_s) +TRANS(xvavgr_d, LASX, gvec_xxx, MO_64, do_vavgr_s) +TRANS(xvavgr_bu, LASX, gvec_xxx, MO_8, do_vavgr_u) +TRANS(xvavgr_hu, LASX, gvec_xxx, MO_16, do_vavgr_u) +TRANS(xvavgr_wu, LASX, gvec_xxx, MO_32, do_vavgr_u) +TRANS(xvavgr_du, LASX, gvec_xxx, MO_64, do_vavgr_u) static void gen_vabsd_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -1313,6 +1657,14 @@ TRANS(vabsd_bu, LSX, gvec_vvv, MO_8, do_vabsd_u) TRANS(vabsd_hu, LSX, gvec_vvv, MO_16, do_vabsd_u) TRANS(vabsd_wu, LSX, gvec_vvv, MO_32, do_vabsd_u) TRANS(vabsd_du, LSX, gvec_vvv, MO_64, do_vabsd_u) +TRANS(xvabsd_b, LASX, gvec_xxx, MO_8, do_vabsd_s) +TRANS(xvabsd_h, LASX, gvec_xxx, MO_16, do_vabsd_s) +TRANS(xvabsd_w, LASX, gvec_xxx, MO_32, do_vabsd_s) +TRANS(xvabsd_d, LASX, gvec_xxx, MO_64, do_vabsd_s) +TRANS(xvabsd_bu, LASX, gvec_xxx, MO_8, do_vabsd_u) +TRANS(xvabsd_hu, LASX, gvec_xxx, MO_16, do_vabsd_u) +TRANS(xvabsd_wu, LASX, gvec_xxx, MO_32, do_vabsd_u) +TRANS(xvabsd_du, LASX, gvec_xxx, MO_64, do_vabsd_u) static void gen_vadda(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -1366,6 +1718,10 @@ TRANS(vadda_b, LSX, gvec_vvv, MO_8, do_vadda) TRANS(vadda_h, LSX, gvec_vvv, MO_16, do_vadda) TRANS(vadda_w, LSX, gvec_vvv, MO_32, do_vadda) TRANS(vadda_d, LSX, gvec_vvv, MO_64, do_vadda) +TRANS(xvadda_b, LASX, gvec_xxx, MO_8, do_vadda) +TRANS(xvadda_h, LASX, gvec_xxx, MO_16, do_vadda) +TRANS(xvadda_w, LASX, gvec_xxx, MO_32, do_vadda) +TRANS(xvadda_d, LASX, gvec_xxx, MO_64, do_vadda) TRANS(vmax_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_smax) TRANS(vmax_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_smax) @@ -1375,6 +1731,14 @@ TRANS(vmax_bu, LSX, gvec_vvv, MO_8, tcg_gen_gvec_umax) TRANS(vmax_hu, LSX, gvec_vvv, MO_16, tcg_gen_gvec_umax) TRANS(vmax_wu, LSX, gvec_vvv, MO_32, tcg_gen_gvec_umax) TRANS(vmax_du, LSX, gvec_vvv, MO_64, tcg_gen_gvec_umax) +TRANS(xvmax_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_smax) +TRANS(xvmax_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_smax) +TRANS(xvmax_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_smax) +TRANS(xvmax_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_smax) +TRANS(xvmax_bu, LASX, gvec_xxx, MO_8, tcg_gen_gvec_umax) +TRANS(xvmax_hu, LASX, gvec_xxx, MO_16, tcg_gen_gvec_umax) +TRANS(xvmax_wu, LASX, gvec_xxx, MO_32, tcg_gen_gvec_umax) +TRANS(xvmax_du, LASX, gvec_xxx, MO_64, tcg_gen_gvec_umax) TRANS(vmin_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_smin) TRANS(vmin_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_smin) @@ -1384,6 +1748,14 @@ TRANS(vmin_bu, LSX, gvec_vvv, MO_8, tcg_gen_gvec_umin) TRANS(vmin_hu, LSX, gvec_vvv, MO_16, tcg_gen_gvec_umin) TRANS(vmin_wu, LSX, gvec_vvv, MO_32, tcg_gen_gvec_umin) TRANS(vmin_du, LSX, gvec_vvv, MO_64, tcg_gen_gvec_umin) +TRANS(xvmin_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_smin) +TRANS(xvmin_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_smin) +TRANS(xvmin_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_smin) +TRANS(xvmin_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_smin) +TRANS(xvmin_bu, LASX, gvec_xxx, MO_8, tcg_gen_gvec_umin) +TRANS(xvmin_hu, LASX, gvec_xxx, MO_16, tcg_gen_gvec_umin) +TRANS(xvmin_wu, LASX, gvec_xxx, MO_32, tcg_gen_gvec_umin) +TRANS(xvmin_du, LASX, gvec_xxx, MO_64, tcg_gen_gvec_umin) static void gen_vmini_s(unsigned vece, TCGv_vec t, TCGv_vec a, int64_t imm) { @@ -1485,6 +1857,14 @@ TRANS(vmini_bu, LSX, gvec_vv_i, MO_8, do_vmini_u) TRANS(vmini_hu, LSX, gvec_vv_i, MO_16, do_vmini_u) TRANS(vmini_wu, LSX, gvec_vv_i, MO_32, do_vmini_u) TRANS(vmini_du, LSX, gvec_vv_i, MO_64, do_vmini_u) +TRANS(xvmini_b, LASX, gvec_xx_i, MO_8, do_vmini_s) +TRANS(xvmini_h, LASX, gvec_xx_i, MO_16, do_vmini_s) +TRANS(xvmini_w, LASX, gvec_xx_i, MO_32, do_vmini_s) +TRANS(xvmini_d, LASX, gvec_xx_i, MO_64, do_vmini_s) +TRANS(xvmini_bu, LASX, gvec_xx_i, MO_8, do_vmini_u) +TRANS(xvmini_hu, LASX, gvec_xx_i, MO_16, do_vmini_u) +TRANS(xvmini_wu, LASX, gvec_xx_i, MO_32, do_vmini_u) +TRANS(xvmini_du, LASX, gvec_xx_i, MO_64, do_vmini_u) static void do_vmaxi_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, int64_t imm, uint32_t oprsz, uint32_t maxsz) @@ -1566,11 +1946,23 @@ TRANS(vmaxi_bu, LSX, gvec_vv_i, MO_8, do_vmaxi_u) TRANS(vmaxi_hu, LSX, gvec_vv_i, MO_16, do_vmaxi_u) TRANS(vmaxi_wu, LSX, gvec_vv_i, MO_32, do_vmaxi_u) TRANS(vmaxi_du, LSX, gvec_vv_i, MO_64, do_vmaxi_u) +TRANS(xvmaxi_b, LASX, gvec_xx_i, MO_8, do_vmaxi_s) +TRANS(xvmaxi_h, LASX, gvec_xx_i, MO_16, do_vmaxi_s) +TRANS(xvmaxi_w, LASX, gvec_xx_i, MO_32, do_vmaxi_s) +TRANS(xvmaxi_d, LASX, gvec_xx_i, MO_64, do_vmaxi_s) +TRANS(xvmaxi_bu, LASX, gvec_xx_i, MO_8, do_vmaxi_u) +TRANS(xvmaxi_hu, LASX, gvec_xx_i, MO_16, do_vmaxi_u) +TRANS(xvmaxi_wu, LASX, gvec_xx_i, MO_32, do_vmaxi_u) +TRANS(xvmaxi_du, LASX, gvec_xx_i, MO_64, do_vmaxi_u) TRANS(vmul_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_mul) TRANS(vmul_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_mul) TRANS(vmul_w, LSX, gvec_vvv, MO_32, tcg_gen_gvec_mul) TRANS(vmul_d, LSX, gvec_vvv, MO_64, tcg_gen_gvec_mul) +TRANS(xvmul_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_mul) +TRANS(xvmul_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_mul) +TRANS(xvmul_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_mul) +TRANS(xvmul_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_mul) static void gen_vmuh_w(TCGv_i32 t, TCGv_i32 a, TCGv_i32 b) { @@ -1615,6 +2007,10 @@ TRANS(vmuh_b, LSX, gvec_vvv, MO_8, do_vmuh_s) TRANS(vmuh_h, LSX, gvec_vvv, MO_16, do_vmuh_s) TRANS(vmuh_w, LSX, gvec_vvv, MO_32, do_vmuh_s) TRANS(vmuh_d, LSX, gvec_vvv, MO_64, do_vmuh_s) +TRANS(xvmuh_b, LASX, gvec_xxx, MO_8, do_vmuh_s) +TRANS(xvmuh_h, LASX, gvec_xxx, MO_16, do_vmuh_s) +TRANS(xvmuh_w, LASX, gvec_xxx, MO_32, do_vmuh_s) +TRANS(xvmuh_d, LASX, gvec_xxx, MO_64, do_vmuh_s) static void gen_vmuh_wu(TCGv_i32 t, TCGv_i32 a, TCGv_i32 b) { @@ -1659,6 +2055,10 @@ TRANS(vmuh_bu, LSX, gvec_vvv, MO_8, do_vmuh_u) TRANS(vmuh_hu, LSX, gvec_vvv, MO_16, do_vmuh_u) TRANS(vmuh_wu, LSX, gvec_vvv, MO_32, do_vmuh_u) TRANS(vmuh_du, LSX, gvec_vvv, MO_64, do_vmuh_u) +TRANS(xvmuh_bu, LASX, gvec_xxx, MO_8, do_vmuh_u) +TRANS(xvmuh_hu, LASX, gvec_xxx, MO_16, do_vmuh_u) +TRANS(xvmuh_wu, LASX, gvec_xxx, MO_32, do_vmuh_u) +TRANS(xvmuh_du, LASX, gvec_xxx, MO_64, do_vmuh_u) static void gen_vmulwev_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -1731,6 +2131,9 @@ static void do_vmulwev_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmulwev_h_b, LSX, gvec_vvv, MO_8, do_vmulwev_s) TRANS(vmulwev_w_h, LSX, gvec_vvv, MO_16, do_vmulwev_s) TRANS(vmulwev_d_w, LSX, gvec_vvv, MO_32, do_vmulwev_s) +TRANS(xvmulwev_h_b, LASX, gvec_xxx, MO_8, do_vmulwev_s) +TRANS(xvmulwev_w_h, LASX, gvec_xxx, MO_16, do_vmulwev_s) +TRANS(xvmulwev_d_w, LASX, gvec_xxx, MO_32, do_vmulwev_s) static void tcg_gen_mulus2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) @@ -1738,37 +2141,62 @@ static void tcg_gen_mulus2_i64(TCGv_i64 rl, TCGv_i64 rh, tcg_gen_mulsu2_i64(rl, rh, arg2, arg1); } -#define VMUL_Q(NAME, FN, idx1, idx2) \ -static bool trans_## NAME (DisasContext *ctx, arg_vvv *a) \ -{ \ - TCGv_i64 rh, rl, arg1, arg2; \ - \ - if (!avail_LSX(ctx)) { \ - return false; \ - } \ - \ - rh = tcg_temp_new_i64(); \ - rl = tcg_temp_new_i64(); \ - arg1 = tcg_temp_new_i64(); \ - arg2 = tcg_temp_new_i64(); \ - \ - get_vreg64(arg1, a->vj, idx1); \ - get_vreg64(arg2, a->vk, idx2); \ - \ - tcg_gen_## FN ##_i64(rl, rh, arg1, arg2); \ - \ - set_vreg64(rh, a->vd, 1); \ - set_vreg64(rl, a->vd, 0); \ - \ - return true; \ -} - -VMUL_Q(vmulwev_q_d, muls2, 0, 0) -VMUL_Q(vmulwod_q_d, muls2, 1, 1) -VMUL_Q(vmulwev_q_du, mulu2, 0, 0) -VMUL_Q(vmulwod_q_du, mulu2, 1, 1) -VMUL_Q(vmulwev_q_du_d, mulus2, 0, 0) -VMUL_Q(vmulwod_q_du_d, mulus2, 1, 1) +static bool gen_vmul_q_vl(DisasContext *ctx, + arg_vvv *a, uint32_t oprsz, int idx1, int idx2, + void (*func)(TCGv_i64, TCGv_i64, + TCGv_i64, TCGv_i64)) +{ + TCGv_i64 rh, rl, arg1, arg2; + int i; + + if (!check_vec(ctx, oprsz)) { + return true; + } + + rh = tcg_temp_new_i64(); + rl = tcg_temp_new_i64(); + arg1 = tcg_temp_new_i64(); + arg2 = tcg_temp_new_i64(); + + for (i = 0; i < oprsz / 16; i++) { + get_vreg64(arg1, a->vj, 2 * i + idx1); + get_vreg64(arg2, a->vk, 2 * i + idx2); + + func(rl, rh, arg1, arg2); + + set_vreg64(rh, a->vd, 2 * i + 1); + set_vreg64(rl, a->vd, 2 * i); + } + + return true; +} + +static bool gen_vmul_q(DisasContext *ctx, arg_vvv *a, int idx1, int idx2, + void (*func)(TCGv_i64, TCGv_i64, + TCGv_i64, TCGv_i64)) +{ + return gen_vmul_q_vl(ctx, a, 16, idx1, idx2, func); +} + +static bool gen_xvmul_q(DisasContext *ctx, arg_vvv *a, int idx1, int idx2, + void (*func)(TCGv_i64, TCGv_i64, + TCGv_i64, TCGv_i64)) +{ + return gen_vmul_q_vl(ctx, a, 32, idx1, idx2, func); +} + +TRANS(vmulwev_q_d, LSX, gen_vmul_q, 0, 0, tcg_gen_muls2_i64) +TRANS(vmulwod_q_d, LSX, gen_vmul_q, 1, 1, tcg_gen_muls2_i64) +TRANS(vmulwev_q_du, LSX, gen_vmul_q, 0, 0, tcg_gen_mulu2_i64) +TRANS(vmulwod_q_du, LSX, gen_vmul_q, 1, 1, tcg_gen_mulu2_i64) +TRANS(vmulwev_q_du_d, LSX, gen_vmul_q, 0, 0, tcg_gen_mulus2_i64) +TRANS(vmulwod_q_du_d, LSX, gen_vmul_q, 1, 1, tcg_gen_mulus2_i64) +TRANS(xvmulwev_q_d, LASX, gen_xvmul_q, 0, 0, tcg_gen_muls2_i64) +TRANS(xvmulwod_q_d, LASX, gen_xvmul_q, 1, 1, tcg_gen_muls2_i64) +TRANS(xvmulwev_q_du, LASX, gen_xvmul_q, 0, 0, tcg_gen_mulu2_i64) +TRANS(xvmulwod_q_du, LASX, gen_xvmul_q, 1, 1, tcg_gen_mulu2_i64) +TRANS(xvmulwev_q_du_d, LASX, gen_xvmul_q, 0, 0, tcg_gen_mulus2_i64) +TRANS(xvmulwod_q_du_d, LASX, gen_xvmul_q, 1, 1, tcg_gen_mulus2_i64) static void gen_vmulwod_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -1839,6 +2267,9 @@ static void do_vmulwod_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmulwod_h_b, LSX, gvec_vvv, MO_8, do_vmulwod_s) TRANS(vmulwod_w_h, LSX, gvec_vvv, MO_16, do_vmulwod_s) TRANS(vmulwod_d_w, LSX, gvec_vvv, MO_32, do_vmulwod_s) +TRANS(xvmulwod_h_b, LASX, gvec_xxx, MO_8, do_vmulwod_s) +TRANS(xvmulwod_w_h, LASX, gvec_xxx, MO_16, do_vmulwod_s) +TRANS(xvmulwod_d_w, LASX, gvec_xxx, MO_32, do_vmulwod_s) static void gen_vmulwev_u(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -1909,6 +2340,9 @@ static void do_vmulwev_u(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmulwev_h_bu, LSX, gvec_vvv, MO_8, do_vmulwev_u) TRANS(vmulwev_w_hu, LSX, gvec_vvv, MO_16, do_vmulwev_u) TRANS(vmulwev_d_wu, LSX, gvec_vvv, MO_32, do_vmulwev_u) +TRANS(xvmulwev_h_bu, LASX, gvec_xxx, MO_8, do_vmulwev_u) +TRANS(xvmulwev_w_hu, LASX, gvec_xxx, MO_16, do_vmulwev_u) +TRANS(xvmulwev_d_wu, LASX, gvec_xxx, MO_32, do_vmulwev_u) static void gen_vmulwod_u(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -1979,6 +2413,9 @@ static void do_vmulwod_u(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmulwod_h_bu, LSX, gvec_vvv, MO_8, do_vmulwod_u) TRANS(vmulwod_w_hu, LSX, gvec_vvv, MO_16, do_vmulwod_u) TRANS(vmulwod_d_wu, LSX, gvec_vvv, MO_32, do_vmulwod_u) +TRANS(xvmulwod_h_bu, LASX, gvec_xxx, MO_8, do_vmulwod_u) +TRANS(xvmulwod_w_hu, LASX, gvec_xxx, MO_16, do_vmulwod_u) +TRANS(xvmulwod_d_wu, LASX, gvec_xxx, MO_32, do_vmulwod_u) static void gen_vmulwev_u_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2051,6 +2488,9 @@ static void do_vmulwev_u_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmulwev_h_bu_b, LSX, gvec_vvv, MO_8, do_vmulwev_u_s) TRANS(vmulwev_w_hu_h, LSX, gvec_vvv, MO_16, do_vmulwev_u_s) TRANS(vmulwev_d_wu_w, LSX, gvec_vvv, MO_32, do_vmulwev_u_s) +TRANS(xvmulwev_h_bu_b, LASX, gvec_xxx, MO_8, do_vmulwev_u_s) +TRANS(xvmulwev_w_hu_h, LASX, gvec_xxx, MO_16, do_vmulwev_u_s) +TRANS(xvmulwev_d_wu_w, LASX, gvec_xxx, MO_32, do_vmulwev_u_s) static void gen_vmulwod_u_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2120,6 +2560,9 @@ static void do_vmulwod_u_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmulwod_h_bu_b, LSX, gvec_vvv, MO_8, do_vmulwod_u_s) TRANS(vmulwod_w_hu_h, LSX, gvec_vvv, MO_16, do_vmulwod_u_s) TRANS(vmulwod_d_wu_w, LSX, gvec_vvv, MO_32, do_vmulwod_u_s) +TRANS(xvmulwod_h_bu_b, LASX, gvec_xxx, MO_8, do_vmulwod_u_s) +TRANS(xvmulwod_w_hu_h, LASX, gvec_xxx, MO_16, do_vmulwod_u_s) +TRANS(xvmulwod_d_wu_w, LASX, gvec_xxx, MO_32, do_vmulwod_u_s) static void gen_vmadd(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2194,6 +2637,10 @@ TRANS(vmadd_b, LSX, gvec_vvv, MO_8, do_vmadd) TRANS(vmadd_h, LSX, gvec_vvv, MO_16, do_vmadd) TRANS(vmadd_w, LSX, gvec_vvv, MO_32, do_vmadd) TRANS(vmadd_d, LSX, gvec_vvv, MO_64, do_vmadd) +TRANS(xvmadd_b, LASX, gvec_xxx, MO_8, do_vmadd) +TRANS(xvmadd_h, LASX, gvec_xxx, MO_16, do_vmadd) +TRANS(xvmadd_w, LASX, gvec_xxx, MO_32, do_vmadd) +TRANS(xvmadd_d, LASX, gvec_xxx, MO_64, do_vmadd) static void gen_vmsub(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2268,6 +2715,10 @@ TRANS(vmsub_b, LSX, gvec_vvv, MO_8, do_vmsub) TRANS(vmsub_h, LSX, gvec_vvv, MO_16, do_vmsub) TRANS(vmsub_w, LSX, gvec_vvv, MO_32, do_vmsub) TRANS(vmsub_d, LSX, gvec_vvv, MO_64, do_vmsub) +TRANS(xvmsub_b, LASX, gvec_xxx, MO_8, do_vmsub) +TRANS(xvmsub_h, LASX, gvec_xxx, MO_16, do_vmsub) +TRANS(xvmsub_w, LASX, gvec_xxx, MO_32, do_vmsub) +TRANS(xvmsub_d, LASX, gvec_xxx, MO_64, do_vmsub) static void gen_vmaddwev_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2342,43 +2793,69 @@ static void do_vmaddwev_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmaddwev_h_b, LSX, gvec_vvv, MO_8, do_vmaddwev_s) TRANS(vmaddwev_w_h, LSX, gvec_vvv, MO_16, do_vmaddwev_s) TRANS(vmaddwev_d_w, LSX, gvec_vvv, MO_32, do_vmaddwev_s) +TRANS(xvmaddwev_h_b, LASX, gvec_xxx, MO_8, do_vmaddwev_s) +TRANS(xvmaddwev_w_h, LASX, gvec_xxx, MO_16, do_vmaddwev_s) +TRANS(xvmaddwev_d_w, LASX, gvec_xxx, MO_32, do_vmaddwev_s) + +static bool gen_vmadd_q_vl(DisasContext * ctx, + arg_vvv *a, uint32_t oprsz, int idx1, int idx2, + void (*func)(TCGv_i64, TCGv_i64, + TCGv_i64, TCGv_i64)) +{ + TCGv_i64 rh, rl, arg1, arg2, th, tl; + int i; -#define VMADD_Q(NAME, FN, idx1, idx2) \ -static bool trans_## NAME (DisasContext *ctx, arg_vvv *a) \ -{ \ - TCGv_i64 rh, rl, arg1, arg2, th, tl; \ - \ - if (!avail_LSX(ctx)) { \ - return false; \ - } \ - \ - rh = tcg_temp_new_i64(); \ - rl = tcg_temp_new_i64(); \ - arg1 = tcg_temp_new_i64(); \ - arg2 = tcg_temp_new_i64(); \ - th = tcg_temp_new_i64(); \ - tl = tcg_temp_new_i64(); \ - \ - get_vreg64(arg1, a->vj, idx1); \ - get_vreg64(arg2, a->vk, idx2); \ - get_vreg64(rh, a->vd, 1); \ - get_vreg64(rl, a->vd, 0); \ - \ - tcg_gen_## FN ##_i64(tl, th, arg1, arg2); \ - tcg_gen_add2_i64(rl, rh, rl, rh, tl, th); \ - \ - set_vreg64(rh, a->vd, 1); \ - set_vreg64(rl, a->vd, 0); \ - \ - return true; \ -} - -VMADD_Q(vmaddwev_q_d, muls2, 0, 0) -VMADD_Q(vmaddwod_q_d, muls2, 1, 1) -VMADD_Q(vmaddwev_q_du, mulu2, 0, 0) -VMADD_Q(vmaddwod_q_du, mulu2, 1, 1) -VMADD_Q(vmaddwev_q_du_d, mulus2, 0, 0) -VMADD_Q(vmaddwod_q_du_d, mulus2, 1, 1) + if (!check_vec(ctx, oprsz)) { + return true; + } + + rh = tcg_temp_new_i64(); + rl = tcg_temp_new_i64(); + arg1 = tcg_temp_new_i64(); + arg2 = tcg_temp_new_i64(); + th = tcg_temp_new_i64(); + tl = tcg_temp_new_i64(); + + for (i = 0; i < oprsz / 16; i++) { + get_vreg64(arg1, a->vj, 2 * i + idx1); + get_vreg64(arg2, a->vk, 2 * i + idx2); + get_vreg64(rh, a->vd, 2 * i + 1); + get_vreg64(rl, a->vd, 2 * i); + + func(tl, th, arg1, arg2); + tcg_gen_add2_i64(rl, rh, rl, rh, tl, th); + + set_vreg64(rh, a->vd, 2 * i + 1); + set_vreg64(rl, a->vd, 2 * i); + } + + return true; +} + +static bool gen_vmadd_q(DisasContext *ctx, arg_vvv *a, int idx1, int idx2, + void (*func)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64)) +{ + return gen_vmadd_q_vl(ctx, a, 16, idx1, idx2, func); +} + +static bool gen_xvmadd_q(DisasContext *ctx, arg_vvv *a, int idx1, int idx2, + void (*func)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64)) +{ + return gen_vmadd_q_vl(ctx, a, 32, idx1, idx2, func); +} + +TRANS(vmaddwev_q_d, LSX, gen_vmadd_q, 0, 0, tcg_gen_muls2_i64) +TRANS(vmaddwod_q_d, LSX, gen_vmadd_q, 1, 1, tcg_gen_muls2_i64) +TRANS(vmaddwev_q_du, LSX, gen_vmadd_q, 0, 0, tcg_gen_mulu2_i64) +TRANS(vmaddwod_q_du, LSX, gen_vmadd_q, 1, 1, tcg_gen_mulu2_i64) +TRANS(vmaddwev_q_du_d, LSX, gen_vmadd_q, 0, 0, tcg_gen_mulus2_i64) +TRANS(vmaddwod_q_du_d, LSX, gen_vmadd_q, 1, 1, tcg_gen_mulus2_i64) +TRANS(xvmaddwev_q_d, LASX, gen_xvmadd_q, 0, 0, tcg_gen_muls2_i64) +TRANS(xvmaddwod_q_d, LASX, gen_xvmadd_q, 1, 1, tcg_gen_muls2_i64) +TRANS(xvmaddwev_q_du, LASX, gen_xvmadd_q, 0, 0, tcg_gen_mulu2_i64) +TRANS(xvmaddwod_q_du, LASX, gen_xvmadd_q, 1, 1, tcg_gen_mulu2_i64) +TRANS(xvmaddwev_q_du_d, LASX, gen_xvmadd_q, 0, 0, tcg_gen_mulus2_i64) +TRANS(xvmaddwod_q_du_d, LASX, gen_xvmadd_q, 1, 1, tcg_gen_mulus2_i64) static void gen_vmaddwod_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2450,6 +2927,9 @@ static void do_vmaddwod_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmaddwod_h_b, LSX, gvec_vvv, MO_8, do_vmaddwod_s) TRANS(vmaddwod_w_h, LSX, gvec_vvv, MO_16, do_vmaddwod_s) TRANS(vmaddwod_d_w, LSX, gvec_vvv, MO_32, do_vmaddwod_s) +TRANS(xvmaddwod_h_b, LASX, gvec_xxx, MO_8, do_vmaddwod_s) +TRANS(xvmaddwod_w_h, LASX, gvec_xxx, MO_16, do_vmaddwod_s) +TRANS(xvmaddwod_d_w, LASX, gvec_xxx, MO_32, do_vmaddwod_s) static void gen_vmaddwev_u(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2520,6 +3000,9 @@ static void do_vmaddwev_u(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmaddwev_h_bu, LSX, gvec_vvv, MO_8, do_vmaddwev_u) TRANS(vmaddwev_w_hu, LSX, gvec_vvv, MO_16, do_vmaddwev_u) TRANS(vmaddwev_d_wu, LSX, gvec_vvv, MO_32, do_vmaddwev_u) +TRANS(xvmaddwev_h_bu, LASX, gvec_xxx, MO_8, do_vmaddwev_u) +TRANS(xvmaddwev_w_hu, LASX, gvec_xxx, MO_16, do_vmaddwev_u) +TRANS(xvmaddwev_d_wu, LASX, gvec_xxx, MO_32, do_vmaddwev_u) static void gen_vmaddwod_u(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2591,6 +3074,9 @@ static void do_vmaddwod_u(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmaddwod_h_bu, LSX, gvec_vvv, MO_8, do_vmaddwod_u) TRANS(vmaddwod_w_hu, LSX, gvec_vvv, MO_16, do_vmaddwod_u) TRANS(vmaddwod_d_wu, LSX, gvec_vvv, MO_32, do_vmaddwod_u) +TRANS(xvmaddwod_h_bu, LASX, gvec_xxx, MO_8, do_vmaddwod_u) +TRANS(xvmaddwod_w_hu, LASX, gvec_xxx, MO_16, do_vmaddwod_u) +TRANS(xvmaddwod_d_wu, LASX, gvec_xxx, MO_32, do_vmaddwod_u) static void gen_vmaddwev_u_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2664,6 +3150,9 @@ static void do_vmaddwev_u_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmaddwev_h_bu_b, LSX, gvec_vvv, MO_8, do_vmaddwev_u_s) TRANS(vmaddwev_w_hu_h, LSX, gvec_vvv, MO_16, do_vmaddwev_u_s) TRANS(vmaddwev_d_wu_w, LSX, gvec_vvv, MO_32, do_vmaddwev_u_s) +TRANS(xvmaddwev_h_bu_b, LASX, gvec_xxx, MO_8, do_vmaddwev_u_s) +TRANS(xvmaddwev_w_hu_h, LASX, gvec_xxx, MO_16, do_vmaddwev_u_s) +TRANS(xvmaddwev_d_wu_w, LASX, gvec_xxx, MO_32, do_vmaddwev_u_s) static void gen_vmaddwod_u_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2736,6 +3225,9 @@ static void do_vmaddwod_u_s(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, TRANS(vmaddwod_h_bu_b, LSX, gvec_vvv, MO_8, do_vmaddwod_u_s) TRANS(vmaddwod_w_hu_h, LSX, gvec_vvv, MO_16, do_vmaddwod_u_s) TRANS(vmaddwod_d_wu_w, LSX, gvec_vvv, MO_32, do_vmaddwod_u_s) +TRANS(xvmaddwod_h_bu_b, LASX, gvec_xxx, MO_8, do_vmaddwod_u_s) +TRANS(xvmaddwod_w_hu_h, LASX, gvec_xxx, MO_16, do_vmaddwod_u_s) +TRANS(xvmaddwod_d_wu_w, LASX, gvec_xxx, MO_32, do_vmaddwod_u_s) TRANS(vdiv_b, LSX, gen_vvv, gen_helper_vdiv_b) TRANS(vdiv_h, LSX, gen_vvv, gen_helper_vdiv_h) @@ -2753,6 +3245,22 @@ TRANS(vmod_bu, LSX, gen_vvv, gen_helper_vmod_bu) TRANS(vmod_hu, LSX, gen_vvv, gen_helper_vmod_hu) TRANS(vmod_wu, LSX, gen_vvv, gen_helper_vmod_wu) TRANS(vmod_du, LSX, gen_vvv, gen_helper_vmod_du) +TRANS(xvdiv_b, LASX, gen_xxx, gen_helper_vdiv_b) +TRANS(xvdiv_h, LASX, gen_xxx, gen_helper_vdiv_h) +TRANS(xvdiv_w, LASX, gen_xxx, gen_helper_vdiv_w) +TRANS(xvdiv_d, LASX, gen_xxx, gen_helper_vdiv_d) +TRANS(xvdiv_bu, LASX, gen_xxx, gen_helper_vdiv_bu) +TRANS(xvdiv_hu, LASX, gen_xxx, gen_helper_vdiv_hu) +TRANS(xvdiv_wu, LASX, gen_xxx, gen_helper_vdiv_wu) +TRANS(xvdiv_du, LASX, gen_xxx, gen_helper_vdiv_du) +TRANS(xvmod_b, LASX, gen_xxx, gen_helper_vmod_b) +TRANS(xvmod_h, LASX, gen_xxx, gen_helper_vmod_h) +TRANS(xvmod_w, LASX, gen_xxx, gen_helper_vmod_w) +TRANS(xvmod_d, LASX, gen_xxx, gen_helper_vmod_d) +TRANS(xvmod_bu, LASX, gen_xxx, gen_helper_vmod_bu) +TRANS(xvmod_hu, LASX, gen_xxx, gen_helper_vmod_hu) +TRANS(xvmod_wu, LASX, gen_xxx, gen_helper_vmod_wu) +TRANS(xvmod_du, LASX, gen_xxx, gen_helper_vmod_du) static void gen_vsat_s(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec max) { @@ -2805,6 +3313,10 @@ TRANS(vsat_b, LSX, gvec_vv_i, MO_8, do_vsat_s) TRANS(vsat_h, LSX, gvec_vv_i, MO_16, do_vsat_s) TRANS(vsat_w, LSX, gvec_vv_i, MO_32, do_vsat_s) TRANS(vsat_d, LSX, gvec_vv_i, MO_64, do_vsat_s) +TRANS(xvsat_b, LASX, gvec_xx_i, MO_8, do_vsat_s) +TRANS(xvsat_h, LASX, gvec_xx_i, MO_16, do_vsat_s) +TRANS(xvsat_w, LASX, gvec_xx_i, MO_32, do_vsat_s) +TRANS(xvsat_d, LASX, gvec_xx_i, MO_64, do_vsat_s) static void gen_vsat_u(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec max) { @@ -2854,6 +3366,10 @@ TRANS(vsat_bu, LSX, gvec_vv_i, MO_8, do_vsat_u) TRANS(vsat_hu, LSX, gvec_vv_i, MO_16, do_vsat_u) TRANS(vsat_wu, LSX, gvec_vv_i, MO_32, do_vsat_u) TRANS(vsat_du, LSX, gvec_vv_i, MO_64, do_vsat_u) +TRANS(xvsat_bu, LASX, gvec_xx_i, MO_8, do_vsat_u) +TRANS(xvsat_hu, LASX, gvec_xx_i, MO_16, do_vsat_u) +TRANS(xvsat_wu, LASX, gvec_xx_i, MO_32, do_vsat_u) +TRANS(xvsat_du, LASX, gvec_xx_i, MO_64, do_vsat_u) TRANS(vexth_h_b, LSX, gen_vv, gen_helper_vexth_h_b) TRANS(vexth_w_h, LSX, gen_vv, gen_helper_vexth_w_h) @@ -2863,6 +3379,27 @@ TRANS(vexth_hu_bu, LSX, gen_vv, gen_helper_vexth_hu_bu) TRANS(vexth_wu_hu, LSX, gen_vv, gen_helper_vexth_wu_hu) TRANS(vexth_du_wu, LSX, gen_vv, gen_helper_vexth_du_wu) TRANS(vexth_qu_du, LSX, gen_vv, gen_helper_vexth_qu_du) +TRANS(xvexth_h_b, LASX, gen_xx, gen_helper_vexth_h_b) +TRANS(xvexth_w_h, LASX, gen_xx, gen_helper_vexth_w_h) +TRANS(xvexth_d_w, LASX, gen_xx, gen_helper_vexth_d_w) +TRANS(xvexth_q_d, LASX, gen_xx, gen_helper_vexth_q_d) +TRANS(xvexth_hu_bu, LASX, gen_xx, gen_helper_vexth_hu_bu) +TRANS(xvexth_wu_hu, LASX, gen_xx, gen_helper_vexth_wu_hu) +TRANS(xvexth_du_wu, LASX, gen_xx, gen_helper_vexth_du_wu) +TRANS(xvexth_qu_du, LASX, gen_xx, gen_helper_vexth_qu_du) + +TRANS(vext2xv_h_b, LASX, gen_xx, gen_helper_vext2xv_h_b) +TRANS(vext2xv_w_b, LASX, gen_xx, gen_helper_vext2xv_w_b) +TRANS(vext2xv_d_b, LASX, gen_xx, gen_helper_vext2xv_d_b) +TRANS(vext2xv_w_h, LASX, gen_xx, gen_helper_vext2xv_w_h) +TRANS(vext2xv_d_h, LASX, gen_xx, gen_helper_vext2xv_d_h) +TRANS(vext2xv_d_w, LASX, gen_xx, gen_helper_vext2xv_d_w) +TRANS(vext2xv_hu_bu, LASX, gen_xx, gen_helper_vext2xv_hu_bu) +TRANS(vext2xv_wu_bu, LASX, gen_xx, gen_helper_vext2xv_wu_bu) +TRANS(vext2xv_du_bu, LASX, gen_xx, gen_helper_vext2xv_du_bu) +TRANS(vext2xv_wu_hu, LASX, gen_xx, gen_helper_vext2xv_wu_hu) +TRANS(vext2xv_du_hu, LASX, gen_xx, gen_helper_vext2xv_du_hu) +TRANS(vext2xv_du_wu, LASX, gen_xx, gen_helper_vext2xv_du_wu) static void gen_vsigncov(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b) { @@ -2916,6 +3453,10 @@ TRANS(vsigncov_b, LSX, gvec_vvv, MO_8, do_vsigncov) TRANS(vsigncov_h, LSX, gvec_vvv, MO_16, do_vsigncov) TRANS(vsigncov_w, LSX, gvec_vvv, MO_32, do_vsigncov) TRANS(vsigncov_d, LSX, gvec_vvv, MO_64, do_vsigncov) +TRANS(xvsigncov_b, LASX, gvec_xxx, MO_8, do_vsigncov) +TRANS(xvsigncov_h, LASX, gvec_xxx, MO_16, do_vsigncov) +TRANS(xvsigncov_w, LASX, gvec_xxx, MO_32, do_vsigncov) +TRANS(xvsigncov_d, LASX, gvec_xxx, MO_64, do_vsigncov) TRANS(vmskltz_b, LSX, gen_vv, gen_helper_vmskltz_b) TRANS(vmskltz_h, LSX, gen_vv, gen_helper_vmskltz_h) @@ -2923,6 +3464,12 @@ TRANS(vmskltz_w, LSX, gen_vv, gen_helper_vmskltz_w) TRANS(vmskltz_d, LSX, gen_vv, gen_helper_vmskltz_d) TRANS(vmskgez_b, LSX, gen_vv, gen_helper_vmskgez_b) TRANS(vmsknz_b, LSX, gen_vv, gen_helper_vmsknz_b) +TRANS(xvmskltz_b, LASX, gen_xx, gen_helper_vmskltz_b) +TRANS(xvmskltz_h, LASX, gen_xx, gen_helper_vmskltz_h) +TRANS(xvmskltz_w, LASX, gen_xx, gen_helper_vmskltz_w) +TRANS(xvmskltz_d, LASX, gen_xx, gen_helper_vmskltz_d) +TRANS(xvmskgez_b, LASX, gen_xx, gen_helper_vmskgez_b) +TRANS(xvmsknz_b, LASX, gen_xx, gen_helper_vmsknz_b) #define EXPAND_BYTE(bit) ((uint64_t)(bit ? 0xff : 0)) @@ -3040,17 +3587,15 @@ static uint64_t vldi_get_value(DisasContext *ctx, uint32_t imm) return data; } -static bool trans_vldi(DisasContext *ctx, arg_vldi *a) +static bool gen_vldi(DisasContext *ctx, arg_vldi *a, uint32_t oprsz) { int sel, vece; uint64_t value; - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, oprsz)) { + return true; } - CHECK_SXE; - sel = (a->imm >> 12) & 0x1; if (sel) { @@ -3061,37 +3606,29 @@ static bool trans_vldi(DisasContext *ctx, arg_vldi *a) vece = (a->imm >> 10) & 0x3; } - tcg_gen_gvec_dup_i64(vece, vec_full_offset(a->vd), 16, ctx->vl/8, + tcg_gen_gvec_dup_i64(vece, vec_full_offset(a->vd), oprsz, ctx->vl/8, tcg_constant_i64(value)); return true; } -TRANS(vand_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_and) -TRANS(vor_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_or) -TRANS(vxor_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_xor) -TRANS(vnor_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_nor) +TRANS(vldi, LSX, gen_vldi, 16) +TRANS(xvldi, LASX, gen_vldi, 32) -static bool trans_vandn_v(DisasContext *ctx, arg_vvv *a) +static bool gen_vandn_v(DisasContext *ctx, arg_vvv *a, uint32_t oprsz) { uint32_t vd_ofs, vj_ofs, vk_ofs; - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, oprsz)) { + return true; } - CHECK_SXE; - vd_ofs = vec_full_offset(a->vd); vj_ofs = vec_full_offset(a->vj); vk_ofs = vec_full_offset(a->vk); - tcg_gen_gvec_andc(MO_64, vd_ofs, vk_ofs, vj_ofs, 16, ctx->vl/8); + tcg_gen_gvec_andc(MO_64, vd_ofs, vk_ofs, vj_ofs, oprsz, ctx->vl / 8); return true; } -TRANS(vorn_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_orc) -TRANS(vandi_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_andi) -TRANS(vori_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_ori) -TRANS(vxori_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_xori) static void gen_vnori(unsigned vece, TCGv_vec t, TCGv_vec a, int64_t imm) { @@ -3124,7 +3661,26 @@ static void do_vnori_b(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, tcg_gen_gvec_2i(vd_ofs, vj_ofs, oprsz, maxsz, imm, &op); } +TRANS(vand_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_and) +TRANS(vor_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_or) +TRANS(vxor_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_xor) +TRANS(vnor_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_nor) +TRANS(vandn_v, LSX, gen_vandn_v, 16) +TRANS(vorn_v, LSX, gvec_vvv, MO_64, tcg_gen_gvec_orc) +TRANS(vandi_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_andi) +TRANS(vori_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_ori) +TRANS(vxori_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_xori) TRANS(vnori_b, LSX, gvec_vv_i, MO_8, do_vnori_b) +TRANS(xvand_v, LASX, gvec_xxx, MO_64, tcg_gen_gvec_and) +TRANS(xvor_v, LASX, gvec_xxx, MO_64, tcg_gen_gvec_or) +TRANS(xvxor_v, LASX, gvec_xxx, MO_64, tcg_gen_gvec_xor) +TRANS(xvnor_v, LASX, gvec_xxx, MO_64, tcg_gen_gvec_nor) +TRANS(xvandn_v, LASX, gen_vandn_v, 32) +TRANS(xvorn_v, LASX, gvec_xxx, MO_64, tcg_gen_gvec_orc) +TRANS(xvandi_b, LASX, gvec_xx_i, MO_8, tcg_gen_gvec_andi) +TRANS(xvori_b, LASX, gvec_xx_i, MO_8, tcg_gen_gvec_ori) +TRANS(xvxori_b, LASX, gvec_xx_i, MO_8, tcg_gen_gvec_xori) +TRANS(xvnori_b, LASX, gvec_xx_i, MO_8, do_vnori_b) TRANS(vsll_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_shlv) TRANS(vsll_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_shlv) @@ -3134,6 +3690,14 @@ TRANS(vslli_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_shli) TRANS(vslli_h, LSX, gvec_vv_i, MO_16, tcg_gen_gvec_shli) TRANS(vslli_w, LSX, gvec_vv_i, MO_32, tcg_gen_gvec_shli) TRANS(vslli_d, LSX, gvec_vv_i, MO_64, tcg_gen_gvec_shli) +TRANS(xvsll_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_shlv) +TRANS(xvsll_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_shlv) +TRANS(xvsll_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_shlv) +TRANS(xvsll_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_shlv) +TRANS(xvslli_b, LASX, gvec_xx_i, MO_8, tcg_gen_gvec_shli) +TRANS(xvslli_h, LASX, gvec_xx_i, MO_16, tcg_gen_gvec_shli) +TRANS(xvslli_w, LASX, gvec_xx_i, MO_32, tcg_gen_gvec_shli) +TRANS(xvslli_d, LASX, gvec_xx_i, MO_64, tcg_gen_gvec_shli) TRANS(vsrl_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_shrv) TRANS(vsrl_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_shrv) @@ -3143,6 +3707,14 @@ TRANS(vsrli_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_shri) TRANS(vsrli_h, LSX, gvec_vv_i, MO_16, tcg_gen_gvec_shri) TRANS(vsrli_w, LSX, gvec_vv_i, MO_32, tcg_gen_gvec_shri) TRANS(vsrli_d, LSX, gvec_vv_i, MO_64, tcg_gen_gvec_shri) +TRANS(xvsrl_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_shrv) +TRANS(xvsrl_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_shrv) +TRANS(xvsrl_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_shrv) +TRANS(xvsrl_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_shrv) +TRANS(xvsrli_b, LASX, gvec_xx_i, MO_8, tcg_gen_gvec_shri) +TRANS(xvsrli_h, LASX, gvec_xx_i, MO_16, tcg_gen_gvec_shri) +TRANS(xvsrli_w, LASX, gvec_xx_i, MO_32, tcg_gen_gvec_shri) +TRANS(xvsrli_d, LASX, gvec_xx_i, MO_64, tcg_gen_gvec_shri) TRANS(vsra_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_sarv) TRANS(vsra_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_sarv) @@ -3152,6 +3724,14 @@ TRANS(vsrai_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_sari) TRANS(vsrai_h, LSX, gvec_vv_i, MO_16, tcg_gen_gvec_sari) TRANS(vsrai_w, LSX, gvec_vv_i, MO_32, tcg_gen_gvec_sari) TRANS(vsrai_d, LSX, gvec_vv_i, MO_64, tcg_gen_gvec_sari) +TRANS(xvsra_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_sarv) +TRANS(xvsra_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_sarv) +TRANS(xvsra_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_sarv) +TRANS(xvsra_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_sarv) +TRANS(xvsrai_b, LASX, gvec_xx_i, MO_8, tcg_gen_gvec_sari) +TRANS(xvsrai_h, LASX, gvec_xx_i, MO_16, tcg_gen_gvec_sari) +TRANS(xvsrai_w, LASX, gvec_xx_i, MO_32, tcg_gen_gvec_sari) +TRANS(xvsrai_d, LASX, gvec_xx_i, MO_64, tcg_gen_gvec_sari) TRANS(vrotr_b, LSX, gvec_vvv, MO_8, tcg_gen_gvec_rotrv) TRANS(vrotr_h, LSX, gvec_vvv, MO_16, tcg_gen_gvec_rotrv) @@ -3161,6 +3741,14 @@ TRANS(vrotri_b, LSX, gvec_vv_i, MO_8, tcg_gen_gvec_rotri) TRANS(vrotri_h, LSX, gvec_vv_i, MO_16, tcg_gen_gvec_rotri) TRANS(vrotri_w, LSX, gvec_vv_i, MO_32, tcg_gen_gvec_rotri) TRANS(vrotri_d, LSX, gvec_vv_i, MO_64, tcg_gen_gvec_rotri) +TRANS(xvrotr_b, LASX, gvec_xxx, MO_8, tcg_gen_gvec_rotrv) +TRANS(xvrotr_h, LASX, gvec_xxx, MO_16, tcg_gen_gvec_rotrv) +TRANS(xvrotr_w, LASX, gvec_xxx, MO_32, tcg_gen_gvec_rotrv) +TRANS(xvrotr_d, LASX, gvec_xxx, MO_64, tcg_gen_gvec_rotrv) +TRANS(xvrotri_b, LASX, gvec_xx_i, MO_8, tcg_gen_gvec_rotri) +TRANS(xvrotri_h, LASX, gvec_xx_i, MO_16, tcg_gen_gvec_rotri) +TRANS(xvrotri_w, LASX, gvec_xx_i, MO_32, tcg_gen_gvec_rotri) +TRANS(xvrotri_d, LASX, gvec_xx_i, MO_64, tcg_gen_gvec_rotri) TRANS(vsllwil_h_b, LSX, gen_vv_i, gen_helper_vsllwil_h_b) TRANS(vsllwil_w_h, LSX, gen_vv_i, gen_helper_vsllwil_w_h) @@ -3170,6 +3758,14 @@ TRANS(vsllwil_hu_bu, LSX, gen_vv_i, gen_helper_vsllwil_hu_bu) TRANS(vsllwil_wu_hu, LSX, gen_vv_i, gen_helper_vsllwil_wu_hu) TRANS(vsllwil_du_wu, LSX, gen_vv_i, gen_helper_vsllwil_du_wu) TRANS(vextl_qu_du, LSX, gen_vv, gen_helper_vextl_qu_du) +TRANS(xvsllwil_h_b, LASX, gen_xx_i, gen_helper_vsllwil_h_b) +TRANS(xvsllwil_w_h, LASX, gen_xx_i, gen_helper_vsllwil_w_h) +TRANS(xvsllwil_d_w, LASX, gen_xx_i, gen_helper_vsllwil_d_w) +TRANS(xvextl_q_d, LASX, gen_xx, gen_helper_vextl_q_d) +TRANS(xvsllwil_hu_bu, LASX, gen_xx_i, gen_helper_vsllwil_hu_bu) +TRANS(xvsllwil_wu_hu, LASX, gen_xx_i, gen_helper_vsllwil_wu_hu) +TRANS(xvsllwil_du_wu, LASX, gen_xx_i, gen_helper_vsllwil_du_wu) +TRANS(xvextl_qu_du, LASX, gen_xx, gen_helper_vextl_qu_du) TRANS(vsrlr_b, LSX, gen_vvv, gen_helper_vsrlr_b) TRANS(vsrlr_h, LSX, gen_vvv, gen_helper_vsrlr_h) @@ -3179,6 +3775,14 @@ TRANS(vsrlri_b, LSX, gen_vv_i, gen_helper_vsrlri_b) TRANS(vsrlri_h, LSX, gen_vv_i, gen_helper_vsrlri_h) TRANS(vsrlri_w, LSX, gen_vv_i, gen_helper_vsrlri_w) TRANS(vsrlri_d, LSX, gen_vv_i, gen_helper_vsrlri_d) +TRANS(xvsrlr_b, LASX, gen_xxx, gen_helper_vsrlr_b) +TRANS(xvsrlr_h, LASX, gen_xxx, gen_helper_vsrlr_h) +TRANS(xvsrlr_w, LASX, gen_xxx, gen_helper_vsrlr_w) +TRANS(xvsrlr_d, LASX, gen_xxx, gen_helper_vsrlr_d) +TRANS(xvsrlri_b, LASX, gen_xx_i, gen_helper_vsrlri_b) +TRANS(xvsrlri_h, LASX, gen_xx_i, gen_helper_vsrlri_h) +TRANS(xvsrlri_w, LASX, gen_xx_i, gen_helper_vsrlri_w) +TRANS(xvsrlri_d, LASX, gen_xx_i, gen_helper_vsrlri_d) TRANS(vsrar_b, LSX, gen_vvv, gen_helper_vsrar_b) TRANS(vsrar_h, LSX, gen_vvv, gen_helper_vsrar_h) @@ -3188,6 +3792,14 @@ TRANS(vsrari_b, LSX, gen_vv_i, gen_helper_vsrari_b) TRANS(vsrari_h, LSX, gen_vv_i, gen_helper_vsrari_h) TRANS(vsrari_w, LSX, gen_vv_i, gen_helper_vsrari_w) TRANS(vsrari_d, LSX, gen_vv_i, gen_helper_vsrari_d) +TRANS(xvsrar_b, LASX, gen_xxx, gen_helper_vsrar_b) +TRANS(xvsrar_h, LASX, gen_xxx, gen_helper_vsrar_h) +TRANS(xvsrar_w, LASX, gen_xxx, gen_helper_vsrar_w) +TRANS(xvsrar_d, LASX, gen_xxx, gen_helper_vsrar_d) +TRANS(xvsrari_b, LASX, gen_xx_i, gen_helper_vsrari_b) +TRANS(xvsrari_h, LASX, gen_xx_i, gen_helper_vsrari_h) +TRANS(xvsrari_w, LASX, gen_xx_i, gen_helper_vsrari_w) +TRANS(xvsrari_d, LASX, gen_xx_i, gen_helper_vsrari_d) TRANS(vsrln_b_h, LSX, gen_vvv, gen_helper_vsrln_b_h) TRANS(vsrln_h_w, LSX, gen_vvv, gen_helper_vsrln_h_w) @@ -3195,6 +3807,12 @@ TRANS(vsrln_w_d, LSX, gen_vvv, gen_helper_vsrln_w_d) TRANS(vsran_b_h, LSX, gen_vvv, gen_helper_vsran_b_h) TRANS(vsran_h_w, LSX, gen_vvv, gen_helper_vsran_h_w) TRANS(vsran_w_d, LSX, gen_vvv, gen_helper_vsran_w_d) +TRANS(xvsrln_b_h, LASX, gen_xxx, gen_helper_vsrln_b_h) +TRANS(xvsrln_h_w, LASX, gen_xxx, gen_helper_vsrln_h_w) +TRANS(xvsrln_w_d, LASX, gen_xxx, gen_helper_vsrln_w_d) +TRANS(xvsran_b_h, LASX, gen_xxx, gen_helper_vsran_b_h) +TRANS(xvsran_h_w, LASX, gen_xxx, gen_helper_vsran_h_w) +TRANS(xvsran_w_d, LASX, gen_xxx, gen_helper_vsran_w_d) TRANS(vsrlni_b_h, LSX, gen_vv_i, gen_helper_vsrlni_b_h) TRANS(vsrlni_h_w, LSX, gen_vv_i, gen_helper_vsrlni_h_w) @@ -3204,6 +3822,14 @@ TRANS(vsrani_b_h, LSX, gen_vv_i, gen_helper_vsrani_b_h) TRANS(vsrani_h_w, LSX, gen_vv_i, gen_helper_vsrani_h_w) TRANS(vsrani_w_d, LSX, gen_vv_i, gen_helper_vsrani_w_d) TRANS(vsrani_d_q, LSX, gen_vv_i, gen_helper_vsrani_d_q) +TRANS(xvsrlni_b_h, LASX, gen_xx_i, gen_helper_vsrlni_b_h) +TRANS(xvsrlni_h_w, LASX, gen_xx_i, gen_helper_vsrlni_h_w) +TRANS(xvsrlni_w_d, LASX, gen_xx_i, gen_helper_vsrlni_w_d) +TRANS(xvsrlni_d_q, LASX, gen_xx_i, gen_helper_vsrlni_d_q) +TRANS(xvsrani_b_h, LASX, gen_xx_i, gen_helper_vsrani_b_h) +TRANS(xvsrani_h_w, LASX, gen_xx_i, gen_helper_vsrani_h_w) +TRANS(xvsrani_w_d, LASX, gen_xx_i, gen_helper_vsrani_w_d) +TRANS(xvsrani_d_q, LASX, gen_xx_i, gen_helper_vsrani_d_q) TRANS(vsrlrn_b_h, LSX, gen_vvv, gen_helper_vsrlrn_b_h) TRANS(vsrlrn_h_w, LSX, gen_vvv, gen_helper_vsrlrn_h_w) @@ -3211,6 +3837,12 @@ TRANS(vsrlrn_w_d, LSX, gen_vvv, gen_helper_vsrlrn_w_d) TRANS(vsrarn_b_h, LSX, gen_vvv, gen_helper_vsrarn_b_h) TRANS(vsrarn_h_w, LSX, gen_vvv, gen_helper_vsrarn_h_w) TRANS(vsrarn_w_d, LSX, gen_vvv, gen_helper_vsrarn_w_d) +TRANS(xvsrlrn_b_h, LASX, gen_xxx, gen_helper_vsrlrn_b_h) +TRANS(xvsrlrn_h_w, LASX, gen_xxx, gen_helper_vsrlrn_h_w) +TRANS(xvsrlrn_w_d, LASX, gen_xxx, gen_helper_vsrlrn_w_d) +TRANS(xvsrarn_b_h, LASX, gen_xxx, gen_helper_vsrarn_b_h) +TRANS(xvsrarn_h_w, LASX, gen_xxx, gen_helper_vsrarn_h_w) +TRANS(xvsrarn_w_d, LASX, gen_xxx, gen_helper_vsrarn_w_d) TRANS(vsrlrni_b_h, LSX, gen_vv_i, gen_helper_vsrlrni_b_h) TRANS(vsrlrni_h_w, LSX, gen_vv_i, gen_helper_vsrlrni_h_w) @@ -3220,6 +3852,14 @@ TRANS(vsrarni_b_h, LSX, gen_vv_i, gen_helper_vsrarni_b_h) TRANS(vsrarni_h_w, LSX, gen_vv_i, gen_helper_vsrarni_h_w) TRANS(vsrarni_w_d, LSX, gen_vv_i, gen_helper_vsrarni_w_d) TRANS(vsrarni_d_q, LSX, gen_vv_i, gen_helper_vsrarni_d_q) +TRANS(xvsrlrni_b_h, LASX, gen_xx_i, gen_helper_vsrlrni_b_h) +TRANS(xvsrlrni_h_w, LASX, gen_xx_i, gen_helper_vsrlrni_h_w) +TRANS(xvsrlrni_w_d, LASX, gen_xx_i, gen_helper_vsrlrni_w_d) +TRANS(xvsrlrni_d_q, LASX, gen_xx_i, gen_helper_vsrlrni_d_q) +TRANS(xvsrarni_b_h, LASX, gen_xx_i, gen_helper_vsrarni_b_h) +TRANS(xvsrarni_h_w, LASX, gen_xx_i, gen_helper_vsrarni_h_w) +TRANS(xvsrarni_w_d, LASX, gen_xx_i, gen_helper_vsrarni_w_d) +TRANS(xvsrarni_d_q, LASX, gen_xx_i, gen_helper_vsrarni_d_q) TRANS(vssrln_b_h, LSX, gen_vvv, gen_helper_vssrln_b_h) TRANS(vssrln_h_w, LSX, gen_vvv, gen_helper_vssrln_h_w) @@ -3233,6 +3873,18 @@ TRANS(vssrln_wu_d, LSX, gen_vvv, gen_helper_vssrln_wu_d) TRANS(vssran_bu_h, LSX, gen_vvv, gen_helper_vssran_bu_h) TRANS(vssran_hu_w, LSX, gen_vvv, gen_helper_vssran_hu_w) TRANS(vssran_wu_d, LSX, gen_vvv, gen_helper_vssran_wu_d) +TRANS(xvssrln_b_h, LASX, gen_xxx, gen_helper_vssrln_b_h) +TRANS(xvssrln_h_w, LASX, gen_xxx, gen_helper_vssrln_h_w) +TRANS(xvssrln_w_d, LASX, gen_xxx, gen_helper_vssrln_w_d) +TRANS(xvssran_b_h, LASX, gen_xxx, gen_helper_vssran_b_h) +TRANS(xvssran_h_w, LASX, gen_xxx, gen_helper_vssran_h_w) +TRANS(xvssran_w_d, LASX, gen_xxx, gen_helper_vssran_w_d) +TRANS(xvssrln_bu_h, LASX, gen_xxx, gen_helper_vssrln_bu_h) +TRANS(xvssrln_hu_w, LASX, gen_xxx, gen_helper_vssrln_hu_w) +TRANS(xvssrln_wu_d, LASX, gen_xxx, gen_helper_vssrln_wu_d) +TRANS(xvssran_bu_h, LASX, gen_xxx, gen_helper_vssran_bu_h) +TRANS(xvssran_hu_w, LASX, gen_xxx, gen_helper_vssran_hu_w) +TRANS(xvssran_wu_d, LASX, gen_xxx, gen_helper_vssran_wu_d) TRANS(vssrlni_b_h, LSX, gen_vv_i, gen_helper_vssrlni_b_h) TRANS(vssrlni_h_w, LSX, gen_vv_i, gen_helper_vssrlni_h_w) @@ -3250,6 +3902,22 @@ TRANS(vssrani_bu_h, LSX, gen_vv_i, gen_helper_vssrani_bu_h) TRANS(vssrani_hu_w, LSX, gen_vv_i, gen_helper_vssrani_hu_w) TRANS(vssrani_wu_d, LSX, gen_vv_i, gen_helper_vssrani_wu_d) TRANS(vssrani_du_q, LSX, gen_vv_i, gen_helper_vssrani_du_q) +TRANS(xvssrlni_b_h, LASX, gen_xx_i, gen_helper_vssrlni_b_h) +TRANS(xvssrlni_h_w, LASX, gen_xx_i, gen_helper_vssrlni_h_w) +TRANS(xvssrlni_w_d, LASX, gen_xx_i, gen_helper_vssrlni_w_d) +TRANS(xvssrlni_d_q, LASX, gen_xx_i, gen_helper_vssrlni_d_q) +TRANS(xvssrani_b_h, LASX, gen_xx_i, gen_helper_vssrani_b_h) +TRANS(xvssrani_h_w, LASX, gen_xx_i, gen_helper_vssrani_h_w) +TRANS(xvssrani_w_d, LASX, gen_xx_i, gen_helper_vssrani_w_d) +TRANS(xvssrani_d_q, LASX, gen_xx_i, gen_helper_vssrani_d_q) +TRANS(xvssrlni_bu_h, LASX, gen_xx_i, gen_helper_vssrlni_bu_h) +TRANS(xvssrlni_hu_w, LASX, gen_xx_i, gen_helper_vssrlni_hu_w) +TRANS(xvssrlni_wu_d, LASX, gen_xx_i, gen_helper_vssrlni_wu_d) +TRANS(xvssrlni_du_q, LASX, gen_xx_i, gen_helper_vssrlni_du_q) +TRANS(xvssrani_bu_h, LASX, gen_xx_i, gen_helper_vssrani_bu_h) +TRANS(xvssrani_hu_w, LASX, gen_xx_i, gen_helper_vssrani_hu_w) +TRANS(xvssrani_wu_d, LASX, gen_xx_i, gen_helper_vssrani_wu_d) +TRANS(xvssrani_du_q, LASX, gen_xx_i, gen_helper_vssrani_du_q) TRANS(vssrlrn_b_h, LSX, gen_vvv, gen_helper_vssrlrn_b_h) TRANS(vssrlrn_h_w, LSX, gen_vvv, gen_helper_vssrlrn_h_w) @@ -3263,6 +3931,18 @@ TRANS(vssrlrn_wu_d, LSX, gen_vvv, gen_helper_vssrlrn_wu_d) TRANS(vssrarn_bu_h, LSX, gen_vvv, gen_helper_vssrarn_bu_h) TRANS(vssrarn_hu_w, LSX, gen_vvv, gen_helper_vssrarn_hu_w) TRANS(vssrarn_wu_d, LSX, gen_vvv, gen_helper_vssrarn_wu_d) +TRANS(xvssrlrn_b_h, LASX, gen_xxx, gen_helper_vssrlrn_b_h) +TRANS(xvssrlrn_h_w, LASX, gen_xxx, gen_helper_vssrlrn_h_w) +TRANS(xvssrlrn_w_d, LASX, gen_xxx, gen_helper_vssrlrn_w_d) +TRANS(xvssrarn_b_h, LASX, gen_xxx, gen_helper_vssrarn_b_h) +TRANS(xvssrarn_h_w, LASX, gen_xxx, gen_helper_vssrarn_h_w) +TRANS(xvssrarn_w_d, LASX, gen_xxx, gen_helper_vssrarn_w_d) +TRANS(xvssrlrn_bu_h, LASX, gen_xxx, gen_helper_vssrlrn_bu_h) +TRANS(xvssrlrn_hu_w, LASX, gen_xxx, gen_helper_vssrlrn_hu_w) +TRANS(xvssrlrn_wu_d, LASX, gen_xxx, gen_helper_vssrlrn_wu_d) +TRANS(xvssrarn_bu_h, LASX, gen_xxx, gen_helper_vssrarn_bu_h) +TRANS(xvssrarn_hu_w, LASX, gen_xxx, gen_helper_vssrarn_hu_w) +TRANS(xvssrarn_wu_d, LASX, gen_xxx, gen_helper_vssrarn_wu_d) TRANS(vssrlrni_b_h, LSX, gen_vv_i, gen_helper_vssrlrni_b_h) TRANS(vssrlrni_h_w, LSX, gen_vv_i, gen_helper_vssrlrni_h_w) @@ -3280,6 +3960,22 @@ TRANS(vssrarni_bu_h, LSX, gen_vv_i, gen_helper_vssrarni_bu_h) TRANS(vssrarni_hu_w, LSX, gen_vv_i, gen_helper_vssrarni_hu_w) TRANS(vssrarni_wu_d, LSX, gen_vv_i, gen_helper_vssrarni_wu_d) TRANS(vssrarni_du_q, LSX, gen_vv_i, gen_helper_vssrarni_du_q) +TRANS(xvssrlrni_b_h, LASX, gen_xx_i, gen_helper_vssrlrni_b_h) +TRANS(xvssrlrni_h_w, LASX, gen_xx_i, gen_helper_vssrlrni_h_w) +TRANS(xvssrlrni_w_d, LASX, gen_xx_i, gen_helper_vssrlrni_w_d) +TRANS(xvssrlrni_d_q, LASX, gen_xx_i, gen_helper_vssrlrni_d_q) +TRANS(xvssrarni_b_h, LASX, gen_xx_i, gen_helper_vssrarni_b_h) +TRANS(xvssrarni_h_w, LASX, gen_xx_i, gen_helper_vssrarni_h_w) +TRANS(xvssrarni_w_d, LASX, gen_xx_i, gen_helper_vssrarni_w_d) +TRANS(xvssrarni_d_q, LASX, gen_xx_i, gen_helper_vssrarni_d_q) +TRANS(xvssrlrni_bu_h, LASX, gen_xx_i, gen_helper_vssrlrni_bu_h) +TRANS(xvssrlrni_hu_w, LASX, gen_xx_i, gen_helper_vssrlrni_hu_w) +TRANS(xvssrlrni_wu_d, LASX, gen_xx_i, gen_helper_vssrlrni_wu_d) +TRANS(xvssrlrni_du_q, LASX, gen_xx_i, gen_helper_vssrlrni_du_q) +TRANS(xvssrarni_bu_h, LASX, gen_xx_i, gen_helper_vssrarni_bu_h) +TRANS(xvssrarni_hu_w, LASX, gen_xx_i, gen_helper_vssrarni_hu_w) +TRANS(xvssrarni_wu_d, LASX, gen_xx_i, gen_helper_vssrarni_wu_d) +TRANS(xvssrarni_du_q, LASX, gen_xx_i, gen_helper_vssrarni_du_q) TRANS(vclo_b, LSX, gen_vv, gen_helper_vclo_b) TRANS(vclo_h, LSX, gen_vv, gen_helper_vclo_h) @@ -3289,11 +3985,23 @@ TRANS(vclz_b, LSX, gen_vv, gen_helper_vclz_b) TRANS(vclz_h, LSX, gen_vv, gen_helper_vclz_h) TRANS(vclz_w, LSX, gen_vv, gen_helper_vclz_w) TRANS(vclz_d, LSX, gen_vv, gen_helper_vclz_d) +TRANS(xvclo_b, LASX, gen_xx, gen_helper_vclo_b) +TRANS(xvclo_h, LASX, gen_xx, gen_helper_vclo_h) +TRANS(xvclo_w, LASX, gen_xx, gen_helper_vclo_w) +TRANS(xvclo_d, LASX, gen_xx, gen_helper_vclo_d) +TRANS(xvclz_b, LASX, gen_xx, gen_helper_vclz_b) +TRANS(xvclz_h, LASX, gen_xx, gen_helper_vclz_h) +TRANS(xvclz_w, LASX, gen_xx, gen_helper_vclz_w) +TRANS(xvclz_d, LASX, gen_xx, gen_helper_vclz_d) TRANS(vpcnt_b, LSX, gen_vv, gen_helper_vpcnt_b) TRANS(vpcnt_h, LSX, gen_vv, gen_helper_vpcnt_h) TRANS(vpcnt_w, LSX, gen_vv, gen_helper_vpcnt_w) TRANS(vpcnt_d, LSX, gen_vv, gen_helper_vpcnt_d) +TRANS(xvpcnt_b, LASX, gen_xx, gen_helper_vpcnt_b) +TRANS(xvpcnt_h, LASX, gen_xx, gen_helper_vpcnt_h) +TRANS(xvpcnt_w, LASX, gen_xx, gen_helper_vpcnt_w) +TRANS(xvpcnt_d, LASX, gen_xx, gen_helper_vpcnt_d) static void do_vbit(unsigned vece, TCGv_vec t, TCGv_vec a, TCGv_vec b, void (*func)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec)) @@ -3365,6 +4073,10 @@ TRANS(vbitclr_b, LSX, gvec_vvv, MO_8, do_vbitclr) TRANS(vbitclr_h, LSX, gvec_vvv, MO_16, do_vbitclr) TRANS(vbitclr_w, LSX, gvec_vvv, MO_32, do_vbitclr) TRANS(vbitclr_d, LSX, gvec_vvv, MO_64, do_vbitclr) +TRANS(xvbitclr_b, LASX, gvec_xxx, MO_8, do_vbitclr) +TRANS(xvbitclr_h, LASX, gvec_xxx, MO_16, do_vbitclr) +TRANS(xvbitclr_w, LASX, gvec_xxx, MO_32, do_vbitclr) +TRANS(xvbitclr_d, LASX, gvec_xxx, MO_64, do_vbitclr) static void do_vbiti(unsigned vece, TCGv_vec t, TCGv_vec a, int64_t imm, void (*func)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec)) @@ -3435,6 +4147,10 @@ TRANS(vbitclri_b, LSX, gvec_vv_i, MO_8, do_vbitclri) TRANS(vbitclri_h, LSX, gvec_vv_i, MO_16, do_vbitclri) TRANS(vbitclri_w, LSX, gvec_vv_i, MO_32, do_vbitclri) TRANS(vbitclri_d, LSX, gvec_vv_i, MO_64, do_vbitclri) +TRANS(xvbitclri_b, LASX, gvec_xx_i, MO_8, do_vbitclri) +TRANS(xvbitclri_h, LASX, gvec_xx_i, MO_16, do_vbitclri) +TRANS(xvbitclri_w, LASX, gvec_xx_i, MO_32, do_vbitclri) +TRANS(xvbitclri_d, LASX, gvec_xx_i, MO_64, do_vbitclri) static void do_vbitset(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, uint32_t vk_ofs, uint32_t oprsz, uint32_t maxsz) @@ -3476,6 +4192,10 @@ TRANS(vbitset_b, LSX, gvec_vvv, MO_8, do_vbitset) TRANS(vbitset_h, LSX, gvec_vvv, MO_16, do_vbitset) TRANS(vbitset_w, LSX, gvec_vvv, MO_32, do_vbitset) TRANS(vbitset_d, LSX, gvec_vvv, MO_64, do_vbitset) +TRANS(xvbitset_b, LASX, gvec_xxx, MO_8, do_vbitset) +TRANS(xvbitset_h, LASX, gvec_xxx, MO_16, do_vbitset) +TRANS(xvbitset_w, LASX, gvec_xxx, MO_32, do_vbitset) +TRANS(xvbitset_d, LASX, gvec_xxx, MO_64, do_vbitset) static void do_vbitseti(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, int64_t imm, uint32_t oprsz, uint32_t maxsz) @@ -3517,6 +4237,10 @@ TRANS(vbitseti_b, LSX, gvec_vv_i, MO_8, do_vbitseti) TRANS(vbitseti_h, LSX, gvec_vv_i, MO_16, do_vbitseti) TRANS(vbitseti_w, LSX, gvec_vv_i, MO_32, do_vbitseti) TRANS(vbitseti_d, LSX, gvec_vv_i, MO_64, do_vbitseti) +TRANS(xvbitseti_b, LASX, gvec_xx_i, MO_8, do_vbitseti) +TRANS(xvbitseti_h, LASX, gvec_xx_i, MO_16, do_vbitseti) +TRANS(xvbitseti_w, LASX, gvec_xx_i, MO_32, do_vbitseti) +TRANS(xvbitseti_d, LASX, gvec_xx_i, MO_64, do_vbitseti) static void do_vbitrev(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, uint32_t vk_ofs, uint32_t oprsz, uint32_t maxsz) @@ -3558,6 +4282,10 @@ TRANS(vbitrev_b, LSX, gvec_vvv, MO_8, do_vbitrev) TRANS(vbitrev_h, LSX, gvec_vvv, MO_16, do_vbitrev) TRANS(vbitrev_w, LSX, gvec_vvv, MO_32, do_vbitrev) TRANS(vbitrev_d, LSX, gvec_vvv, MO_64, do_vbitrev) +TRANS(xvbitrev_b, LASX, gvec_xxx, MO_8, do_vbitrev) +TRANS(xvbitrev_h, LASX, gvec_xxx, MO_16, do_vbitrev) +TRANS(xvbitrev_w, LASX, gvec_xxx, MO_32, do_vbitrev) +TRANS(xvbitrev_d, LASX, gvec_xxx, MO_64, do_vbitrev) static void do_vbitrevi(unsigned vece, uint32_t vd_ofs, uint32_t vj_ofs, int64_t imm, uint32_t oprsz, uint32_t maxsz) @@ -3599,356 +4327,409 @@ TRANS(vbitrevi_b, LSX, gvec_vv_i, MO_8, do_vbitrevi) TRANS(vbitrevi_h, LSX, gvec_vv_i, MO_16, do_vbitrevi) TRANS(vbitrevi_w, LSX, gvec_vv_i, MO_32, do_vbitrevi) TRANS(vbitrevi_d, LSX, gvec_vv_i, MO_64, do_vbitrevi) +TRANS(xvbitrevi_b, LASX, gvec_xx_i, MO_8, do_vbitrevi) +TRANS(xvbitrevi_h, LASX, gvec_xx_i, MO_16, do_vbitrevi) +TRANS(xvbitrevi_w, LASX, gvec_xx_i, MO_32, do_vbitrevi) +TRANS(xvbitrevi_d, LASX, gvec_xx_i, MO_64, do_vbitrevi) TRANS(vfrstp_b, LSX, gen_vvv, gen_helper_vfrstp_b) TRANS(vfrstp_h, LSX, gen_vvv, gen_helper_vfrstp_h) TRANS(vfrstpi_b, LSX, gen_vv_i, gen_helper_vfrstpi_b) TRANS(vfrstpi_h, LSX, gen_vv_i, gen_helper_vfrstpi_h) - -TRANS(vfadd_s, LSX, gen_vvv, gen_helper_vfadd_s) -TRANS(vfadd_d, LSX, gen_vvv, gen_helper_vfadd_d) -TRANS(vfsub_s, LSX, gen_vvv, gen_helper_vfsub_s) -TRANS(vfsub_d, LSX, gen_vvv, gen_helper_vfsub_d) -TRANS(vfmul_s, LSX, gen_vvv, gen_helper_vfmul_s) -TRANS(vfmul_d, LSX, gen_vvv, gen_helper_vfmul_d) -TRANS(vfdiv_s, LSX, gen_vvv, gen_helper_vfdiv_s) -TRANS(vfdiv_d, LSX, gen_vvv, gen_helper_vfdiv_d) - -TRANS(vfmadd_s, LSX, gen_vvvv, gen_helper_vfmadd_s) -TRANS(vfmadd_d, LSX, gen_vvvv, gen_helper_vfmadd_d) -TRANS(vfmsub_s, LSX, gen_vvvv, gen_helper_vfmsub_s) -TRANS(vfmsub_d, LSX, gen_vvvv, gen_helper_vfmsub_d) -TRANS(vfnmadd_s, LSX, gen_vvvv, gen_helper_vfnmadd_s) -TRANS(vfnmadd_d, LSX, gen_vvvv, gen_helper_vfnmadd_d) -TRANS(vfnmsub_s, LSX, gen_vvvv, gen_helper_vfnmsub_s) -TRANS(vfnmsub_d, LSX, gen_vvvv, gen_helper_vfnmsub_d) - -TRANS(vfmax_s, LSX, gen_vvv, gen_helper_vfmax_s) -TRANS(vfmax_d, LSX, gen_vvv, gen_helper_vfmax_d) -TRANS(vfmin_s, LSX, gen_vvv, gen_helper_vfmin_s) -TRANS(vfmin_d, LSX, gen_vvv, gen_helper_vfmin_d) - -TRANS(vfmaxa_s, LSX, gen_vvv, gen_helper_vfmaxa_s) -TRANS(vfmaxa_d, LSX, gen_vvv, gen_helper_vfmaxa_d) -TRANS(vfmina_s, LSX, gen_vvv, gen_helper_vfmina_s) -TRANS(vfmina_d, LSX, gen_vvv, gen_helper_vfmina_d) - -TRANS(vflogb_s, LSX, gen_vv, gen_helper_vflogb_s) -TRANS(vflogb_d, LSX, gen_vv, gen_helper_vflogb_d) - -TRANS(vfclass_s, LSX, gen_vv, gen_helper_vfclass_s) -TRANS(vfclass_d, LSX, gen_vv, gen_helper_vfclass_d) - -TRANS(vfsqrt_s, LSX, gen_vv, gen_helper_vfsqrt_s) -TRANS(vfsqrt_d, LSX, gen_vv, gen_helper_vfsqrt_d) -TRANS(vfrecip_s, LSX, gen_vv, gen_helper_vfrecip_s) -TRANS(vfrecip_d, LSX, gen_vv, gen_helper_vfrecip_d) -TRANS(vfrsqrt_s, LSX, gen_vv, gen_helper_vfrsqrt_s) -TRANS(vfrsqrt_d, LSX, gen_vv, gen_helper_vfrsqrt_d) - -TRANS(vfcvtl_s_h, LSX, gen_vv, gen_helper_vfcvtl_s_h) -TRANS(vfcvth_s_h, LSX, gen_vv, gen_helper_vfcvth_s_h) -TRANS(vfcvtl_d_s, LSX, gen_vv, gen_helper_vfcvtl_d_s) -TRANS(vfcvth_d_s, LSX, gen_vv, gen_helper_vfcvth_d_s) -TRANS(vfcvt_h_s, LSX, gen_vvv, gen_helper_vfcvt_h_s) -TRANS(vfcvt_s_d, LSX, gen_vvv, gen_helper_vfcvt_s_d) - -TRANS(vfrintrne_s, LSX, gen_vv, gen_helper_vfrintrne_s) -TRANS(vfrintrne_d, LSX, gen_vv, gen_helper_vfrintrne_d) -TRANS(vfrintrz_s, LSX, gen_vv, gen_helper_vfrintrz_s) -TRANS(vfrintrz_d, LSX, gen_vv, gen_helper_vfrintrz_d) -TRANS(vfrintrp_s, LSX, gen_vv, gen_helper_vfrintrp_s) -TRANS(vfrintrp_d, LSX, gen_vv, gen_helper_vfrintrp_d) -TRANS(vfrintrm_s, LSX, gen_vv, gen_helper_vfrintrm_s) -TRANS(vfrintrm_d, LSX, gen_vv, gen_helper_vfrintrm_d) -TRANS(vfrint_s, LSX, gen_vv, gen_helper_vfrint_s) -TRANS(vfrint_d, LSX, gen_vv, gen_helper_vfrint_d) - -TRANS(vftintrne_w_s, LSX, gen_vv, gen_helper_vftintrne_w_s) -TRANS(vftintrne_l_d, LSX, gen_vv, gen_helper_vftintrne_l_d) -TRANS(vftintrz_w_s, LSX, gen_vv, gen_helper_vftintrz_w_s) -TRANS(vftintrz_l_d, LSX, gen_vv, gen_helper_vftintrz_l_d) -TRANS(vftintrp_w_s, LSX, gen_vv, gen_helper_vftintrp_w_s) -TRANS(vftintrp_l_d, LSX, gen_vv, gen_helper_vftintrp_l_d) -TRANS(vftintrm_w_s, LSX, gen_vv, gen_helper_vftintrm_w_s) -TRANS(vftintrm_l_d, LSX, gen_vv, gen_helper_vftintrm_l_d) -TRANS(vftint_w_s, LSX, gen_vv, gen_helper_vftint_w_s) -TRANS(vftint_l_d, LSX, gen_vv, gen_helper_vftint_l_d) -TRANS(vftintrz_wu_s, LSX, gen_vv, gen_helper_vftintrz_wu_s) -TRANS(vftintrz_lu_d, LSX, gen_vv, gen_helper_vftintrz_lu_d) -TRANS(vftint_wu_s, LSX, gen_vv, gen_helper_vftint_wu_s) -TRANS(vftint_lu_d, LSX, gen_vv, gen_helper_vftint_lu_d) -TRANS(vftintrne_w_d, LSX, gen_vvv, gen_helper_vftintrne_w_d) -TRANS(vftintrz_w_d, LSX, gen_vvv, gen_helper_vftintrz_w_d) -TRANS(vftintrp_w_d, LSX, gen_vvv, gen_helper_vftintrp_w_d) -TRANS(vftintrm_w_d, LSX, gen_vvv, gen_helper_vftintrm_w_d) -TRANS(vftint_w_d, LSX, gen_vvv, gen_helper_vftint_w_d) -TRANS(vftintrnel_l_s, LSX, gen_vv, gen_helper_vftintrnel_l_s) -TRANS(vftintrneh_l_s, LSX, gen_vv, gen_helper_vftintrneh_l_s) -TRANS(vftintrzl_l_s, LSX, gen_vv, gen_helper_vftintrzl_l_s) -TRANS(vftintrzh_l_s, LSX, gen_vv, gen_helper_vftintrzh_l_s) -TRANS(vftintrpl_l_s, LSX, gen_vv, gen_helper_vftintrpl_l_s) -TRANS(vftintrph_l_s, LSX, gen_vv, gen_helper_vftintrph_l_s) -TRANS(vftintrml_l_s, LSX, gen_vv, gen_helper_vftintrml_l_s) -TRANS(vftintrmh_l_s, LSX, gen_vv, gen_helper_vftintrmh_l_s) -TRANS(vftintl_l_s, LSX, gen_vv, gen_helper_vftintl_l_s) -TRANS(vftinth_l_s, LSX, gen_vv, gen_helper_vftinth_l_s) - -TRANS(vffint_s_w, LSX, gen_vv, gen_helper_vffint_s_w) -TRANS(vffint_d_l, LSX, gen_vv, gen_helper_vffint_d_l) -TRANS(vffint_s_wu, LSX, gen_vv, gen_helper_vffint_s_wu) -TRANS(vffint_d_lu, LSX, gen_vv, gen_helper_vffint_d_lu) -TRANS(vffintl_d_w, LSX, gen_vv, gen_helper_vffintl_d_w) -TRANS(vffinth_d_w, LSX, gen_vv, gen_helper_vffinth_d_w) -TRANS(vffint_s_l, LSX, gen_vvv, gen_helper_vffint_s_l) - -static bool do_cmp(DisasContext *ctx, arg_vvv *a, MemOp mop, TCGCond cond) +TRANS(xvfrstp_b, LASX, gen_xxx, gen_helper_vfrstp_b) +TRANS(xvfrstp_h, LASX, gen_xxx, gen_helper_vfrstp_h) +TRANS(xvfrstpi_b, LASX, gen_xx_i, gen_helper_vfrstpi_b) +TRANS(xvfrstpi_h, LASX, gen_xx_i, gen_helper_vfrstpi_h) + +TRANS(vfadd_s, LSX, gen_vvv_ptr, gen_helper_vfadd_s) +TRANS(vfadd_d, LSX, gen_vvv_ptr, gen_helper_vfadd_d) +TRANS(vfsub_s, LSX, gen_vvv_ptr, gen_helper_vfsub_s) +TRANS(vfsub_d, LSX, gen_vvv_ptr, gen_helper_vfsub_d) +TRANS(vfmul_s, LSX, gen_vvv_ptr, gen_helper_vfmul_s) +TRANS(vfmul_d, LSX, gen_vvv_ptr, gen_helper_vfmul_d) +TRANS(vfdiv_s, LSX, gen_vvv_ptr, gen_helper_vfdiv_s) +TRANS(vfdiv_d, LSX, gen_vvv_ptr, gen_helper_vfdiv_d) +TRANS(xvfadd_s, LASX, gen_xxx_ptr, gen_helper_vfadd_s) +TRANS(xvfadd_d, LASX, gen_xxx_ptr, gen_helper_vfadd_d) +TRANS(xvfsub_s, LASX, gen_xxx_ptr, gen_helper_vfsub_s) +TRANS(xvfsub_d, LASX, gen_xxx_ptr, gen_helper_vfsub_d) +TRANS(xvfmul_s, LASX, gen_xxx_ptr, gen_helper_vfmul_s) +TRANS(xvfmul_d, LASX, gen_xxx_ptr, gen_helper_vfmul_d) +TRANS(xvfdiv_s, LASX, gen_xxx_ptr, gen_helper_vfdiv_s) +TRANS(xvfdiv_d, LASX, gen_xxx_ptr, gen_helper_vfdiv_d) + +TRANS(vfmadd_s, LSX, gen_vvvv_ptr, gen_helper_vfmadd_s) +TRANS(vfmadd_d, LSX, gen_vvvv_ptr, gen_helper_vfmadd_d) +TRANS(vfmsub_s, LSX, gen_vvvv_ptr, gen_helper_vfmsub_s) +TRANS(vfmsub_d, LSX, gen_vvvv_ptr, gen_helper_vfmsub_d) +TRANS(vfnmadd_s, LSX, gen_vvvv_ptr, gen_helper_vfnmadd_s) +TRANS(vfnmadd_d, LSX, gen_vvvv_ptr, gen_helper_vfnmadd_d) +TRANS(vfnmsub_s, LSX, gen_vvvv_ptr, gen_helper_vfnmsub_s) +TRANS(vfnmsub_d, LSX, gen_vvvv_ptr, gen_helper_vfnmsub_d) +TRANS(xvfmadd_s, LASX, gen_xxxx_ptr, gen_helper_vfmadd_s) +TRANS(xvfmadd_d, LASX, gen_xxxx_ptr, gen_helper_vfmadd_d) +TRANS(xvfmsub_s, LASX, gen_xxxx_ptr, gen_helper_vfmsub_s) +TRANS(xvfmsub_d, LASX, gen_xxxx_ptr, gen_helper_vfmsub_d) +TRANS(xvfnmadd_s, LASX, gen_xxxx_ptr, gen_helper_vfnmadd_s) +TRANS(xvfnmadd_d, LASX, gen_xxxx_ptr, gen_helper_vfnmadd_d) +TRANS(xvfnmsub_s, LASX, gen_xxxx_ptr, gen_helper_vfnmsub_s) +TRANS(xvfnmsub_d, LASX, gen_xxxx_ptr, gen_helper_vfnmsub_d) + +TRANS(vfmax_s, LSX, gen_vvv_ptr, gen_helper_vfmax_s) +TRANS(vfmax_d, LSX, gen_vvv_ptr, gen_helper_vfmax_d) +TRANS(vfmin_s, LSX, gen_vvv_ptr, gen_helper_vfmin_s) +TRANS(vfmin_d, LSX, gen_vvv_ptr, gen_helper_vfmin_d) +TRANS(xvfmax_s, LASX, gen_xxx_ptr, gen_helper_vfmax_s) +TRANS(xvfmax_d, LASX, gen_xxx_ptr, gen_helper_vfmax_d) +TRANS(xvfmin_s, LASX, gen_xxx_ptr, gen_helper_vfmin_s) +TRANS(xvfmin_d, LASX, gen_xxx_ptr, gen_helper_vfmin_d) + +TRANS(vfmaxa_s, LSX, gen_vvv_ptr, gen_helper_vfmaxa_s) +TRANS(vfmaxa_d, LSX, gen_vvv_ptr, gen_helper_vfmaxa_d) +TRANS(vfmina_s, LSX, gen_vvv_ptr, gen_helper_vfmina_s) +TRANS(vfmina_d, LSX, gen_vvv_ptr, gen_helper_vfmina_d) +TRANS(xvfmaxa_s, LASX, gen_xxx_ptr, gen_helper_vfmaxa_s) +TRANS(xvfmaxa_d, LASX, gen_xxx_ptr, gen_helper_vfmaxa_d) +TRANS(xvfmina_s, LASX, gen_xxx_ptr, gen_helper_vfmina_s) +TRANS(xvfmina_d, LASX, gen_xxx_ptr, gen_helper_vfmina_d) + +TRANS(vflogb_s, LSX, gen_vv_ptr, gen_helper_vflogb_s) +TRANS(vflogb_d, LSX, gen_vv_ptr, gen_helper_vflogb_d) +TRANS(xvflogb_s, LASX, gen_xx_ptr, gen_helper_vflogb_s) +TRANS(xvflogb_d, LASX, gen_xx_ptr, gen_helper_vflogb_d) + +TRANS(vfclass_s, LSX, gen_vv_ptr, gen_helper_vfclass_s) +TRANS(vfclass_d, LSX, gen_vv_ptr, gen_helper_vfclass_d) +TRANS(xvfclass_s, LASX, gen_xx_ptr, gen_helper_vfclass_s) +TRANS(xvfclass_d, LASX, gen_xx_ptr, gen_helper_vfclass_d) + +TRANS(vfsqrt_s, LSX, gen_vv_ptr, gen_helper_vfsqrt_s) +TRANS(vfsqrt_d, LSX, gen_vv_ptr, gen_helper_vfsqrt_d) +TRANS(vfrecip_s, LSX, gen_vv_ptr, gen_helper_vfrecip_s) +TRANS(vfrecip_d, LSX, gen_vv_ptr, gen_helper_vfrecip_d) +TRANS(vfrsqrt_s, LSX, gen_vv_ptr, gen_helper_vfrsqrt_s) +TRANS(vfrsqrt_d, LSX, gen_vv_ptr, gen_helper_vfrsqrt_d) +TRANS(xvfsqrt_s, LASX, gen_xx_ptr, gen_helper_vfsqrt_s) +TRANS(xvfsqrt_d, LASX, gen_xx_ptr, gen_helper_vfsqrt_d) +TRANS(xvfrecip_s, LASX, gen_xx_ptr, gen_helper_vfrecip_s) +TRANS(xvfrecip_d, LASX, gen_xx_ptr, gen_helper_vfrecip_d) +TRANS(xvfrsqrt_s, LASX, gen_xx_ptr, gen_helper_vfrsqrt_s) +TRANS(xvfrsqrt_d, LASX, gen_xx_ptr, gen_helper_vfrsqrt_d) + +TRANS(vfcvtl_s_h, LSX, gen_vv_ptr, gen_helper_vfcvtl_s_h) +TRANS(vfcvth_s_h, LSX, gen_vv_ptr, gen_helper_vfcvth_s_h) +TRANS(vfcvtl_d_s, LSX, gen_vv_ptr, gen_helper_vfcvtl_d_s) +TRANS(vfcvth_d_s, LSX, gen_vv_ptr, gen_helper_vfcvth_d_s) +TRANS(vfcvt_h_s, LSX, gen_vvv_ptr, gen_helper_vfcvt_h_s) +TRANS(vfcvt_s_d, LSX, gen_vvv_ptr, gen_helper_vfcvt_s_d) +TRANS(xvfcvtl_s_h, LASX, gen_xx_ptr, gen_helper_vfcvtl_s_h) +TRANS(xvfcvth_s_h, LASX, gen_xx_ptr, gen_helper_vfcvth_s_h) +TRANS(xvfcvtl_d_s, LASX, gen_xx_ptr, gen_helper_vfcvtl_d_s) +TRANS(xvfcvth_d_s, LASX, gen_xx_ptr, gen_helper_vfcvth_d_s) +TRANS(xvfcvt_h_s, LASX, gen_xxx_ptr, gen_helper_vfcvt_h_s) +TRANS(xvfcvt_s_d, LASX, gen_xxx_ptr, gen_helper_vfcvt_s_d) + +TRANS(vfrintrne_s, LSX, gen_vv_ptr, gen_helper_vfrintrne_s) +TRANS(vfrintrne_d, LSX, gen_vv_ptr, gen_helper_vfrintrne_d) +TRANS(vfrintrz_s, LSX, gen_vv_ptr, gen_helper_vfrintrz_s) +TRANS(vfrintrz_d, LSX, gen_vv_ptr, gen_helper_vfrintrz_d) +TRANS(vfrintrp_s, LSX, gen_vv_ptr, gen_helper_vfrintrp_s) +TRANS(vfrintrp_d, LSX, gen_vv_ptr, gen_helper_vfrintrp_d) +TRANS(vfrintrm_s, LSX, gen_vv_ptr, gen_helper_vfrintrm_s) +TRANS(vfrintrm_d, LSX, gen_vv_ptr, gen_helper_vfrintrm_d) +TRANS(vfrint_s, LSX, gen_vv_ptr, gen_helper_vfrint_s) +TRANS(vfrint_d, LSX, gen_vv_ptr, gen_helper_vfrint_d) +TRANS(xvfrintrne_s, LASX, gen_xx_ptr, gen_helper_vfrintrne_s) +TRANS(xvfrintrne_d, LASX, gen_xx_ptr, gen_helper_vfrintrne_d) +TRANS(xvfrintrz_s, LASX, gen_xx_ptr, gen_helper_vfrintrz_s) +TRANS(xvfrintrz_d, LASX, gen_xx_ptr, gen_helper_vfrintrz_d) +TRANS(xvfrintrp_s, LASX, gen_xx_ptr, gen_helper_vfrintrp_s) +TRANS(xvfrintrp_d, LASX, gen_xx_ptr, gen_helper_vfrintrp_d) +TRANS(xvfrintrm_s, LASX, gen_xx_ptr, gen_helper_vfrintrm_s) +TRANS(xvfrintrm_d, LASX, gen_xx_ptr, gen_helper_vfrintrm_d) +TRANS(xvfrint_s, LASX, gen_xx_ptr, gen_helper_vfrint_s) +TRANS(xvfrint_d, LASX, gen_xx_ptr, gen_helper_vfrint_d) + +TRANS(vftintrne_w_s, LSX, gen_vv_ptr, gen_helper_vftintrne_w_s) +TRANS(vftintrne_l_d, LSX, gen_vv_ptr, gen_helper_vftintrne_l_d) +TRANS(vftintrz_w_s, LSX, gen_vv_ptr, gen_helper_vftintrz_w_s) +TRANS(vftintrz_l_d, LSX, gen_vv_ptr, gen_helper_vftintrz_l_d) +TRANS(vftintrp_w_s, LSX, gen_vv_ptr, gen_helper_vftintrp_w_s) +TRANS(vftintrp_l_d, LSX, gen_vv_ptr, gen_helper_vftintrp_l_d) +TRANS(vftintrm_w_s, LSX, gen_vv_ptr, gen_helper_vftintrm_w_s) +TRANS(vftintrm_l_d, LSX, gen_vv_ptr, gen_helper_vftintrm_l_d) +TRANS(vftint_w_s, LSX, gen_vv_ptr, gen_helper_vftint_w_s) +TRANS(vftint_l_d, LSX, gen_vv_ptr, gen_helper_vftint_l_d) +TRANS(vftintrz_wu_s, LSX, gen_vv_ptr, gen_helper_vftintrz_wu_s) +TRANS(vftintrz_lu_d, LSX, gen_vv_ptr, gen_helper_vftintrz_lu_d) +TRANS(vftint_wu_s, LSX, gen_vv_ptr, gen_helper_vftint_wu_s) +TRANS(vftint_lu_d, LSX, gen_vv_ptr, gen_helper_vftint_lu_d) +TRANS(vftintrne_w_d, LSX, gen_vvv_ptr, gen_helper_vftintrne_w_d) +TRANS(vftintrz_w_d, LSX, gen_vvv_ptr, gen_helper_vftintrz_w_d) +TRANS(vftintrp_w_d, LSX, gen_vvv_ptr, gen_helper_vftintrp_w_d) +TRANS(vftintrm_w_d, LSX, gen_vvv_ptr, gen_helper_vftintrm_w_d) +TRANS(vftint_w_d, LSX, gen_vvv_ptr, gen_helper_vftint_w_d) +TRANS(vftintrnel_l_s, LSX, gen_vv_ptr, gen_helper_vftintrnel_l_s) +TRANS(vftintrneh_l_s, LSX, gen_vv_ptr, gen_helper_vftintrneh_l_s) +TRANS(vftintrzl_l_s, LSX, gen_vv_ptr, gen_helper_vftintrzl_l_s) +TRANS(vftintrzh_l_s, LSX, gen_vv_ptr, gen_helper_vftintrzh_l_s) +TRANS(vftintrpl_l_s, LSX, gen_vv_ptr, gen_helper_vftintrpl_l_s) +TRANS(vftintrph_l_s, LSX, gen_vv_ptr, gen_helper_vftintrph_l_s) +TRANS(vftintrml_l_s, LSX, gen_vv_ptr, gen_helper_vftintrml_l_s) +TRANS(vftintrmh_l_s, LSX, gen_vv_ptr, gen_helper_vftintrmh_l_s) +TRANS(vftintl_l_s, LSX, gen_vv_ptr, gen_helper_vftintl_l_s) +TRANS(vftinth_l_s, LSX, gen_vv_ptr, gen_helper_vftinth_l_s) +TRANS(xvftintrne_w_s, LASX, gen_xx_ptr, gen_helper_vftintrne_w_s) +TRANS(xvftintrne_l_d, LASX, gen_xx_ptr, gen_helper_vftintrne_l_d) +TRANS(xvftintrz_w_s, LASX, gen_xx_ptr, gen_helper_vftintrz_w_s) +TRANS(xvftintrz_l_d, LASX, gen_xx_ptr, gen_helper_vftintrz_l_d) +TRANS(xvftintrp_w_s, LASX, gen_xx_ptr, gen_helper_vftintrp_w_s) +TRANS(xvftintrp_l_d, LASX, gen_xx_ptr, gen_helper_vftintrp_l_d) +TRANS(xvftintrm_w_s, LASX, gen_xx_ptr, gen_helper_vftintrm_w_s) +TRANS(xvftintrm_l_d, LASX, gen_xx_ptr, gen_helper_vftintrm_l_d) +TRANS(xvftint_w_s, LASX, gen_xx_ptr, gen_helper_vftint_w_s) +TRANS(xvftint_l_d, LASX, gen_xx_ptr, gen_helper_vftint_l_d) +TRANS(xvftintrz_wu_s, LASX, gen_xx_ptr, gen_helper_vftintrz_wu_s) +TRANS(xvftintrz_lu_d, LASX, gen_xx_ptr, gen_helper_vftintrz_lu_d) +TRANS(xvftint_wu_s, LASX, gen_xx_ptr, gen_helper_vftint_wu_s) +TRANS(xvftint_lu_d, LASX, gen_xx_ptr, gen_helper_vftint_lu_d) +TRANS(xvftintrne_w_d, LASX, gen_xxx_ptr, gen_helper_vftintrne_w_d) +TRANS(xvftintrz_w_d, LASX, gen_xxx_ptr, gen_helper_vftintrz_w_d) +TRANS(xvftintrp_w_d, LASX, gen_xxx_ptr, gen_helper_vftintrp_w_d) +TRANS(xvftintrm_w_d, LASX, gen_xxx_ptr, gen_helper_vftintrm_w_d) +TRANS(xvftint_w_d, LASX, gen_xxx_ptr, gen_helper_vftint_w_d) +TRANS(xvftintrnel_l_s, LASX, gen_xx_ptr, gen_helper_vftintrnel_l_s) +TRANS(xvftintrneh_l_s, LASX, gen_xx_ptr, gen_helper_vftintrneh_l_s) +TRANS(xvftintrzl_l_s, LASX, gen_xx_ptr, gen_helper_vftintrzl_l_s) +TRANS(xvftintrzh_l_s, LASX, gen_xx_ptr, gen_helper_vftintrzh_l_s) +TRANS(xvftintrpl_l_s, LASX, gen_xx_ptr, gen_helper_vftintrpl_l_s) +TRANS(xvftintrph_l_s, LASX, gen_xx_ptr, gen_helper_vftintrph_l_s) +TRANS(xvftintrml_l_s, LASX, gen_xx_ptr, gen_helper_vftintrml_l_s) +TRANS(xvftintrmh_l_s, LASX, gen_xx_ptr, gen_helper_vftintrmh_l_s) +TRANS(xvftintl_l_s, LASX, gen_xx_ptr, gen_helper_vftintl_l_s) +TRANS(xvftinth_l_s, LASX, gen_xx_ptr, gen_helper_vftinth_l_s) + +TRANS(vffint_s_w, LSX, gen_vv_ptr, gen_helper_vffint_s_w) +TRANS(vffint_d_l, LSX, gen_vv_ptr, gen_helper_vffint_d_l) +TRANS(vffint_s_wu, LSX, gen_vv_ptr, gen_helper_vffint_s_wu) +TRANS(vffint_d_lu, LSX, gen_vv_ptr, gen_helper_vffint_d_lu) +TRANS(vffintl_d_w, LSX, gen_vv_ptr, gen_helper_vffintl_d_w) +TRANS(vffinth_d_w, LSX, gen_vv_ptr, gen_helper_vffinth_d_w) +TRANS(vffint_s_l, LSX, gen_vvv_ptr, gen_helper_vffint_s_l) +TRANS(xvffint_s_w, LASX, gen_xx_ptr, gen_helper_vffint_s_w) +TRANS(xvffint_d_l, LASX, gen_xx_ptr, gen_helper_vffint_d_l) +TRANS(xvffint_s_wu, LASX, gen_xx_ptr, gen_helper_vffint_s_wu) +TRANS(xvffint_d_lu, LASX, gen_xx_ptr, gen_helper_vffint_d_lu) +TRANS(xvffintl_d_w, LASX, gen_xx_ptr, gen_helper_vffintl_d_w) +TRANS(xvffinth_d_w, LASX, gen_xx_ptr, gen_helper_vffinth_d_w) +TRANS(xvffint_s_l, LASX, gen_xxx_ptr, gen_helper_vffint_s_l) + +static bool do_cmp_vl(DisasContext *ctx, arg_vvv *a, + uint32_t oprsz, MemOp mop, TCGCond cond) { uint32_t vd_ofs, vj_ofs, vk_ofs; - CHECK_SXE; + if (!check_vec(ctx, oprsz)) { + return true; + } vd_ofs = vec_full_offset(a->vd); vj_ofs = vec_full_offset(a->vj); vk_ofs = vec_full_offset(a->vk); - tcg_gen_gvec_cmp(cond, mop, vd_ofs, vj_ofs, vk_ofs, 16, ctx->vl/8); + tcg_gen_gvec_cmp(cond, mop, vd_ofs, vj_ofs, vk_ofs, oprsz, ctx->vl / 8); + return true; +} + +static bool do_cmp(DisasContext *ctx, arg_vvv *a, + MemOp mop, TCGCond cond) +{ + return do_cmp_vl(ctx, a, 16, mop, cond); +} + +static bool do_xcmp(DisasContext *ctx, arg_vvv *a, + MemOp mop, TCGCond cond) +{ + return do_cmp_vl(ctx, a, 32, mop, cond); +} + +static bool do_cmpi_vl(DisasContext *ctx, arg_vv_i *a, + uint32_t oprsz, MemOp mop, TCGCond cond) +{ + uint32_t vd_ofs, vj_ofs; + + if (!check_vec(ctx, oprsz)) { + return true; + } + + vd_ofs = vec_full_offset(a->vd); + vj_ofs = vec_full_offset(a->vj); + + tcg_gen_gvec_cmpi(cond, mop, vd_ofs, vj_ofs, a->imm, oprsz, ctx->vl / 8); return true; } -static void do_cmpi_vec(TCGCond cond, - unsigned vece, TCGv_vec t, TCGv_vec a, int64_t imm) -{ - tcg_gen_cmp_vec(cond, vece, t, a, tcg_constant_vec_matching(t, vece, imm)); -} - -static void gen_vseqi_s_vec(unsigned vece, TCGv_vec t, TCGv_vec a, int64_t imm) -{ - do_cmpi_vec(TCG_COND_EQ, vece, t, a, imm); -} - -static void gen_vslei_s_vec(unsigned vece, TCGv_vec t, TCGv_vec a, int64_t imm) -{ - do_cmpi_vec(TCG_COND_LE, vece, t, a, imm); -} - -static void gen_vslti_s_vec(unsigned vece, TCGv_vec t, TCGv_vec a, int64_t imm) -{ - do_cmpi_vec(TCG_COND_LT, vece, t, a, imm); -} - -static void gen_vslei_u_vec(unsigned vece, TCGv_vec t, TCGv_vec a, int64_t imm) -{ - do_cmpi_vec(TCG_COND_LEU, vece, t, a, imm); -} - -static void gen_vslti_u_vec(unsigned vece, TCGv_vec t, TCGv_vec a, int64_t imm) -{ - do_cmpi_vec(TCG_COND_LTU, vece, t, a, imm); -} - -#define DO_CMPI_S(NAME) \ -static bool do_## NAME ##_s(DisasContext *ctx, arg_vv_i *a, MemOp mop) \ -{ \ - uint32_t vd_ofs, vj_ofs; \ - \ - CHECK_SXE; \ - \ - static const TCGOpcode vecop_list[] = { \ - INDEX_op_cmp_vec, 0 \ - }; \ - static const GVecGen2i op[4] = { \ - { \ - .fniv = gen_## NAME ##_s_vec, \ - .fnoi = gen_helper_## NAME ##_b, \ - .opt_opc = vecop_list, \ - .vece = MO_8 \ - }, \ - { \ - .fniv = gen_## NAME ##_s_vec, \ - .fnoi = gen_helper_## NAME ##_h, \ - .opt_opc = vecop_list, \ - .vece = MO_16 \ - }, \ - { \ - .fniv = gen_## NAME ##_s_vec, \ - .fnoi = gen_helper_## NAME ##_w, \ - .opt_opc = vecop_list, \ - .vece = MO_32 \ - }, \ - { \ - .fniv = gen_## NAME ##_s_vec, \ - .fnoi = gen_helper_## NAME ##_d, \ - .opt_opc = vecop_list, \ - .vece = MO_64 \ - } \ - }; \ - \ - vd_ofs = vec_full_offset(a->vd); \ - vj_ofs = vec_full_offset(a->vj); \ - \ - tcg_gen_gvec_2i(vd_ofs, vj_ofs, 16, ctx->vl/8, a->imm, &op[mop]); \ - \ - return true; \ -} - -DO_CMPI_S(vseqi) -DO_CMPI_S(vslei) -DO_CMPI_S(vslti) - -#define DO_CMPI_U(NAME) \ -static bool do_## NAME ##_u(DisasContext *ctx, arg_vv_i *a, MemOp mop) \ -{ \ - uint32_t vd_ofs, vj_ofs; \ - \ - CHECK_SXE; \ - \ - static const TCGOpcode vecop_list[] = { \ - INDEX_op_cmp_vec, 0 \ - }; \ - static const GVecGen2i op[4] = { \ - { \ - .fniv = gen_## NAME ##_u_vec, \ - .fnoi = gen_helper_## NAME ##_bu, \ - .opt_opc = vecop_list, \ - .vece = MO_8 \ - }, \ - { \ - .fniv = gen_## NAME ##_u_vec, \ - .fnoi = gen_helper_## NAME ##_hu, \ - .opt_opc = vecop_list, \ - .vece = MO_16 \ - }, \ - { \ - .fniv = gen_## NAME ##_u_vec, \ - .fnoi = gen_helper_## NAME ##_wu, \ - .opt_opc = vecop_list, \ - .vece = MO_32 \ - }, \ - { \ - .fniv = gen_## NAME ##_u_vec, \ - .fnoi = gen_helper_## NAME ##_du, \ - .opt_opc = vecop_list, \ - .vece = MO_64 \ - } \ - }; \ - \ - vd_ofs = vec_full_offset(a->vd); \ - vj_ofs = vec_full_offset(a->vj); \ - \ - tcg_gen_gvec_2i(vd_ofs, vj_ofs, 16, ctx->vl/8, a->imm, &op[mop]); \ - \ - return true; \ -} - -DO_CMPI_U(vslei) -DO_CMPI_U(vslti) +static bool do_cmpi(DisasContext *ctx, arg_vv_i *a, + MemOp mop, TCGCond cond) +{ + return do_cmpi_vl(ctx, a, 16, mop, cond); +} + +static bool do_xcmpi(DisasContext *ctx, arg_vv_i *a, + MemOp mop, TCGCond cond) +{ + return do_cmpi_vl(ctx, a, 32, mop, cond); +} TRANS(vseq_b, LSX, do_cmp, MO_8, TCG_COND_EQ) TRANS(vseq_h, LSX, do_cmp, MO_16, TCG_COND_EQ) TRANS(vseq_w, LSX, do_cmp, MO_32, TCG_COND_EQ) TRANS(vseq_d, LSX, do_cmp, MO_64, TCG_COND_EQ) -TRANS(vseqi_b, LSX, do_vseqi_s, MO_8) -TRANS(vseqi_h, LSX, do_vseqi_s, MO_16) -TRANS(vseqi_w, LSX, do_vseqi_s, MO_32) -TRANS(vseqi_d, LSX, do_vseqi_s, MO_64) +TRANS(vseqi_b, LSX, do_cmpi, MO_8, TCG_COND_EQ) +TRANS(vseqi_h, LSX, do_cmpi, MO_16, TCG_COND_EQ) +TRANS(vseqi_w, LSX, do_cmpi, MO_32, TCG_COND_EQ) +TRANS(vseqi_d, LSX, do_cmpi, MO_64, TCG_COND_EQ) +TRANS(xvseq_b, LASX, do_xcmp, MO_8, TCG_COND_EQ) +TRANS(xvseq_h, LASX, do_xcmp, MO_16, TCG_COND_EQ) +TRANS(xvseq_w, LASX, do_xcmp, MO_32, TCG_COND_EQ) +TRANS(xvseq_d, LASX, do_xcmp, MO_64, TCG_COND_EQ) +TRANS(xvseqi_b, LASX, do_xcmpi, MO_8, TCG_COND_EQ) +TRANS(xvseqi_h, LASX, do_xcmpi, MO_16, TCG_COND_EQ) +TRANS(xvseqi_w, LASX, do_xcmpi, MO_32, TCG_COND_EQ) +TRANS(xvseqi_d, LASX, do_xcmpi, MO_64, TCG_COND_EQ) TRANS(vsle_b, LSX, do_cmp, MO_8, TCG_COND_LE) TRANS(vsle_h, LSX, do_cmp, MO_16, TCG_COND_LE) TRANS(vsle_w, LSX, do_cmp, MO_32, TCG_COND_LE) TRANS(vsle_d, LSX, do_cmp, MO_64, TCG_COND_LE) -TRANS(vslei_b, LSX, do_vslei_s, MO_8) -TRANS(vslei_h, LSX, do_vslei_s, MO_16) -TRANS(vslei_w, LSX, do_vslei_s, MO_32) -TRANS(vslei_d, LSX, do_vslei_s, MO_64) +TRANS(vslei_b, LSX, do_cmpi, MO_8, TCG_COND_LE) +TRANS(vslei_h, LSX, do_cmpi, MO_16, TCG_COND_LE) +TRANS(vslei_w, LSX, do_cmpi, MO_32, TCG_COND_LE) +TRANS(vslei_d, LSX, do_cmpi, MO_64, TCG_COND_LE) TRANS(vsle_bu, LSX, do_cmp, MO_8, TCG_COND_LEU) TRANS(vsle_hu, LSX, do_cmp, MO_16, TCG_COND_LEU) TRANS(vsle_wu, LSX, do_cmp, MO_32, TCG_COND_LEU) TRANS(vsle_du, LSX, do_cmp, MO_64, TCG_COND_LEU) -TRANS(vslei_bu, LSX, do_vslei_u, MO_8) -TRANS(vslei_hu, LSX, do_vslei_u, MO_16) -TRANS(vslei_wu, LSX, do_vslei_u, MO_32) -TRANS(vslei_du, LSX, do_vslei_u, MO_64) +TRANS(vslei_bu, LSX, do_cmpi, MO_8, TCG_COND_LEU) +TRANS(vslei_hu, LSX, do_cmpi, MO_16, TCG_COND_LEU) +TRANS(vslei_wu, LSX, do_cmpi, MO_32, TCG_COND_LEU) +TRANS(vslei_du, LSX, do_cmpi, MO_64, TCG_COND_LEU) +TRANS(xvsle_b, LASX, do_xcmp, MO_8, TCG_COND_LE) +TRANS(xvsle_h, LASX, do_xcmp, MO_16, TCG_COND_LE) +TRANS(xvsle_w, LASX, do_xcmp, MO_32, TCG_COND_LE) +TRANS(xvsle_d, LASX, do_xcmp, MO_64, TCG_COND_LE) +TRANS(xvslei_b, LASX, do_xcmpi, MO_8, TCG_COND_LE) +TRANS(xvslei_h, LASX, do_xcmpi, MO_16, TCG_COND_LE) +TRANS(xvslei_w, LASX, do_xcmpi, MO_32, TCG_COND_LE) +TRANS(xvslei_d, LASX, do_xcmpi, MO_64, TCG_COND_LE) +TRANS(xvsle_bu, LASX, do_xcmp, MO_8, TCG_COND_LEU) +TRANS(xvsle_hu, LASX, do_xcmp, MO_16, TCG_COND_LEU) +TRANS(xvsle_wu, LASX, do_xcmp, MO_32, TCG_COND_LEU) +TRANS(xvsle_du, LASX, do_xcmp, MO_64, TCG_COND_LEU) +TRANS(xvslei_bu, LASX, do_xcmpi, MO_8, TCG_COND_LEU) +TRANS(xvslei_hu, LASX, do_xcmpi, MO_16, TCG_COND_LEU) +TRANS(xvslei_wu, LASX, do_xcmpi, MO_32, TCG_COND_LEU) +TRANS(xvslei_du, LASX, do_xcmpi, MO_64, TCG_COND_LEU) TRANS(vslt_b, LSX, do_cmp, MO_8, TCG_COND_LT) TRANS(vslt_h, LSX, do_cmp, MO_16, TCG_COND_LT) TRANS(vslt_w, LSX, do_cmp, MO_32, TCG_COND_LT) TRANS(vslt_d, LSX, do_cmp, MO_64, TCG_COND_LT) -TRANS(vslti_b, LSX, do_vslti_s, MO_8) -TRANS(vslti_h, LSX, do_vslti_s, MO_16) -TRANS(vslti_w, LSX, do_vslti_s, MO_32) -TRANS(vslti_d, LSX, do_vslti_s, MO_64) +TRANS(vslti_b, LSX, do_cmpi, MO_8, TCG_COND_LT) +TRANS(vslti_h, LSX, do_cmpi, MO_16, TCG_COND_LT) +TRANS(vslti_w, LSX, do_cmpi, MO_32, TCG_COND_LT) +TRANS(vslti_d, LSX, do_cmpi, MO_64, TCG_COND_LT) TRANS(vslt_bu, LSX, do_cmp, MO_8, TCG_COND_LTU) TRANS(vslt_hu, LSX, do_cmp, MO_16, TCG_COND_LTU) TRANS(vslt_wu, LSX, do_cmp, MO_32, TCG_COND_LTU) TRANS(vslt_du, LSX, do_cmp, MO_64, TCG_COND_LTU) -TRANS(vslti_bu, LSX, do_vslti_u, MO_8) -TRANS(vslti_hu, LSX, do_vslti_u, MO_16) -TRANS(vslti_wu, LSX, do_vslti_u, MO_32) -TRANS(vslti_du, LSX, do_vslti_u, MO_64) - -static bool trans_vfcmp_cond_s(DisasContext *ctx, arg_vvv_fcond *a) +TRANS(vslti_bu, LSX, do_cmpi, MO_8, TCG_COND_LTU) +TRANS(vslti_hu, LSX, do_cmpi, MO_16, TCG_COND_LTU) +TRANS(vslti_wu, LSX, do_cmpi, MO_32, TCG_COND_LTU) +TRANS(vslti_du, LSX, do_cmpi, MO_64, TCG_COND_LTU) +TRANS(xvslt_b, LASX, do_xcmp, MO_8, TCG_COND_LT) +TRANS(xvslt_h, LASX, do_xcmp, MO_16, TCG_COND_LT) +TRANS(xvslt_w, LASX, do_xcmp, MO_32, TCG_COND_LT) +TRANS(xvslt_d, LASX, do_xcmp, MO_64, TCG_COND_LT) +TRANS(xvslti_b, LASX, do_xcmpi, MO_8, TCG_COND_LT) +TRANS(xvslti_h, LASX, do_xcmpi, MO_16, TCG_COND_LT) +TRANS(xvslti_w, LASX, do_xcmpi, MO_32, TCG_COND_LT) +TRANS(xvslti_d, LASX, do_xcmpi, MO_64, TCG_COND_LT) +TRANS(xvslt_bu, LASX, do_xcmp, MO_8, TCG_COND_LTU) +TRANS(xvslt_hu, LASX, do_xcmp, MO_16, TCG_COND_LTU) +TRANS(xvslt_wu, LASX, do_xcmp, MO_32, TCG_COND_LTU) +TRANS(xvslt_du, LASX, do_xcmp, MO_64, TCG_COND_LTU) +TRANS(xvslti_bu, LASX, do_xcmpi, MO_8, TCG_COND_LTU) +TRANS(xvslti_hu, LASX, do_xcmpi, MO_16, TCG_COND_LTU) +TRANS(xvslti_wu, LASX, do_xcmpi, MO_32, TCG_COND_LTU) +TRANS(xvslti_du, LASX, do_xcmpi, MO_64, TCG_COND_LTU) + +static bool do_vfcmp_cond_s(DisasContext *ctx, arg_vvv_fcond *a, uint32_t sz) { uint32_t flags; - void (*fn)(TCGv_env, TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32); + void (*fn)(TCGv_env, TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32); TCGv_i32 vd = tcg_constant_i32(a->vd); TCGv_i32 vj = tcg_constant_i32(a->vj); TCGv_i32 vk = tcg_constant_i32(a->vk); + TCGv_i32 oprsz = tcg_constant_i32(sz); - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, sz)) { + return true; } - CHECK_SXE; - fn = (a->fcond & 1 ? gen_helper_vfcmp_s_s : gen_helper_vfcmp_c_s); flags = get_fcmp_flags(a->fcond >> 1); - fn(cpu_env, vd, vj, vk, tcg_constant_i32(flags)); + fn(cpu_env, oprsz, vd, vj, vk, tcg_constant_i32(flags)); return true; } -static bool trans_vfcmp_cond_d(DisasContext *ctx, arg_vvv_fcond *a) +static bool do_vfcmp_cond_d(DisasContext *ctx, arg_vvv_fcond *a, uint32_t sz) { uint32_t flags; - void (*fn)(TCGv_env, TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32); + void (*fn)(TCGv_env, TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32); TCGv_i32 vd = tcg_constant_i32(a->vd); TCGv_i32 vj = tcg_constant_i32(a->vj); TCGv_i32 vk = tcg_constant_i32(a->vk); + TCGv_i32 oprsz = tcg_constant_i32(sz); - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, sz)) { + return true; } - CHECK_SXE; - fn = (a->fcond & 1 ? gen_helper_vfcmp_s_d : gen_helper_vfcmp_c_d); flags = get_fcmp_flags(a->fcond >> 1); - fn(cpu_env, vd, vj, vk, tcg_constant_i32(flags)); + fn(cpu_env, oprsz, vd, vj, vk, tcg_constant_i32(flags)); return true; } -static bool trans_vbitsel_v(DisasContext *ctx, arg_vvvv *a) +TRANS(vfcmp_cond_s, LSX, do_vfcmp_cond_s, 16) +TRANS(vfcmp_cond_d, LSX, do_vfcmp_cond_d, 16) +TRANS(xvfcmp_cond_s, LASX, do_vfcmp_cond_s, 32) +TRANS(xvfcmp_cond_d, LASX, do_vfcmp_cond_d, 32) + +static bool do_vbitsel_v(DisasContext *ctx, arg_vvvv *a, uint32_t oprsz) { - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, oprsz)) { + return true; } - CHECK_SXE; - tcg_gen_gvec_bitsel(MO_64, vec_full_offset(a->vd), vec_full_offset(a->va), vec_full_offset(a->vk), vec_full_offset(a->vj), - 16, ctx->vl/8); + oprsz, ctx->vl / 8); return true; } +TRANS(vbitsel_v, LSX, do_vbitsel_v, 16) +TRANS(xvbitsel_v, LASX, do_vbitsel_v, 32) + static void gen_vbitseli(unsigned vece, TCGv_vec a, TCGv_vec b, int64_t imm) { tcg_gen_bitsel_vec(vece, a, a, tcg_constant_vec_matching(a, vece, imm), b); } -static bool trans_vbitseli_b(DisasContext *ctx, arg_vv_i *a) +static bool do_vbitseli_b(DisasContext *ctx, arg_vv_i *a, uint32_t oprsz) { static const GVecGen2i op = { .fniv = gen_vbitseli, @@ -3957,17 +4738,18 @@ static bool trans_vbitseli_b(DisasContext *ctx, arg_vv_i *a) .load_dest = true }; - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, oprsz)) { + return true; } - CHECK_SXE; - tcg_gen_gvec_2i(vec_full_offset(a->vd), vec_full_offset(a->vj), - 16, ctx->vl/8, a->imm, &op); + oprsz, ctx->vl / 8, a->imm , &op); return true; } +TRANS(vbitseli_b, LSX, do_vbitseli_b, 16) +TRANS(xvbitseli_b, LASX, do_vbitseli_b, 32) + #define VSET(NAME, COND) \ static bool trans_## NAME (DisasContext *ctx, arg_cv *a) \ { \ @@ -3984,7 +4766,10 @@ static bool trans_## NAME (DisasContext *ctx, arg_cv *a) \ return false; \ } \ \ - CHECK_SXE; \ + if (!check_vec(ctx, 16)) { \ + return true; \ + } \ + \ tcg_gen_or_i64(t1, al, ah); \ tcg_gen_setcondi_i64(COND, t1, t1, 0); \ tcg_gen_st8_tl(t1, cpu_env, offsetof(CPULoongArchState, cf[a->cd & 0x7])); \ @@ -4004,193 +4789,156 @@ TRANS(vsetallnez_h, LSX, gen_cv, gen_helper_vsetallnez_h) TRANS(vsetallnez_w, LSX, gen_cv, gen_helper_vsetallnez_w) TRANS(vsetallnez_d, LSX, gen_cv, gen_helper_vsetallnez_d) -static bool trans_vinsgr2vr_b(DisasContext *ctx, arg_vr_i *a) -{ - TCGv src = gpr_src(ctx, a->rj, EXT_NONE); - - if (!avail_LSX(ctx)) { - return false; - } - - CHECK_SXE; - tcg_gen_st8_i64(src, cpu_env, - offsetof(CPULoongArchState, fpr[a->vd].vreg.B(a->imm))); - return true; +#define XVSET(NAME, COND) \ +static bool trans_## NAME(DisasContext *ctx, arg_cv * a) \ +{ \ + TCGv_i64 t1, t2, d[4]; \ + \ + d[0] = tcg_temp_new_i64(); \ + d[1] = tcg_temp_new_i64(); \ + d[2] = tcg_temp_new_i64(); \ + d[3] = tcg_temp_new_i64(); \ + t1 = tcg_temp_new_i64(); \ + t2 = tcg_temp_new_i64(); \ + \ + get_vreg64(d[0], a->vj, 0); \ + get_vreg64(d[1], a->vj, 1); \ + get_vreg64(d[2], a->vj, 2); \ + get_vreg64(d[3], a->vj, 3); \ + \ + if (!avail_LASX(ctx)) { \ + return false; \ + } \ + \ + if (!check_vec(ctx, 32)) { \ + return true; \ + } \ + \ + tcg_gen_or_i64(t1, d[0], d[1]); \ + tcg_gen_or_i64(t2, d[2], d[3]); \ + tcg_gen_or_i64(t1, t2, t1); \ + tcg_gen_setcondi_i64(COND, t1, t1, 0); \ + tcg_gen_st8_tl(t1, cpu_env, offsetof(CPULoongArchState, cf[a->cd & 0x7])); \ + \ + return true; \ } -static bool trans_vinsgr2vr_h(DisasContext *ctx, arg_vr_i *a) -{ - TCGv src = gpr_src(ctx, a->rj, EXT_NONE); +XVSET(xvseteqz_v, TCG_COND_EQ) +XVSET(xvsetnez_v, TCG_COND_NE) - if (!avail_LSX(ctx)) { - return false; - } +TRANS(xvsetanyeqz_b, LASX, gen_cx, gen_helper_vsetanyeqz_b) +TRANS(xvsetanyeqz_h, LASX, gen_cx, gen_helper_vsetanyeqz_h) +TRANS(xvsetanyeqz_w, LASX, gen_cx, gen_helper_vsetanyeqz_w) +TRANS(xvsetanyeqz_d, LASX, gen_cx, gen_helper_vsetanyeqz_d) +TRANS(xvsetallnez_b, LASX, gen_cx, gen_helper_vsetallnez_b) +TRANS(xvsetallnez_h, LASX, gen_cx, gen_helper_vsetallnez_h) +TRANS(xvsetallnez_w, LASX, gen_cx, gen_helper_vsetallnez_w) +TRANS(xvsetallnez_d, LASX, gen_cx, gen_helper_vsetallnez_d) - CHECK_SXE; - tcg_gen_st16_i64(src, cpu_env, - offsetof(CPULoongArchState, fpr[a->vd].vreg.H(a->imm))); - return true; -} - -static bool trans_vinsgr2vr_w(DisasContext *ctx, arg_vr_i *a) +static bool gen_g2v_vl(DisasContext *ctx, arg_vr_i *a, uint32_t oprsz, MemOp mop, + void (*func)(TCGv, TCGv_ptr, tcg_target_long)) { TCGv src = gpr_src(ctx, a->rj, EXT_NONE); - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, oprsz)) { + return true; } - CHECK_SXE; - tcg_gen_st32_i64(src, cpu_env, - offsetof(CPULoongArchState, fpr[a->vd].vreg.W(a->imm))); - return true; -} - -static bool trans_vinsgr2vr_d(DisasContext *ctx, arg_vr_i *a) -{ - TCGv src = gpr_src(ctx, a->rj, EXT_NONE); - - if (!avail_LSX(ctx)) { - return false; - } + func(src, cpu_env, vec_reg_offset(a->vd, a->imm, mop)); - CHECK_SXE; - tcg_gen_st_i64(src, cpu_env, - offsetof(CPULoongArchState, fpr[a->vd].vreg.D(a->imm))); return true; } -static bool trans_vpickve2gr_b(DisasContext *ctx, arg_rv_i *a) +static bool gen_g2v(DisasContext *ctx, arg_vr_i *a, MemOp mop, + void (*func)(TCGv, TCGv_ptr, tcg_target_long)) { - TCGv dst = gpr_dst(ctx, a->rd, EXT_NONE); - - if (!avail_LSX(ctx)) { - return false; - } - - CHECK_SXE; - tcg_gen_ld8s_i64(dst, cpu_env, - offsetof(CPULoongArchState, fpr[a->vj].vreg.B(a->imm))); - return true; + return gen_g2v_vl(ctx, a, 16, mop, func); } -static bool trans_vpickve2gr_h(DisasContext *ctx, arg_rv_i *a) +static bool gen_g2x(DisasContext *ctx, arg_vr_i *a, MemOp mop, + void (*func)(TCGv, TCGv_ptr, tcg_target_long)) { - TCGv dst = gpr_dst(ctx, a->rd, EXT_NONE); - - if (!avail_LSX(ctx)) { - return false; - } - - CHECK_SXE; - tcg_gen_ld16s_i64(dst, cpu_env, - offsetof(CPULoongArchState, fpr[a->vj].vreg.H(a->imm))); - return true; + return gen_g2v_vl(ctx, a, 32, mop, func); } -static bool trans_vpickve2gr_w(DisasContext *ctx, arg_rv_i *a) -{ - TCGv dst = gpr_dst(ctx, a->rd, EXT_NONE); +TRANS(vinsgr2vr_b, LSX, gen_g2v, MO_8, tcg_gen_st8_i64) +TRANS(vinsgr2vr_h, LSX, gen_g2v, MO_16, tcg_gen_st16_i64) +TRANS(vinsgr2vr_w, LSX, gen_g2v, MO_32, tcg_gen_st32_i64) +TRANS(vinsgr2vr_d, LSX, gen_g2v, MO_64, tcg_gen_st_i64) +TRANS(xvinsgr2vr_w, LASX, gen_g2x, MO_32, tcg_gen_st32_i64) +TRANS(xvinsgr2vr_d, LASX, gen_g2x, MO_64, tcg_gen_st_i64) - if (!avail_LSX(ctx)) { - return false; - } - - CHECK_SXE; - tcg_gen_ld32s_i64(dst, cpu_env, - offsetof(CPULoongArchState, fpr[a->vj].vreg.W(a->imm))); - return true; -} - -static bool trans_vpickve2gr_d(DisasContext *ctx, arg_rv_i *a) +static bool gen_v2g_vl(DisasContext *ctx, arg_rv_i *a, uint32_t oprsz, MemOp mop, + void (*func)(TCGv, TCGv_ptr, tcg_target_long)) { TCGv dst = gpr_dst(ctx, a->rd, EXT_NONE); - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, oprsz)) { + return true; } - CHECK_SXE; - tcg_gen_ld_i64(dst, cpu_env, - offsetof(CPULoongArchState, fpr[a->vj].vreg.D(a->imm))); - return true; -} - -static bool trans_vpickve2gr_bu(DisasContext *ctx, arg_rv_i *a) -{ - TCGv dst = gpr_dst(ctx, a->rd, EXT_NONE); - - if (!avail_LSX(ctx)) { - return false; - } + func(dst, cpu_env, vec_reg_offset(a->vj, a->imm, mop)); - CHECK_SXE; - tcg_gen_ld8u_i64(dst, cpu_env, - offsetof(CPULoongArchState, fpr[a->vj].vreg.B(a->imm))); return true; } -static bool trans_vpickve2gr_hu(DisasContext *ctx, arg_rv_i *a) +static bool gen_v2g(DisasContext *ctx, arg_rv_i *a, MemOp mop, + void (*func)(TCGv, TCGv_ptr, tcg_target_long)) { - TCGv dst = gpr_dst(ctx, a->rd, EXT_NONE); - - if (!avail_LSX(ctx)) { - return false; - } - - CHECK_SXE; - tcg_gen_ld16u_i64(dst, cpu_env, - offsetof(CPULoongArchState, fpr[a->vj].vreg.H(a->imm))); - return true; + return gen_v2g_vl(ctx, a, 16, mop, func); } -static bool trans_vpickve2gr_wu(DisasContext *ctx, arg_rv_i *a) +static bool gen_x2g(DisasContext *ctx, arg_rv_i *a, MemOp mop, + void (*func)(TCGv, TCGv_ptr, tcg_target_long)) { - TCGv dst = gpr_dst(ctx, a->rd, EXT_NONE); - - if (!avail_LSX(ctx)) { - return false; - } - - CHECK_SXE; - tcg_gen_ld32u_i64(dst, cpu_env, - offsetof(CPULoongArchState, fpr[a->vj].vreg.W(a->imm))); - return true; + return gen_v2g_vl(ctx, a, 32, mop, func); } -static bool trans_vpickve2gr_du(DisasContext *ctx, arg_rv_i *a) +TRANS(vpickve2gr_b, LSX, gen_v2g, MO_8, tcg_gen_ld8s_i64) +TRANS(vpickve2gr_h, LSX, gen_v2g, MO_16, tcg_gen_ld16s_i64) +TRANS(vpickve2gr_w, LSX, gen_v2g, MO_32, tcg_gen_ld32s_i64) +TRANS(vpickve2gr_d, LSX, gen_v2g, MO_64, tcg_gen_ld_i64) +TRANS(vpickve2gr_bu, LSX, gen_v2g, MO_8, tcg_gen_ld8u_i64) +TRANS(vpickve2gr_hu, LSX, gen_v2g, MO_16, tcg_gen_ld16u_i64) +TRANS(vpickve2gr_wu, LSX, gen_v2g, MO_32, tcg_gen_ld32u_i64) +TRANS(vpickve2gr_du, LSX, gen_v2g, MO_64, tcg_gen_ld_i64) +TRANS(xvpickve2gr_w, LASX, gen_x2g, MO_32, tcg_gen_ld32s_i64) +TRANS(xvpickve2gr_d, LASX, gen_x2g, MO_64, tcg_gen_ld_i64) +TRANS(xvpickve2gr_wu, LASX, gen_x2g, MO_32, tcg_gen_ld32u_i64) +TRANS(xvpickve2gr_du, LASX, gen_x2g, MO_64, tcg_gen_ld_i64) + +static bool gvec_dup_vl(DisasContext *ctx, arg_vr *a, + uint32_t oprsz, MemOp mop) { - TCGv dst = gpr_dst(ctx, a->rd, EXT_NONE); + TCGv src = gpr_src(ctx, a->rj, EXT_NONE); - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, oprsz)) { + return true; } - CHECK_SXE; - tcg_gen_ld_i64(dst, cpu_env, - offsetof(CPULoongArchState, fpr[a->vj].vreg.D(a->imm))); + tcg_gen_gvec_dup_i64(mop, vec_full_offset(a->vd), + oprsz, ctx->vl/8, src); return true; } static bool gvec_dup(DisasContext *ctx, arg_vr *a, MemOp mop) { - TCGv src = gpr_src(ctx, a->rj, EXT_NONE); - - if (!avail_LSX(ctx)) { - return false; - } - - CHECK_SXE; + return gvec_dup_vl(ctx, a, 16, mop); +} - tcg_gen_gvec_dup_i64(mop, vec_full_offset(a->vd), - 16, ctx->vl/8, src); - return true; +static bool gvec_dupx(DisasContext *ctx, arg_vr *a, MemOp mop) +{ + return gvec_dup_vl(ctx, a, 32, mop); } TRANS(vreplgr2vr_b, LSX, gvec_dup, MO_8) TRANS(vreplgr2vr_h, LSX, gvec_dup, MO_16) TRANS(vreplgr2vr_w, LSX, gvec_dup, MO_32) TRANS(vreplgr2vr_d, LSX, gvec_dup, MO_64) +TRANS(xvreplgr2vr_b, LASX, gvec_dupx, MO_8) +TRANS(xvreplgr2vr_h, LASX, gvec_dupx, MO_16) +TRANS(xvreplgr2vr_w, LASX, gvec_dupx, MO_32) +TRANS(xvreplgr2vr_d, LASX, gvec_dupx, MO_64) static bool trans_vreplvei_b(DisasContext *ctx, arg_vv_i *a) { @@ -4198,7 +4946,10 @@ static bool trans_vreplvei_b(DisasContext *ctx, arg_vv_i *a) return false; } - CHECK_SXE; + if (!check_vec(ctx, 16)) { + return true; + } + tcg_gen_gvec_dup_mem(MO_8,vec_full_offset(a->vd), offsetof(CPULoongArchState, fpr[a->vj].vreg.B((a->imm))), @@ -4212,7 +4963,10 @@ static bool trans_vreplvei_h(DisasContext *ctx, arg_vv_i *a) return false; } - CHECK_SXE; + if (!check_vec(ctx, 16)) { + return true; + } + tcg_gen_gvec_dup_mem(MO_16, vec_full_offset(a->vd), offsetof(CPULoongArchState, fpr[a->vj].vreg.H((a->imm))), @@ -4225,7 +4979,10 @@ static bool trans_vreplvei_w(DisasContext *ctx, arg_vv_i *a) return false; } - CHECK_SXE; + if (!check_vec(ctx, 16)) { + return true; + } + tcg_gen_gvec_dup_mem(MO_32, vec_full_offset(a->vd), offsetof(CPULoongArchState, fpr[a->vj].vreg.W((a->imm))), @@ -4238,7 +4995,10 @@ static bool trans_vreplvei_d(DisasContext *ctx, arg_vv_i *a) return false; } - CHECK_SXE; + if (!check_vec(ctx, 16)) { + return true; + } + tcg_gen_gvec_dup_mem(MO_64, vec_full_offset(a->vd), offsetof(CPULoongArchState, fpr[a->vj].vreg.D((a->imm))), @@ -4246,106 +5006,169 @@ static bool trans_vreplvei_d(DisasContext *ctx, arg_vv_i *a) return true; } -static bool gen_vreplve(DisasContext *ctx, arg_vvr *a, int vece, int bit, - void (*func)(TCGv_i64, TCGv_ptr, tcg_target_long)) +static bool gen_vreplve_vl(DisasContext *ctx, arg_vvr *a, + uint32_t oprsz, int vece, int bit, + void (*func)(TCGv_i64, TCGv_ptr, tcg_target_long)) { + int i; TCGv_i64 t0 = tcg_temp_new_i64(); TCGv_ptr t1 = tcg_temp_new_ptr(); TCGv_i64 t2 = tcg_temp_new_i64(); - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, oprsz)) { + return true; } - CHECK_SXE; - - tcg_gen_andi_i64(t0, gpr_src(ctx, a->rk, EXT_NONE), (LSX_LEN/bit) -1); + tcg_gen_andi_i64(t0, gpr_src(ctx, a->rk, EXT_NONE), (LSX_LEN / bit) - 1); tcg_gen_shli_i64(t0, t0, vece); if (HOST_BIG_ENDIAN) { - tcg_gen_xori_i64(t0, t0, vece << ((LSX_LEN/bit) -1)); + tcg_gen_xori_i64(t0, t0, vece << ((LSX_LEN / bit) - 1)); } tcg_gen_trunc_i64_ptr(t1, t0); tcg_gen_add_ptr(t1, t1, cpu_env); - func(t2, t1, vec_full_offset(a->vj)); - tcg_gen_gvec_dup_i64(vece, vec_full_offset(a->vd), 16, ctx->vl/8, t2); + + for (i = 0; i < oprsz; i += 16) { + func(t2, t1, vec_full_offset(a->vj) + i); + tcg_gen_gvec_dup_i64(vece, vec_full_offset(a->vd) + i, 16, 16, t2); + } return true; } +static bool gen_vreplve(DisasContext *ctx, arg_vvr *a, int vece, int bit, + void (*func)(TCGv_i64, TCGv_ptr, tcg_target_long)) +{ + return gen_vreplve_vl(ctx, a, 16, vece, bit, func); +} + +static bool gen_xvreplve(DisasContext *ctx, arg_vvr *a, int vece, int bit, + void (*func)(TCGv_i64, TCGv_ptr, tcg_target_long)) +{ + return gen_vreplve_vl(ctx, a, 32, vece, bit, func); +} + TRANS(vreplve_b, LSX, gen_vreplve, MO_8, 8, tcg_gen_ld8u_i64) TRANS(vreplve_h, LSX, gen_vreplve, MO_16, 16, tcg_gen_ld16u_i64) TRANS(vreplve_w, LSX, gen_vreplve, MO_32, 32, tcg_gen_ld32u_i64) TRANS(vreplve_d, LSX, gen_vreplve, MO_64, 64, tcg_gen_ld_i64) +TRANS(xvreplve_b, LASX, gen_xvreplve, MO_8, 8, tcg_gen_ld8u_i64) +TRANS(xvreplve_h, LASX, gen_xvreplve, MO_16, 16, tcg_gen_ld16u_i64) +TRANS(xvreplve_w, LASX, gen_xvreplve, MO_32, 32, tcg_gen_ld32u_i64) +TRANS(xvreplve_d, LASX, gen_xvreplve, MO_64, 64, tcg_gen_ld_i64) -static bool trans_vbsll_v(DisasContext *ctx, arg_vv_i *a) +static bool gen_xvrepl128(DisasContext *ctx, arg_vv_i *a, MemOp mop) { - int ofs; - TCGv_i64 desthigh, destlow, high, low; + int i; - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, 32)) { + return true; } - CHECK_SXE; + for (i = 0; i < 32; i += 16) { + tcg_gen_gvec_dup_mem(mop, vec_full_offset(a->vd) + i, + vec_reg_offset(a->vj, a->imm, mop) + i, 16, 16); - desthigh = tcg_temp_new_i64(); - destlow = tcg_temp_new_i64(); - high = tcg_temp_new_i64(); - low = tcg_temp_new_i64(); + } + return true; +} - get_vreg64(low, a->vj, 0); +TRANS(xvrepl128vei_b, LASX, gen_xvrepl128, MO_8) +TRANS(xvrepl128vei_h, LASX, gen_xvrepl128, MO_16) +TRANS(xvrepl128vei_w, LASX, gen_xvrepl128, MO_32) +TRANS(xvrepl128vei_d, LASX, gen_xvrepl128, MO_64) - ofs = ((a->imm) & 0xf) * 8; - if (ofs < 64) { - get_vreg64(high, a->vj, 1); - tcg_gen_extract2_i64(desthigh, low, high, 64 - ofs); - tcg_gen_shli_i64(destlow, low, ofs); - } else { - tcg_gen_shli_i64(desthigh, low, ofs - 64); - destlow = tcg_constant_i64(0); +static bool gen_xvreplve0(DisasContext *ctx, arg_vv *a, MemOp mop) +{ + if (!check_vec(ctx, 32)) { + return true; } - set_vreg64(desthigh, a->vd, 1); - set_vreg64(destlow, a->vd, 0); - + tcg_gen_gvec_dup_mem(mop, vec_full_offset(a->vd), + vec_full_offset(a->vj), 32, 32); return true; } -static bool trans_vbsrl_v(DisasContext *ctx, arg_vv_i *a) +TRANS(xvreplve0_b, LASX, gen_xvreplve0, MO_8) +TRANS(xvreplve0_h, LASX, gen_xvreplve0, MO_16) +TRANS(xvreplve0_w, LASX, gen_xvreplve0, MO_32) +TRANS(xvreplve0_d, LASX, gen_xvreplve0, MO_64) +TRANS(xvreplve0_q, LASX, gen_xvreplve0, MO_128) + +TRANS(xvinsve0_w, LASX, gen_xx_i, gen_helper_xvinsve0_w) +TRANS(xvinsve0_d, LASX, gen_xx_i, gen_helper_xvinsve0_d) + +TRANS(xvpickve_w, LASX, gen_xx_i, gen_helper_xvpickve_w) +TRANS(xvpickve_d, LASX, gen_xx_i, gen_helper_xvpickve_d) + +static bool do_vbsll_v(DisasContext *ctx, arg_vv_i *a, uint32_t oprsz) { - TCGv_i64 desthigh, destlow, high, low; - int ofs; + int i, ofs; - if (!avail_LSX(ctx)) { - return false; + if (!check_vec(ctx, oprsz)) { + return true; } - CHECK_SXE; + for (i = 0; i < oprsz / 16; i++) { + TCGv desthigh = tcg_temp_new_i64(); + TCGv destlow = tcg_temp_new_i64(); + TCGv high = tcg_temp_new_i64(); + TCGv low = tcg_temp_new_i64(); + + get_vreg64(low, a->vj, 2 * i); + + ofs = ((a->imm) & 0xf) * 8; + if (ofs < 64) { + get_vreg64(high, a->vj, 2 * i + 1); + tcg_gen_extract2_i64(desthigh, low, high, 64 - ofs); + tcg_gen_shli_i64(destlow, low, ofs); + } else { + tcg_gen_shli_i64(desthigh, low, ofs - 64); + destlow = tcg_constant_i64(0); + } + set_vreg64(desthigh, a->vd, 2 * i + 1); + set_vreg64(destlow, a->vd, 2 * i); + } - desthigh = tcg_temp_new_i64(); - destlow = tcg_temp_new_i64(); - high = tcg_temp_new_i64(); - low = tcg_temp_new_i64(); + return true; +} - get_vreg64(high, a->vj, 1); +static bool do_vbsrl_v(DisasContext *ctx, arg_vv_i *a, uint32_t oprsz) +{ + int i, ofs; - ofs = ((a->imm) & 0xf) * 8; - if (ofs < 64) { - get_vreg64(low, a->vj, 0); - tcg_gen_extract2_i64(destlow, low, high, ofs); - tcg_gen_shri_i64(desthigh, high, ofs); - } else { - tcg_gen_shri_i64(destlow, high, ofs - 64); - desthigh = tcg_constant_i64(0); + if (!check_vec(ctx, 32)) { + return true; } - set_vreg64(desthigh, a->vd, 1); - set_vreg64(destlow, a->vd, 0); + for (i = 0; i < oprsz / 16; i++) { + TCGv desthigh = tcg_temp_new_i64(); + TCGv destlow = tcg_temp_new_i64(); + TCGv high = tcg_temp_new_i64(); + TCGv low = tcg_temp_new_i64(); + get_vreg64(high, a->vj, 2 * i + 1); + + ofs = ((a->imm) & 0xf) * 8; + if (ofs < 64) { + get_vreg64(low, a->vj, 2 * i); + tcg_gen_extract2_i64(destlow, low, high, ofs); + tcg_gen_shri_i64(desthigh, high, ofs); + } else { + tcg_gen_shri_i64(destlow, high, ofs - 64); + desthigh = tcg_constant_i64(0); + } + set_vreg64(desthigh, a->vd, 2 * i + 1); + set_vreg64(destlow, a->vd, 2 * i); + } return true; } +TRANS(vbsll_v, LSX, do_vbsll_v, 16) +TRANS(vbsrl_v, LSX, do_vbsrl_v, 16) +TRANS(xvbsll_v, LASX, do_vbsll_v, 32) +TRANS(xvbsrl_v, LASX, do_vbsrl_v, 32) + TRANS(vpackev_b, LSX, gen_vvv, gen_helper_vpackev_b) TRANS(vpackev_h, LSX, gen_vvv, gen_helper_vpackev_h) TRANS(vpackev_w, LSX, gen_vvv, gen_helper_vpackev_w) @@ -4354,6 +5177,14 @@ TRANS(vpackod_b, LSX, gen_vvv, gen_helper_vpackod_b) TRANS(vpackod_h, LSX, gen_vvv, gen_helper_vpackod_h) TRANS(vpackod_w, LSX, gen_vvv, gen_helper_vpackod_w) TRANS(vpackod_d, LSX, gen_vvv, gen_helper_vpackod_d) +TRANS(xvpackev_b, LASX, gen_xxx, gen_helper_vpackev_b) +TRANS(xvpackev_h, LASX, gen_xxx, gen_helper_vpackev_h) +TRANS(xvpackev_w, LASX, gen_xxx, gen_helper_vpackev_w) +TRANS(xvpackev_d, LASX, gen_xxx, gen_helper_vpackev_d) +TRANS(xvpackod_b, LASX, gen_xxx, gen_helper_vpackod_b) +TRANS(xvpackod_h, LASX, gen_xxx, gen_helper_vpackod_h) +TRANS(xvpackod_w, LASX, gen_xxx, gen_helper_vpackod_w) +TRANS(xvpackod_d, LASX, gen_xxx, gen_helper_vpackod_d) TRANS(vpickev_b, LSX, gen_vvv, gen_helper_vpickev_b) TRANS(vpickev_h, LSX, gen_vvv, gen_helper_vpickev_h) @@ -4363,6 +5194,14 @@ TRANS(vpickod_b, LSX, gen_vvv, gen_helper_vpickod_b) TRANS(vpickod_h, LSX, gen_vvv, gen_helper_vpickod_h) TRANS(vpickod_w, LSX, gen_vvv, gen_helper_vpickod_w) TRANS(vpickod_d, LSX, gen_vvv, gen_helper_vpickod_d) +TRANS(xvpickev_b, LASX, gen_xxx, gen_helper_vpickev_b) +TRANS(xvpickev_h, LASX, gen_xxx, gen_helper_vpickev_h) +TRANS(xvpickev_w, LASX, gen_xxx, gen_helper_vpickev_w) +TRANS(xvpickev_d, LASX, gen_xxx, gen_helper_vpickev_d) +TRANS(xvpickod_b, LASX, gen_xxx, gen_helper_vpickod_b) +TRANS(xvpickod_h, LASX, gen_xxx, gen_helper_vpickod_h) +TRANS(xvpickod_w, LASX, gen_xxx, gen_helper_vpickod_w) +TRANS(xvpickod_d, LASX, gen_xxx, gen_helper_vpickod_d) TRANS(vilvl_b, LSX, gen_vvv, gen_helper_vilvl_b) TRANS(vilvl_h, LSX, gen_vvv, gen_helper_vilvl_h) @@ -4372,22 +5211,46 @@ TRANS(vilvh_b, LSX, gen_vvv, gen_helper_vilvh_b) TRANS(vilvh_h, LSX, gen_vvv, gen_helper_vilvh_h) TRANS(vilvh_w, LSX, gen_vvv, gen_helper_vilvh_w) TRANS(vilvh_d, LSX, gen_vvv, gen_helper_vilvh_d) +TRANS(xvilvl_b, LASX, gen_xxx, gen_helper_vilvl_b) +TRANS(xvilvl_h, LASX, gen_xxx, gen_helper_vilvl_h) +TRANS(xvilvl_w, LASX, gen_xxx, gen_helper_vilvl_w) +TRANS(xvilvl_d, LASX, gen_xxx, gen_helper_vilvl_d) +TRANS(xvilvh_b, LASX, gen_xxx, gen_helper_vilvh_b) +TRANS(xvilvh_h, LASX, gen_xxx, gen_helper_vilvh_h) +TRANS(xvilvh_w, LASX, gen_xxx, gen_helper_vilvh_w) +TRANS(xvilvh_d, LASX, gen_xxx, gen_helper_vilvh_d) TRANS(vshuf_b, LSX, gen_vvvv, gen_helper_vshuf_b) TRANS(vshuf_h, LSX, gen_vvv, gen_helper_vshuf_h) TRANS(vshuf_w, LSX, gen_vvv, gen_helper_vshuf_w) TRANS(vshuf_d, LSX, gen_vvv, gen_helper_vshuf_d) +TRANS(xvshuf_b, LASX, gen_xxxx, gen_helper_vshuf_b) +TRANS(xvshuf_h, LASX, gen_xxx, gen_helper_vshuf_h) +TRANS(xvshuf_w, LASX, gen_xxx, gen_helper_vshuf_w) +TRANS(xvshuf_d, LASX, gen_xxx, gen_helper_vshuf_d) TRANS(vshuf4i_b, LSX, gen_vv_i, gen_helper_vshuf4i_b) TRANS(vshuf4i_h, LSX, gen_vv_i, gen_helper_vshuf4i_h) TRANS(vshuf4i_w, LSX, gen_vv_i, gen_helper_vshuf4i_w) TRANS(vshuf4i_d, LSX, gen_vv_i, gen_helper_vshuf4i_d) +TRANS(xvshuf4i_b, LASX, gen_xx_i, gen_helper_vshuf4i_b) +TRANS(xvshuf4i_h, LASX, gen_xx_i, gen_helper_vshuf4i_h) +TRANS(xvshuf4i_w, LASX, gen_xx_i, gen_helper_vshuf4i_w) +TRANS(xvshuf4i_d, LASX, gen_xx_i, gen_helper_vshuf4i_d) +TRANS(xvperm_w, LASX, gen_xxx, gen_helper_vperm_w) TRANS(vpermi_w, LSX, gen_vv_i, gen_helper_vpermi_w) +TRANS(xvpermi_w, LASX, gen_xx_i, gen_helper_vpermi_w) +TRANS(xvpermi_d, LASX, gen_xx_i, gen_helper_vpermi_d) +TRANS(xvpermi_q, LASX, gen_xx_i, gen_helper_vpermi_q) TRANS(vextrins_b, LSX, gen_vv_i, gen_helper_vextrins_b) TRANS(vextrins_h, LSX, gen_vv_i, gen_helper_vextrins_h) TRANS(vextrins_w, LSX, gen_vv_i, gen_helper_vextrins_w) TRANS(vextrins_d, LSX, gen_vv_i, gen_helper_vextrins_d) +TRANS(xvextrins_b, LASX, gen_xx_i, gen_helper_vextrins_b) +TRANS(xvextrins_h, LASX, gen_xx_i, gen_helper_vextrins_h) +TRANS(xvextrins_w, LASX, gen_xx_i, gen_helper_vextrins_w) +TRANS(xvextrins_d, LASX, gen_xx_i, gen_helper_vextrins_d) static bool trans_vld(DisasContext *ctx, arg_vr_i *a) { @@ -4399,7 +5262,9 @@ static bool trans_vld(DisasContext *ctx, arg_vr_i *a) return false; } - CHECK_SXE; + if (!check_vec(ctx, 16)) { + return true; + } addr = gpr_src(ctx, a->rj, EXT_NONE); val = tcg_temp_new_i128(); @@ -4426,7 +5291,9 @@ static bool trans_vst(DisasContext *ctx, arg_vr_i *a) return false; } - CHECK_SXE; + if (!check_vec(ctx, 16)) { + return true; + } addr = gpr_src(ctx, a->rj, EXT_NONE); val = tcg_temp_new_i128(); @@ -4453,7 +5320,9 @@ static bool trans_vldx(DisasContext *ctx, arg_vrr *a) return false; } - CHECK_SXE; + if (!check_vec(ctx, 16)) { + return true; + } src1 = gpr_src(ctx, a->rj, EXT_NONE); src2 = gpr_src(ctx, a->rk, EXT_NONE); @@ -4480,7 +5349,9 @@ static bool trans_vstx(DisasContext *ctx, arg_vrr *a) return false; } - CHECK_SXE; + if (!check_vec(ctx, 16)) { + return true; + } src1 = gpr_src(ctx, a->rj, EXT_NONE); src2 = gpr_src(ctx, a->rk, EXT_NONE); @@ -4497,59 +5368,155 @@ static bool trans_vstx(DisasContext *ctx, arg_vrr *a) return true; } -#define VLDREPL(NAME, MO) \ -static bool trans_## NAME (DisasContext *ctx, arg_vr_i *a) \ -{ \ - TCGv addr; \ - TCGv_i64 val; \ - \ - if (!avail_LSX(ctx)) { \ - return false; \ - } \ - \ - CHECK_SXE; \ - \ - addr = gpr_src(ctx, a->rj, EXT_NONE); \ - val = tcg_temp_new_i64(); \ - \ - addr = make_address_i(ctx, addr, a->imm); \ - \ - tcg_gen_qemu_ld_i64(val, addr, ctx->mem_idx, MO); \ - tcg_gen_gvec_dup_i64(MO, vec_full_offset(a->vd), 16, ctx->vl/8, val); \ - \ - return true; \ -} - -VLDREPL(vldrepl_b, MO_8) -VLDREPL(vldrepl_h, MO_16) -VLDREPL(vldrepl_w, MO_32) -VLDREPL(vldrepl_d, MO_64) - -#define VSTELM(NAME, MO, E) \ -static bool trans_## NAME (DisasContext *ctx, arg_vr_ii *a) \ -{ \ - TCGv addr; \ - TCGv_i64 val; \ - \ - if (!avail_LSX(ctx)) { \ - return false; \ - } \ - \ - CHECK_SXE; \ - \ - addr = gpr_src(ctx, a->rj, EXT_NONE); \ - val = tcg_temp_new_i64(); \ - \ - addr = make_address_i(ctx, addr, a->imm); \ - \ - tcg_gen_ld_i64(val, cpu_env, \ - offsetof(CPULoongArchState, fpr[a->vd].vreg.E(a->imm2))); \ - tcg_gen_qemu_st_i64(val, addr, ctx->mem_idx, MO); \ - \ - return true; \ -} - -VSTELM(vstelm_b, MO_8, B) -VSTELM(vstelm_h, MO_16, H) -VSTELM(vstelm_w, MO_32, W) -VSTELM(vstelm_d, MO_64, D) +static bool do_vldrepl_vl(DisasContext *ctx, arg_vr_i *a, + uint32_t oprsz, MemOp mop) +{ + TCGv addr; + TCGv_i64 val; + + if (!check_vec(ctx, oprsz)) { + return true; + } + + addr = gpr_src(ctx, a->rj, EXT_NONE); + val = tcg_temp_new_i64(); + + addr = make_address_i(ctx, addr, a->imm); + + tcg_gen_qemu_ld_i64(val, addr, ctx->mem_idx, mop); + tcg_gen_gvec_dup_i64(mop, vec_full_offset(a->vd), oprsz, ctx->vl / 8, val); + + return true; +} + +static bool do_vldrepl(DisasContext *ctx, arg_vr_i *a, MemOp mop) +{ + return do_vldrepl_vl(ctx, a, 16, mop); +} + +static bool do_xvldrepl(DisasContext *ctx, arg_vr_i *a, MemOp mop) +{ + return do_vldrepl_vl(ctx, a, 32, mop); +} + +TRANS(vldrepl_b, LSX, do_vldrepl, MO_8) +TRANS(vldrepl_h, LSX, do_vldrepl, MO_16) +TRANS(vldrepl_w, LSX, do_vldrepl, MO_32) +TRANS(vldrepl_d, LSX, do_vldrepl, MO_64) +TRANS(xvldrepl_b, LASX, do_xvldrepl, MO_8) +TRANS(xvldrepl_h, LASX, do_xvldrepl, MO_16) +TRANS(xvldrepl_w, LASX, do_xvldrepl, MO_32) +TRANS(xvldrepl_d, LASX, do_xvldrepl, MO_64) + +static bool do_vstelm_vl(DisasContext *ctx, + arg_vr_ii *a, uint32_t oprsz, MemOp mop) +{ + TCGv addr; + TCGv_i64 val; + + if (!check_vec(ctx, oprsz)) { + return true; + } + + addr = gpr_src(ctx, a->rj, EXT_NONE); + val = tcg_temp_new_i64(); + + addr = make_address_i(ctx, addr, a->imm); + tcg_gen_ld_i64(val, cpu_env, vec_reg_offset(a->vd, a->imm2, mop)); + tcg_gen_qemu_st_i64(val, addr, ctx->mem_idx, mop); + return true; +} + +static bool do_vstelm(DisasContext *ctx, arg_vr_ii *a, MemOp mop) +{ + return do_vstelm_vl(ctx, a, 16, mop); +} + +static bool do_xvstelm(DisasContext *ctx, arg_vr_ii *a, MemOp mop) +{ + return do_vstelm_vl(ctx, a, 32, mop); +} + +TRANS(vstelm_b, LSX, do_vstelm, MO_8) +TRANS(vstelm_h, LSX, do_vstelm, MO_16) +TRANS(vstelm_w, LSX, do_vstelm, MO_32) +TRANS(vstelm_d, LSX, do_vstelm, MO_64) +TRANS(xvstelm_b, LASX, do_xvstelm, MO_8) +TRANS(xvstelm_h, LASX, do_xvstelm, MO_16) +TRANS(xvstelm_w, LASX, do_xvstelm, MO_32) +TRANS(xvstelm_d, LASX, do_xvstelm, MO_64) + +static bool gen_lasx_memory(DisasContext *ctx, arg_vr_i *a, + void (*func)(DisasContext *, int, TCGv)) +{ + TCGv addr = gpr_src(ctx, a->rj, EXT_NONE); + TCGv temp = NULL; + + if (!check_vec(ctx, 32)) { + return true; + } + + if (a->imm) { + temp = tcg_temp_new(); + tcg_gen_addi_tl(temp, addr, a->imm); + addr = temp; + } + + func(ctx, a->vd, addr); + return true; +} + +static void gen_xvld(DisasContext *ctx, int vreg, TCGv addr) +{ + int i; + TCGv temp = tcg_temp_new(); + TCGv dest = tcg_temp_new(); + + tcg_gen_qemu_ld_i64(dest, addr, ctx->mem_idx, MO_TEUQ); + set_vreg64(dest, vreg, 0); + + for (i = 1; i < 4; i++) { + tcg_gen_addi_tl(temp, addr, 8 * i); + tcg_gen_qemu_ld_i64(dest, temp, ctx->mem_idx, MO_TEUQ); + set_vreg64(dest, vreg, i); + } +} + +static void gen_xvst(DisasContext * ctx, int vreg, TCGv addr) +{ + int i; + TCGv temp = tcg_temp_new(); + TCGv dest = tcg_temp_new(); + + get_vreg64(dest, vreg, 0); + tcg_gen_qemu_st_i64(dest, addr, ctx->mem_idx, MO_TEUQ); + + for (i = 1; i < 4; i++) { + tcg_gen_addi_tl(temp, addr, 8 * i); + get_vreg64(dest, vreg, i); + tcg_gen_qemu_st_i64(dest, temp, ctx->mem_idx, MO_TEUQ); + } +} + +TRANS(xvld, LASX, gen_lasx_memory, gen_xvld) +TRANS(xvst, LASX, gen_lasx_memory, gen_xvst) + +static bool gen_lasx_memoryx(DisasContext *ctx, arg_vrr *a, + void (*func)(DisasContext*, int, TCGv)) +{ + TCGv src1 = gpr_src(ctx, a->rj, EXT_NONE); + TCGv src2 = gpr_src(ctx, a->rk, EXT_NONE); + TCGv addr = tcg_temp_new(); + + if (!check_vec(ctx, 32)) { + return true; + } + + tcg_gen_add_tl(addr, src1, src2); + func(ctx, a->vd, addr); + + return true; +} + +TRANS(xvldx, LASX, gen_lasx_memoryx, gen_xvld) +TRANS(xvstx, LASX, gen_lasx_memoryx, gen_xvst) diff --git a/target/loongarch/insns.decode b/target/loongarch/insns.decode index c9c3bc2c73..64b308f9fb 100644 --- a/target/loongarch/insns.decode +++ b/target/loongarch/insns.decode @@ -550,6 +550,10 @@ dbcl 0000 00000010 10101 ............... @i15 @vr_i8i2 .... ........ imm2:2 ........ rj:5 vd:5 &vr_ii imm=%i8s2 @vr_i8i3 .... ....... imm2:3 ........ rj:5 vd:5 &vr_ii imm=%i8s1 @vr_i8i4 .... ...... imm2:4 imm:s8 rj:5 vd:5 &vr_ii +@vr_i8i2x .... ........ imm2:2 ........ rj:5 vd:5 &vr_ii imm=%i8s3 +@vr_i8i3x .... ....... imm2:3 ........ rj:5 vd:5 &vr_ii imm=%i8s2 +@vr_i8i4x .... ...... imm2:4 ........ rj:5 vd:5 &vr_ii imm=%i8s1 +@vr_i8i5x .... ..... imm2:5 imm:s8 rj:5 vd:5 &vr_ii @vrr .... ........ ..... rk:5 rj:5 vd:5 &vrr @v_i13 .... ........ .. imm:13 vd:5 &v_i @@ -1296,3 +1300,781 @@ vstelm_d 0011 00010001 0 . ........ ..... ..... @vr_i8i1 vstelm_w 0011 00010010 .. ........ ..... ..... @vr_i8i2 vstelm_h 0011 0001010 ... ........ ..... ..... @vr_i8i3 vstelm_b 0011 000110 .... ........ ..... ..... @vr_i8i4 + +# +# LoongArch LASX instructions +# +xvadd_b 0111 01000000 10100 ..... ..... ..... @vvv +xvadd_h 0111 01000000 10101 ..... ..... ..... @vvv +xvadd_w 0111 01000000 10110 ..... ..... ..... @vvv +xvadd_d 0111 01000000 10111 ..... ..... ..... @vvv +xvadd_q 0111 01010010 11010 ..... ..... ..... @vvv +xvsub_b 0111 01000000 11000 ..... ..... ..... @vvv +xvsub_h 0111 01000000 11001 ..... ..... ..... @vvv +xvsub_w 0111 01000000 11010 ..... ..... ..... @vvv +xvsub_d 0111 01000000 11011 ..... ..... ..... @vvv +xvsub_q 0111 01010010 11011 ..... ..... ..... @vvv + +xvaddi_bu 0111 01101000 10100 ..... ..... ..... @vv_ui5 +xvaddi_hu 0111 01101000 10101 ..... ..... ..... @vv_ui5 +xvaddi_wu 0111 01101000 10110 ..... ..... ..... @vv_ui5 +xvaddi_du 0111 01101000 10111 ..... ..... ..... @vv_ui5 +xvsubi_bu 0111 01101000 11000 ..... ..... ..... @vv_ui5 +xvsubi_hu 0111 01101000 11001 ..... ..... ..... @vv_ui5 +xvsubi_wu 0111 01101000 11010 ..... ..... ..... @vv_ui5 +xvsubi_du 0111 01101000 11011 ..... ..... ..... @vv_ui5 + +xvneg_b 0111 01101001 11000 01100 ..... ..... @vv +xvneg_h 0111 01101001 11000 01101 ..... ..... @vv +xvneg_w 0111 01101001 11000 01110 ..... ..... @vv +xvneg_d 0111 01101001 11000 01111 ..... ..... @vv + +xvsadd_b 0111 01000100 01100 ..... ..... ..... @vvv +xvsadd_h 0111 01000100 01101 ..... ..... ..... @vvv +xvsadd_w 0111 01000100 01110 ..... ..... ..... @vvv +xvsadd_d 0111 01000100 01111 ..... ..... ..... @vvv +xvsadd_bu 0111 01000100 10100 ..... ..... ..... @vvv +xvsadd_hu 0111 01000100 10101 ..... ..... ..... @vvv +xvsadd_wu 0111 01000100 10110 ..... ..... ..... @vvv +xvsadd_du 0111 01000100 10111 ..... ..... ..... @vvv + +xvssub_b 0111 01000100 10000 ..... ..... ..... @vvv +xvssub_h 0111 01000100 10001 ..... ..... ..... @vvv +xvssub_w 0111 01000100 10010 ..... ..... ..... @vvv +xvssub_d 0111 01000100 10011 ..... ..... ..... @vvv +xvssub_bu 0111 01000100 11000 ..... ..... ..... @vvv +xvssub_hu 0111 01000100 11001 ..... ..... ..... @vvv +xvssub_wu 0111 01000100 11010 ..... ..... ..... @vvv +xvssub_du 0111 01000100 11011 ..... ..... ..... @vvv + +xvhaddw_h_b 0111 01000101 01000 ..... ..... ..... @vvv +xvhaddw_w_h 0111 01000101 01001 ..... ..... ..... @vvv +xvhaddw_d_w 0111 01000101 01010 ..... ..... ..... @vvv +xvhaddw_q_d 0111 01000101 01011 ..... ..... ..... @vvv +xvhaddw_hu_bu 0111 01000101 10000 ..... ..... ..... @vvv +xvhaddw_wu_hu 0111 01000101 10001 ..... ..... ..... @vvv +xvhaddw_du_wu 0111 01000101 10010 ..... ..... ..... @vvv +xvhaddw_qu_du 0111 01000101 10011 ..... ..... ..... @vvv + +xvhsubw_h_b 0111 01000101 01100 ..... ..... ..... @vvv +xvhsubw_w_h 0111 01000101 01101 ..... ..... ..... @vvv +xvhsubw_d_w 0111 01000101 01110 ..... ..... ..... @vvv +xvhsubw_q_d 0111 01000101 01111 ..... ..... ..... @vvv +xvhsubw_hu_bu 0111 01000101 10100 ..... ..... ..... @vvv +xvhsubw_wu_hu 0111 01000101 10101 ..... ..... ..... @vvv +xvhsubw_du_wu 0111 01000101 10110 ..... ..... ..... @vvv +xvhsubw_qu_du 0111 01000101 10111 ..... ..... ..... @vvv + +xvaddwev_h_b 0111 01000001 11100 ..... ..... ..... @vvv +xvaddwev_w_h 0111 01000001 11101 ..... ..... ..... @vvv +xvaddwev_d_w 0111 01000001 11110 ..... ..... ..... @vvv +xvaddwev_q_d 0111 01000001 11111 ..... ..... ..... @vvv +xvaddwod_h_b 0111 01000010 00100 ..... ..... ..... @vvv +xvaddwod_w_h 0111 01000010 00101 ..... ..... ..... @vvv +xvaddwod_d_w 0111 01000010 00110 ..... ..... ..... @vvv +xvaddwod_q_d 0111 01000010 00111 ..... ..... ..... @vvv + +xvsubwev_h_b 0111 01000010 00000 ..... ..... ..... @vvv +xvsubwev_w_h 0111 01000010 00001 ..... ..... ..... @vvv +xvsubwev_d_w 0111 01000010 00010 ..... ..... ..... @vvv +xvsubwev_q_d 0111 01000010 00011 ..... ..... ..... @vvv +xvsubwod_h_b 0111 01000010 01000 ..... ..... ..... @vvv +xvsubwod_w_h 0111 01000010 01001 ..... ..... ..... @vvv +xvsubwod_d_w 0111 01000010 01010 ..... ..... ..... @vvv +xvsubwod_q_d 0111 01000010 01011 ..... ..... ..... @vvv + +xvaddwev_h_bu 0111 01000010 11100 ..... ..... ..... @vvv +xvaddwev_w_hu 0111 01000010 11101 ..... ..... ..... @vvv +xvaddwev_d_wu 0111 01000010 11110 ..... ..... ..... @vvv +xvaddwev_q_du 0111 01000010 11111 ..... ..... ..... @vvv +xvaddwod_h_bu 0111 01000011 00100 ..... ..... ..... @vvv +xvaddwod_w_hu 0111 01000011 00101 ..... ..... ..... @vvv +xvaddwod_d_wu 0111 01000011 00110 ..... ..... ..... @vvv +xvaddwod_q_du 0111 01000011 00111 ..... ..... ..... @vvv + +xvsubwev_h_bu 0111 01000011 00000 ..... ..... ..... @vvv +xvsubwev_w_hu 0111 01000011 00001 ..... ..... ..... @vvv +xvsubwev_d_wu 0111 01000011 00010 ..... ..... ..... @vvv +xvsubwev_q_du 0111 01000011 00011 ..... ..... ..... @vvv +xvsubwod_h_bu 0111 01000011 01000 ..... ..... ..... @vvv +xvsubwod_w_hu 0111 01000011 01001 ..... ..... ..... @vvv +xvsubwod_d_wu 0111 01000011 01010 ..... ..... ..... @vvv +xvsubwod_q_du 0111 01000011 01011 ..... ..... ..... @vvv + +xvaddwev_h_bu_b 0111 01000011 11100 ..... ..... ..... @vvv +xvaddwev_w_hu_h 0111 01000011 11101 ..... ..... ..... @vvv +xvaddwev_d_wu_w 0111 01000011 11110 ..... ..... ..... @vvv +xvaddwev_q_du_d 0111 01000011 11111 ..... ..... ..... @vvv +xvaddwod_h_bu_b 0111 01000100 00000 ..... ..... ..... @vvv +xvaddwod_w_hu_h 0111 01000100 00001 ..... ..... ..... @vvv +xvaddwod_d_wu_w 0111 01000100 00010 ..... ..... ..... @vvv +xvaddwod_q_du_d 0111 01000100 00011 ..... ..... ..... @vvv + +xvavg_b 0111 01000110 01000 ..... ..... ..... @vvv +xvavg_h 0111 01000110 01001 ..... ..... ..... @vvv +xvavg_w 0111 01000110 01010 ..... ..... ..... @vvv +xvavg_d 0111 01000110 01011 ..... ..... ..... @vvv +xvavg_bu 0111 01000110 01100 ..... ..... ..... @vvv +xvavg_hu 0111 01000110 01101 ..... ..... ..... @vvv +xvavg_wu 0111 01000110 01110 ..... ..... ..... @vvv +xvavg_du 0111 01000110 01111 ..... ..... ..... @vvv +xvavgr_b 0111 01000110 10000 ..... ..... ..... @vvv +xvavgr_h 0111 01000110 10001 ..... ..... ..... @vvv +xvavgr_w 0111 01000110 10010 ..... ..... ..... @vvv +xvavgr_d 0111 01000110 10011 ..... ..... ..... @vvv +xvavgr_bu 0111 01000110 10100 ..... ..... ..... @vvv +xvavgr_hu 0111 01000110 10101 ..... ..... ..... @vvv +xvavgr_wu 0111 01000110 10110 ..... ..... ..... @vvv +xvavgr_du 0111 01000110 10111 ..... ..... ..... @vvv + +xvabsd_b 0111 01000110 00000 ..... ..... ..... @vvv +xvabsd_h 0111 01000110 00001 ..... ..... ..... @vvv +xvabsd_w 0111 01000110 00010 ..... ..... ..... @vvv +xvabsd_d 0111 01000110 00011 ..... ..... ..... @vvv +xvabsd_bu 0111 01000110 00100 ..... ..... ..... @vvv +xvabsd_hu 0111 01000110 00101 ..... ..... ..... @vvv +xvabsd_wu 0111 01000110 00110 ..... ..... ..... @vvv +xvabsd_du 0111 01000110 00111 ..... ..... ..... @vvv + +xvadda_b 0111 01000101 11000 ..... ..... ..... @vvv +xvadda_h 0111 01000101 11001 ..... ..... ..... @vvv +xvadda_w 0111 01000101 11010 ..... ..... ..... @vvv +xvadda_d 0111 01000101 11011 ..... ..... ..... @vvv + +xvmax_b 0111 01000111 00000 ..... ..... ..... @vvv +xvmax_h 0111 01000111 00001 ..... ..... ..... @vvv +xvmax_w 0111 01000111 00010 ..... ..... ..... @vvv +xvmax_d 0111 01000111 00011 ..... ..... ..... @vvv +xvmax_bu 0111 01000111 01000 ..... ..... ..... @vvv +xvmax_hu 0111 01000111 01001 ..... ..... ..... @vvv +xvmax_wu 0111 01000111 01010 ..... ..... ..... @vvv +xvmax_du 0111 01000111 01011 ..... ..... ..... @vvv + +xvmaxi_b 0111 01101001 00000 ..... ..... ..... @vv_i5 +xvmaxi_h 0111 01101001 00001 ..... ..... ..... @vv_i5 +xvmaxi_w 0111 01101001 00010 ..... ..... ..... @vv_i5 +xvmaxi_d 0111 01101001 00011 ..... ..... ..... @vv_i5 +xvmaxi_bu 0111 01101001 01000 ..... ..... ..... @vv_ui5 +xvmaxi_hu 0111 01101001 01001 ..... ..... ..... @vv_ui5 +xvmaxi_wu 0111 01101001 01010 ..... ..... ..... @vv_ui5 +xvmaxi_du 0111 01101001 01011 ..... ..... ..... @vv_ui5 + +xvmin_b 0111 01000111 00100 ..... ..... ..... @vvv +xvmin_h 0111 01000111 00101 ..... ..... ..... @vvv +xvmin_w 0111 01000111 00110 ..... ..... ..... @vvv +xvmin_d 0111 01000111 00111 ..... ..... ..... @vvv +xvmin_bu 0111 01000111 01100 ..... ..... ..... @vvv +xvmin_hu 0111 01000111 01101 ..... ..... ..... @vvv +xvmin_wu 0111 01000111 01110 ..... ..... ..... @vvv +xvmin_du 0111 01000111 01111 ..... ..... ..... @vvv + +xvmini_b 0111 01101001 00100 ..... ..... ..... @vv_i5 +xvmini_h 0111 01101001 00101 ..... ..... ..... @vv_i5 +xvmini_w 0111 01101001 00110 ..... ..... ..... @vv_i5 +xvmini_d 0111 01101001 00111 ..... ..... ..... @vv_i5 +xvmini_bu 0111 01101001 01100 ..... ..... ..... @vv_ui5 +xvmini_hu 0111 01101001 01101 ..... ..... ..... @vv_ui5 +xvmini_wu 0111 01101001 01110 ..... ..... ..... @vv_ui5 +xvmini_du 0111 01101001 01111 ..... ..... ..... @vv_ui5 + +xvmul_b 0111 01001000 01000 ..... ..... ..... @vvv +xvmul_h 0111 01001000 01001 ..... ..... ..... @vvv +xvmul_w 0111 01001000 01010 ..... ..... ..... @vvv +xvmul_d 0111 01001000 01011 ..... ..... ..... @vvv +xvmuh_b 0111 01001000 01100 ..... ..... ..... @vvv +xvmuh_h 0111 01001000 01101 ..... ..... ..... @vvv +xvmuh_w 0111 01001000 01110 ..... ..... ..... @vvv +xvmuh_d 0111 01001000 01111 ..... ..... ..... @vvv +xvmuh_bu 0111 01001000 10000 ..... ..... ..... @vvv +xvmuh_hu 0111 01001000 10001 ..... ..... ..... @vvv +xvmuh_wu 0111 01001000 10010 ..... ..... ..... @vvv +xvmuh_du 0111 01001000 10011 ..... ..... ..... @vvv + +xvmulwev_h_b 0111 01001001 00000 ..... ..... ..... @vvv +xvmulwev_w_h 0111 01001001 00001 ..... ..... ..... @vvv +xvmulwev_d_w 0111 01001001 00010 ..... ..... ..... @vvv +xvmulwev_q_d 0111 01001001 00011 ..... ..... ..... @vvv +xvmulwod_h_b 0111 01001001 00100 ..... ..... ..... @vvv +xvmulwod_w_h 0111 01001001 00101 ..... ..... ..... @vvv +xvmulwod_d_w 0111 01001001 00110 ..... ..... ..... @vvv +xvmulwod_q_d 0111 01001001 00111 ..... ..... ..... @vvv +xvmulwev_h_bu 0111 01001001 10000 ..... ..... ..... @vvv +xvmulwev_w_hu 0111 01001001 10001 ..... ..... ..... @vvv +xvmulwev_d_wu 0111 01001001 10010 ..... ..... ..... @vvv +xvmulwev_q_du 0111 01001001 10011 ..... ..... ..... @vvv +xvmulwod_h_bu 0111 01001001 10100 ..... ..... ..... @vvv +xvmulwod_w_hu 0111 01001001 10101 ..... ..... ..... @vvv +xvmulwod_d_wu 0111 01001001 10110 ..... ..... ..... @vvv +xvmulwod_q_du 0111 01001001 10111 ..... ..... ..... @vvv +xvmulwev_h_bu_b 0111 01001010 00000 ..... ..... ..... @vvv +xvmulwev_w_hu_h 0111 01001010 00001 ..... ..... ..... @vvv +xvmulwev_d_wu_w 0111 01001010 00010 ..... ..... ..... @vvv +xvmulwev_q_du_d 0111 01001010 00011 ..... ..... ..... @vvv +xvmulwod_h_bu_b 0111 01001010 00100 ..... ..... ..... @vvv +xvmulwod_w_hu_h 0111 01001010 00101 ..... ..... ..... @vvv +xvmulwod_d_wu_w 0111 01001010 00110 ..... ..... ..... @vvv +xvmulwod_q_du_d 0111 01001010 00111 ..... ..... ..... @vvv + +xvmadd_b 0111 01001010 10000 ..... ..... ..... @vvv +xvmadd_h 0111 01001010 10001 ..... ..... ..... @vvv +xvmadd_w 0111 01001010 10010 ..... ..... ..... @vvv +xvmadd_d 0111 01001010 10011 ..... ..... ..... @vvv +xvmsub_b 0111 01001010 10100 ..... ..... ..... @vvv +xvmsub_h 0111 01001010 10101 ..... ..... ..... @vvv +xvmsub_w 0111 01001010 10110 ..... ..... ..... @vvv +xvmsub_d 0111 01001010 10111 ..... ..... ..... @vvv + +xvmaddwev_h_b 0111 01001010 11000 ..... ..... ..... @vvv +xvmaddwev_w_h 0111 01001010 11001 ..... ..... ..... @vvv +xvmaddwev_d_w 0111 01001010 11010 ..... ..... ..... @vvv +xvmaddwev_q_d 0111 01001010 11011 ..... ..... ..... @vvv +xvmaddwod_h_b 0111 01001010 11100 ..... ..... ..... @vvv +xvmaddwod_w_h 0111 01001010 11101 ..... ..... ..... @vvv +xvmaddwod_d_w 0111 01001010 11110 ..... ..... ..... @vvv +xvmaddwod_q_d 0111 01001010 11111 ..... ..... ..... @vvv +xvmaddwev_h_bu 0111 01001011 01000 ..... ..... ..... @vvv +xvmaddwev_w_hu 0111 01001011 01001 ..... ..... ..... @vvv +xvmaddwev_d_wu 0111 01001011 01010 ..... ..... ..... @vvv +xvmaddwev_q_du 0111 01001011 01011 ..... ..... ..... @vvv +xvmaddwod_h_bu 0111 01001011 01100 ..... ..... ..... @vvv +xvmaddwod_w_hu 0111 01001011 01101 ..... ..... ..... @vvv +xvmaddwod_d_wu 0111 01001011 01110 ..... ..... ..... @vvv +xvmaddwod_q_du 0111 01001011 01111 ..... ..... ..... @vvv +xvmaddwev_h_bu_b 0111 01001011 11000 ..... ..... ..... @vvv +xvmaddwev_w_hu_h 0111 01001011 11001 ..... ..... ..... @vvv +xvmaddwev_d_wu_w 0111 01001011 11010 ..... ..... ..... @vvv +xvmaddwev_q_du_d 0111 01001011 11011 ..... ..... ..... @vvv +xvmaddwod_h_bu_b 0111 01001011 11100 ..... ..... ..... @vvv +xvmaddwod_w_hu_h 0111 01001011 11101 ..... ..... ..... @vvv +xvmaddwod_d_wu_w 0111 01001011 11110 ..... ..... ..... @vvv +xvmaddwod_q_du_d 0111 01001011 11111 ..... ..... ..... @vvv + +xvdiv_b 0111 01001110 00000 ..... ..... ..... @vvv +xvdiv_h 0111 01001110 00001 ..... ..... ..... @vvv +xvdiv_w 0111 01001110 00010 ..... ..... ..... @vvv +xvdiv_d 0111 01001110 00011 ..... ..... ..... @vvv +xvmod_b 0111 01001110 00100 ..... ..... ..... @vvv +xvmod_h 0111 01001110 00101 ..... ..... ..... @vvv +xvmod_w 0111 01001110 00110 ..... ..... ..... @vvv +xvmod_d 0111 01001110 00111 ..... ..... ..... @vvv +xvdiv_bu 0111 01001110 01000 ..... ..... ..... @vvv +xvdiv_hu 0111 01001110 01001 ..... ..... ..... @vvv +xvdiv_wu 0111 01001110 01010 ..... ..... ..... @vvv +xvdiv_du 0111 01001110 01011 ..... ..... ..... @vvv +xvmod_bu 0111 01001110 01100 ..... ..... ..... @vvv +xvmod_hu 0111 01001110 01101 ..... ..... ..... @vvv +xvmod_wu 0111 01001110 01110 ..... ..... ..... @vvv +xvmod_du 0111 01001110 01111 ..... ..... ..... @vvv + +xvsat_b 0111 01110010 01000 01 ... ..... ..... @vv_ui3 +xvsat_h 0111 01110010 01000 1 .... ..... ..... @vv_ui4 +xvsat_w 0111 01110010 01001 ..... ..... ..... @vv_ui5 +xvsat_d 0111 01110010 0101 ...... ..... ..... @vv_ui6 +xvsat_bu 0111 01110010 10000 01 ... ..... ..... @vv_ui3 +xvsat_hu 0111 01110010 10000 1 .... ..... ..... @vv_ui4 +xvsat_wu 0111 01110010 10001 ..... ..... ..... @vv_ui5 +xvsat_du 0111 01110010 1001 ...... ..... ..... @vv_ui6 + +xvexth_h_b 0111 01101001 11101 11000 ..... ..... @vv +xvexth_w_h 0111 01101001 11101 11001 ..... ..... @vv +xvexth_d_w 0111 01101001 11101 11010 ..... ..... @vv +xvexth_q_d 0111 01101001 11101 11011 ..... ..... @vv +xvexth_hu_bu 0111 01101001 11101 11100 ..... ..... @vv +xvexth_wu_hu 0111 01101001 11101 11101 ..... ..... @vv +xvexth_du_wu 0111 01101001 11101 11110 ..... ..... @vv +xvexth_qu_du 0111 01101001 11101 11111 ..... ..... @vv + +vext2xv_h_b 0111 01101001 11110 00100 ..... ..... @vv +vext2xv_w_b 0111 01101001 11110 00101 ..... ..... @vv +vext2xv_d_b 0111 01101001 11110 00110 ..... ..... @vv +vext2xv_w_h 0111 01101001 11110 00111 ..... ..... @vv +vext2xv_d_h 0111 01101001 11110 01000 ..... ..... @vv +vext2xv_d_w 0111 01101001 11110 01001 ..... ..... @vv +vext2xv_hu_bu 0111 01101001 11110 01010 ..... ..... @vv +vext2xv_wu_bu 0111 01101001 11110 01011 ..... ..... @vv +vext2xv_du_bu 0111 01101001 11110 01100 ..... ..... @vv +vext2xv_wu_hu 0111 01101001 11110 01101 ..... ..... @vv +vext2xv_du_hu 0111 01101001 11110 01110 ..... ..... @vv +vext2xv_du_wu 0111 01101001 11110 01111 ..... ..... @vv + +xvsigncov_b 0111 01010010 11100 ..... ..... ..... @vvv +xvsigncov_h 0111 01010010 11101 ..... ..... ..... @vvv +xvsigncov_w 0111 01010010 11110 ..... ..... ..... @vvv +xvsigncov_d 0111 01010010 11111 ..... ..... ..... @vvv + +xvmskltz_b 0111 01101001 11000 10000 ..... ..... @vv +xvmskltz_h 0111 01101001 11000 10001 ..... ..... @vv +xvmskltz_w 0111 01101001 11000 10010 ..... ..... @vv +xvmskltz_d 0111 01101001 11000 10011 ..... ..... @vv +xvmskgez_b 0111 01101001 11000 10100 ..... ..... @vv +xvmsknz_b 0111 01101001 11000 11000 ..... ..... @vv + +xvldi 0111 01111110 00 ............. ..... @v_i13 + +xvand_v 0111 01010010 01100 ..... ..... ..... @vvv +xvor_v 0111 01010010 01101 ..... ..... ..... @vvv +xvxor_v 0111 01010010 01110 ..... ..... ..... @vvv +xvnor_v 0111 01010010 01111 ..... ..... ..... @vvv +xvandn_v 0111 01010010 10000 ..... ..... ..... @vvv +xvorn_v 0111 01010010 10001 ..... ..... ..... @vvv + +xvandi_b 0111 01111101 00 ........ ..... ..... @vv_ui8 +xvori_b 0111 01111101 01 ........ ..... ..... @vv_ui8 +xvxori_b 0111 01111101 10 ........ ..... ..... @vv_ui8 +xvnori_b 0111 01111101 11 ........ ..... ..... @vv_ui8 + +xvsll_b 0111 01001110 10000 ..... ..... ..... @vvv +xvsll_h 0111 01001110 10001 ..... ..... ..... @vvv +xvsll_w 0111 01001110 10010 ..... ..... ..... @vvv +xvsll_d 0111 01001110 10011 ..... ..... ..... @vvv +xvslli_b 0111 01110010 11000 01 ... ..... ..... @vv_ui3 +xvslli_h 0111 01110010 11000 1 .... ..... ..... @vv_ui4 +xvslli_w 0111 01110010 11001 ..... ..... ..... @vv_ui5 +xvslli_d 0111 01110010 1101 ...... ..... ..... @vv_ui6 +xvsrl_b 0111 01001110 10100 ..... ..... ..... @vvv +xvsrl_h 0111 01001110 10101 ..... ..... ..... @vvv +xvsrl_w 0111 01001110 10110 ..... ..... ..... @vvv +xvsrl_d 0111 01001110 10111 ..... ..... ..... @vvv +xvsrli_b 0111 01110011 00000 01 ... ..... ..... @vv_ui3 +xvsrli_h 0111 01110011 00000 1 .... ..... ..... @vv_ui4 +xvsrli_w 0111 01110011 00001 ..... ..... ..... @vv_ui5 +xvsrli_d 0111 01110011 0001 ...... ..... ..... @vv_ui6 +xvsra_b 0111 01001110 11000 ..... ..... ..... @vvv +xvsra_h 0111 01001110 11001 ..... ..... ..... @vvv +xvsra_w 0111 01001110 11010 ..... ..... ..... @vvv +xvsra_d 0111 01001110 11011 ..... ..... ..... @vvv +xvsrai_b 0111 01110011 01000 01 ... ..... ..... @vv_ui3 +xvsrai_h 0111 01110011 01000 1 .... ..... ..... @vv_ui4 +xvsrai_w 0111 01110011 01001 ..... ..... ..... @vv_ui5 +xvsrai_d 0111 01110011 0101 ...... ..... ..... @vv_ui6 +xvrotr_b 0111 01001110 11100 ..... ..... ..... @vvv +xvrotr_h 0111 01001110 11101 ..... ..... ..... @vvv +xvrotr_w 0111 01001110 11110 ..... ..... ..... @vvv +xvrotr_d 0111 01001110 11111 ..... ..... ..... @vvv +xvrotri_b 0111 01101010 00000 01 ... ..... ..... @vv_ui3 +xvrotri_h 0111 01101010 00000 1 .... ..... ..... @vv_ui4 +xvrotri_w 0111 01101010 00001 ..... ..... ..... @vv_ui5 +xvrotri_d 0111 01101010 0001 ...... ..... ..... @vv_ui6 + +xvsllwil_h_b 0111 01110000 10000 01 ... ..... ..... @vv_ui3 +xvsllwil_w_h 0111 01110000 10000 1 .... ..... ..... @vv_ui4 +xvsllwil_d_w 0111 01110000 10001 ..... ..... ..... @vv_ui5 +xvextl_q_d 0111 01110000 10010 00000 ..... ..... @vv +xvsllwil_hu_bu 0111 01110000 11000 01 ... ..... ..... @vv_ui3 +xvsllwil_wu_hu 0111 01110000 11000 1 .... ..... ..... @vv_ui4 +xvsllwil_du_wu 0111 01110000 11001 ..... ..... ..... @vv_ui5 +xvextl_qu_du 0111 01110000 11010 00000 ..... ..... @vv + +xvsrlr_b 0111 01001111 00000 ..... ..... ..... @vvv +xvsrlr_h 0111 01001111 00001 ..... ..... ..... @vvv +xvsrlr_w 0111 01001111 00010 ..... ..... ..... @vvv +xvsrlr_d 0111 01001111 00011 ..... ..... ..... @vvv +xvsrlri_b 0111 01101010 01000 01 ... ..... ..... @vv_ui3 +xvsrlri_h 0111 01101010 01000 1 .... ..... ..... @vv_ui4 +xvsrlri_w 0111 01101010 01001 ..... ..... ..... @vv_ui5 +xvsrlri_d 0111 01101010 0101 ...... ..... ..... @vv_ui6 +xvsrar_b 0111 01001111 00100 ..... ..... ..... @vvv +xvsrar_h 0111 01001111 00101 ..... ..... ..... @vvv +xvsrar_w 0111 01001111 00110 ..... ..... ..... @vvv +xvsrar_d 0111 01001111 00111 ..... ..... ..... @vvv +xvsrari_b 0111 01101010 10000 01 ... ..... ..... @vv_ui3 +xvsrari_h 0111 01101010 10000 1 .... ..... ..... @vv_ui4 +xvsrari_w 0111 01101010 10001 ..... ..... ..... @vv_ui5 +xvsrari_d 0111 01101010 1001 ...... ..... ..... @vv_ui6 + +xvsrln_b_h 0111 01001111 01001 ..... ..... ..... @vvv +xvsrln_h_w 0111 01001111 01010 ..... ..... ..... @vvv +xvsrln_w_d 0111 01001111 01011 ..... ..... ..... @vvv +xvsran_b_h 0111 01001111 01101 ..... ..... ..... @vvv +xvsran_h_w 0111 01001111 01110 ..... ..... ..... @vvv +xvsran_w_d 0111 01001111 01111 ..... ..... ..... @vvv + +xvsrlni_b_h 0111 01110100 00000 1 .... ..... ..... @vv_ui4 +xvsrlni_h_w 0111 01110100 00001 ..... ..... ..... @vv_ui5 +xvsrlni_w_d 0111 01110100 0001 ...... ..... ..... @vv_ui6 +xvsrlni_d_q 0111 01110100 001 ....... ..... ..... @vv_ui7 +xvsrani_b_h 0111 01110101 10000 1 .... ..... ..... @vv_ui4 +xvsrani_h_w 0111 01110101 10001 ..... ..... ..... @vv_ui5 +xvsrani_w_d 0111 01110101 1001 ...... ..... ..... @vv_ui6 +xvsrani_d_q 0111 01110101 101 ....... ..... ..... @vv_ui7 + +xvsrlrn_b_h 0111 01001111 10001 ..... ..... ..... @vvv +xvsrlrn_h_w 0111 01001111 10010 ..... ..... ..... @vvv +xvsrlrn_w_d 0111 01001111 10011 ..... ..... ..... @vvv +xvsrarn_b_h 0111 01001111 10101 ..... ..... ..... @vvv +xvsrarn_h_w 0111 01001111 10110 ..... ..... ..... @vvv +xvsrarn_w_d 0111 01001111 10111 ..... ..... ..... @vvv + +xvsrlrni_b_h 0111 01110100 01000 1 .... ..... ..... @vv_ui4 +xvsrlrni_h_w 0111 01110100 01001 ..... ..... ..... @vv_ui5 +xvsrlrni_w_d 0111 01110100 0101 ...... ..... ..... @vv_ui6 +xvsrlrni_d_q 0111 01110100 011 ....... ..... ..... @vv_ui7 +xvsrarni_b_h 0111 01110101 11000 1 .... ..... ..... @vv_ui4 +xvsrarni_h_w 0111 01110101 11001 ..... ..... ..... @vv_ui5 +xvsrarni_w_d 0111 01110101 1101 ...... ..... ..... @vv_ui6 +xvsrarni_d_q 0111 01110101 111 ....... ..... ..... @vv_ui7 + +xvssrln_b_h 0111 01001111 11001 ..... ..... ..... @vvv +xvssrln_h_w 0111 01001111 11010 ..... ..... ..... @vvv +xvssrln_w_d 0111 01001111 11011 ..... ..... ..... @vvv +xvssran_b_h 0111 01001111 11101 ..... ..... ..... @vvv +xvssran_h_w 0111 01001111 11110 ..... ..... ..... @vvv +xvssran_w_d 0111 01001111 11111 ..... ..... ..... @vvv +xvssrln_bu_h 0111 01010000 01001 ..... ..... ..... @vvv +xvssrln_hu_w 0111 01010000 01010 ..... ..... ..... @vvv +xvssrln_wu_d 0111 01010000 01011 ..... ..... ..... @vvv +xvssran_bu_h 0111 01010000 01101 ..... ..... ..... @vvv +xvssran_hu_w 0111 01010000 01110 ..... ..... ..... @vvv +xvssran_wu_d 0111 01010000 01111 ..... ..... ..... @vvv + +xvssrlni_b_h 0111 01110100 10000 1 .... ..... ..... @vv_ui4 +xvssrlni_h_w 0111 01110100 10001 ..... ..... ..... @vv_ui5 +xvssrlni_w_d 0111 01110100 1001 ...... ..... ..... @vv_ui6 +xvssrlni_d_q 0111 01110100 101 ....... ..... ..... @vv_ui7 +xvssrani_b_h 0111 01110110 00000 1 .... ..... ..... @vv_ui4 +xvssrani_h_w 0111 01110110 00001 ..... ..... ..... @vv_ui5 +xvssrani_w_d 0111 01110110 0001 ...... ..... ..... @vv_ui6 +xvssrani_d_q 0111 01110110 001 ....... ..... ..... @vv_ui7 +xvssrlni_bu_h 0111 01110100 11000 1 .... ..... ..... @vv_ui4 +xvssrlni_hu_w 0111 01110100 11001 ..... ..... ..... @vv_ui5 +xvssrlni_wu_d 0111 01110100 1101 ...... ..... ..... @vv_ui6 +xvssrlni_du_q 0111 01110100 111 ....... ..... ..... @vv_ui7 +xvssrani_bu_h 0111 01110110 01000 1 .... ..... ..... @vv_ui4 +xvssrani_hu_w 0111 01110110 01001 ..... ..... ..... @vv_ui5 +xvssrani_wu_d 0111 01110110 0101 ...... ..... ..... @vv_ui6 +xvssrani_du_q 0111 01110110 011 ....... ..... ..... @vv_ui7 + +xvssrlrn_b_h 0111 01010000 00001 ..... ..... ..... @vvv +xvssrlrn_h_w 0111 01010000 00010 ..... ..... ..... @vvv +xvssrlrn_w_d 0111 01010000 00011 ..... ..... ..... @vvv +xvssrarn_b_h 0111 01010000 00101 ..... ..... ..... @vvv +xvssrarn_h_w 0111 01010000 00110 ..... ..... ..... @vvv +xvssrarn_w_d 0111 01010000 00111 ..... ..... ..... @vvv +xvssrlrn_bu_h 0111 01010000 10001 ..... ..... ..... @vvv +xvssrlrn_hu_w 0111 01010000 10010 ..... ..... ..... @vvv +xvssrlrn_wu_d 0111 01010000 10011 ..... ..... ..... @vvv +xvssrarn_bu_h 0111 01010000 10101 ..... ..... ..... @vvv +xvssrarn_hu_w 0111 01010000 10110 ..... ..... ..... @vvv +xvssrarn_wu_d 0111 01010000 10111 ..... ..... ..... @vvv + +xvssrlrni_b_h 0111 01110101 00000 1 .... ..... ..... @vv_ui4 +xvssrlrni_h_w 0111 01110101 00001 ..... ..... ..... @vv_ui5 +xvssrlrni_w_d 0111 01110101 0001 ...... ..... ..... @vv_ui6 +xvssrlrni_d_q 0111 01110101 001 ....... ..... ..... @vv_ui7 +xvssrarni_b_h 0111 01110110 10000 1 .... ..... ..... @vv_ui4 +xvssrarni_h_w 0111 01110110 10001 ..... ..... ..... @vv_ui5 +xvssrarni_w_d 0111 01110110 1001 ...... ..... ..... @vv_ui6 +xvssrarni_d_q 0111 01110110 101 ....... ..... ..... @vv_ui7 +xvssrlrni_bu_h 0111 01110101 01000 1 .... ..... ..... @vv_ui4 +xvssrlrni_hu_w 0111 01110101 01001 ..... ..... ..... @vv_ui5 +xvssrlrni_wu_d 0111 01110101 0101 ...... ..... ..... @vv_ui6 +xvssrlrni_du_q 0111 01110101 011 ....... ..... ..... @vv_ui7 +xvssrarni_bu_h 0111 01110110 11000 1 .... ..... ..... @vv_ui4 +xvssrarni_hu_w 0111 01110110 11001 ..... ..... ..... @vv_ui5 +xvssrarni_wu_d 0111 01110110 1101 ...... ..... ..... @vv_ui6 +xvssrarni_du_q 0111 01110110 111 ....... ..... ..... @vv_ui7 + +xvclo_b 0111 01101001 11000 00000 ..... ..... @vv +xvclo_h 0111 01101001 11000 00001 ..... ..... @vv +xvclo_w 0111 01101001 11000 00010 ..... ..... @vv +xvclo_d 0111 01101001 11000 00011 ..... ..... @vv +xvclz_b 0111 01101001 11000 00100 ..... ..... @vv +xvclz_h 0111 01101001 11000 00101 ..... ..... @vv +xvclz_w 0111 01101001 11000 00110 ..... ..... @vv +xvclz_d 0111 01101001 11000 00111 ..... ..... @vv + +xvpcnt_b 0111 01101001 11000 01000 ..... ..... @vv +xvpcnt_h 0111 01101001 11000 01001 ..... ..... @vv +xvpcnt_w 0111 01101001 11000 01010 ..... ..... @vv +xvpcnt_d 0111 01101001 11000 01011 ..... ..... @vv + +xvbitclr_b 0111 01010000 11000 ..... ..... ..... @vvv +xvbitclr_h 0111 01010000 11001 ..... ..... ..... @vvv +xvbitclr_w 0111 01010000 11010 ..... ..... ..... @vvv +xvbitclr_d 0111 01010000 11011 ..... ..... ..... @vvv +xvbitclri_b 0111 01110001 00000 01 ... ..... ..... @vv_ui3 +xvbitclri_h 0111 01110001 00000 1 .... ..... ..... @vv_ui4 +xvbitclri_w 0111 01110001 00001 ..... ..... ..... @vv_ui5 +xvbitclri_d 0111 01110001 0001 ...... ..... ..... @vv_ui6 + +xvbitset_b 0111 01010000 11100 ..... ..... ..... @vvv +xvbitset_h 0111 01010000 11101 ..... ..... ..... @vvv +xvbitset_w 0111 01010000 11110 ..... ..... ..... @vvv +xvbitset_d 0111 01010000 11111 ..... ..... ..... @vvv +xvbitseti_b 0111 01110001 01000 01 ... ..... ..... @vv_ui3 +xvbitseti_h 0111 01110001 01000 1 .... ..... ..... @vv_ui4 +xvbitseti_w 0111 01110001 01001 ..... ..... ..... @vv_ui5 +xvbitseti_d 0111 01110001 0101 ...... ..... ..... @vv_ui6 + +xvbitrev_b 0111 01010001 00000 ..... ..... ..... @vvv +xvbitrev_h 0111 01010001 00001 ..... ..... ..... @vvv +xvbitrev_w 0111 01010001 00010 ..... ..... ..... @vvv +xvbitrev_d 0111 01010001 00011 ..... ..... ..... @vvv +xvbitrevi_b 0111 01110001 10000 01 ... ..... ..... @vv_ui3 +xvbitrevi_h 0111 01110001 10000 1 .... ..... ..... @vv_ui4 +xvbitrevi_w 0111 01110001 10001 ..... ..... ..... @vv_ui5 +xvbitrevi_d 0111 01110001 1001 ...... ..... ..... @vv_ui6 + +xvfrstp_b 0111 01010010 10110 ..... ..... ..... @vvv +xvfrstp_h 0111 01010010 10111 ..... ..... ..... @vvv +xvfrstpi_b 0111 01101001 10100 ..... ..... ..... @vv_ui5 +xvfrstpi_h 0111 01101001 10101 ..... ..... ..... @vv_ui5 + +xvfadd_s 0111 01010011 00001 ..... ..... ..... @vvv +xvfadd_d 0111 01010011 00010 ..... ..... ..... @vvv +xvfsub_s 0111 01010011 00101 ..... ..... ..... @vvv +xvfsub_d 0111 01010011 00110 ..... ..... ..... @vvv +xvfmul_s 0111 01010011 10001 ..... ..... ..... @vvv +xvfmul_d 0111 01010011 10010 ..... ..... ..... @vvv +xvfdiv_s 0111 01010011 10101 ..... ..... ..... @vvv +xvfdiv_d 0111 01010011 10110 ..... ..... ..... @vvv + +xvfmadd_s 0000 10100001 ..... ..... ..... ..... @vvvv +xvfmadd_d 0000 10100010 ..... ..... ..... ..... @vvvv +xvfmsub_s 0000 10100101 ..... ..... ..... ..... @vvvv +xvfmsub_d 0000 10100110 ..... ..... ..... ..... @vvvv +xvfnmadd_s 0000 10101001 ..... ..... ..... ..... @vvvv +xvfnmadd_d 0000 10101010 ..... ..... ..... ..... @vvvv +xvfnmsub_s 0000 10101101 ..... ..... ..... ..... @vvvv +xvfnmsub_d 0000 10101110 ..... ..... ..... ..... @vvvv + +xvfmax_s 0111 01010011 11001 ..... ..... ..... @vvv +xvfmax_d 0111 01010011 11010 ..... ..... ..... @vvv +xvfmin_s 0111 01010011 11101 ..... ..... ..... @vvv +xvfmin_d 0111 01010011 11110 ..... ..... ..... @vvv + +xvfmaxa_s 0111 01010100 00001 ..... ..... ..... @vvv +xvfmaxa_d 0111 01010100 00010 ..... ..... ..... @vvv +xvfmina_s 0111 01010100 00101 ..... ..... ..... @vvv +xvfmina_d 0111 01010100 00110 ..... ..... ..... @vvv + +xvflogb_s 0111 01101001 11001 10001 ..... ..... @vv +xvflogb_d 0111 01101001 11001 10010 ..... ..... @vv + +xvfclass_s 0111 01101001 11001 10101 ..... ..... @vv +xvfclass_d 0111 01101001 11001 10110 ..... ..... @vv + +xvfsqrt_s 0111 01101001 11001 11001 ..... ..... @vv +xvfsqrt_d 0111 01101001 11001 11010 ..... ..... @vv +xvfrecip_s 0111 01101001 11001 11101 ..... ..... @vv +xvfrecip_d 0111 01101001 11001 11110 ..... ..... @vv +xvfrsqrt_s 0111 01101001 11010 00001 ..... ..... @vv +xvfrsqrt_d 0111 01101001 11010 00010 ..... ..... @vv + +xvfcvtl_s_h 0111 01101001 11011 11010 ..... ..... @vv +xvfcvth_s_h 0111 01101001 11011 11011 ..... ..... @vv +xvfcvtl_d_s 0111 01101001 11011 11100 ..... ..... @vv +xvfcvth_d_s 0111 01101001 11011 11101 ..... ..... @vv +xvfcvt_h_s 0111 01010100 01100 ..... ..... ..... @vvv +xvfcvt_s_d 0111 01010100 01101 ..... ..... ..... @vvv + +xvfrintrne_s 0111 01101001 11010 11101 ..... ..... @vv +xvfrintrne_d 0111 01101001 11010 11110 ..... ..... @vv +xvfrintrz_s 0111 01101001 11010 11001 ..... ..... @vv +xvfrintrz_d 0111 01101001 11010 11010 ..... ..... @vv +xvfrintrp_s 0111 01101001 11010 10101 ..... ..... @vv +xvfrintrp_d 0111 01101001 11010 10110 ..... ..... @vv +xvfrintrm_s 0111 01101001 11010 10001 ..... ..... @vv +xvfrintrm_d 0111 01101001 11010 10010 ..... ..... @vv +xvfrint_s 0111 01101001 11010 01101 ..... ..... @vv +xvfrint_d 0111 01101001 11010 01110 ..... ..... @vv + +xvftintrne_w_s 0111 01101001 11100 10100 ..... ..... @vv +xvftintrne_l_d 0111 01101001 11100 10101 ..... ..... @vv +xvftintrz_w_s 0111 01101001 11100 10010 ..... ..... @vv +xvftintrz_l_d 0111 01101001 11100 10011 ..... ..... @vv +xvftintrp_w_s 0111 01101001 11100 10000 ..... ..... @vv +xvftintrp_l_d 0111 01101001 11100 10001 ..... ..... @vv +xvftintrm_w_s 0111 01101001 11100 01110 ..... ..... @vv +xvftintrm_l_d 0111 01101001 11100 01111 ..... ..... @vv +xvftint_w_s 0111 01101001 11100 01100 ..... ..... @vv +xvftint_l_d 0111 01101001 11100 01101 ..... ..... @vv +xvftintrz_wu_s 0111 01101001 11100 11100 ..... ..... @vv +xvftintrz_lu_d 0111 01101001 11100 11101 ..... ..... @vv +xvftint_wu_s 0111 01101001 11100 10110 ..... ..... @vv +xvftint_lu_d 0111 01101001 11100 10111 ..... ..... @vv + +xvftintrne_w_d 0111 01010100 10111 ..... ..... ..... @vvv +xvftintrz_w_d 0111 01010100 10110 ..... ..... ..... @vvv +xvftintrp_w_d 0111 01010100 10101 ..... ..... ..... @vvv +xvftintrm_w_d 0111 01010100 10100 ..... ..... ..... @vvv +xvftint_w_d 0111 01010100 10011 ..... ..... ..... @vvv + +xvftintrnel_l_s 0111 01101001 11101 01000 ..... ..... @vv +xvftintrneh_l_s 0111 01101001 11101 01001 ..... ..... @vv +xvftintrzl_l_s 0111 01101001 11101 00110 ..... ..... @vv +xvftintrzh_l_s 0111 01101001 11101 00111 ..... ..... @vv +xvftintrpl_l_s 0111 01101001 11101 00100 ..... ..... @vv +xvftintrph_l_s 0111 01101001 11101 00101 ..... ..... @vv +xvftintrml_l_s 0111 01101001 11101 00010 ..... ..... @vv +xvftintrmh_l_s 0111 01101001 11101 00011 ..... ..... @vv +xvftintl_l_s 0111 01101001 11101 00000 ..... ..... @vv +xvftinth_l_s 0111 01101001 11101 00001 ..... ..... @vv + +xvffint_s_w 0111 01101001 11100 00000 ..... ..... @vv +xvffint_d_l 0111 01101001 11100 00010 ..... ..... @vv +xvffint_s_wu 0111 01101001 11100 00001 ..... ..... @vv +xvffint_d_lu 0111 01101001 11100 00011 ..... ..... @vv +xvffintl_d_w 0111 01101001 11100 00100 ..... ..... @vv +xvffinth_d_w 0111 01101001 11100 00101 ..... ..... @vv +xvffint_s_l 0111 01010100 10000 ..... ..... ..... @vvv + +xvseq_b 0111 01000000 00000 ..... ..... ..... @vvv +xvseq_h 0111 01000000 00001 ..... ..... ..... @vvv +xvseq_w 0111 01000000 00010 ..... ..... ..... @vvv +xvseq_d 0111 01000000 00011 ..... ..... ..... @vvv +xvseqi_b 0111 01101000 00000 ..... ..... ..... @vv_i5 +xvseqi_h 0111 01101000 00001 ..... ..... ..... @vv_i5 +xvseqi_w 0111 01101000 00010 ..... ..... ..... @vv_i5 +xvseqi_d 0111 01101000 00011 ..... ..... ..... @vv_i5 + +xvsle_b 0111 01000000 00100 ..... ..... ..... @vvv +xvsle_h 0111 01000000 00101 ..... ..... ..... @vvv +xvsle_w 0111 01000000 00110 ..... ..... ..... @vvv +xvsle_d 0111 01000000 00111 ..... ..... ..... @vvv +xvslei_b 0111 01101000 00100 ..... ..... ..... @vv_i5 +xvslei_h 0111 01101000 00101 ..... ..... ..... @vv_i5 +xvslei_w 0111 01101000 00110 ..... ..... ..... @vv_i5 +xvslei_d 0111 01101000 00111 ..... ..... ..... @vv_i5 +xvsle_bu 0111 01000000 01000 ..... ..... ..... @vvv +xvsle_hu 0111 01000000 01001 ..... ..... ..... @vvv +xvsle_wu 0111 01000000 01010 ..... ..... ..... @vvv +xvsle_du 0111 01000000 01011 ..... ..... ..... @vvv +xvslei_bu 0111 01101000 01000 ..... ..... ..... @vv_ui5 +xvslei_hu 0111 01101000 01001 ..... ..... ..... @vv_ui5 +xvslei_wu 0111 01101000 01010 ..... ..... ..... @vv_ui5 +xvslei_du 0111 01101000 01011 ..... ..... ..... @vv_ui5 + +xvslt_b 0111 01000000 01100 ..... ..... ..... @vvv +xvslt_h 0111 01000000 01101 ..... ..... ..... @vvv +xvslt_w 0111 01000000 01110 ..... ..... ..... @vvv +xvslt_d 0111 01000000 01111 ..... ..... ..... @vvv +xvslti_b 0111 01101000 01100 ..... ..... ..... @vv_i5 +xvslti_h 0111 01101000 01101 ..... ..... ..... @vv_i5 +xvslti_w 0111 01101000 01110 ..... ..... ..... @vv_i5 +xvslti_d 0111 01101000 01111 ..... ..... ..... @vv_i5 +xvslt_bu 0111 01000000 10000 ..... ..... ..... @vvv +xvslt_hu 0111 01000000 10001 ..... ..... ..... @vvv +xvslt_wu 0111 01000000 10010 ..... ..... ..... @vvv +xvslt_du 0111 01000000 10011 ..... ..... ..... @vvv +xvslti_bu 0111 01101000 10000 ..... ..... ..... @vv_ui5 +xvslti_hu 0111 01101000 10001 ..... ..... ..... @vv_ui5 +xvslti_wu 0111 01101000 10010 ..... ..... ..... @vv_ui5 +xvslti_du 0111 01101000 10011 ..... ..... ..... @vv_ui5 + +xvfcmp_cond_s 0000 11001001 ..... ..... ..... ..... @vvv_fcond +xvfcmp_cond_d 0000 11001010 ..... ..... ..... ..... @vvv_fcond + +xvbitsel_v 0000 11010010 ..... ..... ..... ..... @vvvv + +xvbitseli_b 0111 01111100 01 ........ ..... ..... @vv_ui8 + +xvseteqz_v 0111 01101001 11001 00110 ..... 00 ... @cv +xvsetnez_v 0111 01101001 11001 00111 ..... 00 ... @cv +xvsetanyeqz_b 0111 01101001 11001 01000 ..... 00 ... @cv +xvsetanyeqz_h 0111 01101001 11001 01001 ..... 00 ... @cv +xvsetanyeqz_w 0111 01101001 11001 01010 ..... 00 ... @cv +xvsetanyeqz_d 0111 01101001 11001 01011 ..... 00 ... @cv +xvsetallnez_b 0111 01101001 11001 01100 ..... 00 ... @cv +xvsetallnez_h 0111 01101001 11001 01101 ..... 00 ... @cv +xvsetallnez_w 0111 01101001 11001 01110 ..... 00 ... @cv +xvsetallnez_d 0111 01101001 11001 01111 ..... 00 ... @cv + +xvinsgr2vr_w 0111 01101110 10111 10 ... ..... ..... @vr_ui3 +xvinsgr2vr_d 0111 01101110 10111 110 .. ..... ..... @vr_ui2 +xvpickve2gr_w 0111 01101110 11111 10 ... ..... ..... @rv_ui3 +xvpickve2gr_d 0111 01101110 11111 110 .. ..... ..... @rv_ui2 +xvpickve2gr_wu 0111 01101111 00111 10 ... ..... ..... @rv_ui3 +xvpickve2gr_du 0111 01101111 00111 110 .. ..... ..... @rv_ui2 + +xvreplgr2vr_b 0111 01101001 11110 00000 ..... ..... @vr +xvreplgr2vr_h 0111 01101001 11110 00001 ..... ..... @vr +xvreplgr2vr_w 0111 01101001 11110 00010 ..... ..... @vr +xvreplgr2vr_d 0111 01101001 11110 00011 ..... ..... @vr + +xvreplve_b 0111 01010010 00100 ..... ..... ..... @vvr +xvreplve_h 0111 01010010 00101 ..... ..... ..... @vvr +xvreplve_w 0111 01010010 00110 ..... ..... ..... @vvr +xvreplve_d 0111 01010010 00111 ..... ..... ..... @vvr + +xvrepl128vei_b 0111 01101111 01111 0 .... ..... ..... @vv_ui4 +xvrepl128vei_h 0111 01101111 01111 10 ... ..... ..... @vv_ui3 +xvrepl128vei_w 0111 01101111 01111 110 .. ..... ..... @vv_ui2 +xvrepl128vei_d 0111 01101111 01111 1110 . ..... ..... @vv_ui1 + +xvreplve0_b 0111 01110000 01110 00000 ..... ..... @vv +xvreplve0_h 0111 01110000 01111 00000 ..... ..... @vv +xvreplve0_w 0111 01110000 01111 10000 ..... ..... @vv +xvreplve0_d 0111 01110000 01111 11000 ..... ..... @vv +xvreplve0_q 0111 01110000 01111 11100 ..... ..... @vv + +xvinsve0_w 0111 01101111 11111 10 ... ..... ..... @vv_ui3 +xvinsve0_d 0111 01101111 11111 110 .. ..... ..... @vv_ui2 + +xvpickve_w 0111 01110000 00111 10 ... ..... ..... @vv_ui3 +xvpickve_d 0111 01110000 00111 110 .. ..... ..... @vv_ui2 + +xvbsll_v 0111 01101000 11100 ..... ..... ..... @vv_ui5 +xvbsrl_v 0111 01101000 11101 ..... ..... ..... @vv_ui5 + +xvpackev_b 0111 01010001 01100 ..... ..... ..... @vvv +xvpackev_h 0111 01010001 01101 ..... ..... ..... @vvv +xvpackev_w 0111 01010001 01110 ..... ..... ..... @vvv +xvpackev_d 0111 01010001 01111 ..... ..... ..... @vvv +xvpackod_b 0111 01010001 10000 ..... ..... ..... @vvv +xvpackod_h 0111 01010001 10001 ..... ..... ..... @vvv +xvpackod_w 0111 01010001 10010 ..... ..... ..... @vvv +xvpackod_d 0111 01010001 10011 ..... ..... ..... @vvv + +xvpickev_b 0111 01010001 11100 ..... ..... ..... @vvv +xvpickev_h 0111 01010001 11101 ..... ..... ..... @vvv +xvpickev_w 0111 01010001 11110 ..... ..... ..... @vvv +xvpickev_d 0111 01010001 11111 ..... ..... ..... @vvv +xvpickod_b 0111 01010010 00000 ..... ..... ..... @vvv +xvpickod_h 0111 01010010 00001 ..... ..... ..... @vvv +xvpickod_w 0111 01010010 00010 ..... ..... ..... @vvv +xvpickod_d 0111 01010010 00011 ..... ..... ..... @vvv + +xvilvl_b 0111 01010001 10100 ..... ..... ..... @vvv +xvilvl_h 0111 01010001 10101 ..... ..... ..... @vvv +xvilvl_w 0111 01010001 10110 ..... ..... ..... @vvv +xvilvl_d 0111 01010001 10111 ..... ..... ..... @vvv +xvilvh_b 0111 01010001 11000 ..... ..... ..... @vvv +xvilvh_h 0111 01010001 11001 ..... ..... ..... @vvv +xvilvh_w 0111 01010001 11010 ..... ..... ..... @vvv +xvilvh_d 0111 01010001 11011 ..... ..... ..... @vvv + +xvshuf_b 0000 11010110 ..... ..... ..... ..... @vvvv +xvshuf_h 0111 01010111 10101 ..... ..... ..... @vvv +xvshuf_w 0111 01010111 10110 ..... ..... ..... @vvv +xvshuf_d 0111 01010111 10111 ..... ..... ..... @vvv + +xvperm_w 0111 01010111 11010 ..... ..... ..... @vvv + +xvshuf4i_b 0111 01111001 00 ........ ..... ..... @vv_ui8 +xvshuf4i_h 0111 01111001 01 ........ ..... ..... @vv_ui8 +xvshuf4i_w 0111 01111001 10 ........ ..... ..... @vv_ui8 +xvshuf4i_d 0111 01111001 11 ........ ..... ..... @vv_ui8 + +xvpermi_w 0111 01111110 01 ........ ..... ..... @vv_ui8 +xvpermi_d 0111 01111110 10 ........ ..... ..... @vv_ui8 +xvpermi_q 0111 01111110 11 ........ ..... ..... @vv_ui8 + +xvextrins_d 0111 01111000 00 ........ ..... ..... @vv_ui8 +xvextrins_w 0111 01111000 01 ........ ..... ..... @vv_ui8 +xvextrins_h 0111 01111000 10 ........ ..... ..... @vv_ui8 +xvextrins_b 0111 01111000 11 ........ ..... ..... @vv_ui8 + +xvld 0010 110010 ............ ..... ..... @vr_i12 +xvst 0010 110011 ............ ..... ..... @vr_i12 +xvldx 0011 10000100 10000 ..... ..... ..... @vrr +xvstx 0011 10000100 11000 ..... ..... ..... @vrr + +xvldrepl_d 0011 00100001 0 ......... ..... ..... @vr_i9 +xvldrepl_w 0011 00100010 .......... ..... ..... @vr_i10 +xvldrepl_h 0011 0010010 ........... ..... ..... @vr_i11 +xvldrepl_b 0011 001010 ............ ..... ..... @vr_i12 +xvstelm_d 0011 00110001 .. ........ ..... ..... @vr_i8i2x +xvstelm_w 0011 0011001 ... ........ ..... ..... @vr_i8i3x +xvstelm_h 0011 001101 .... ........ ..... ..... @vr_i8i4x +xvstelm_b 0011 00111 ..... ........ ..... ..... @vr_i8i5x diff --git a/target/loongarch/internals.h b/target/loongarch/internals.h index 7b0f29c942..c492863cc5 100644 --- a/target/loongarch/internals.h +++ b/target/loongarch/internals.h @@ -21,28 +21,6 @@ /* Global bit for huge page */ #define LOONGARCH_HGLOBAL_SHIFT 12 -#if HOST_BIG_ENDIAN -#define B(x) B[15 - (x)] -#define H(x) H[7 - (x)] -#define W(x) W[3 - (x)] -#define D(x) D[1 - (x)] -#define UB(x) UB[15 - (x)] -#define UH(x) UH[7 - (x)] -#define UW(x) UW[3 - (x)] -#define UD(x) UD[1 -(x)] -#define Q(x) Q[x] -#else -#define B(x) B[x] -#define H(x) H[x] -#define W(x) W[x] -#define D(x) D[x] -#define UB(x) UB[x] -#define UH(x) UH[x] -#define UW(x) UW[x] -#define UD(x) UD[x] -#define Q(x) Q[x] -#endif - void loongarch_translate_init(void); void loongarch_cpu_dump_state(CPUState *cpu, FILE *f, int flags); diff --git a/target/loongarch/lsx_helper.c b/target/loongarch/lsx_helper.c deleted file mode 100644 index 9571f0aef0..0000000000 --- a/target/loongarch/lsx_helper.c +++ /dev/null @@ -1,3004 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * QEMU LoongArch LSX helper functions. - * - * Copyright (c) 2022-2023 Loongson Technology Corporation Limited - */ - -#include "qemu/osdep.h" -#include "cpu.h" -#include "exec/exec-all.h" -#include "exec/helper-proto.h" -#include "fpu/softfloat.h" -#include "internals.h" -#include "tcg/tcg.h" - -#define DO_ADD(a, b) (a + b) -#define DO_SUB(a, b) (a - b) - -#define DO_ODD_EVEN(NAME, BIT, E1, E2, DO_OP) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - typedef __typeof(Vd->E1(0)) TD; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = DO_OP((TD)Vj->E2(2 * i + 1), (TD)Vk->E2(2 * i)); \ - } \ -} - -DO_ODD_EVEN(vhaddw_h_b, 16, H, B, DO_ADD) -DO_ODD_EVEN(vhaddw_w_h, 32, W, H, DO_ADD) -DO_ODD_EVEN(vhaddw_d_w, 64, D, W, DO_ADD) - -void HELPER(vhaddw_q_d)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t vk) -{ - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - VReg *Vk = &(env->fpr[vk].vreg); - - Vd->Q(0) = int128_add(int128_makes64(Vj->D(1)), int128_makes64(Vk->D(0))); -} - -DO_ODD_EVEN(vhsubw_h_b, 16, H, B, DO_SUB) -DO_ODD_EVEN(vhsubw_w_h, 32, W, H, DO_SUB) -DO_ODD_EVEN(vhsubw_d_w, 64, D, W, DO_SUB) - -void HELPER(vhsubw_q_d)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t vk) -{ - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - VReg *Vk = &(env->fpr[vk].vreg); - - Vd->Q(0) = int128_sub(int128_makes64(Vj->D(1)), int128_makes64(Vk->D(0))); -} - -DO_ODD_EVEN(vhaddw_hu_bu, 16, UH, UB, DO_ADD) -DO_ODD_EVEN(vhaddw_wu_hu, 32, UW, UH, DO_ADD) -DO_ODD_EVEN(vhaddw_du_wu, 64, UD, UW, DO_ADD) - -void HELPER(vhaddw_qu_du)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t vk) -{ - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - VReg *Vk = &(env->fpr[vk].vreg); - - Vd->Q(0) = int128_add(int128_make64((uint64_t)Vj->D(1)), - int128_make64((uint64_t)Vk->D(0))); -} - -DO_ODD_EVEN(vhsubw_hu_bu, 16, UH, UB, DO_SUB) -DO_ODD_EVEN(vhsubw_wu_hu, 32, UW, UH, DO_SUB) -DO_ODD_EVEN(vhsubw_du_wu, 64, UD, UW, DO_SUB) - -void HELPER(vhsubw_qu_du)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t vk) -{ - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - VReg *Vk = &(env->fpr[vk].vreg); - - Vd->Q(0) = int128_sub(int128_make64((uint64_t)Vj->D(1)), - int128_make64((uint64_t)Vk->D(0))); -} - -#define DO_EVEN(NAME, BIT, E1, E2, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - typedef __typeof(Vd->E1(0)) TD; \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = DO_OP((TD)Vj->E2(2 * i) ,(TD)Vk->E2(2 * i)); \ - } \ -} - -#define DO_ODD(NAME, BIT, E1, E2, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - typedef __typeof(Vd->E1(0)) TD; \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = DO_OP((TD)Vj->E2(2 * i + 1), (TD)Vk->E2(2 * i + 1)); \ - } \ -} - -void HELPER(vaddwev_q_d)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_add(int128_makes64(Vj->D(0)), int128_makes64(Vk->D(0))); -} - -DO_EVEN(vaddwev_h_b, 16, H, B, DO_ADD) -DO_EVEN(vaddwev_w_h, 32, W, H, DO_ADD) -DO_EVEN(vaddwev_d_w, 64, D, W, DO_ADD) - -void HELPER(vaddwod_q_d)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_add(int128_makes64(Vj->D(1)), int128_makes64(Vk->D(1))); -} - -DO_ODD(vaddwod_h_b, 16, H, B, DO_ADD) -DO_ODD(vaddwod_w_h, 32, W, H, DO_ADD) -DO_ODD(vaddwod_d_w, 64, D, W, DO_ADD) - -void HELPER(vsubwev_q_d)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_sub(int128_makes64(Vj->D(0)), int128_makes64(Vk->D(0))); -} - -DO_EVEN(vsubwev_h_b, 16, H, B, DO_SUB) -DO_EVEN(vsubwev_w_h, 32, W, H, DO_SUB) -DO_EVEN(vsubwev_d_w, 64, D, W, DO_SUB) - -void HELPER(vsubwod_q_d)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_sub(int128_makes64(Vj->D(1)), int128_makes64(Vk->D(1))); -} - -DO_ODD(vsubwod_h_b, 16, H, B, DO_SUB) -DO_ODD(vsubwod_w_h, 32, W, H, DO_SUB) -DO_ODD(vsubwod_d_w, 64, D, W, DO_SUB) - -void HELPER(vaddwev_q_du)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_add(int128_make64((uint64_t)Vj->D(0)), - int128_make64((uint64_t)Vk->D(0))); -} - -DO_EVEN(vaddwev_h_bu, 16, UH, UB, DO_ADD) -DO_EVEN(vaddwev_w_hu, 32, UW, UH, DO_ADD) -DO_EVEN(vaddwev_d_wu, 64, UD, UW, DO_ADD) - -void HELPER(vaddwod_q_du)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_add(int128_make64((uint64_t)Vj->D(1)), - int128_make64((uint64_t)Vk->D(1))); -} - -DO_ODD(vaddwod_h_bu, 16, UH, UB, DO_ADD) -DO_ODD(vaddwod_w_hu, 32, UW, UH, DO_ADD) -DO_ODD(vaddwod_d_wu, 64, UD, UW, DO_ADD) - -void HELPER(vsubwev_q_du)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_sub(int128_make64((uint64_t)Vj->D(0)), - int128_make64((uint64_t)Vk->D(0))); -} - -DO_EVEN(vsubwev_h_bu, 16, UH, UB, DO_SUB) -DO_EVEN(vsubwev_w_hu, 32, UW, UH, DO_SUB) -DO_EVEN(vsubwev_d_wu, 64, UD, UW, DO_SUB) - -void HELPER(vsubwod_q_du)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_sub(int128_make64((uint64_t)Vj->D(1)), - int128_make64((uint64_t)Vk->D(1))); -} - -DO_ODD(vsubwod_h_bu, 16, UH, UB, DO_SUB) -DO_ODD(vsubwod_w_hu, 32, UW, UH, DO_SUB) -DO_ODD(vsubwod_d_wu, 64, UD, UW, DO_SUB) - -#define DO_EVEN_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - typedef __typeof(Vd->ES1(0)) TDS; \ - typedef __typeof(Vd->EU1(0)) TDU; \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->ES1(i) = DO_OP((TDU)Vj->EU2(2 * i) ,(TDS)Vk->ES2(2 * i)); \ - } \ -} - -#define DO_ODD_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - typedef __typeof(Vd->ES1(0)) TDS; \ - typedef __typeof(Vd->EU1(0)) TDU; \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->ES1(i) = DO_OP((TDU)Vj->EU2(2 * i + 1), (TDS)Vk->ES2(2 * i + 1)); \ - } \ -} - -void HELPER(vaddwev_q_du_d)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_add(int128_make64((uint64_t)Vj->D(0)), - int128_makes64(Vk->D(0))); -} - -DO_EVEN_U_S(vaddwev_h_bu_b, 16, H, UH, B, UB, DO_ADD) -DO_EVEN_U_S(vaddwev_w_hu_h, 32, W, UW, H, UH, DO_ADD) -DO_EVEN_U_S(vaddwev_d_wu_w, 64, D, UD, W, UW, DO_ADD) - -void HELPER(vaddwod_q_du_d)(void *vd, void *vj, void *vk, uint32_t v) -{ - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - Vd->Q(0) = int128_add(int128_make64((uint64_t)Vj->D(1)), - int128_makes64(Vk->D(1))); -} - -DO_ODD_U_S(vaddwod_h_bu_b, 16, H, UH, B, UB, DO_ADD) -DO_ODD_U_S(vaddwod_w_hu_h, 32, W, UW, H, UH, DO_ADD) -DO_ODD_U_S(vaddwod_d_wu_w, 64, D, UD, W, UW, DO_ADD) - -#define DO_VAVG(a, b) ((a >> 1) + (b >> 1) + (a & b & 1)) -#define DO_VAVGR(a, b) ((a >> 1) + (b >> 1) + ((a | b) & 1)) - -#define DO_3OP(NAME, BIT, E, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = DO_OP(Vj->E(i), Vk->E(i)); \ - } \ -} - -DO_3OP(vavg_b, 8, B, DO_VAVG) -DO_3OP(vavg_h, 16, H, DO_VAVG) -DO_3OP(vavg_w, 32, W, DO_VAVG) -DO_3OP(vavg_d, 64, D, DO_VAVG) -DO_3OP(vavgr_b, 8, B, DO_VAVGR) -DO_3OP(vavgr_h, 16, H, DO_VAVGR) -DO_3OP(vavgr_w, 32, W, DO_VAVGR) -DO_3OP(vavgr_d, 64, D, DO_VAVGR) -DO_3OP(vavg_bu, 8, UB, DO_VAVG) -DO_3OP(vavg_hu, 16, UH, DO_VAVG) -DO_3OP(vavg_wu, 32, UW, DO_VAVG) -DO_3OP(vavg_du, 64, UD, DO_VAVG) -DO_3OP(vavgr_bu, 8, UB, DO_VAVGR) -DO_3OP(vavgr_hu, 16, UH, DO_VAVGR) -DO_3OP(vavgr_wu, 32, UW, DO_VAVGR) -DO_3OP(vavgr_du, 64, UD, DO_VAVGR) - -#define DO_VABSD(a, b) ((a > b) ? (a -b) : (b-a)) - -DO_3OP(vabsd_b, 8, B, DO_VABSD) -DO_3OP(vabsd_h, 16, H, DO_VABSD) -DO_3OP(vabsd_w, 32, W, DO_VABSD) -DO_3OP(vabsd_d, 64, D, DO_VABSD) -DO_3OP(vabsd_bu, 8, UB, DO_VABSD) -DO_3OP(vabsd_hu, 16, UH, DO_VABSD) -DO_3OP(vabsd_wu, 32, UW, DO_VABSD) -DO_3OP(vabsd_du, 64, UD, DO_VABSD) - -#define DO_VABS(a) ((a < 0) ? (-a) : (a)) - -#define DO_VADDA(NAME, BIT, E, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = DO_OP(Vj->E(i)) + DO_OP(Vk->E(i)); \ - } \ -} - -DO_VADDA(vadda_b, 8, B, DO_VABS) -DO_VADDA(vadda_h, 16, H, DO_VABS) -DO_VADDA(vadda_w, 32, W, DO_VABS) -DO_VADDA(vadda_d, 64, D, DO_VABS) - -#define DO_MIN(a, b) (a < b ? a : b) -#define DO_MAX(a, b) (a > b ? a : b) - -#define VMINMAXI(NAME, BIT, E, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - typedef __typeof(Vd->E(0)) TD; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = DO_OP(Vj->E(i), (TD)imm); \ - } \ -} - -VMINMAXI(vmini_b, 8, B, DO_MIN) -VMINMAXI(vmini_h, 16, H, DO_MIN) -VMINMAXI(vmini_w, 32, W, DO_MIN) -VMINMAXI(vmini_d, 64, D, DO_MIN) -VMINMAXI(vmaxi_b, 8, B, DO_MAX) -VMINMAXI(vmaxi_h, 16, H, DO_MAX) -VMINMAXI(vmaxi_w, 32, W, DO_MAX) -VMINMAXI(vmaxi_d, 64, D, DO_MAX) -VMINMAXI(vmini_bu, 8, UB, DO_MIN) -VMINMAXI(vmini_hu, 16, UH, DO_MIN) -VMINMAXI(vmini_wu, 32, UW, DO_MIN) -VMINMAXI(vmini_du, 64, UD, DO_MIN) -VMINMAXI(vmaxi_bu, 8, UB, DO_MAX) -VMINMAXI(vmaxi_hu, 16, UH, DO_MAX) -VMINMAXI(vmaxi_wu, 32, UW, DO_MAX) -VMINMAXI(vmaxi_du, 64, UD, DO_MAX) - -#define DO_VMUH(NAME, BIT, E1, E2, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - typedef __typeof(Vd->E1(0)) T; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E2(i) = ((T)Vj->E2(i)) * ((T)Vk->E2(i)) >> BIT; \ - } \ -} - -void HELPER(vmuh_d)(void *vd, void *vj, void *vk, uint32_t v) -{ - uint64_t l, h1, h2; - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - muls64(&l, &h1, Vj->D(0), Vk->D(0)); - muls64(&l, &h2, Vj->D(1), Vk->D(1)); - - Vd->D(0) = h1; - Vd->D(1) = h2; -} - -DO_VMUH(vmuh_b, 8, H, B, DO_MUH) -DO_VMUH(vmuh_h, 16, W, H, DO_MUH) -DO_VMUH(vmuh_w, 32, D, W, DO_MUH) - -void HELPER(vmuh_du)(void *vd, void *vj, void *vk, uint32_t v) -{ - uint64_t l, h1, h2; - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - VReg *Vk = (VReg *)vk; - - mulu64(&l, &h1, Vj->D(0), Vk->D(0)); - mulu64(&l, &h2, Vj->D(1), Vk->D(1)); - - Vd->D(0) = h1; - Vd->D(1) = h2; -} - -DO_VMUH(vmuh_bu, 8, UH, UB, DO_MUH) -DO_VMUH(vmuh_hu, 16, UW, UH, DO_MUH) -DO_VMUH(vmuh_wu, 32, UD, UW, DO_MUH) - -#define DO_MUL(a, b) (a * b) - -DO_EVEN(vmulwev_h_b, 16, H, B, DO_MUL) -DO_EVEN(vmulwev_w_h, 32, W, H, DO_MUL) -DO_EVEN(vmulwev_d_w, 64, D, W, DO_MUL) - -DO_ODD(vmulwod_h_b, 16, H, B, DO_MUL) -DO_ODD(vmulwod_w_h, 32, W, H, DO_MUL) -DO_ODD(vmulwod_d_w, 64, D, W, DO_MUL) - -DO_EVEN(vmulwev_h_bu, 16, UH, UB, DO_MUL) -DO_EVEN(vmulwev_w_hu, 32, UW, UH, DO_MUL) -DO_EVEN(vmulwev_d_wu, 64, UD, UW, DO_MUL) - -DO_ODD(vmulwod_h_bu, 16, UH, UB, DO_MUL) -DO_ODD(vmulwod_w_hu, 32, UW, UH, DO_MUL) -DO_ODD(vmulwod_d_wu, 64, UD, UW, DO_MUL) - -DO_EVEN_U_S(vmulwev_h_bu_b, 16, H, UH, B, UB, DO_MUL) -DO_EVEN_U_S(vmulwev_w_hu_h, 32, W, UW, H, UH, DO_MUL) -DO_EVEN_U_S(vmulwev_d_wu_w, 64, D, UD, W, UW, DO_MUL) - -DO_ODD_U_S(vmulwod_h_bu_b, 16, H, UH, B, UB, DO_MUL) -DO_ODD_U_S(vmulwod_w_hu_h, 32, W, UW, H, UH, DO_MUL) -DO_ODD_U_S(vmulwod_d_wu_w, 64, D, UD, W, UW, DO_MUL) - -#define DO_MADD(a, b, c) (a + b * c) -#define DO_MSUB(a, b, c) (a - b * c) - -#define VMADDSUB(NAME, BIT, E, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = DO_OP(Vd->E(i), Vj->E(i) ,Vk->E(i)); \ - } \ -} - -VMADDSUB(vmadd_b, 8, B, DO_MADD) -VMADDSUB(vmadd_h, 16, H, DO_MADD) -VMADDSUB(vmadd_w, 32, W, DO_MADD) -VMADDSUB(vmadd_d, 64, D, DO_MADD) -VMADDSUB(vmsub_b, 8, B, DO_MSUB) -VMADDSUB(vmsub_h, 16, H, DO_MSUB) -VMADDSUB(vmsub_w, 32, W, DO_MSUB) -VMADDSUB(vmsub_d, 64, D, DO_MSUB) - -#define VMADDWEV(NAME, BIT, E1, E2, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - typedef __typeof(Vd->E1(0)) TD; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) += DO_OP((TD)Vj->E2(2 * i), (TD)Vk->E2(2 * i)); \ - } \ -} - -VMADDWEV(vmaddwev_h_b, 16, H, B, DO_MUL) -VMADDWEV(vmaddwev_w_h, 32, W, H, DO_MUL) -VMADDWEV(vmaddwev_d_w, 64, D, W, DO_MUL) -VMADDWEV(vmaddwev_h_bu, 16, UH, UB, DO_MUL) -VMADDWEV(vmaddwev_w_hu, 32, UW, UH, DO_MUL) -VMADDWEV(vmaddwev_d_wu, 64, UD, UW, DO_MUL) - -#define VMADDWOD(NAME, BIT, E1, E2, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - typedef __typeof(Vd->E1(0)) TD; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) += DO_OP((TD)Vj->E2(2 * i + 1), \ - (TD)Vk->E2(2 * i + 1)); \ - } \ -} - -VMADDWOD(vmaddwod_h_b, 16, H, B, DO_MUL) -VMADDWOD(vmaddwod_w_h, 32, W, H, DO_MUL) -VMADDWOD(vmaddwod_d_w, 64, D, W, DO_MUL) -VMADDWOD(vmaddwod_h_bu, 16, UH, UB, DO_MUL) -VMADDWOD(vmaddwod_w_hu, 32, UW, UH, DO_MUL) -VMADDWOD(vmaddwod_d_wu, 64, UD, UW, DO_MUL) - -#define VMADDWEV_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - typedef __typeof(Vd->ES1(0)) TS1; \ - typedef __typeof(Vd->EU1(0)) TU1; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->ES1(i) += DO_OP((TU1)Vj->EU2(2 * i), \ - (TS1)Vk->ES2(2 * i)); \ - } \ -} - -VMADDWEV_U_S(vmaddwev_h_bu_b, 16, H, UH, B, UB, DO_MUL) -VMADDWEV_U_S(vmaddwev_w_hu_h, 32, W, UW, H, UH, DO_MUL) -VMADDWEV_U_S(vmaddwev_d_wu_w, 64, D, UD, W, UW, DO_MUL) - -#define VMADDWOD_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - typedef __typeof(Vd->ES1(0)) TS1; \ - typedef __typeof(Vd->EU1(0)) TU1; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->ES1(i) += DO_OP((TU1)Vj->EU2(2 * i + 1), \ - (TS1)Vk->ES2(2 * i + 1)); \ - } \ -} - -VMADDWOD_U_S(vmaddwod_h_bu_b, 16, H, UH, B, UB, DO_MUL) -VMADDWOD_U_S(vmaddwod_w_hu_h, 32, W, UW, H, UH, DO_MUL) -VMADDWOD_U_S(vmaddwod_d_wu_w, 64, D, UD, W, UW, DO_MUL) - -#define DO_DIVU(N, M) (unlikely(M == 0) ? 0 : N / M) -#define DO_REMU(N, M) (unlikely(M == 0) ? 0 : N % M) -#define DO_DIV(N, M) (unlikely(M == 0) ? 0 :\ - unlikely((N == -N) && (M == (__typeof(N))(-1))) ? N : N / M) -#define DO_REM(N, M) (unlikely(M == 0) ? 0 :\ - unlikely((N == -N) && (M == (__typeof(N))(-1))) ? 0 : N % M) - -#define VDIV(NAME, BIT, E, DO_OP) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = DO_OP(Vj->E(i), Vk->E(i)); \ - } \ -} - -VDIV(vdiv_b, 8, B, DO_DIV) -VDIV(vdiv_h, 16, H, DO_DIV) -VDIV(vdiv_w, 32, W, DO_DIV) -VDIV(vdiv_d, 64, D, DO_DIV) -VDIV(vdiv_bu, 8, UB, DO_DIVU) -VDIV(vdiv_hu, 16, UH, DO_DIVU) -VDIV(vdiv_wu, 32, UW, DO_DIVU) -VDIV(vdiv_du, 64, UD, DO_DIVU) -VDIV(vmod_b, 8, B, DO_REM) -VDIV(vmod_h, 16, H, DO_REM) -VDIV(vmod_w, 32, W, DO_REM) -VDIV(vmod_d, 64, D, DO_REM) -VDIV(vmod_bu, 8, UB, DO_REMU) -VDIV(vmod_hu, 16, UH, DO_REMU) -VDIV(vmod_wu, 32, UW, DO_REMU) -VDIV(vmod_du, 64, UD, DO_REMU) - -#define VSAT_S(NAME, BIT, E) \ -void HELPER(NAME)(void *vd, void *vj, uint64_t max, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - typedef __typeof(Vd->E(0)) TD; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = Vj->E(i) > (TD)max ? (TD)max : \ - Vj->E(i) < (TD)~max ? (TD)~max: Vj->E(i); \ - } \ -} - -VSAT_S(vsat_b, 8, B) -VSAT_S(vsat_h, 16, H) -VSAT_S(vsat_w, 32, W) -VSAT_S(vsat_d, 64, D) - -#define VSAT_U(NAME, BIT, E) \ -void HELPER(NAME)(void *vd, void *vj, uint64_t max, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - typedef __typeof(Vd->E(0)) TD; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = Vj->E(i) > (TD)max ? (TD)max : Vj->E(i); \ - } \ -} - -VSAT_U(vsat_bu, 8, UB) -VSAT_U(vsat_hu, 16, UH) -VSAT_U(vsat_wu, 32, UW) -VSAT_U(vsat_du, 64, UD) - -#define VEXTH(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t vd, uint32_t vj) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = Vj->E2(i + LSX_LEN/BIT); \ - } \ -} - -void HELPER(vexth_q_d)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - Vd->Q(0) = int128_makes64(Vj->D(1)); -} - -void HELPER(vexth_qu_du)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - Vd->Q(0) = int128_make64((uint64_t)Vj->D(1)); -} - -VEXTH(vexth_h_b, 16, H, B) -VEXTH(vexth_w_h, 32, W, H) -VEXTH(vexth_d_w, 64, D, W) -VEXTH(vexth_hu_bu, 16, UH, UB) -VEXTH(vexth_wu_hu, 32, UW, UH) -VEXTH(vexth_du_wu, 64, UD, UW) - -#define DO_SIGNCOV(a, b) (a == 0 ? 0 : a < 0 ? -b : b) - -DO_3OP(vsigncov_b, 8, B, DO_SIGNCOV) -DO_3OP(vsigncov_h, 16, H, DO_SIGNCOV) -DO_3OP(vsigncov_w, 32, W, DO_SIGNCOV) -DO_3OP(vsigncov_d, 64, D, DO_SIGNCOV) - -static uint64_t do_vmskltz_b(int64_t val) -{ - uint64_t m = 0x8080808080808080ULL; - uint64_t c = val & m; - c |= c << 7; - c |= c << 14; - c |= c << 28; - return c >> 56; -} - -void HELPER(vmskltz_b)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - uint16_t temp = 0; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - temp = do_vmskltz_b(Vj->D(0)); - temp |= (do_vmskltz_b(Vj->D(1)) << 8); - Vd->D(0) = temp; - Vd->D(1) = 0; -} - -static uint64_t do_vmskltz_h(int64_t val) -{ - uint64_t m = 0x8000800080008000ULL; - uint64_t c = val & m; - c |= c << 15; - c |= c << 30; - return c >> 60; -} - -void HELPER(vmskltz_h)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - uint16_t temp = 0; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - temp = do_vmskltz_h(Vj->D(0)); - temp |= (do_vmskltz_h(Vj->D(1)) << 4); - Vd->D(0) = temp; - Vd->D(1) = 0; -} - -static uint64_t do_vmskltz_w(int64_t val) -{ - uint64_t m = 0x8000000080000000ULL; - uint64_t c = val & m; - c |= c << 31; - return c >> 62; -} - -void HELPER(vmskltz_w)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - uint16_t temp = 0; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - temp = do_vmskltz_w(Vj->D(0)); - temp |= (do_vmskltz_w(Vj->D(1)) << 2); - Vd->D(0) = temp; - Vd->D(1) = 0; -} - -static uint64_t do_vmskltz_d(int64_t val) -{ - return (uint64_t)val >> 63; -} -void HELPER(vmskltz_d)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - uint16_t temp = 0; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - temp = do_vmskltz_d(Vj->D(0)); - temp |= (do_vmskltz_d(Vj->D(1)) << 1); - Vd->D(0) = temp; - Vd->D(1) = 0; -} - -void HELPER(vmskgez_b)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - uint16_t temp = 0; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - temp = do_vmskltz_b(Vj->D(0)); - temp |= (do_vmskltz_b(Vj->D(1)) << 8); - Vd->D(0) = (uint16_t)(~temp); - Vd->D(1) = 0; -} - -static uint64_t do_vmskez_b(uint64_t a) -{ - uint64_t m = 0x7f7f7f7f7f7f7f7fULL; - uint64_t c = ~(((a & m) + m) | a | m); - c |= c << 7; - c |= c << 14; - c |= c << 28; - return c >> 56; -} - -void HELPER(vmsknz_b)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - uint16_t temp = 0; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - temp = do_vmskez_b(Vj->D(0)); - temp |= (do_vmskez_b(Vj->D(1)) << 8); - Vd->D(0) = (uint16_t)(~temp); - Vd->D(1) = 0; -} - -void HELPER(vnori_b)(void *vd, void *vj, uint64_t imm, uint32_t v) -{ - int i; - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - - for (i = 0; i < LSX_LEN/8; i++) { - Vd->B(i) = ~(Vj->B(i) | (uint8_t)imm); - } -} - -#define VSLLWIL(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - typedef __typeof(temp.E1(0)) TD; \ - \ - temp.D(0) = 0; \ - temp.D(1) = 0; \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E1(i) = (TD)Vj->E2(i) << (imm % BIT); \ - } \ - *Vd = temp; \ -} - -void HELPER(vextl_q_d)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - Vd->Q(0) = int128_makes64(Vj->D(0)); -} - -void HELPER(vextl_qu_du)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - Vd->Q(0) = int128_make64(Vj->D(0)); -} - -VSLLWIL(vsllwil_h_b, 16, H, B) -VSLLWIL(vsllwil_w_h, 32, W, H) -VSLLWIL(vsllwil_d_w, 64, D, W) -VSLLWIL(vsllwil_hu_bu, 16, UH, UB) -VSLLWIL(vsllwil_wu_hu, 32, UW, UH) -VSLLWIL(vsllwil_du_wu, 64, UD, UW) - -#define do_vsrlr(E, T) \ -static T do_vsrlr_ ##E(T s1, int sh) \ -{ \ - if (sh == 0) { \ - return s1; \ - } else { \ - return (s1 >> sh) + ((s1 >> (sh - 1)) & 0x1); \ - } \ -} - -do_vsrlr(B, uint8_t) -do_vsrlr(H, uint16_t) -do_vsrlr(W, uint32_t) -do_vsrlr(D, uint64_t) - -#define VSRLR(NAME, BIT, T, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = do_vsrlr_ ## E(Vj->E(i), ((T)Vk->E(i))%BIT); \ - } \ -} - -VSRLR(vsrlr_b, 8, uint8_t, B) -VSRLR(vsrlr_h, 16, uint16_t, H) -VSRLR(vsrlr_w, 32, uint32_t, W) -VSRLR(vsrlr_d, 64, uint64_t, D) - -#define VSRLRI(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = do_vsrlr_ ## E(Vj->E(i), imm); \ - } \ -} - -VSRLRI(vsrlri_b, 8, B) -VSRLRI(vsrlri_h, 16, H) -VSRLRI(vsrlri_w, 32, W) -VSRLRI(vsrlri_d, 64, D) - -#define do_vsrar(E, T) \ -static T do_vsrar_ ##E(T s1, int sh) \ -{ \ - if (sh == 0) { \ - return s1; \ - } else { \ - return (s1 >> sh) + ((s1 >> (sh - 1)) & 0x1); \ - } \ -} - -do_vsrar(B, int8_t) -do_vsrar(H, int16_t) -do_vsrar(W, int32_t) -do_vsrar(D, int64_t) - -#define VSRAR(NAME, BIT, T, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = do_vsrar_ ## E(Vj->E(i), ((T)Vk->E(i))%BIT); \ - } \ -} - -VSRAR(vsrar_b, 8, uint8_t, B) -VSRAR(vsrar_h, 16, uint16_t, H) -VSRAR(vsrar_w, 32, uint32_t, W) -VSRAR(vsrar_d, 64, uint64_t, D) - -#define VSRARI(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = do_vsrar_ ## E(Vj->E(i), imm); \ - } \ -} - -VSRARI(vsrari_b, 8, B) -VSRARI(vsrari_h, 16, H) -VSRARI(vsrari_w, 32, W) -VSRARI(vsrari_d, 64, D) - -#define R_SHIFT(a, b) (a >> b) - -#define VSRLN(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = R_SHIFT((T)Vj->E2(i),((T)Vk->E2(i)) % BIT); \ - } \ - Vd->D(1) = 0; \ -} - -VSRLN(vsrln_b_h, 16, uint16_t, B, H) -VSRLN(vsrln_h_w, 32, uint32_t, H, W) -VSRLN(vsrln_w_d, 64, uint64_t, W, D) - -#define VSRAN(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = R_SHIFT(Vj->E2(i), ((T)Vk->E2(i)) % BIT); \ - } \ - Vd->D(1) = 0; \ -} - -VSRAN(vsran_b_h, 16, uint16_t, B, H) -VSRAN(vsran_h_w, 32, uint32_t, H, W) -VSRAN(vsran_w_d, 64, uint64_t, W, D) - -#define VSRLNI(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i, max; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - temp.D(0) = 0; \ - temp.D(1) = 0; \ - max = LSX_LEN/BIT; \ - for (i = 0; i < max; i++) { \ - temp.E1(i) = R_SHIFT((T)Vj->E2(i), imm); \ - temp.E1(i + max) = R_SHIFT((T)Vd->E2(i), imm); \ - } \ - *Vd = temp; \ -} - -void HELPER(vsrlni_d_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - temp.D(0) = 0; - temp.D(1) = 0; - temp.D(0) = int128_getlo(int128_urshift(Vj->Q(0), imm % 128)); - temp.D(1) = int128_getlo(int128_urshift(Vd->Q(0), imm % 128)); - *Vd = temp; -} - -VSRLNI(vsrlni_b_h, 16, uint16_t, B, H) -VSRLNI(vsrlni_h_w, 32, uint32_t, H, W) -VSRLNI(vsrlni_w_d, 64, uint64_t, W, D) - -#define VSRANI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i, max; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - temp.D(0) = 0; \ - temp.D(1) = 0; \ - max = LSX_LEN/BIT; \ - for (i = 0; i < max; i++) { \ - temp.E1(i) = R_SHIFT(Vj->E2(i), imm); \ - temp.E1(i + max) = R_SHIFT(Vd->E2(i), imm); \ - } \ - *Vd = temp; \ -} - -void HELPER(vsrani_d_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - temp.D(0) = 0; - temp.D(1) = 0; - temp.D(0) = int128_getlo(int128_rshift(Vj->Q(0), imm % 128)); - temp.D(1) = int128_getlo(int128_rshift(Vd->Q(0), imm % 128)); - *Vd = temp; -} - -VSRANI(vsrani_b_h, 16, B, H) -VSRANI(vsrani_h_w, 32, H, W) -VSRANI(vsrani_w_d, 64, W, D) - -#define VSRLRN(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_vsrlr_ ## E2(Vj->E2(i), ((T)Vk->E2(i))%BIT); \ - } \ - Vd->D(1) = 0; \ -} - -VSRLRN(vsrlrn_b_h, 16, uint16_t, B, H) -VSRLRN(vsrlrn_h_w, 32, uint32_t, H, W) -VSRLRN(vsrlrn_w_d, 64, uint64_t, W, D) - -#define VSRARN(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_vsrar_ ## E2(Vj->E2(i), ((T)Vk->E2(i))%BIT); \ - } \ - Vd->D(1) = 0; \ -} - -VSRARN(vsrarn_b_h, 16, uint8_t, B, H) -VSRARN(vsrarn_h_w, 32, uint16_t, H, W) -VSRARN(vsrarn_w_d, 64, uint32_t, W, D) - -#define VSRLRNI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i, max; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - temp.D(0) = 0; \ - temp.D(1) = 0; \ - max = LSX_LEN/BIT; \ - for (i = 0; i < max; i++) { \ - temp.E1(i) = do_vsrlr_ ## E2(Vj->E2(i), imm); \ - temp.E1(i + max) = do_vsrlr_ ## E2(Vd->E2(i), imm); \ - } \ - *Vd = temp; \ -} - -void HELPER(vsrlrni_d_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - Int128 r1, r2; - - if (imm == 0) { - temp.D(0) = int128_getlo(Vj->Q(0)); - temp.D(1) = int128_getlo(Vd->Q(0)); - } else { - r1 = int128_and(int128_urshift(Vj->Q(0), (imm -1)), int128_one()); - r2 = int128_and(int128_urshift(Vd->Q(0), (imm -1)), int128_one()); - - temp.D(0) = int128_getlo(int128_add(int128_urshift(Vj->Q(0), imm), r1)); - temp.D(1) = int128_getlo(int128_add(int128_urshift(Vd->Q(0), imm), r2)); - } - *Vd = temp; -} - -VSRLRNI(vsrlrni_b_h, 16, B, H) -VSRLRNI(vsrlrni_h_w, 32, H, W) -VSRLRNI(vsrlrni_w_d, 64, W, D) - -#define VSRARNI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i, max; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - temp.D(0) = 0; \ - temp.D(1) = 0; \ - max = LSX_LEN/BIT; \ - for (i = 0; i < max; i++) { \ - temp.E1(i) = do_vsrar_ ## E2(Vj->E2(i), imm); \ - temp.E1(i + max) = do_vsrar_ ## E2(Vd->E2(i), imm); \ - } \ - *Vd = temp; \ -} - -void HELPER(vsrarni_d_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - Int128 r1, r2; - - if (imm == 0) { - temp.D(0) = int128_getlo(Vj->Q(0)); - temp.D(1) = int128_getlo(Vd->Q(0)); - } else { - r1 = int128_and(int128_rshift(Vj->Q(0), (imm -1)), int128_one()); - r2 = int128_and(int128_rshift(Vd->Q(0), (imm -1)), int128_one()); - - temp.D(0) = int128_getlo(int128_add(int128_rshift(Vj->Q(0), imm), r1)); - temp.D(1) = int128_getlo(int128_add(int128_rshift(Vd->Q(0), imm), r2)); - } - *Vd = temp; -} - -VSRARNI(vsrarni_b_h, 16, B, H) -VSRARNI(vsrarni_h_w, 32, H, W) -VSRARNI(vsrarni_w_d, 64, W, D) - -#define SSRLNS(NAME, T1, T2, T3) \ -static T1 do_ssrlns_ ## NAME(T2 e2, int sa, int sh) \ -{ \ - T1 shft_res; \ - if (sa == 0) { \ - shft_res = e2; \ - } else { \ - shft_res = (((T1)e2) >> sa); \ - } \ - T3 mask; \ - mask = (1ull << sh) -1; \ - if (shft_res > mask) { \ - return mask; \ - } else { \ - return shft_res; \ - } \ -} - -SSRLNS(B, uint16_t, int16_t, uint8_t) -SSRLNS(H, uint32_t, int32_t, uint16_t) -SSRLNS(W, uint64_t, int64_t, uint32_t) - -#define VSSRLN(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_ssrlns_ ## E1(Vj->E2(i), (T)Vk->E2(i)% BIT, BIT/2 -1); \ - } \ - Vd->D(1) = 0; \ -} - -VSSRLN(vssrln_b_h, 16, uint16_t, B, H) -VSSRLN(vssrln_h_w, 32, uint32_t, H, W) -VSSRLN(vssrln_w_d, 64, uint64_t, W, D) - -#define SSRANS(E, T1, T2) \ -static T1 do_ssrans_ ## E(T1 e2, int sa, int sh) \ -{ \ - T1 shft_res; \ - if (sa == 0) { \ - shft_res = e2; \ - } else { \ - shft_res = e2 >> sa; \ - } \ - T2 mask; \ - mask = (1ll << sh) -1; \ - if (shft_res > mask) { \ - return mask; \ - } else if (shft_res < -(mask +1)) { \ - return ~mask; \ - } else { \ - return shft_res; \ - } \ -} - -SSRANS(B, int16_t, int8_t) -SSRANS(H, int32_t, int16_t) -SSRANS(W, int64_t, int32_t) - -#define VSSRAN(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_ssrans_ ## E1(Vj->E2(i), (T)Vk->E2(i)%BIT, BIT/2 -1); \ - } \ - Vd->D(1) = 0; \ -} - -VSSRAN(vssran_b_h, 16, uint16_t, B, H) -VSSRAN(vssran_h_w, 32, uint32_t, H, W) -VSSRAN(vssran_w_d, 64, uint64_t, W, D) - -#define SSRLNU(E, T1, T2, T3) \ -static T1 do_ssrlnu_ ## E(T3 e2, int sa, int sh) \ -{ \ - T1 shft_res; \ - if (sa == 0) { \ - shft_res = e2; \ - } else { \ - shft_res = (((T1)e2) >> sa); \ - } \ - T2 mask; \ - mask = (1ull << sh) -1; \ - if (shft_res > mask) { \ - return mask; \ - } else { \ - return shft_res; \ - } \ -} - -SSRLNU(B, uint16_t, uint8_t, int16_t) -SSRLNU(H, uint32_t, uint16_t, int32_t) -SSRLNU(W, uint64_t, uint32_t, int64_t) - -#define VSSRLNU(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_ssrlnu_ ## E1(Vj->E2(i), (T)Vk->E2(i)%BIT, BIT/2); \ - } \ - Vd->D(1) = 0; \ -} - -VSSRLNU(vssrln_bu_h, 16, uint16_t, B, H) -VSSRLNU(vssrln_hu_w, 32, uint32_t, H, W) -VSSRLNU(vssrln_wu_d, 64, uint64_t, W, D) - -#define SSRANU(E, T1, T2, T3) \ -static T1 do_ssranu_ ## E(T3 e2, int sa, int sh) \ -{ \ - T1 shft_res; \ - if (sa == 0) { \ - shft_res = e2; \ - } else { \ - shft_res = e2 >> sa; \ - } \ - if (e2 < 0) { \ - shft_res = 0; \ - } \ - T2 mask; \ - mask = (1ull << sh) -1; \ - if (shft_res > mask) { \ - return mask; \ - } else { \ - return shft_res; \ - } \ -} - -SSRANU(B, uint16_t, uint8_t, int16_t) -SSRANU(H, uint32_t, uint16_t, int32_t) -SSRANU(W, uint64_t, uint32_t, int64_t) - -#define VSSRANU(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_ssranu_ ## E1(Vj->E2(i), (T)Vk->E2(i)%BIT, BIT/2); \ - } \ - Vd->D(1) = 0; \ -} - -VSSRANU(vssran_bu_h, 16, uint16_t, B, H) -VSSRANU(vssran_hu_w, 32, uint32_t, H, W) -VSSRANU(vssran_wu_d, 64, uint64_t, W, D) - -#define VSSRLNI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E1(i) = do_ssrlns_ ## E1(Vj->E2(i), imm, BIT/2 -1); \ - temp.E1(i + LSX_LEN/BIT) = do_ssrlns_ ## E1(Vd->E2(i), imm, BIT/2 -1);\ - } \ - *Vd = temp; \ -} - -void HELPER(vssrlni_d_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - Int128 shft_res1, shft_res2, mask; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - if (imm == 0) { - shft_res1 = Vj->Q(0); - shft_res2 = Vd->Q(0); - } else { - shft_res1 = int128_urshift(Vj->Q(0), imm); - shft_res2 = int128_urshift(Vd->Q(0), imm); - } - mask = int128_sub(int128_lshift(int128_one(), 63), int128_one()); - - if (int128_ult(mask, shft_res1)) { - Vd->D(0) = int128_getlo(mask); - }else { - Vd->D(0) = int128_getlo(shft_res1); - } - - if (int128_ult(mask, shft_res2)) { - Vd->D(1) = int128_getlo(mask); - }else { - Vd->D(1) = int128_getlo(shft_res2); - } -} - -VSSRLNI(vssrlni_b_h, 16, B, H) -VSSRLNI(vssrlni_h_w, 32, H, W) -VSSRLNI(vssrlni_w_d, 64, W, D) - -#define VSSRANI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E1(i) = do_ssrans_ ## E1(Vj->E2(i), imm, BIT/2 -1); \ - temp.E1(i + LSX_LEN/BIT) = do_ssrans_ ## E1(Vd->E2(i), imm, BIT/2 -1); \ - } \ - *Vd = temp; \ -} - -void HELPER(vssrani_d_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - Int128 shft_res1, shft_res2, mask, min; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - if (imm == 0) { - shft_res1 = Vj->Q(0); - shft_res2 = Vd->Q(0); - } else { - shft_res1 = int128_rshift(Vj->Q(0), imm); - shft_res2 = int128_rshift(Vd->Q(0), imm); - } - mask = int128_sub(int128_lshift(int128_one(), 63), int128_one()); - min = int128_lshift(int128_one(), 63); - - if (int128_gt(shft_res1, mask)) { - Vd->D(0) = int128_getlo(mask); - } else if (int128_lt(shft_res1, int128_neg(min))) { - Vd->D(0) = int128_getlo(min); - } else { - Vd->D(0) = int128_getlo(shft_res1); - } - - if (int128_gt(shft_res2, mask)) { - Vd->D(1) = int128_getlo(mask); - } else if (int128_lt(shft_res2, int128_neg(min))) { - Vd->D(1) = int128_getlo(min); - } else { - Vd->D(1) = int128_getlo(shft_res2); - } -} - -VSSRANI(vssrani_b_h, 16, B, H) -VSSRANI(vssrani_h_w, 32, H, W) -VSSRANI(vssrani_w_d, 64, W, D) - -#define VSSRLNUI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E1(i) = do_ssrlnu_ ## E1(Vj->E2(i), imm, BIT/2); \ - temp.E1(i + LSX_LEN/BIT) = do_ssrlnu_ ## E1(Vd->E2(i), imm, BIT/2); \ - } \ - *Vd = temp; \ -} - -void HELPER(vssrlni_du_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - Int128 shft_res1, shft_res2, mask; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - if (imm == 0) { - shft_res1 = Vj->Q(0); - shft_res2 = Vd->Q(0); - } else { - shft_res1 = int128_urshift(Vj->Q(0), imm); - shft_res2 = int128_urshift(Vd->Q(0), imm); - } - mask = int128_sub(int128_lshift(int128_one(), 64), int128_one()); - - if (int128_ult(mask, shft_res1)) { - Vd->D(0) = int128_getlo(mask); - }else { - Vd->D(0) = int128_getlo(shft_res1); - } - - if (int128_ult(mask, shft_res2)) { - Vd->D(1) = int128_getlo(mask); - }else { - Vd->D(1) = int128_getlo(shft_res2); - } -} - -VSSRLNUI(vssrlni_bu_h, 16, B, H) -VSSRLNUI(vssrlni_hu_w, 32, H, W) -VSSRLNUI(vssrlni_wu_d, 64, W, D) - -#define VSSRANUI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E1(i) = do_ssranu_ ## E1(Vj->E2(i), imm, BIT/2); \ - temp.E1(i + LSX_LEN/BIT) = do_ssranu_ ## E1(Vd->E2(i), imm, BIT/2); \ - } \ - *Vd = temp; \ -} - -void HELPER(vssrani_du_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - Int128 shft_res1, shft_res2, mask; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - if (imm == 0) { - shft_res1 = Vj->Q(0); - shft_res2 = Vd->Q(0); - } else { - shft_res1 = int128_rshift(Vj->Q(0), imm); - shft_res2 = int128_rshift(Vd->Q(0), imm); - } - - if (int128_lt(Vj->Q(0), int128_zero())) { - shft_res1 = int128_zero(); - } - - if (int128_lt(Vd->Q(0), int128_zero())) { - shft_res2 = int128_zero(); - } - - mask = int128_sub(int128_lshift(int128_one(), 64), int128_one()); - - if (int128_ult(mask, shft_res1)) { - Vd->D(0) = int128_getlo(mask); - }else { - Vd->D(0) = int128_getlo(shft_res1); - } - - if (int128_ult(mask, shft_res2)) { - Vd->D(1) = int128_getlo(mask); - }else { - Vd->D(1) = int128_getlo(shft_res2); - } -} - -VSSRANUI(vssrani_bu_h, 16, B, H) -VSSRANUI(vssrani_hu_w, 32, H, W) -VSSRANUI(vssrani_wu_d, 64, W, D) - -#define SSRLRNS(E1, E2, T1, T2, T3) \ -static T1 do_ssrlrns_ ## E1(T2 e2, int sa, int sh) \ -{ \ - T1 shft_res; \ - \ - shft_res = do_vsrlr_ ## E2(e2, sa); \ - T1 mask; \ - mask = (1ull << sh) -1; \ - if (shft_res > mask) { \ - return mask; \ - } else { \ - return shft_res; \ - } \ -} - -SSRLRNS(B, H, uint16_t, int16_t, uint8_t) -SSRLRNS(H, W, uint32_t, int32_t, uint16_t) -SSRLRNS(W, D, uint64_t, int64_t, uint32_t) - -#define VSSRLRN(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_ssrlrns_ ## E1(Vj->E2(i), (T)Vk->E2(i)%BIT, BIT/2 -1); \ - } \ - Vd->D(1) = 0; \ -} - -VSSRLRN(vssrlrn_b_h, 16, uint16_t, B, H) -VSSRLRN(vssrlrn_h_w, 32, uint32_t, H, W) -VSSRLRN(vssrlrn_w_d, 64, uint64_t, W, D) - -#define SSRARNS(E1, E2, T1, T2) \ -static T1 do_ssrarns_ ## E1(T1 e2, int sa, int sh) \ -{ \ - T1 shft_res; \ - \ - shft_res = do_vsrar_ ## E2(e2, sa); \ - T2 mask; \ - mask = (1ll << sh) -1; \ - if (shft_res > mask) { \ - return mask; \ - } else if (shft_res < -(mask +1)) { \ - return ~mask; \ - } else { \ - return shft_res; \ - } \ -} - -SSRARNS(B, H, int16_t, int8_t) -SSRARNS(H, W, int32_t, int16_t) -SSRARNS(W, D, int64_t, int32_t) - -#define VSSRARN(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_ssrarns_ ## E1(Vj->E2(i), (T)Vk->E2(i)%BIT, BIT/2 -1); \ - } \ - Vd->D(1) = 0; \ -} - -VSSRARN(vssrarn_b_h, 16, uint16_t, B, H) -VSSRARN(vssrarn_h_w, 32, uint32_t, H, W) -VSSRARN(vssrarn_w_d, 64, uint64_t, W, D) - -#define SSRLRNU(E1, E2, T1, T2, T3) \ -static T1 do_ssrlrnu_ ## E1(T3 e2, int sa, int sh) \ -{ \ - T1 shft_res; \ - \ - shft_res = do_vsrlr_ ## E2(e2, sa); \ - \ - T2 mask; \ - mask = (1ull << sh) -1; \ - if (shft_res > mask) { \ - return mask; \ - } else { \ - return shft_res; \ - } \ -} - -SSRLRNU(B, H, uint16_t, uint8_t, int16_t) -SSRLRNU(H, W, uint32_t, uint16_t, int32_t) -SSRLRNU(W, D, uint64_t, uint32_t, int64_t) - -#define VSSRLRNU(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_ssrlrnu_ ## E1(Vj->E2(i), (T)Vk->E2(i)%BIT, BIT/2); \ - } \ - Vd->D(1) = 0; \ -} - -VSSRLRNU(vssrlrn_bu_h, 16, uint16_t, B, H) -VSSRLRNU(vssrlrn_hu_w, 32, uint32_t, H, W) -VSSRLRNU(vssrlrn_wu_d, 64, uint64_t, W, D) - -#define SSRARNU(E1, E2, T1, T2, T3) \ -static T1 do_ssrarnu_ ## E1(T3 e2, int sa, int sh) \ -{ \ - T1 shft_res; \ - \ - if (e2 < 0) { \ - shft_res = 0; \ - } else { \ - shft_res = do_vsrar_ ## E2(e2, sa); \ - } \ - T2 mask; \ - mask = (1ull << sh) -1; \ - if (shft_res > mask) { \ - return mask; \ - } else { \ - return shft_res; \ - } \ -} - -SSRARNU(B, H, uint16_t, uint8_t, int16_t) -SSRARNU(H, W, uint32_t, uint16_t, int32_t) -SSRARNU(W, D, uint64_t, uint32_t, int64_t) - -#define VSSRARNU(NAME, BIT, T, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E1(i) = do_ssrarnu_ ## E1(Vj->E2(i), (T)Vk->E2(i)%BIT, BIT/2); \ - } \ - Vd->D(1) = 0; \ -} - -VSSRARNU(vssrarn_bu_h, 16, uint16_t, B, H) -VSSRARNU(vssrarn_hu_w, 32, uint32_t, H, W) -VSSRARNU(vssrarn_wu_d, 64, uint64_t, W, D) - -#define VSSRLRNI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E1(i) = do_ssrlrns_ ## E1(Vj->E2(i), imm, BIT/2 -1); \ - temp.E1(i + LSX_LEN/BIT) = do_ssrlrns_ ## E1(Vd->E2(i), imm, BIT/2 -1);\ - } \ - *Vd = temp; \ -} - -#define VSSRLRNI_Q(NAME, sh) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - Int128 shft_res1, shft_res2, mask, r1, r2; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - if (imm == 0) { \ - shft_res1 = Vj->Q(0); \ - shft_res2 = Vd->Q(0); \ - } else { \ - r1 = int128_and(int128_urshift(Vj->Q(0), (imm -1)), int128_one()); \ - r2 = int128_and(int128_urshift(Vd->Q(0), (imm -1)), int128_one()); \ - \ - shft_res1 = (int128_add(int128_urshift(Vj->Q(0), imm), r1)); \ - shft_res2 = (int128_add(int128_urshift(Vd->Q(0), imm), r2)); \ - } \ - \ - mask = int128_sub(int128_lshift(int128_one(), sh), int128_one()); \ - \ - if (int128_ult(mask, shft_res1)) { \ - Vd->D(0) = int128_getlo(mask); \ - }else { \ - Vd->D(0) = int128_getlo(shft_res1); \ - } \ - \ - if (int128_ult(mask, shft_res2)) { \ - Vd->D(1) = int128_getlo(mask); \ - }else { \ - Vd->D(1) = int128_getlo(shft_res2); \ - } \ -} - -VSSRLRNI(vssrlrni_b_h, 16, B, H) -VSSRLRNI(vssrlrni_h_w, 32, H, W) -VSSRLRNI(vssrlrni_w_d, 64, W, D) -VSSRLRNI_Q(vssrlrni_d_q, 63) - -#define VSSRARNI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E1(i) = do_ssrarns_ ## E1(Vj->E2(i), imm, BIT/2 -1); \ - temp.E1(i + LSX_LEN/BIT) = do_ssrarns_ ## E1(Vd->E2(i), imm, BIT/2 -1); \ - } \ - *Vd = temp; \ -} - -void HELPER(vssrarni_d_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - Int128 shft_res1, shft_res2, mask1, mask2, r1, r2; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - if (imm == 0) { - shft_res1 = Vj->Q(0); - shft_res2 = Vd->Q(0); - } else { - r1 = int128_and(int128_rshift(Vj->Q(0), (imm -1)), int128_one()); - r2 = int128_and(int128_rshift(Vd->Q(0), (imm -1)), int128_one()); - - shft_res1 = int128_add(int128_rshift(Vj->Q(0), imm), r1); - shft_res2 = int128_add(int128_rshift(Vd->Q(0), imm), r2); - } - - mask1 = int128_sub(int128_lshift(int128_one(), 63), int128_one()); - mask2 = int128_lshift(int128_one(), 63); - - if (int128_gt(shft_res1, mask1)) { - Vd->D(0) = int128_getlo(mask1); - } else if (int128_lt(shft_res1, int128_neg(mask2))) { - Vd->D(0) = int128_getlo(mask2); - } else { - Vd->D(0) = int128_getlo(shft_res1); - } - - if (int128_gt(shft_res2, mask1)) { - Vd->D(1) = int128_getlo(mask1); - } else if (int128_lt(shft_res2, int128_neg(mask2))) { - Vd->D(1) = int128_getlo(mask2); - } else { - Vd->D(1) = int128_getlo(shft_res2); - } -} - -VSSRARNI(vssrarni_b_h, 16, B, H) -VSSRARNI(vssrarni_h_w, 32, H, W) -VSSRARNI(vssrarni_w_d, 64, W, D) - -#define VSSRLRNUI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E1(i) = do_ssrlrnu_ ## E1(Vj->E2(i), imm, BIT/2); \ - temp.E1(i + LSX_LEN/BIT) = do_ssrlrnu_ ## E1(Vd->E2(i), imm, BIT/2); \ - } \ - *Vd = temp; \ -} - -VSSRLRNUI(vssrlrni_bu_h, 16, B, H) -VSSRLRNUI(vssrlrni_hu_w, 32, H, W) -VSSRLRNUI(vssrlrni_wu_d, 64, W, D) -VSSRLRNI_Q(vssrlrni_du_q, 64) - -#define VSSRARNUI(NAME, BIT, E1, E2) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E1(i) = do_ssrarnu_ ## E1(Vj->E2(i), imm, BIT/2); \ - temp.E1(i + LSX_LEN/BIT) = do_ssrarnu_ ## E1(Vd->E2(i), imm, BIT/2); \ - } \ - *Vd = temp; \ -} - -void HELPER(vssrarni_du_q)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - Int128 shft_res1, shft_res2, mask1, mask2, r1, r2; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - if (imm == 0) { - shft_res1 = Vj->Q(0); - shft_res2 = Vd->Q(0); - } else { - r1 = int128_and(int128_rshift(Vj->Q(0), (imm -1)), int128_one()); - r2 = int128_and(int128_rshift(Vd->Q(0), (imm -1)), int128_one()); - - shft_res1 = int128_add(int128_rshift(Vj->Q(0), imm), r1); - shft_res2 = int128_add(int128_rshift(Vd->Q(0), imm), r2); - } - - if (int128_lt(Vj->Q(0), int128_zero())) { - shft_res1 = int128_zero(); - } - if (int128_lt(Vd->Q(0), int128_zero())) { - shft_res2 = int128_zero(); - } - - mask1 = int128_sub(int128_lshift(int128_one(), 64), int128_one()); - mask2 = int128_lshift(int128_one(), 64); - - if (int128_gt(shft_res1, mask1)) { - Vd->D(0) = int128_getlo(mask1); - } else if (int128_lt(shft_res1, int128_neg(mask2))) { - Vd->D(0) = int128_getlo(mask2); - } else { - Vd->D(0) = int128_getlo(shft_res1); - } - - if (int128_gt(shft_res2, mask1)) { - Vd->D(1) = int128_getlo(mask1); - } else if (int128_lt(shft_res2, int128_neg(mask2))) { - Vd->D(1) = int128_getlo(mask2); - } else { - Vd->D(1) = int128_getlo(shft_res2); - } -} - -VSSRARNUI(vssrarni_bu_h, 16, B, H) -VSSRARNUI(vssrarni_hu_w, 32, H, W) -VSSRARNUI(vssrarni_wu_d, 64, W, D) - -#define DO_2OP(NAME, BIT, E, DO_OP) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t vd, uint32_t vj) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) \ - { \ - Vd->E(i) = DO_OP(Vj->E(i)); \ - } \ -} - -#define DO_CLO_B(N) (clz32(~N & 0xff) - 24) -#define DO_CLO_H(N) (clz32(~N & 0xffff) - 16) -#define DO_CLO_W(N) (clz32(~N)) -#define DO_CLO_D(N) (clz64(~N)) -#define DO_CLZ_B(N) (clz32(N) - 24) -#define DO_CLZ_H(N) (clz32(N) - 16) -#define DO_CLZ_W(N) (clz32(N)) -#define DO_CLZ_D(N) (clz64(N)) - -DO_2OP(vclo_b, 8, UB, DO_CLO_B) -DO_2OP(vclo_h, 16, UH, DO_CLO_H) -DO_2OP(vclo_w, 32, UW, DO_CLO_W) -DO_2OP(vclo_d, 64, UD, DO_CLO_D) -DO_2OP(vclz_b, 8, UB, DO_CLZ_B) -DO_2OP(vclz_h, 16, UH, DO_CLZ_H) -DO_2OP(vclz_w, 32, UW, DO_CLZ_W) -DO_2OP(vclz_d, 64, UD, DO_CLZ_D) - -#define VPCNT(NAME, BIT, E, FN) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t vd, uint32_t vj) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) \ - { \ - Vd->E(i) = FN(Vj->E(i)); \ - } \ -} - -VPCNT(vpcnt_b, 8, UB, ctpop8) -VPCNT(vpcnt_h, 16, UH, ctpop16) -VPCNT(vpcnt_w, 32, UW, ctpop32) -VPCNT(vpcnt_d, 64, UD, ctpop64) - -#define DO_BITCLR(a, bit) (a & ~(1ull << bit)) -#define DO_BITSET(a, bit) (a | 1ull << bit) -#define DO_BITREV(a, bit) (a ^ (1ull << bit)) - -#define DO_BIT(NAME, BIT, E, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - VReg *Vk = (VReg *)vk; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = DO_OP(Vj->E(i), Vk->E(i)%BIT); \ - } \ -} - -DO_BIT(vbitclr_b, 8, UB, DO_BITCLR) -DO_BIT(vbitclr_h, 16, UH, DO_BITCLR) -DO_BIT(vbitclr_w, 32, UW, DO_BITCLR) -DO_BIT(vbitclr_d, 64, UD, DO_BITCLR) -DO_BIT(vbitset_b, 8, UB, DO_BITSET) -DO_BIT(vbitset_h, 16, UH, DO_BITSET) -DO_BIT(vbitset_w, 32, UW, DO_BITSET) -DO_BIT(vbitset_d, 64, UD, DO_BITSET) -DO_BIT(vbitrev_b, 8, UB, DO_BITREV) -DO_BIT(vbitrev_h, 16, UH, DO_BITREV) -DO_BIT(vbitrev_w, 32, UW, DO_BITREV) -DO_BIT(vbitrev_d, 64, UD, DO_BITREV) - -#define DO_BITI(NAME, BIT, E, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = DO_OP(Vj->E(i), imm); \ - } \ -} - -DO_BITI(vbitclri_b, 8, UB, DO_BITCLR) -DO_BITI(vbitclri_h, 16, UH, DO_BITCLR) -DO_BITI(vbitclri_w, 32, UW, DO_BITCLR) -DO_BITI(vbitclri_d, 64, UD, DO_BITCLR) -DO_BITI(vbitseti_b, 8, UB, DO_BITSET) -DO_BITI(vbitseti_h, 16, UH, DO_BITSET) -DO_BITI(vbitseti_w, 32, UW, DO_BITSET) -DO_BITI(vbitseti_d, 64, UD, DO_BITSET) -DO_BITI(vbitrevi_b, 8, UB, DO_BITREV) -DO_BITI(vbitrevi_h, 16, UH, DO_BITREV) -DO_BITI(vbitrevi_w, 32, UW, DO_BITREV) -DO_BITI(vbitrevi_d, 64, UD, DO_BITREV) - -#define VFRSTP(NAME, BIT, MASK, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i, m; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - if (Vj->E(i) < 0) { \ - break; \ - } \ - } \ - m = Vk->E(0) & MASK; \ - Vd->E(m) = i; \ -} - -VFRSTP(vfrstp_b, 8, 0xf, B) -VFRSTP(vfrstp_h, 16, 0x7, H) - -#define VFRSTPI(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i, m; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - if (Vj->E(i) < 0) { \ - break; \ - } \ - } \ - m = imm % (LSX_LEN/BIT); \ - Vd->E(m) = i; \ -} - -VFRSTPI(vfrstpi_b, 8, B) -VFRSTPI(vfrstpi_h, 16, H) - -static void vec_update_fcsr0_mask(CPULoongArchState *env, - uintptr_t pc, int mask) -{ - int flags = get_float_exception_flags(&env->fp_status); - - set_float_exception_flags(0, &env->fp_status); - - flags &= ~mask; - - if (flags) { - flags = ieee_ex_to_loongarch(flags); - UPDATE_FP_CAUSE(env->fcsr0, flags); - } - - if (GET_FP_ENABLES(env->fcsr0) & flags) { - do_raise_exception(env, EXCCODE_FPE, pc); - } else { - UPDATE_FP_FLAGS(env->fcsr0, flags); - } -} - -static void vec_update_fcsr0(CPULoongArchState *env, uintptr_t pc) -{ - vec_update_fcsr0_mask(env, pc, 0); -} - -static inline void vec_clear_cause(CPULoongArchState *env) -{ - SET_FP_CAUSE(env->fcsr0, 0); -} - -#define DO_3OP_F(NAME, BIT, E, FN) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - vec_clear_cause(env); \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = FN(Vj->E(i), Vk->E(i), &env->fp_status); \ - vec_update_fcsr0(env, GETPC()); \ - } \ -} - -DO_3OP_F(vfadd_s, 32, UW, float32_add) -DO_3OP_F(vfadd_d, 64, UD, float64_add) -DO_3OP_F(vfsub_s, 32, UW, float32_sub) -DO_3OP_F(vfsub_d, 64, UD, float64_sub) -DO_3OP_F(vfmul_s, 32, UW, float32_mul) -DO_3OP_F(vfmul_d, 64, UD, float64_mul) -DO_3OP_F(vfdiv_s, 32, UW, float32_div) -DO_3OP_F(vfdiv_d, 64, UD, float64_div) -DO_3OP_F(vfmax_s, 32, UW, float32_maxnum) -DO_3OP_F(vfmax_d, 64, UD, float64_maxnum) -DO_3OP_F(vfmin_s, 32, UW, float32_minnum) -DO_3OP_F(vfmin_d, 64, UD, float64_minnum) -DO_3OP_F(vfmaxa_s, 32, UW, float32_maxnummag) -DO_3OP_F(vfmaxa_d, 64, UD, float64_maxnummag) -DO_3OP_F(vfmina_s, 32, UW, float32_minnummag) -DO_3OP_F(vfmina_d, 64, UD, float64_minnummag) - -#define DO_4OP_F(NAME, BIT, E, FN, flags) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk, uint32_t va) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - VReg *Va = &(env->fpr[va].vreg); \ - \ - vec_clear_cause(env); \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = FN(Vj->E(i), Vk->E(i), Va->E(i), flags, &env->fp_status); \ - vec_update_fcsr0(env, GETPC()); \ - } \ -} - -DO_4OP_F(vfmadd_s, 32, UW, float32_muladd, 0) -DO_4OP_F(vfmadd_d, 64, UD, float64_muladd, 0) -DO_4OP_F(vfmsub_s, 32, UW, float32_muladd, float_muladd_negate_c) -DO_4OP_F(vfmsub_d, 64, UD, float64_muladd, float_muladd_negate_c) -DO_4OP_F(vfnmadd_s, 32, UW, float32_muladd, float_muladd_negate_result) -DO_4OP_F(vfnmadd_d, 64, UD, float64_muladd, float_muladd_negate_result) -DO_4OP_F(vfnmsub_s, 32, UW, float32_muladd, - float_muladd_negate_c | float_muladd_negate_result) -DO_4OP_F(vfnmsub_d, 64, UD, float64_muladd, - float_muladd_negate_c | float_muladd_negate_result) - -#define DO_2OP_F(NAME, BIT, E, FN) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t vd, uint32_t vj) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - vec_clear_cause(env); \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = FN(env, Vj->E(i)); \ - } \ -} - -#define FLOGB(BIT, T) \ -static T do_flogb_## BIT(CPULoongArchState *env, T fj) \ -{ \ - T fp, fd; \ - float_status *status = &env->fp_status; \ - FloatRoundMode old_mode = get_float_rounding_mode(status); \ - \ - set_float_rounding_mode(float_round_down, status); \ - fp = float ## BIT ##_log2(fj, status); \ - fd = float ## BIT ##_round_to_int(fp, status); \ - set_float_rounding_mode(old_mode, status); \ - vec_update_fcsr0_mask(env, GETPC(), float_flag_inexact); \ - return fd; \ -} - -FLOGB(32, uint32_t) -FLOGB(64, uint64_t) - -#define FCLASS(NAME, BIT, E, FN) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t vd, uint32_t vj) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = FN(env, Vj->E(i)); \ - } \ -} - -FCLASS(vfclass_s, 32, UW, helper_fclass_s) -FCLASS(vfclass_d, 64, UD, helper_fclass_d) - -#define FSQRT(BIT, T) \ -static T do_fsqrt_## BIT(CPULoongArchState *env, T fj) \ -{ \ - T fd; \ - fd = float ## BIT ##_sqrt(fj, &env->fp_status); \ - vec_update_fcsr0(env, GETPC()); \ - return fd; \ -} - -FSQRT(32, uint32_t) -FSQRT(64, uint64_t) - -#define FRECIP(BIT, T) \ -static T do_frecip_## BIT(CPULoongArchState *env, T fj) \ -{ \ - T fd; \ - fd = float ## BIT ##_div(float ## BIT ##_one, fj, &env->fp_status); \ - vec_update_fcsr0(env, GETPC()); \ - return fd; \ -} - -FRECIP(32, uint32_t) -FRECIP(64, uint64_t) - -#define FRSQRT(BIT, T) \ -static T do_frsqrt_## BIT(CPULoongArchState *env, T fj) \ -{ \ - T fd, fp; \ - fp = float ## BIT ##_sqrt(fj, &env->fp_status); \ - fd = float ## BIT ##_div(float ## BIT ##_one, fp, &env->fp_status); \ - vec_update_fcsr0(env, GETPC()); \ - return fd; \ -} - -FRSQRT(32, uint32_t) -FRSQRT(64, uint64_t) - -DO_2OP_F(vflogb_s, 32, UW, do_flogb_32) -DO_2OP_F(vflogb_d, 64, UD, do_flogb_64) -DO_2OP_F(vfsqrt_s, 32, UW, do_fsqrt_32) -DO_2OP_F(vfsqrt_d, 64, UD, do_fsqrt_64) -DO_2OP_F(vfrecip_s, 32, UW, do_frecip_32) -DO_2OP_F(vfrecip_d, 64, UD, do_frecip_64) -DO_2OP_F(vfrsqrt_s, 32, UW, do_frsqrt_32) -DO_2OP_F(vfrsqrt_d, 64, UD, do_frsqrt_64) - -static uint32_t float16_cvt_float32(uint16_t h, float_status *status) -{ - return float16_to_float32(h, true, status); -} -static uint64_t float32_cvt_float64(uint32_t s, float_status *status) -{ - return float32_to_float64(s, status); -} - -static uint16_t float32_cvt_float16(uint32_t s, float_status *status) -{ - return float32_to_float16(s, true, status); -} -static uint32_t float64_cvt_float32(uint64_t d, float_status *status) -{ - return float64_to_float32(d, status); -} - -void HELPER(vfcvtl_s_h)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - int i; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - vec_clear_cause(env); - for (i = 0; i < LSX_LEN/32; i++) { - temp.UW(i) = float16_cvt_float32(Vj->UH(i), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } - *Vd = temp; -} - -void HELPER(vfcvtl_d_s)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - int i; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - vec_clear_cause(env); - for (i = 0; i < LSX_LEN/64; i++) { - temp.UD(i) = float32_cvt_float64(Vj->UW(i), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } - *Vd = temp; -} - -void HELPER(vfcvth_s_h)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - int i; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - vec_clear_cause(env); - for (i = 0; i < LSX_LEN/32; i++) { - temp.UW(i) = float16_cvt_float32(Vj->UH(i + 4), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } - *Vd = temp; -} - -void HELPER(vfcvth_d_s)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - int i; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - vec_clear_cause(env); - for (i = 0; i < LSX_LEN/64; i++) { - temp.UD(i) = float32_cvt_float64(Vj->UW(i + 2), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } - *Vd = temp; -} - -void HELPER(vfcvt_h_s)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t vk) -{ - int i; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - VReg *Vk = &(env->fpr[vk].vreg); - - vec_clear_cause(env); - for(i = 0; i < LSX_LEN/32; i++) { - temp.UH(i + 4) = float32_cvt_float16(Vj->UW(i), &env->fp_status); - temp.UH(i) = float32_cvt_float16(Vk->UW(i), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } - *Vd = temp; -} - -void HELPER(vfcvt_s_d)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t vk) -{ - int i; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - VReg *Vk = &(env->fpr[vk].vreg); - - vec_clear_cause(env); - for(i = 0; i < LSX_LEN/64; i++) { - temp.UW(i + 2) = float64_cvt_float32(Vj->UD(i), &env->fp_status); - temp.UW(i) = float64_cvt_float32(Vk->UD(i), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } - *Vd = temp; -} - -void HELPER(vfrint_s)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - int i; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - vec_clear_cause(env); - for (i = 0; i < 4; i++) { - Vd->W(i) = float32_round_to_int(Vj->UW(i), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } -} - -void HELPER(vfrint_d)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - int i; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - vec_clear_cause(env); - for (i = 0; i < 2; i++) { - Vd->D(i) = float64_round_to_int(Vj->UD(i), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } -} - -#define FCVT_2OP(NAME, BIT, E, MODE) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t vd, uint32_t vj) \ -{ \ - int i; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - vec_clear_cause(env); \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - FloatRoundMode old_mode = get_float_rounding_mode(&env->fp_status); \ - set_float_rounding_mode(MODE, &env->fp_status); \ - Vd->E(i) = float## BIT ## _round_to_int(Vj->E(i), &env->fp_status); \ - set_float_rounding_mode(old_mode, &env->fp_status); \ - vec_update_fcsr0(env, GETPC()); \ - } \ -} - -FCVT_2OP(vfrintrne_s, 32, UW, float_round_nearest_even) -FCVT_2OP(vfrintrne_d, 64, UD, float_round_nearest_even) -FCVT_2OP(vfrintrz_s, 32, UW, float_round_to_zero) -FCVT_2OP(vfrintrz_d, 64, UD, float_round_to_zero) -FCVT_2OP(vfrintrp_s, 32, UW, float_round_up) -FCVT_2OP(vfrintrp_d, 64, UD, float_round_up) -FCVT_2OP(vfrintrm_s, 32, UW, float_round_down) -FCVT_2OP(vfrintrm_d, 64, UD, float_round_down) - -#define FTINT(NAME, FMT1, FMT2, T1, T2, MODE) \ -static T2 do_ftint ## NAME(CPULoongArchState *env, T1 fj) \ -{ \ - T2 fd; \ - FloatRoundMode old_mode = get_float_rounding_mode(&env->fp_status); \ - \ - set_float_rounding_mode(MODE, &env->fp_status); \ - fd = do_## FMT1 ##_to_## FMT2(env, fj); \ - set_float_rounding_mode(old_mode, &env->fp_status); \ - return fd; \ -} - -#define DO_FTINT(FMT1, FMT2, T1, T2) \ -static T2 do_## FMT1 ##_to_## FMT2(CPULoongArchState *env, T1 fj) \ -{ \ - T2 fd; \ - \ - fd = FMT1 ##_to_## FMT2(fj, &env->fp_status); \ - if (get_float_exception_flags(&env->fp_status) & (float_flag_invalid)) { \ - if (FMT1 ##_is_any_nan(fj)) { \ - fd = 0; \ - } \ - } \ - vec_update_fcsr0(env, GETPC()); \ - return fd; \ -} - -DO_FTINT(float32, int32, uint32_t, uint32_t) -DO_FTINT(float64, int64, uint64_t, uint64_t) -DO_FTINT(float32, uint32, uint32_t, uint32_t) -DO_FTINT(float64, uint64, uint64_t, uint64_t) -DO_FTINT(float64, int32, uint64_t, uint32_t) -DO_FTINT(float32, int64, uint32_t, uint64_t) - -FTINT(rne_w_s, float32, int32, uint32_t, uint32_t, float_round_nearest_even) -FTINT(rne_l_d, float64, int64, uint64_t, uint64_t, float_round_nearest_even) -FTINT(rp_w_s, float32, int32, uint32_t, uint32_t, float_round_up) -FTINT(rp_l_d, float64, int64, uint64_t, uint64_t, float_round_up) -FTINT(rz_w_s, float32, int32, uint32_t, uint32_t, float_round_to_zero) -FTINT(rz_l_d, float64, int64, uint64_t, uint64_t, float_round_to_zero) -FTINT(rm_w_s, float32, int32, uint32_t, uint32_t, float_round_down) -FTINT(rm_l_d, float64, int64, uint64_t, uint64_t, float_round_down) - -DO_2OP_F(vftintrne_w_s, 32, UW, do_ftintrne_w_s) -DO_2OP_F(vftintrne_l_d, 64, UD, do_ftintrne_l_d) -DO_2OP_F(vftintrp_w_s, 32, UW, do_ftintrp_w_s) -DO_2OP_F(vftintrp_l_d, 64, UD, do_ftintrp_l_d) -DO_2OP_F(vftintrz_w_s, 32, UW, do_ftintrz_w_s) -DO_2OP_F(vftintrz_l_d, 64, UD, do_ftintrz_l_d) -DO_2OP_F(vftintrm_w_s, 32, UW, do_ftintrm_w_s) -DO_2OP_F(vftintrm_l_d, 64, UD, do_ftintrm_l_d) -DO_2OP_F(vftint_w_s, 32, UW, do_float32_to_int32) -DO_2OP_F(vftint_l_d, 64, UD, do_float64_to_int64) - -FTINT(rz_wu_s, float32, uint32, uint32_t, uint32_t, float_round_to_zero) -FTINT(rz_lu_d, float64, uint64, uint64_t, uint64_t, float_round_to_zero) - -DO_2OP_F(vftintrz_wu_s, 32, UW, do_ftintrz_wu_s) -DO_2OP_F(vftintrz_lu_d, 64, UD, do_ftintrz_lu_d) -DO_2OP_F(vftint_wu_s, 32, UW, do_float32_to_uint32) -DO_2OP_F(vftint_lu_d, 64, UD, do_float64_to_uint64) - -FTINT(rm_w_d, float64, int32, uint64_t, uint32_t, float_round_down) -FTINT(rp_w_d, float64, int32, uint64_t, uint32_t, float_round_up) -FTINT(rz_w_d, float64, int32, uint64_t, uint32_t, float_round_to_zero) -FTINT(rne_w_d, float64, int32, uint64_t, uint32_t, float_round_nearest_even) - -#define FTINT_W_D(NAME, FN) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - vec_clear_cause(env); \ - for (i = 0; i < 2; i++) { \ - temp.W(i + 2) = FN(env, Vj->UD(i)); \ - temp.W(i) = FN(env, Vk->UD(i)); \ - } \ - *Vd = temp; \ -} - -FTINT_W_D(vftint_w_d, do_float64_to_int32) -FTINT_W_D(vftintrm_w_d, do_ftintrm_w_d) -FTINT_W_D(vftintrp_w_d, do_ftintrp_w_d) -FTINT_W_D(vftintrz_w_d, do_ftintrz_w_d) -FTINT_W_D(vftintrne_w_d, do_ftintrne_w_d) - -FTINT(rml_l_s, float32, int64, uint32_t, uint64_t, float_round_down) -FTINT(rpl_l_s, float32, int64, uint32_t, uint64_t, float_round_up) -FTINT(rzl_l_s, float32, int64, uint32_t, uint64_t, float_round_to_zero) -FTINT(rnel_l_s, float32, int64, uint32_t, uint64_t, float_round_nearest_even) -FTINT(rmh_l_s, float32, int64, uint32_t, uint64_t, float_round_down) -FTINT(rph_l_s, float32, int64, uint32_t, uint64_t, float_round_up) -FTINT(rzh_l_s, float32, int64, uint32_t, uint64_t, float_round_to_zero) -FTINT(rneh_l_s, float32, int64, uint32_t, uint64_t, float_round_nearest_even) - -#define FTINTL_L_S(NAME, FN) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t vd, uint32_t vj) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - vec_clear_cause(env); \ - for (i = 0; i < 2; i++) { \ - temp.D(i) = FN(env, Vj->UW(i)); \ - } \ - *Vd = temp; \ -} - -FTINTL_L_S(vftintl_l_s, do_float32_to_int64) -FTINTL_L_S(vftintrml_l_s, do_ftintrml_l_s) -FTINTL_L_S(vftintrpl_l_s, do_ftintrpl_l_s) -FTINTL_L_S(vftintrzl_l_s, do_ftintrzl_l_s) -FTINTL_L_S(vftintrnel_l_s, do_ftintrnel_l_s) - -#define FTINTH_L_S(NAME, FN) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t vd, uint32_t vj) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - vec_clear_cause(env); \ - for (i = 0; i < 2; i++) { \ - temp.D(i) = FN(env, Vj->UW(i + 2)); \ - } \ - *Vd = temp; \ -} - -FTINTH_L_S(vftinth_l_s, do_float32_to_int64) -FTINTH_L_S(vftintrmh_l_s, do_ftintrmh_l_s) -FTINTH_L_S(vftintrph_l_s, do_ftintrph_l_s) -FTINTH_L_S(vftintrzh_l_s, do_ftintrzh_l_s) -FTINTH_L_S(vftintrneh_l_s, do_ftintrneh_l_s) - -#define FFINT(NAME, FMT1, FMT2, T1, T2) \ -static T2 do_ffint_ ## NAME(CPULoongArchState *env, T1 fj) \ -{ \ - T2 fd; \ - \ - fd = FMT1 ##_to_## FMT2(fj, &env->fp_status); \ - vec_update_fcsr0(env, GETPC()); \ - return fd; \ -} - -FFINT(s_w, int32, float32, int32_t, uint32_t) -FFINT(d_l, int64, float64, int64_t, uint64_t) -FFINT(s_wu, uint32, float32, uint32_t, uint32_t) -FFINT(d_lu, uint64, float64, uint64_t, uint64_t) - -DO_2OP_F(vffint_s_w, 32, W, do_ffint_s_w) -DO_2OP_F(vffint_d_l, 64, D, do_ffint_d_l) -DO_2OP_F(vffint_s_wu, 32, UW, do_ffint_s_wu) -DO_2OP_F(vffint_d_lu, 64, UD, do_ffint_d_lu) - -void HELPER(vffintl_d_w)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - int i; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - vec_clear_cause(env); - for (i = 0; i < 2; i++) { - temp.D(i) = int32_to_float64(Vj->W(i), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } - *Vd = temp; -} - -void HELPER(vffinth_d_w)(CPULoongArchState *env, uint32_t vd, uint32_t vj) -{ - int i; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - vec_clear_cause(env); - for (i = 0; i < 2; i++) { - temp.D(i) = int32_to_float64(Vj->W(i + 2), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } - *Vd = temp; -} - -void HELPER(vffint_s_l)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t vk) -{ - int i; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - VReg *Vk = &(env->fpr[vk].vreg); - - vec_clear_cause(env); - for (i = 0; i < 2; i++) { - temp.W(i + 2) = int64_to_float32(Vj->D(i), &env->fp_status); - temp.W(i) = int64_to_float32(Vk->D(i), &env->fp_status); - vec_update_fcsr0(env, GETPC()); - } - *Vd = temp; -} - -#define VSEQ(a, b) (a == b ? -1 : 0) -#define VSLE(a, b) (a <= b ? -1 : 0) -#define VSLT(a, b) (a < b ? -1 : 0) - -#define VCMPI(NAME, BIT, E, DO_OP) \ -void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t v) \ -{ \ - int i; \ - VReg *Vd = (VReg *)vd; \ - VReg *Vj = (VReg *)vj; \ - typedef __typeof(Vd->E(0)) TD; \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - Vd->E(i) = DO_OP(Vj->E(i), (TD)imm); \ - } \ -} - -VCMPI(vseqi_b, 8, B, VSEQ) -VCMPI(vseqi_h, 16, H, VSEQ) -VCMPI(vseqi_w, 32, W, VSEQ) -VCMPI(vseqi_d, 64, D, VSEQ) -VCMPI(vslei_b, 8, B, VSLE) -VCMPI(vslei_h, 16, H, VSLE) -VCMPI(vslei_w, 32, W, VSLE) -VCMPI(vslei_d, 64, D, VSLE) -VCMPI(vslei_bu, 8, UB, VSLE) -VCMPI(vslei_hu, 16, UH, VSLE) -VCMPI(vslei_wu, 32, UW, VSLE) -VCMPI(vslei_du, 64, UD, VSLE) -VCMPI(vslti_b, 8, B, VSLT) -VCMPI(vslti_h, 16, H, VSLT) -VCMPI(vslti_w, 32, W, VSLT) -VCMPI(vslti_d, 64, D, VSLT) -VCMPI(vslti_bu, 8, UB, VSLT) -VCMPI(vslti_hu, 16, UH, VSLT) -VCMPI(vslti_wu, 32, UW, VSLT) -VCMPI(vslti_du, 64, UD, VSLT) - -static uint64_t vfcmp_common(CPULoongArchState *env, - FloatRelation cmp, uint32_t flags) -{ - uint64_t ret = 0; - - switch (cmp) { - case float_relation_less: - ret = (flags & FCMP_LT); - break; - case float_relation_equal: - ret = (flags & FCMP_EQ); - break; - case float_relation_greater: - ret = (flags & FCMP_GT); - break; - case float_relation_unordered: - ret = (flags & FCMP_UN); - break; - default: - g_assert_not_reached(); - } - - if (ret) { - ret = -1; - } - - return ret; -} - -#define VFCMP(NAME, BIT, E, FN) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk, uint32_t flags) \ -{ \ - int i; \ - VReg t; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - vec_clear_cause(env); \ - for (i = 0; i < LSX_LEN/BIT ; i++) { \ - FloatRelation cmp; \ - cmp = FN(Vj->E(i), Vk->E(i), &env->fp_status); \ - t.E(i) = vfcmp_common(env, cmp, flags); \ - vec_update_fcsr0(env, GETPC()); \ - } \ - *Vd = t; \ -} - -VFCMP(vfcmp_c_s, 32, UW, float32_compare_quiet) -VFCMP(vfcmp_s_s, 32, UW, float32_compare) -VFCMP(vfcmp_c_d, 64, UD, float64_compare_quiet) -VFCMP(vfcmp_s_d, 64, UD, float64_compare) - -void HELPER(vbitseli_b)(void *vd, void *vj, uint64_t imm, uint32_t v) -{ - int i; - VReg *Vd = (VReg *)vd; - VReg *Vj = (VReg *)vj; - - for (i = 0; i < 16; i++) { - Vd->B(i) = (~Vd->B(i) & Vj->B(i)) | (Vd->B(i) & imm); - } -} - -/* Copy from target/arm/tcg/sve_helper.c */ -static inline bool do_match2(uint64_t n, uint64_t m0, uint64_t m1, int esz) -{ - uint64_t bits = 8 << esz; - uint64_t ones = dup_const(esz, 1); - uint64_t signs = ones << (bits - 1); - uint64_t cmp0, cmp1; - - cmp1 = dup_const(esz, n); - cmp0 = cmp1 ^ m0; - cmp1 = cmp1 ^ m1; - cmp0 = (cmp0 - ones) & ~cmp0; - cmp1 = (cmp1 - ones) & ~cmp1; - return (cmp0 | cmp1) & signs; -} - -#define SETANYEQZ(NAME, MO) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t cd, uint32_t vj) \ -{ \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - env->cf[cd & 0x7] = do_match2(0, Vj->D(0), Vj->D(1), MO); \ -} -SETANYEQZ(vsetanyeqz_b, MO_8) -SETANYEQZ(vsetanyeqz_h, MO_16) -SETANYEQZ(vsetanyeqz_w, MO_32) -SETANYEQZ(vsetanyeqz_d, MO_64) - -#define SETALLNEZ(NAME, MO) \ -void HELPER(NAME)(CPULoongArchState *env, uint32_t cd, uint32_t vj) \ -{ \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - env->cf[cd & 0x7]= !do_match2(0, Vj->D(0), Vj->D(1), MO); \ -} -SETALLNEZ(vsetallnez_b, MO_8) -SETALLNEZ(vsetallnez_h, MO_16) -SETALLNEZ(vsetallnez_w, MO_32) -SETALLNEZ(vsetallnez_d, MO_64) - -#define VPACKEV(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E(2 * i + 1) = Vj->E(2 * i); \ - temp.E(2 *i) = Vk->E(2 * i); \ - } \ - *Vd = temp; \ -} - -VPACKEV(vpackev_b, 16, B) -VPACKEV(vpackev_h, 32, H) -VPACKEV(vpackev_w, 64, W) -VPACKEV(vpackev_d, 128, D) - -#define VPACKOD(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E(2 * i + 1) = Vj->E(2 * i + 1); \ - temp.E(2 * i) = Vk->E(2 * i + 1); \ - } \ - *Vd = temp; \ -} - -VPACKOD(vpackod_b, 16, B) -VPACKOD(vpackod_h, 32, H) -VPACKOD(vpackod_w, 64, W) -VPACKOD(vpackod_d, 128, D) - -#define VPICKEV(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E(i + LSX_LEN/BIT) = Vj->E(2 * i); \ - temp.E(i) = Vk->E(2 * i); \ - } \ - *Vd = temp; \ -} - -VPICKEV(vpickev_b, 16, B) -VPICKEV(vpickev_h, 32, H) -VPICKEV(vpickev_w, 64, W) -VPICKEV(vpickev_d, 128, D) - -#define VPICKOD(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E(i + LSX_LEN/BIT) = Vj->E(2 * i + 1); \ - temp.E(i) = Vk->E(2 * i + 1); \ - } \ - *Vd = temp; \ -} - -VPICKOD(vpickod_b, 16, B) -VPICKOD(vpickod_h, 32, H) -VPICKOD(vpickod_w, 64, W) -VPICKOD(vpickod_d, 128, D) - -#define VILVL(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E(2 * i + 1) = Vj->E(i); \ - temp.E(2 * i) = Vk->E(i); \ - } \ - *Vd = temp; \ -} - -VILVL(vilvl_b, 16, B) -VILVL(vilvl_h, 32, H) -VILVL(vilvl_w, 64, W) -VILVL(vilvl_d, 128, D) - -#define VILVH(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E(2 * i + 1) = Vj->E(i + LSX_LEN/BIT); \ - temp.E(2 * i) = Vk->E(i + LSX_LEN/BIT); \ - } \ - *Vd = temp; \ -} - -VILVH(vilvh_b, 16, B) -VILVH(vilvh_h, 32, H) -VILVH(vilvh_w, 64, W) -VILVH(vilvh_d, 128, D) - -void HELPER(vshuf_b)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t vk, uint32_t va) -{ - int i, m; - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - VReg *Vk = &(env->fpr[vk].vreg); - VReg *Va = &(env->fpr[va].vreg); - - m = LSX_LEN/8; - for (i = 0; i < m ; i++) { - uint64_t k = (uint8_t)Va->B(i) % (2 * m); - temp.B(i) = k < m ? Vk->B(k) : Vj->B(k - m); - } - *Vd = temp; -} - -#define VSHUF(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t vk) \ -{ \ - int i, m; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - VReg *Vk = &(env->fpr[vk].vreg); \ - \ - m = LSX_LEN/BIT; \ - for (i = 0; i < m; i++) { \ - uint64_t k = ((uint8_t) Vd->E(i)) % (2 * m); \ - temp.E(i) = k < m ? Vk->E(k) : Vj->E(k - m); \ - } \ - *Vd = temp; \ -} - -VSHUF(vshuf_h, 16, H) -VSHUF(vshuf_w, 32, W) -VSHUF(vshuf_d, 64, D) - -#define VSHUF4I(NAME, BIT, E) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int i; \ - VReg temp; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - for (i = 0; i < LSX_LEN/BIT; i++) { \ - temp.E(i) = Vj->E(((i) & 0xfc) + (((imm) >> \ - (2 * ((i) & 0x03))) & 0x03)); \ - } \ - *Vd = temp; \ -} - -VSHUF4I(vshuf4i_b, 8, B) -VSHUF4I(vshuf4i_h, 16, H) -VSHUF4I(vshuf4i_w, 32, W) - -void HELPER(vshuf4i_d)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - VReg temp; - temp.D(0) = (imm & 2 ? Vj : Vd)->D(imm & 1); - temp.D(1) = (imm & 8 ? Vj : Vd)->D((imm >> 2) & 1); - *Vd = temp; -} - -void HELPER(vpermi_w)(CPULoongArchState *env, - uint32_t vd, uint32_t vj, uint32_t imm) -{ - VReg temp; - VReg *Vd = &(env->fpr[vd].vreg); - VReg *Vj = &(env->fpr[vj].vreg); - - temp.W(0) = Vj->W(imm & 0x3); - temp.W(1) = Vj->W((imm >> 2) & 0x3); - temp.W(2) = Vd->W((imm >> 4) & 0x3); - temp.W(3) = Vd->W((imm >> 6) & 0x3); - *Vd = temp; -} - -#define VEXTRINS(NAME, BIT, E, MASK) \ -void HELPER(NAME)(CPULoongArchState *env, \ - uint32_t vd, uint32_t vj, uint32_t imm) \ -{ \ - int ins, extr; \ - VReg *Vd = &(env->fpr[vd].vreg); \ - VReg *Vj = &(env->fpr[vj].vreg); \ - \ - ins = (imm >> 4) & MASK; \ - extr = imm & MASK; \ - Vd->E(ins) = Vj->E(extr); \ -} - -VEXTRINS(vextrins_b, 8, B, 0xf) -VEXTRINS(vextrins_h, 16, H, 0x7) -VEXTRINS(vextrins_w, 32, W, 0x3) -VEXTRINS(vextrins_d, 64, D, 0x1) diff --git a/target/loongarch/machine.c b/target/loongarch/machine.c index d8ac99c9a4..1c4e01d076 100644 --- a/target/loongarch/machine.c +++ b/target/loongarch/machine.c @@ -8,7 +8,7 @@ #include "qemu/osdep.h" #include "cpu.h" #include "migration/cpu.h" -#include "internals.h" +#include "vec.h" static const VMStateDescription vmstate_fpu_reg = { .name = "fpu_reg", @@ -76,6 +76,39 @@ static const VMStateDescription vmstate_lsx = { }, }; +static const VMStateDescription vmstate_lasxh_reg = { + .name = "lasxh_reg", + .version_id = 1, + .minimum_version_id = 1, + .fields = (VMStateField[]) { + VMSTATE_UINT64(UD(2), VReg), + VMSTATE_UINT64(UD(3), VReg), + VMSTATE_END_OF_LIST() + } +}; + +#define VMSTATE_LASXH_REGS(_field, _state, _start) \ + VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, 32, 0, \ + vmstate_lasxh_reg, fpr_t) + +static bool lasx_needed(void *opaque) +{ + LoongArchCPU *cpu = opaque; + + return FIELD_EX64(cpu->env.cpucfg[2], CPUCFG2, LASX); +} + +static const VMStateDescription vmstate_lasx = { + .name = "cpu/lasx", + .version_id = 1, + .minimum_version_id = 1, + .needed = lasx_needed, + .fields = (VMStateField[]) { + VMSTATE_LASXH_REGS(env.fpr, LoongArchCPU, 0), + VMSTATE_END_OF_LIST() + }, +}; + /* TLB state */ const VMStateDescription vmstate_tlb = { .name = "cpu/tlb", @@ -163,6 +196,7 @@ const VMStateDescription vmstate_loongarch_cpu = { .subsections = (const VMStateDescription*[]) { &vmstate_fpu, &vmstate_lsx, + &vmstate_lasx, NULL } }; diff --git a/target/loongarch/meson.build b/target/loongarch/meson.build index b7a27df5a9..7fbf045a5d 100644 --- a/target/loongarch/meson.build +++ b/target/loongarch/meson.build @@ -11,7 +11,7 @@ loongarch_tcg_ss.add(files( 'op_helper.c', 'translate.c', 'gdbstub.c', - 'lsx_helper.c', + 'vec_helper.c', )) loongarch_tcg_ss.add(zlib) diff --git a/target/loongarch/translate.c b/target/loongarch/translate.c index fd393ed76d..f6038fc567 100644 --- a/target/loongarch/translate.c +++ b/target/loongarch/translate.c @@ -18,6 +18,7 @@ #include "fpu/softfloat.h" #include "translate.h" #include "internals.h" +#include "vec.h" /* Global register indices */ TCGv cpu_gpr[32], cpu_pc; @@ -36,6 +37,18 @@ static inline int vec_full_offset(int regno) return offsetof(CPULoongArchState, fpr[regno]); } +static inline int vec_reg_offset(int regno, int index, MemOp mop) +{ + const uint8_t size = 1 << mop; + int offs = index * size; + + if (HOST_BIG_ENDIAN && size < 8 ) { + offs ^= (8 - size); + } + + return offs + vec_full_offset(regno); +} + static inline void get_vreg64(TCGv_i64 dest, int regno, int index) { tcg_gen_ld_i64(dest, cpu_env, @@ -123,6 +136,10 @@ static void loongarch_tr_init_disas_context(DisasContextBase *dcbase, ctx->vl = LSX_LEN; } + if (FIELD_EX64(env->cpucfg[2], CPUCFG2, LASX)) { + ctx->vl = LASX_LEN; + } + ctx->la64 = is_la64(env); ctx->va32 = (ctx->base.tb->flags & HW_FLAGS_VA32) != 0; @@ -261,7 +278,7 @@ static uint64_t make_address_pc(DisasContext *ctx, uint64_t addr) #include "insn_trans/trans_fmemory.c.inc" #include "insn_trans/trans_branch.c.inc" #include "insn_trans/trans_privileged.c.inc" -#include "insn_trans/trans_lsx.c.inc" +#include "insn_trans/trans_vec.c.inc" static void loongarch_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs) { diff --git a/target/loongarch/translate.h b/target/loongarch/translate.h index 89b49a859e..195f53573a 100644 --- a/target/loongarch/translate.h +++ b/target/loongarch/translate.h @@ -23,6 +23,7 @@ #define avail_LSPW(C) (FIELD_EX32((C)->cpucfg2, CPUCFG2, LSPW)) #define avail_LAM(C) (FIELD_EX32((C)->cpucfg2, CPUCFG2, LAM)) #define avail_LSX(C) (FIELD_EX32((C)->cpucfg2, CPUCFG2, LSX)) +#define avail_LASX(C) (FIELD_EX32((C)->cpucfg2, CPUCFG2, LASX)) #define avail_IOCSR(C) (FIELD_EX32((C)->cpucfg1, CPUCFG1, IOCSR)) /* diff --git a/target/loongarch/vec.h b/target/loongarch/vec.h new file mode 100644 index 0000000000..3c9adf8427 --- /dev/null +++ b/target/loongarch/vec.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * QEMU LoongArch vector utilitites + * + * Copyright (c) 2023 Loongson Technology Corporation Limited + */ + +#ifndef LOONGARCH_VEC_H +#define LOONGARCH_VEC_H + +#if HOST_BIG_ENDIAN +#define B(x) B[(x) ^ 15] +#define H(x) H[(x) ^ 7] +#define W(x) W[(x) ^ 3] +#define D(x) D[(x) ^ 1] +#define UB(x) UB[(x) ^ 15] +#define UH(x) UH[(x) ^ 7] +#define UW(x) UW[(x) ^ 3] +#define UD(x) UD[(x) ^ 1] +#define Q(x) Q[x] +#else +#define B(x) B[x] +#define H(x) H[x] +#define W(x) W[x] +#define D(x) D[x] +#define UB(x) UB[x] +#define UH(x) UH[x] +#define UW(x) UW[x] +#define UD(x) UD[x] +#define Q(x) Q[x] +#endif /* HOST_BIG_ENDIAN */ + +#define DO_ADD(a, b) (a + b) +#define DO_SUB(a, b) (a - b) +#define DO_VAVG(a, b) ((a >> 1) + (b >> 1) + (a & b & 1)) +#define DO_VAVGR(a, b) ((a >> 1) + (b >> 1) + ((a | b) & 1)) +#define DO_VABSD(a, b) ((a > b) ? (a -b) : (b-a)) +#define DO_VABS(a) ((a < 0) ? (-a) : (a)) +#define DO_MIN(a, b) (a < b ? a : b) +#define DO_MAX(a, b) (a > b ? a : b) +#define DO_MUL(a, b) (a * b) +#define DO_MADD(a, b, c) (a + b * c) +#define DO_MSUB(a, b, c) (a - b * c) + +#define DO_DIVU(N, M) (unlikely(M == 0) ? 0 : N / M) +#define DO_REMU(N, M) (unlikely(M == 0) ? 0 : N % M) +#define DO_DIV(N, M) (unlikely(M == 0) ? 0 :\ + unlikely((N == -N) && (M == (__typeof(N))(-1))) ? N : N / M) +#define DO_REM(N, M) (unlikely(M == 0) ? 0 :\ + unlikely((N == -N) && (M == (__typeof(N))(-1))) ? 0 : N % M) + +#define DO_SIGNCOV(a, b) (a == 0 ? 0 : a < 0 ? -b : b) + +#define R_SHIFT(a, b) (a >> b) + +#define DO_CLO_B(N) (clz32(~N & 0xff) - 24) +#define DO_CLO_H(N) (clz32(~N & 0xffff) - 16) +#define DO_CLO_W(N) (clz32(~N)) +#define DO_CLO_D(N) (clz64(~N)) +#define DO_CLZ_B(N) (clz32(N) - 24) +#define DO_CLZ_H(N) (clz32(N) - 16) +#define DO_CLZ_W(N) (clz32(N)) +#define DO_CLZ_D(N) (clz64(N)) + +#define DO_BITCLR(a, bit) (a & ~(1ull << bit)) +#define DO_BITSET(a, bit) (a | 1ull << bit) +#define DO_BITREV(a, bit) (a ^ (1ull << bit)) + +#define VSEQ(a, b) (a == b ? -1 : 0) +#define VSLE(a, b) (a <= b ? -1 : 0) +#define VSLT(a, b) (a < b ? -1 : 0) + +#define SHF_POS(i, imm) (((i) & 0xfc) + (((imm) >> (2 * ((i) & 0x03))) & 0x03)) + +#endif /* LOONGARCH_VEC_H */ diff --git a/target/loongarch/vec_helper.c b/target/loongarch/vec_helper.c new file mode 100644 index 0000000000..3faf52cbc4 --- /dev/null +++ b/target/loongarch/vec_helper.c @@ -0,0 +1,3494 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * QEMU LoongArch vector helper functions. + * + * Copyright (c) 2022-2023 Loongson Technology Corporation Limited + */ + +#include "qemu/osdep.h" +#include "cpu.h" +#include "exec/exec-all.h" +#include "exec/helper-proto.h" +#include "fpu/softfloat.h" +#include "internals.h" +#include "tcg/tcg.h" +#include "vec.h" +#include "tcg/tcg-gvec-desc.h" + +#define DO_ODD_EVEN(NAME, BIT, E1, E2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->E1(0)) TD; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E1(i) = DO_OP((TD)Vj->E2(2 * i + 1), (TD)Vk->E2(2 * i)); \ + } \ +} + +DO_ODD_EVEN(vhaddw_h_b, 16, H, B, DO_ADD) +DO_ODD_EVEN(vhaddw_w_h, 32, W, H, DO_ADD) +DO_ODD_EVEN(vhaddw_d_w, 64, D, W, DO_ADD) + +void HELPER(vhaddw_q_d)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16 ; i++) { + Vd->Q(i) = int128_add(int128_makes64(Vj->D(2 * i + 1)), + int128_makes64(Vk->D(2 * i))); + } +} + +DO_ODD_EVEN(vhsubw_h_b, 16, H, B, DO_SUB) +DO_ODD_EVEN(vhsubw_w_h, 32, W, H, DO_SUB) +DO_ODD_EVEN(vhsubw_d_w, 64, D, W, DO_SUB) + +void HELPER(vhsubw_q_d)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_sub(int128_makes64(Vj->D(2 * i + 1)), + int128_makes64(Vk->D(2 * i))); + } +} + +DO_ODD_EVEN(vhaddw_hu_bu, 16, UH, UB, DO_ADD) +DO_ODD_EVEN(vhaddw_wu_hu, 32, UW, UH, DO_ADD) +DO_ODD_EVEN(vhaddw_du_wu, 64, UD, UW, DO_ADD) + +void HELPER(vhaddw_qu_du)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i ++) { + Vd->Q(i) = int128_add(int128_make64(Vj->UD(2 * i + 1)), + int128_make64(Vk->UD(2 * i))); + } +} + +DO_ODD_EVEN(vhsubw_hu_bu, 16, UH, UB, DO_SUB) +DO_ODD_EVEN(vhsubw_wu_hu, 32, UW, UH, DO_SUB) +DO_ODD_EVEN(vhsubw_du_wu, 64, UD, UW, DO_SUB) + +void HELPER(vhsubw_qu_du)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_sub(int128_make64(Vj->UD(2 * i + 1)), + int128_make64(Vk->UD(2 * i))); + } +} + +#define DO_EVEN(NAME, BIT, E1, E2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->E1(0)) TD; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E1(i) = DO_OP((TD)Vj->E2(2 * i) ,(TD)Vk->E2(2 * i)); \ + } \ +} + +#define DO_ODD(NAME, BIT, E1, E2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->E1(0)) TD; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E1(i) = DO_OP((TD)Vj->E2(2 * i + 1), (TD)Vk->E2(2 * i + 1)); \ + } \ +} + +void HELPER(vaddwev_q_d)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_add(int128_makes64(Vj->D(2 * i)), + int128_makes64(Vk->D(2 * i))); + } +} + +DO_EVEN(vaddwev_h_b, 16, H, B, DO_ADD) +DO_EVEN(vaddwev_w_h, 32, W, H, DO_ADD) +DO_EVEN(vaddwev_d_w, 64, D, W, DO_ADD) + +void HELPER(vaddwod_q_d)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_add(int128_makes64(Vj->D(2 * i +1)), + int128_makes64(Vk->D(2 * i +1))); + } +} + +DO_ODD(vaddwod_h_b, 16, H, B, DO_ADD) +DO_ODD(vaddwod_w_h, 32, W, H, DO_ADD) +DO_ODD(vaddwod_d_w, 64, D, W, DO_ADD) + +void HELPER(vsubwev_q_d)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_sub(int128_makes64(Vj->D(2 * i)), + int128_makes64(Vk->D(2 * i))); + } +} + +DO_EVEN(vsubwev_h_b, 16, H, B, DO_SUB) +DO_EVEN(vsubwev_w_h, 32, W, H, DO_SUB) +DO_EVEN(vsubwev_d_w, 64, D, W, DO_SUB) + +void HELPER(vsubwod_q_d)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_sub(int128_makes64(Vj->D(2 * i + 1)), + int128_makes64(Vk->D(2 * i + 1))); + } +} + +DO_ODD(vsubwod_h_b, 16, H, B, DO_SUB) +DO_ODD(vsubwod_w_h, 32, W, H, DO_SUB) +DO_ODD(vsubwod_d_w, 64, D, W, DO_SUB) + +void HELPER(vaddwev_q_du)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_add(int128_make64(Vj->UD(2 * i)), + int128_make64(Vk->UD(2 * i))); + } +} + +DO_EVEN(vaddwev_h_bu, 16, UH, UB, DO_ADD) +DO_EVEN(vaddwev_w_hu, 32, UW, UH, DO_ADD) +DO_EVEN(vaddwev_d_wu, 64, UD, UW, DO_ADD) + +void HELPER(vaddwod_q_du)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_add(int128_make64(Vj->UD(2 * i + 1)), + int128_make64(Vk->UD(2 * i + 1))); + } +} + +DO_ODD(vaddwod_h_bu, 16, UH, UB, DO_ADD) +DO_ODD(vaddwod_w_hu, 32, UW, UH, DO_ADD) +DO_ODD(vaddwod_d_wu, 64, UD, UW, DO_ADD) + +void HELPER(vsubwev_q_du)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_sub(int128_make64(Vj->UD(2 * i)), + int128_make64(Vk->UD(2 * i))); + } +} + +DO_EVEN(vsubwev_h_bu, 16, UH, UB, DO_SUB) +DO_EVEN(vsubwev_w_hu, 32, UW, UH, DO_SUB) +DO_EVEN(vsubwev_d_wu, 64, UD, UW, DO_SUB) + +void HELPER(vsubwod_q_du)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_sub(int128_make64(Vj->UD(2 * i + 1)), + int128_make64(Vk->UD(2 * i + 1))); + } +} + +DO_ODD(vsubwod_h_bu, 16, UH, UB, DO_SUB) +DO_ODD(vsubwod_w_hu, 32, UW, UH, DO_SUB) +DO_ODD(vsubwod_d_wu, 64, UD, UW, DO_SUB) + +#define DO_EVEN_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->ES1(0)) TDS; \ + typedef __typeof(Vd->EU1(0)) TDU; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->ES1(i) = DO_OP((TDU)Vj->EU2(2 * i) ,(TDS)Vk->ES2(2 * i)); \ + } \ +} + +#define DO_ODD_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->ES1(0)) TDS; \ + typedef __typeof(Vd->EU1(0)) TDU; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->ES1(i) = DO_OP((TDU)Vj->EU2(2 * i + 1), (TDS)Vk->ES2(2 * i + 1)); \ + } \ +} + +void HELPER(vaddwev_q_du_d)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_add(int128_make64(Vj->UD(2 * i)), + int128_makes64(Vk->D(2 * i))); + } +} + +DO_EVEN_U_S(vaddwev_h_bu_b, 16, H, UH, B, UB, DO_ADD) +DO_EVEN_U_S(vaddwev_w_hu_h, 32, W, UW, H, UH, DO_ADD) +DO_EVEN_U_S(vaddwev_d_wu_w, 64, D, UD, W, UW, DO_ADD) + +void HELPER(vaddwod_q_du_d)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_add(int128_make64(Vj->UD(2 * i + 1)), + int128_makes64(Vk->D(2 * i + 1))); + } +} + +DO_ODD_U_S(vaddwod_h_bu_b, 16, H, UH, B, UB, DO_ADD) +DO_ODD_U_S(vaddwod_w_hu_h, 32, W, UW, H, UH, DO_ADD) +DO_ODD_U_S(vaddwod_d_wu_w, 64, D, UD, W, UW, DO_ADD) + +#define DO_3OP(NAME, BIT, E, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = DO_OP(Vj->E(i), Vk->E(i)); \ + } \ +} + +DO_3OP(vavg_b, 8, B, DO_VAVG) +DO_3OP(vavg_h, 16, H, DO_VAVG) +DO_3OP(vavg_w, 32, W, DO_VAVG) +DO_3OP(vavg_d, 64, D, DO_VAVG) +DO_3OP(vavgr_b, 8, B, DO_VAVGR) +DO_3OP(vavgr_h, 16, H, DO_VAVGR) +DO_3OP(vavgr_w, 32, W, DO_VAVGR) +DO_3OP(vavgr_d, 64, D, DO_VAVGR) +DO_3OP(vavg_bu, 8, UB, DO_VAVG) +DO_3OP(vavg_hu, 16, UH, DO_VAVG) +DO_3OP(vavg_wu, 32, UW, DO_VAVG) +DO_3OP(vavg_du, 64, UD, DO_VAVG) +DO_3OP(vavgr_bu, 8, UB, DO_VAVGR) +DO_3OP(vavgr_hu, 16, UH, DO_VAVGR) +DO_3OP(vavgr_wu, 32, UW, DO_VAVGR) +DO_3OP(vavgr_du, 64, UD, DO_VAVGR) + +DO_3OP(vabsd_b, 8, B, DO_VABSD) +DO_3OP(vabsd_h, 16, H, DO_VABSD) +DO_3OP(vabsd_w, 32, W, DO_VABSD) +DO_3OP(vabsd_d, 64, D, DO_VABSD) +DO_3OP(vabsd_bu, 8, UB, DO_VABSD) +DO_3OP(vabsd_hu, 16, UH, DO_VABSD) +DO_3OP(vabsd_wu, 32, UW, DO_VABSD) +DO_3OP(vabsd_du, 64, UD, DO_VABSD) + +#define DO_VADDA(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = DO_VABS(Vj->E(i)) + DO_VABS(Vk->E(i)); \ + } \ +} + +DO_VADDA(vadda_b, 8, B) +DO_VADDA(vadda_h, 16, H) +DO_VADDA(vadda_w, 32, W) +DO_VADDA(vadda_d, 64, D) + +#define VMINMAXI(NAME, BIT, E, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + typedef __typeof(Vd->E(0)) TD; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = DO_OP(Vj->E(i), (TD)imm); \ + } \ +} + +VMINMAXI(vmini_b, 8, B, DO_MIN) +VMINMAXI(vmini_h, 16, H, DO_MIN) +VMINMAXI(vmini_w, 32, W, DO_MIN) +VMINMAXI(vmini_d, 64, D, DO_MIN) +VMINMAXI(vmaxi_b, 8, B, DO_MAX) +VMINMAXI(vmaxi_h, 16, H, DO_MAX) +VMINMAXI(vmaxi_w, 32, W, DO_MAX) +VMINMAXI(vmaxi_d, 64, D, DO_MAX) +VMINMAXI(vmini_bu, 8, UB, DO_MIN) +VMINMAXI(vmini_hu, 16, UH, DO_MIN) +VMINMAXI(vmini_wu, 32, UW, DO_MIN) +VMINMAXI(vmini_du, 64, UD, DO_MIN) +VMINMAXI(vmaxi_bu, 8, UB, DO_MAX) +VMINMAXI(vmaxi_hu, 16, UH, DO_MAX) +VMINMAXI(vmaxi_wu, 32, UW, DO_MAX) +VMINMAXI(vmaxi_du, 64, UD, DO_MAX) + +#define DO_VMUH(NAME, BIT, E1, E2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->E1(0)) T; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E2(i) = ((T)Vj->E2(i)) * ((T)Vk->E2(i)) >> BIT; \ + } \ +} + +void HELPER(vmuh_d)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + uint64_t l, h; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 8; i++) { + muls64(&l, &h, Vj->D(i), Vk->D(i)); + Vd->D(i) = h; + } +} + +DO_VMUH(vmuh_b, 8, H, B, DO_MUH) +DO_VMUH(vmuh_h, 16, W, H, DO_MUH) +DO_VMUH(vmuh_w, 32, D, W, DO_MUH) + +void HELPER(vmuh_du)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i; + uint64_t l, h; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 8; i++) { + mulu64(&l, &h, Vj->D(i), Vk->D(i)); + Vd->D(i) = h; + } +} + +DO_VMUH(vmuh_bu, 8, UH, UB, DO_MUH) +DO_VMUH(vmuh_hu, 16, UW, UH, DO_MUH) +DO_VMUH(vmuh_wu, 32, UD, UW, DO_MUH) + +DO_EVEN(vmulwev_h_b, 16, H, B, DO_MUL) +DO_EVEN(vmulwev_w_h, 32, W, H, DO_MUL) +DO_EVEN(vmulwev_d_w, 64, D, W, DO_MUL) + +DO_ODD(vmulwod_h_b, 16, H, B, DO_MUL) +DO_ODD(vmulwod_w_h, 32, W, H, DO_MUL) +DO_ODD(vmulwod_d_w, 64, D, W, DO_MUL) + +DO_EVEN(vmulwev_h_bu, 16, UH, UB, DO_MUL) +DO_EVEN(vmulwev_w_hu, 32, UW, UH, DO_MUL) +DO_EVEN(vmulwev_d_wu, 64, UD, UW, DO_MUL) + +DO_ODD(vmulwod_h_bu, 16, UH, UB, DO_MUL) +DO_ODD(vmulwod_w_hu, 32, UW, UH, DO_MUL) +DO_ODD(vmulwod_d_wu, 64, UD, UW, DO_MUL) + +DO_EVEN_U_S(vmulwev_h_bu_b, 16, H, UH, B, UB, DO_MUL) +DO_EVEN_U_S(vmulwev_w_hu_h, 32, W, UW, H, UH, DO_MUL) +DO_EVEN_U_S(vmulwev_d_wu_w, 64, D, UD, W, UW, DO_MUL) + +DO_ODD_U_S(vmulwod_h_bu_b, 16, H, UH, B, UB, DO_MUL) +DO_ODD_U_S(vmulwod_w_hu_h, 32, W, UW, H, UH, DO_MUL) +DO_ODD_U_S(vmulwod_d_wu_w, 64, D, UD, W, UW, DO_MUL) + +#define VMADDSUB(NAME, BIT, E, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = DO_OP(Vd->E(i), Vj->E(i) ,Vk->E(i)); \ + } \ +} + +VMADDSUB(vmadd_b, 8, B, DO_MADD) +VMADDSUB(vmadd_h, 16, H, DO_MADD) +VMADDSUB(vmadd_w, 32, W, DO_MADD) +VMADDSUB(vmadd_d, 64, D, DO_MADD) +VMADDSUB(vmsub_b, 8, B, DO_MSUB) +VMADDSUB(vmsub_h, 16, H, DO_MSUB) +VMADDSUB(vmsub_w, 32, W, DO_MSUB) +VMADDSUB(vmsub_d, 64, D, DO_MSUB) + +#define VMADDWEV(NAME, BIT, E1, E2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->E1(0)) TD; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E1(i) += DO_OP((TD)Vj->E2(2 * i), (TD)Vk->E2(2 * i)); \ + } \ +} + +VMADDWEV(vmaddwev_h_b, 16, H, B, DO_MUL) +VMADDWEV(vmaddwev_w_h, 32, W, H, DO_MUL) +VMADDWEV(vmaddwev_d_w, 64, D, W, DO_MUL) +VMADDWEV(vmaddwev_h_bu, 16, UH, UB, DO_MUL) +VMADDWEV(vmaddwev_w_hu, 32, UW, UH, DO_MUL) +VMADDWEV(vmaddwev_d_wu, 64, UD, UW, DO_MUL) + +#define VMADDWOD(NAME, BIT, E1, E2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->E1(0)) TD; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E1(i) += DO_OP((TD)Vj->E2(2 * i + 1), \ + (TD)Vk->E2(2 * i + 1)); \ + } \ +} + +VMADDWOD(vmaddwod_h_b, 16, H, B, DO_MUL) +VMADDWOD(vmaddwod_w_h, 32, W, H, DO_MUL) +VMADDWOD(vmaddwod_d_w, 64, D, W, DO_MUL) +VMADDWOD(vmaddwod_h_bu, 16, UH, UB, DO_MUL) +VMADDWOD(vmaddwod_w_hu, 32, UW, UH, DO_MUL) +VMADDWOD(vmaddwod_d_wu, 64, UD, UW, DO_MUL) + +#define VMADDWEV_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->ES1(0)) TS1; \ + typedef __typeof(Vd->EU1(0)) TU1; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->ES1(i) += DO_OP((TU1)Vj->EU2(2 * i), \ + (TS1)Vk->ES2(2 * i)); \ + } \ +} + +VMADDWEV_U_S(vmaddwev_h_bu_b, 16, H, UH, B, UB, DO_MUL) +VMADDWEV_U_S(vmaddwev_w_hu_h, 32, W, UW, H, UH, DO_MUL) +VMADDWEV_U_S(vmaddwev_d_wu_w, 64, D, UD, W, UW, DO_MUL) + +#define VMADDWOD_U_S(NAME, BIT, ES1, EU1, ES2, EU2, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + typedef __typeof(Vd->ES1(0)) TS1; \ + typedef __typeof(Vd->EU1(0)) TU1; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->ES1(i) += DO_OP((TU1)Vj->EU2(2 * i + 1), \ + (TS1)Vk->ES2(2 * i + 1)); \ + } \ +} + +VMADDWOD_U_S(vmaddwod_h_bu_b, 16, H, UH, B, UB, DO_MUL) +VMADDWOD_U_S(vmaddwod_w_hu_h, 32, W, UW, H, UH, DO_MUL) +VMADDWOD_U_S(vmaddwod_d_wu_w, 64, D, UD, W, UW, DO_MUL) + +#define VDIV(NAME, BIT, E, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = DO_OP(Vj->E(i), Vk->E(i)); \ + } \ +} + +VDIV(vdiv_b, 8, B, DO_DIV) +VDIV(vdiv_h, 16, H, DO_DIV) +VDIV(vdiv_w, 32, W, DO_DIV) +VDIV(vdiv_d, 64, D, DO_DIV) +VDIV(vdiv_bu, 8, UB, DO_DIVU) +VDIV(vdiv_hu, 16, UH, DO_DIVU) +VDIV(vdiv_wu, 32, UW, DO_DIVU) +VDIV(vdiv_du, 64, UD, DO_DIVU) +VDIV(vmod_b, 8, B, DO_REM) +VDIV(vmod_h, 16, H, DO_REM) +VDIV(vmod_w, 32, W, DO_REM) +VDIV(vmod_d, 64, D, DO_REM) +VDIV(vmod_bu, 8, UB, DO_REMU) +VDIV(vmod_hu, 16, UH, DO_REMU) +VDIV(vmod_wu, 32, UW, DO_REMU) +VDIV(vmod_du, 64, UD, DO_REMU) + +#define VSAT_S(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t max, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + typedef __typeof(Vd->E(0)) TD; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = Vj->E(i) > (TD)max ? (TD)max : \ + Vj->E(i) < (TD)~max ? (TD)~max: Vj->E(i); \ + } \ +} + +VSAT_S(vsat_b, 8, B) +VSAT_S(vsat_h, 16, H) +VSAT_S(vsat_w, 32, W) +VSAT_S(vsat_d, 64, D) + +#define VSAT_U(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t max, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + typedef __typeof(Vd->E(0)) TD; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = Vj->E(i) > (TD)max ? (TD)max : Vj->E(i); \ + } \ +} + +VSAT_U(vsat_bu, 8, UB) +VSAT_U(vsat_hu, 16, UH) +VSAT_U(vsat_wu, 32, UW) +VSAT_U(vsat_du, 64, UD) + +#define VEXTH(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + i * ofs) = Vj->E2(j + ofs + ofs * 2 * i); \ + } \ + } \ +} + +void HELPER(vexth_q_d)(void *vd, void *vj, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_makes64(Vj->D(2 * i + 1)); + } +} + +void HELPER(vexth_qu_du)(void *vd, void *vj, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_make64(Vj->UD(2 * i + 1)); + } +} + +VEXTH(vexth_h_b, 16, H, B) +VEXTH(vexth_w_h, 32, W, H) +VEXTH(vexth_d_w, 64, D, W) +VEXTH(vexth_hu_bu, 16, UH, UB) +VEXTH(vexth_wu_hu, 32, UW, UH) +VEXTH(vexth_du_wu, 64, UD, UW) + +#define VEXT2XV(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint32_t desc) \ +{ \ + int i; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + temp.E1(i) = Vj->E2(i); \ + } \ + *Vd = temp; \ +} + +VEXT2XV(vext2xv_h_b, 16, H, B) +VEXT2XV(vext2xv_w_b, 32, W, B) +VEXT2XV(vext2xv_d_b, 64, D, B) +VEXT2XV(vext2xv_w_h, 32, W, H) +VEXT2XV(vext2xv_d_h, 64, D, H) +VEXT2XV(vext2xv_d_w, 64, D, W) +VEXT2XV(vext2xv_hu_bu, 16, UH, UB) +VEXT2XV(vext2xv_wu_bu, 32, UW, UB) +VEXT2XV(vext2xv_du_bu, 64, UD, UB) +VEXT2XV(vext2xv_wu_hu, 32, UW, UH) +VEXT2XV(vext2xv_du_hu, 64, UD, UH) +VEXT2XV(vext2xv_du_wu, 64, UD, UW) + +DO_3OP(vsigncov_b, 8, B, DO_SIGNCOV) +DO_3OP(vsigncov_h, 16, H, DO_SIGNCOV) +DO_3OP(vsigncov_w, 32, W, DO_SIGNCOV) +DO_3OP(vsigncov_d, 64, D, DO_SIGNCOV) + +static uint64_t do_vmskltz_b(int64_t val) +{ + uint64_t m = 0x8080808080808080ULL; + uint64_t c = val & m; + c |= c << 7; + c |= c << 14; + c |= c << 28; + return c >> 56; +} + +void HELPER(vmskltz_b)(void *vd, void *vj, uint32_t desc) +{ + int i; + uint16_t temp = 0; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + temp = 0; + temp = do_vmskltz_b(Vj->D(2 * i)); + temp |= (do_vmskltz_b(Vj->D(2 * i + 1)) << 8); + Vd->D(2 * i) = temp; + Vd->D(2 * i + 1) = 0; + } +} + +static uint64_t do_vmskltz_h(int64_t val) +{ + uint64_t m = 0x8000800080008000ULL; + uint64_t c = val & m; + c |= c << 15; + c |= c << 30; + return c >> 60; +} + +void HELPER(vmskltz_h)(void *vd, void *vj, uint32_t desc) +{ + int i; + uint16_t temp = 0; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + temp = 0; + temp = do_vmskltz_h(Vj->D(2 * i)); + temp |= (do_vmskltz_h(Vj->D(2 * i + 1)) << 4); + Vd->D(2 * i) = temp; + Vd->D(2 * i + 1) = 0; + } +} + +static uint64_t do_vmskltz_w(int64_t val) +{ + uint64_t m = 0x8000000080000000ULL; + uint64_t c = val & m; + c |= c << 31; + return c >> 62; +} + +void HELPER(vmskltz_w)(void *vd, void *vj, uint32_t desc) +{ + int i; + uint16_t temp = 0; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + temp = 0; + temp = do_vmskltz_w(Vj->D(2 * i)); + temp |= (do_vmskltz_w(Vj->D(2 * i + 1)) << 2); + Vd->D(2 * i) = temp; + Vd->D(2 * i + 1) = 0; + } +} + +static uint64_t do_vmskltz_d(int64_t val) +{ + return (uint64_t)val >> 63; +} +void HELPER(vmskltz_d)(void *vd, void *vj, uint32_t desc) +{ + int i; + uint16_t temp = 0; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + temp = 0; + temp = do_vmskltz_d(Vj->D(2 * i)); + temp |= (do_vmskltz_d(Vj->D(2 * i + 1)) << 1); + Vd->D(2 * i) = temp; + Vd->D(2 * i + 1) = 0; + } +} + +void HELPER(vmskgez_b)(void *vd, void *vj, uint32_t desc) +{ + int i; + uint16_t temp = 0; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + temp = 0; + temp = do_vmskltz_b(Vj->D(2 * i)); + temp |= (do_vmskltz_b(Vj->D(2 * i + 1)) << 8); + Vd->D(2 * i) = (uint16_t)(~temp); + Vd->D(2 * i + 1) = 0; + } +} + +static uint64_t do_vmskez_b(uint64_t a) +{ + uint64_t m = 0x7f7f7f7f7f7f7f7fULL; + uint64_t c = ~(((a & m) + m) | a | m); + c |= c << 7; + c |= c << 14; + c |= c << 28; + return c >> 56; +} + +void HELPER(vmsknz_b)(void *vd, void *vj, uint32_t desc) +{ + int i; + uint16_t temp = 0; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + temp = 0; + temp = do_vmskez_b(Vj->D(2 * i)); + temp |= (do_vmskez_b(Vj->D(2 * i + 1)) << 8); + Vd->D(2 * i) = (uint16_t)(~temp); + Vd->D(2 * i + 1) = 0; + } +} + +void HELPER(vnori_b)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + + for (i = 0; i < simd_oprsz(desc); i++) { + Vd->B(i) = ~(Vj->B(i) | (uint8_t)imm); + } +} + +#define VSLLWIL(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + typedef __typeof(temp.E1(0)) TD; \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * i) = (TD)Vj->E2(j + ofs * 2 * i) << (imm % BIT); \ + } \ + } \ + *Vd = temp; \ +} + + +void HELPER(vextl_q_d)(void *vd, void *vj, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_makes64(Vj->D(2 * i)); + } +} + +void HELPER(vextl_qu_du)(void *vd, void *vj, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + Vd->Q(i) = int128_make64(Vj->UD(2 * i)); + } +} + +VSLLWIL(vsllwil_h_b, 16, H, B) +VSLLWIL(vsllwil_w_h, 32, W, H) +VSLLWIL(vsllwil_d_w, 64, D, W) +VSLLWIL(vsllwil_hu_bu, 16, UH, UB) +VSLLWIL(vsllwil_wu_hu, 32, UW, UH) +VSLLWIL(vsllwil_du_wu, 64, UD, UW) + +#define do_vsrlr(E, T) \ +static T do_vsrlr_ ##E(T s1, int sh) \ +{ \ + if (sh == 0) { \ + return s1; \ + } else { \ + return (s1 >> sh) + ((s1 >> (sh - 1)) & 0x1); \ + } \ +} + +do_vsrlr(B, uint8_t) +do_vsrlr(H, uint16_t) +do_vsrlr(W, uint32_t) +do_vsrlr(D, uint64_t) + +#define VSRLR(NAME, BIT, T, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = do_vsrlr_ ## E(Vj->E(i), ((T)Vk->E(i))%BIT); \ + } \ +} + +VSRLR(vsrlr_b, 8, uint8_t, B) +VSRLR(vsrlr_h, 16, uint16_t, H) +VSRLR(vsrlr_w, 32, uint32_t, W) +VSRLR(vsrlr_d, 64, uint64_t, D) + +#define VSRLRI(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = do_vsrlr_ ## E(Vj->E(i), imm); \ + } \ +} + +VSRLRI(vsrlri_b, 8, B) +VSRLRI(vsrlri_h, 16, H) +VSRLRI(vsrlri_w, 32, W) +VSRLRI(vsrlri_d, 64, D) + +#define do_vsrar(E, T) \ +static T do_vsrar_ ##E(T s1, int sh) \ +{ \ + if (sh == 0) { \ + return s1; \ + } else { \ + return (s1 >> sh) + ((s1 >> (sh - 1)) & 0x1); \ + } \ +} + +do_vsrar(B, int8_t) +do_vsrar(H, int16_t) +do_vsrar(W, int32_t) +do_vsrar(D, int64_t) + +#define VSRAR(NAME, BIT, T, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = do_vsrar_ ## E(Vj->E(i), ((T)Vk->E(i))%BIT); \ + } \ +} + +VSRAR(vsrar_b, 8, uint8_t, B) +VSRAR(vsrar_h, 16, uint16_t, H) +VSRAR(vsrar_w, 32, uint32_t, W) +VSRAR(vsrar_d, 64, uint64_t, D) + +#define VSRARI(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = do_vsrar_ ## E(Vj->E(i), imm); \ + } \ +} + +VSRARI(vsrari_b, 8, B) +VSRARI(vsrari_h, 16, H) +VSRARI(vsrari_w, 32, W) +VSRARI(vsrari_d, 64, D) + +#define VSRLN(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = R_SHIFT(Vj->E2(j + ofs * i), \ + Vk->E2(j + ofs * i) % BIT); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSRLN(vsrln_b_h, 16, B, UH) +VSRLN(vsrln_h_w, 32, H, UW) +VSRLN(vsrln_w_d, 64, W, UD) + +#define VSRAN(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = R_SHIFT(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSRAN(vsran_b_h, 16, B, H, UH) +VSRAN(vsran_h_w, 32, H, W, UW) +VSRAN(vsran_w_d, 64, W, D, UD) + +#define VSRLNI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = R_SHIFT(Vj->E2(j + ofs * i), imm); \ + temp.E1(j + ofs * (2 * i + 1)) = R_SHIFT(Vd->E2(j + ofs * i), \ + imm); \ + } \ + } \ + *Vd = temp; \ +} + +void HELPER(vsrlni_d_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + + for (i = 0; i < 2; i++) { + temp.D(2 * i) = int128_getlo(int128_urshift(Vj->Q(i), imm % 128)); + temp.D(2 * i +1) = int128_getlo(int128_urshift(Vd->Q(i), imm % 128)); + } + *Vd = temp; +} + +VSRLNI(vsrlni_b_h, 16, B, UH) +VSRLNI(vsrlni_h_w, 32, H, UW) +VSRLNI(vsrlni_w_d, 64, W, UD) + +#define VSRANI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = R_SHIFT(Vj->E2(j + ofs * i), imm); \ + temp.E1(j + ofs * (2 * i + 1)) = R_SHIFT(Vd->E2(j + ofs * i), \ + imm); \ + } \ + } \ + *Vd = temp; \ +} + +void HELPER(vsrani_d_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + + for (i = 0; i < 2; i++) { + temp.D(2 * i) = int128_getlo(int128_rshift(Vj->Q(i), imm % 128)); + temp.D(2 * i + 1) = int128_getlo(int128_rshift(Vd->Q(i), imm % 128)); + } + *Vd = temp; +} + +VSRANI(vsrani_b_h, 16, B, H) +VSRANI(vsrani_h_w, 32, H, W) +VSRANI(vsrani_w_d, 64, W, D) + +#define VSRLRN(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_vsrlr_ ##E2(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSRLRN(vsrlrn_b_h, 16, B, H, UH) +VSRLRN(vsrlrn_h_w, 32, H, W, UW) +VSRLRN(vsrlrn_w_d, 64, W, D, UD) + +#define VSRARN(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_vsrar_ ## E2(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSRARN(vsrarn_b_h, 16, B, H, UH) +VSRARN(vsrarn_h_w, 32, H, W, UW) +VSRARN(vsrarn_w_d, 64, W, D, UD) + +#define VSRLRNI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_vsrlr_ ## E2(Vj->E2(j + ofs * i), imm); \ + temp.E1(j + ofs * (2 * i + 1)) = do_vsrlr_ ## E2(Vd->E2(j + ofs * i), \ + imm); \ + } \ + } \ + *Vd = temp; \ +} + +void HELPER(vsrlrni_d_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + Int128 r[4]; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + if (imm == 0) { + temp.D(2 * i) = int128_getlo(Vj->Q(i)); + temp.D(2 * i + 1) = int128_getlo(Vd->Q(i)); + } else { + r[2 * i] = int128_and(int128_urshift(Vj->Q(i), (imm - 1)), + int128_one()); + r[2 * i + 1] = int128_and(int128_urshift(Vd->Q(i), (imm - 1)), + int128_one()); + temp.D(2 * i) = int128_getlo(int128_add(int128_urshift(Vj->Q(i), + imm), r[2 * i])); + temp.D(2 * i + 1) = int128_getlo(int128_add(int128_urshift(Vd->Q(i), + imm), r[ 2 * i + 1])); + } + } + *Vd = temp; +} + +VSRLRNI(vsrlrni_b_h, 16, B, H) +VSRLRNI(vsrlrni_h_w, 32, H, W) +VSRLRNI(vsrlrni_w_d, 64, W, D) + +#define VSRARNI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_vsrar_ ## E2(Vj->E2(j + ofs * i), imm); \ + temp.E1(j + ofs * (2 * i + 1)) = do_vsrar_ ## E2(Vd->E2(j + ofs * i), \ + imm); \ + } \ + } \ + *Vd = temp; \ +} + +void HELPER(vsrarni_d_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + Int128 r[4]; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + if (imm == 0) { + temp.D(2 * i) = int128_getlo(Vj->Q(i)); + temp.D(2 * i + 1) = int128_getlo(Vd->Q(i)); + } else { + r[2 * i] = int128_and(int128_rshift(Vj->Q(i), (imm - 1)), + int128_one()); + r[2 * i + 1] = int128_and(int128_rshift(Vd->Q(i), (imm - 1)), + int128_one()); + temp.D(2 * i) = int128_getlo(int128_add(int128_rshift(Vj->Q(i), + imm), r[2 * i])); + temp.D(2 * i + 1) = int128_getlo(int128_add(int128_rshift(Vd->Q(i), + imm), r[2 * i + 1])); + } + } + *Vd = temp; +} + +VSRARNI(vsrarni_b_h, 16, B, H) +VSRARNI(vsrarni_h_w, 32, H, W) +VSRARNI(vsrarni_w_d, 64, W, D) + +#define SSRLNS(NAME, T1, T2, T3) \ +static T1 do_ssrlns_ ## NAME(T2 e2, int sa, int sh) \ +{ \ + T1 shft_res; \ + if (sa == 0) { \ + shft_res = e2; \ + } else { \ + shft_res = (((T1)e2) >> sa); \ + } \ + T3 mask; \ + mask = (1ull << sh) -1; \ + if (shft_res > mask) { \ + return mask; \ + } else { \ + return shft_res; \ + } \ +} + +SSRLNS(B, uint16_t, int16_t, uint8_t) +SSRLNS(H, uint32_t, int32_t, uint16_t) +SSRLNS(W, uint64_t, int64_t, uint32_t) + +#define VSSRLN(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_ssrlns_ ## E1(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT, \ + BIT / 2 - 1); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSSRLN(vssrln_b_h, 16, B, H, UH) +VSSRLN(vssrln_h_w, 32, H, W, UW) +VSSRLN(vssrln_w_d, 64, W, D, UD) + +#define SSRANS(E, T1, T2) \ +static T1 do_ssrans_ ## E(T1 e2, int sa, int sh) \ +{ \ + T1 shft_res; \ + if (sa == 0) { \ + shft_res = e2; \ + } else { \ + shft_res = e2 >> sa; \ + } \ + T2 mask; \ + mask = (1ll << sh) - 1; \ + if (shft_res > mask) { \ + return mask; \ + } else if (shft_res < -(mask + 1)) { \ + return ~mask; \ + } else { \ + return shft_res; \ + } \ +} + +SSRANS(B, int16_t, int8_t) +SSRANS(H, int32_t, int16_t) +SSRANS(W, int64_t, int32_t) + +#define VSSRAN(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_ssrans_ ## E1(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT, \ + BIT / 2 - 1); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSSRAN(vssran_b_h, 16, B, H, UH) +VSSRAN(vssran_h_w, 32, H, W, UW) +VSSRAN(vssran_w_d, 64, W, D, UD) + +#define SSRLNU(E, T1, T2, T3) \ +static T1 do_ssrlnu_ ## E(T3 e2, int sa, int sh) \ +{ \ + T1 shft_res; \ + if (sa == 0) { \ + shft_res = e2; \ + } else { \ + shft_res = (((T1)e2) >> sa); \ + } \ + T2 mask; \ + mask = (1ull << sh) - 1; \ + if (shft_res > mask) { \ + return mask; \ + } else { \ + return shft_res; \ + } \ +} + +SSRLNU(B, uint16_t, uint8_t, int16_t) +SSRLNU(H, uint32_t, uint16_t, int32_t) +SSRLNU(W, uint64_t, uint32_t, int64_t) + +#define VSSRLNU(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_ssrlnu_ ## E1(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT, \ + BIT / 2); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSSRLNU(vssrln_bu_h, 16, B, H, UH) +VSSRLNU(vssrln_hu_w, 32, H, W, UW) +VSSRLNU(vssrln_wu_d, 64, W, D, UD) + +#define SSRANU(E, T1, T2, T3) \ +static T1 do_ssranu_ ## E(T3 e2, int sa, int sh) \ +{ \ + T1 shft_res; \ + if (sa == 0) { \ + shft_res = e2; \ + } else { \ + shft_res = e2 >> sa; \ + } \ + if (e2 < 0) { \ + shft_res = 0; \ + } \ + T2 mask; \ + mask = (1ull << sh) - 1; \ + if (shft_res > mask) { \ + return mask; \ + } else { \ + return shft_res; \ + } \ +} + +SSRANU(B, uint16_t, uint8_t, int16_t) +SSRANU(H, uint32_t, uint16_t, int32_t) +SSRANU(W, uint64_t, uint32_t, int64_t) + +#define VSSRANU(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_ssranu_ ## E1(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT, \ + BIT / 2); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSSRANU(vssran_bu_h, 16, B, H, UH) +VSSRANU(vssran_hu_w, 32, H, W, UW) +VSSRANU(vssran_wu_d, 64, W, D, UD) + +#define VSSRLNI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_ssrlns_ ## E1(Vj->E2(j + ofs * i), \ + imm, BIT / 2 - 1); \ + temp.E1(j + ofs * (2 * i + 1)) = do_ssrlns_ ## E1(Vd->E2(j + ofs * i), \ + imm, BIT / 2 - 1); \ + } \ + } \ + *Vd = temp; \ +} + +static void do_vssrlni_q(VReg *Vd, VReg *Vj, + uint64_t imm, int idx, Int128 mask) +{ + Int128 shft_res1, shft_res2; + + if (imm == 0) { + shft_res1 = Vj->Q(idx); + shft_res2 = Vd->Q(idx); + } else { + shft_res1 = int128_urshift(Vj->Q(idx), imm); + shft_res2 = int128_urshift(Vd->Q(idx), imm); + } + + if (int128_ult(mask, shft_res1)) { + Vd->D(idx * 2) = int128_getlo(mask); + }else { + Vd->D(idx * 2) = int128_getlo(shft_res1); + } + + if (int128_ult(mask, shft_res2)) { + Vd->D(idx * 2 + 1) = int128_getlo(mask); + }else { + Vd->D(idx * 2 + 1) = int128_getlo(shft_res2); + } +} + +void HELPER(vssrlni_d_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + Int128 mask; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + mask = int128_sub(int128_lshift(int128_one(), 63), int128_one()); + + for (i = 0; i < oprsz / 16; i++) { + do_vssrlni_q(Vd, Vj, imm, i, mask); + } +} + +VSSRLNI(vssrlni_b_h, 16, B, H) +VSSRLNI(vssrlni_h_w, 32, H, W) +VSSRLNI(vssrlni_w_d, 64, W, D) + +#define VSSRANI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_ssrans_ ## E1(Vj->E2(j + ofs * i), \ + imm, BIT / 2 - 1); \ + temp.E1(j + ofs * (2 * i + 1)) = do_ssrans_ ## E1(Vd->E2(j + ofs * i), \ + imm, BIT / 2 - 1); \ + } \ + } \ + *Vd = temp; \ +} + +static void do_vssrani_d_q(VReg *Vd, VReg *Vj, + uint64_t imm, int idx, Int128 mask, Int128 min) +{ + Int128 shft_res1, shft_res2; + + if (imm == 0) { + shft_res1 = Vj->Q(idx); + shft_res2 = Vd->Q(idx); + } else { + shft_res1 = int128_rshift(Vj->Q(idx), imm); + shft_res2 = int128_rshift(Vd->Q(idx), imm); + } + + if (int128_gt(shft_res1, mask)) { + Vd->D(idx * 2) = int128_getlo(mask); + } else if (int128_lt(shft_res1, int128_neg(min))) { + Vd->D(idx * 2) = int128_getlo(min); + } else { + Vd->D(idx * 2) = int128_getlo(shft_res1); + } + + if (int128_gt(shft_res2, mask)) { + Vd->D(idx * 2 + 1) = int128_getlo(mask); + } else if (int128_lt(shft_res2, int128_neg(min))) { + Vd->D(idx * 2 + 1) = int128_getlo(min); + } else { + Vd->D(idx * 2 + 1) = int128_getlo(shft_res2); + } +} + +void HELPER(vssrani_d_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + Int128 mask, min; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + mask = int128_sub(int128_lshift(int128_one(), 63), int128_one()); + min = int128_lshift(int128_one(), 63); + + for (i = 0; i < oprsz / 16; i++) { + do_vssrani_d_q(Vd, Vj, imm, i, mask, min); + } +} + + +VSSRANI(vssrani_b_h, 16, B, H) +VSSRANI(vssrani_h_w, 32, H, W) +VSSRANI(vssrani_w_d, 64, W, D) + +#define VSSRLNUI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_ssrlnu_ ## E1(Vj->E2(j + ofs * i), \ + imm, BIT / 2); \ + temp.E1(j + ofs * (2 * i + 1)) = do_ssrlnu_ ## E1(Vd->E2(j + ofs * i), \ + imm, BIT / 2); \ + } \ + } \ + *Vd = temp; \ +} + +void HELPER(vssrlni_du_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + Int128 mask; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + mask = int128_sub(int128_lshift(int128_one(), 64), int128_one()); + + for (i = 0; i < oprsz / 16; i++) { + do_vssrlni_q(Vd, Vj, imm, i, mask); + } +} + +VSSRLNUI(vssrlni_bu_h, 16, B, H) +VSSRLNUI(vssrlni_hu_w, 32, H, W) +VSSRLNUI(vssrlni_wu_d, 64, W, D) + +#define VSSRANUI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_ssranu_ ## E1(Vj->E2(j + ofs * i), \ + imm, BIT / 2); \ + temp.E1(j + ofs * (2 * i + 1)) = do_ssranu_ ## E1(Vd->E2(j + ofs * i), \ + imm, BIT / 2); \ + } \ + } \ + *Vd = temp; \ +} + +static void do_vssrani_du_q(VReg *Vd, VReg *Vj, + uint64_t imm, int idx, Int128 mask) +{ + Int128 shft_res1, shft_res2; + + if (imm == 0) { + shft_res1 = Vj->Q(idx); + shft_res2 = Vd->Q(idx); + } else { + shft_res1 = int128_rshift(Vj->Q(idx), imm); + shft_res2 = int128_rshift(Vd->Q(idx), imm); + } + + if (int128_lt(Vj->Q(idx), int128_zero())) { + shft_res1 = int128_zero(); + } + + if (int128_lt(Vd->Q(idx), int128_zero())) { + shft_res2 = int128_zero(); + } + if (int128_ult(mask, shft_res1)) { + Vd->D(idx * 2) = int128_getlo(mask); + }else { + Vd->D(idx * 2) = int128_getlo(shft_res1); + } + + if (int128_ult(mask, shft_res2)) { + Vd->D(idx * 2 + 1) = int128_getlo(mask); + }else { + Vd->D(idx * 2 + 1) = int128_getlo(shft_res2); + } + +} + +void HELPER(vssrani_du_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + Int128 mask; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + mask = int128_sub(int128_lshift(int128_one(), 64), int128_one()); + + for (i = 0; i < oprsz / 16; i++) { + do_vssrani_du_q(Vd, Vj, imm, i, mask); + } +} + +VSSRANUI(vssrani_bu_h, 16, B, H) +VSSRANUI(vssrani_hu_w, 32, H, W) +VSSRANUI(vssrani_wu_d, 64, W, D) + +#define SSRLRNS(E1, E2, T1, T2, T3) \ +static T1 do_ssrlrns_ ## E1(T2 e2, int sa, int sh) \ +{ \ + T1 shft_res; \ + \ + shft_res = do_vsrlr_ ## E2(e2, sa); \ + T1 mask; \ + mask = (1ull << sh) - 1; \ + if (shft_res > mask) { \ + return mask; \ + } else { \ + return shft_res; \ + } \ +} + +SSRLRNS(B, H, uint16_t, int16_t, uint8_t) +SSRLRNS(H, W, uint32_t, int32_t, uint16_t) +SSRLRNS(W, D, uint64_t, int64_t, uint32_t) + +#define VSSRLRN(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_ssrlrns_ ## E1(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT, \ + BIT / 2 - 1); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSSRLRN(vssrlrn_b_h, 16, B, H, UH) +VSSRLRN(vssrlrn_h_w, 32, H, W, UW) +VSSRLRN(vssrlrn_w_d, 64, W, D, UD) + +#define SSRARNS(E1, E2, T1, T2) \ +static T1 do_ssrarns_ ## E1(T1 e2, int sa, int sh) \ +{ \ + T1 shft_res; \ + \ + shft_res = do_vsrar_ ## E2(e2, sa); \ + T2 mask; \ + mask = (1ll << sh) - 1; \ + if (shft_res > mask) { \ + return mask; \ + } else if (shft_res < -(mask +1)) { \ + return ~mask; \ + } else { \ + return shft_res; \ + } \ +} + +SSRARNS(B, H, int16_t, int8_t) +SSRARNS(H, W, int32_t, int16_t) +SSRARNS(W, D, int64_t, int32_t) + +#define VSSRARN(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_ssrarns_ ## E1(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT, \ + BIT/ 2 - 1); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSSRARN(vssrarn_b_h, 16, B, H, UH) +VSSRARN(vssrarn_h_w, 32, H, W, UW) +VSSRARN(vssrarn_w_d, 64, W, D, UD) + +#define SSRLRNU(E1, E2, T1, T2, T3) \ +static T1 do_ssrlrnu_ ## E1(T3 e2, int sa, int sh) \ +{ \ + T1 shft_res; \ + \ + shft_res = do_vsrlr_ ## E2(e2, sa); \ + \ + T2 mask; \ + mask = (1ull << sh) - 1; \ + if (shft_res > mask) { \ + return mask; \ + } else { \ + return shft_res; \ + } \ +} + +SSRLRNU(B, H, uint16_t, uint8_t, int16_t) +SSRLRNU(H, W, uint32_t, uint16_t, int32_t) +SSRLRNU(W, D, uint64_t, uint32_t, int64_t) + +#define VSSRLRNU(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_ssrlrnu_ ## E1(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT, \ + BIT / 2); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSSRLRNU(vssrlrn_bu_h, 16, B, H, UH) +VSSRLRNU(vssrlrn_hu_w, 32, H, W, UW) +VSSRLRNU(vssrlrn_wu_d, 64, W, D, UD) + +#define SSRARNU(E1, E2, T1, T2, T3) \ +static T1 do_ssrarnu_ ## E1(T3 e2, int sa, int sh) \ +{ \ + T1 shft_res; \ + \ + if (e2 < 0) { \ + shft_res = 0; \ + } else { \ + shft_res = do_vsrar_ ## E2(e2, sa); \ + } \ + T2 mask; \ + mask = (1ull << sh) - 1; \ + if (shft_res > mask) { \ + return mask; \ + } else { \ + return shft_res; \ + } \ +} + +SSRARNU(B, H, uint16_t, uint8_t, int16_t) +SSRARNU(H, W, uint32_t, uint16_t, int32_t) +SSRARNU(W, D, uint64_t, uint32_t, int64_t) + +#define VSSRARNU(NAME, BIT, E1, E2, E3) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + Vd->E1(j + ofs * 2 * i) = do_ssrarnu_ ## E1(Vj->E2(j + ofs * i), \ + Vk->E3(j + ofs * i) % BIT, \ + BIT / 2); \ + } \ + Vd->D(2 * i + 1) = 0; \ + } \ +} + +VSSRARNU(vssrarn_bu_h, 16, B, H, UH) +VSSRARNU(vssrarn_hu_w, 32, H, W, UW) +VSSRARNU(vssrarn_wu_d, 64, W, D, UD) + +#define VSSRLRNI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_ssrlrns_ ## E1(Vj->E2(j + ofs * i), \ + imm, BIT / 2 - 1); \ + temp.E1(j + ofs * (2 * i + 1)) = do_ssrlrns_ ## E1(Vd->E2(j + ofs * i), \ + imm, BIT / 2 - 1); \ + } \ + } \ + *Vd = temp; \ +} + +static void do_vssrlrni_q(VReg *Vd, VReg * Vj, + uint64_t imm, int idx, Int128 mask) +{ + Int128 shft_res1, shft_res2, r1, r2; + if (imm == 0) { + shft_res1 = Vj->Q(idx); + shft_res2 = Vd->Q(idx); + } else { + r1 = int128_and(int128_urshift(Vj->Q(idx), (imm - 1)), int128_one()); + r2 = int128_and(int128_urshift(Vd->Q(idx), (imm - 1)), int128_one()); + shft_res1 = (int128_add(int128_urshift(Vj->Q(idx), imm), r1)); + shft_res2 = (int128_add(int128_urshift(Vd->Q(idx), imm), r2)); + } + + if (int128_ult(mask, shft_res1)) { + Vd->D(idx * 2) = int128_getlo(mask); + }else { + Vd->D(idx * 2) = int128_getlo(shft_res1); + } + + if (int128_ult(mask, shft_res2)) { + Vd->D(idx * 2 + 1) = int128_getlo(mask); + }else { + Vd->D(idx * 2 + 1) = int128_getlo(shft_res2); + } +} + +void HELPER(vssrlrni_d_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + Int128 mask; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + mask = int128_sub(int128_lshift(int128_one(), 63), int128_one()); + + for (i = 0; i < oprsz / 16; i++) { + do_vssrlrni_q(Vd, Vj, imm, i, mask); + } +} + +VSSRLRNI(vssrlrni_b_h, 16, B, H) +VSSRLRNI(vssrlrni_h_w, 32, H, W) +VSSRLRNI(vssrlrni_w_d, 64, W, D) + +#define VSSRARNI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_ssrarns_ ## E1(Vj->E2(j + ofs * i), \ + imm, BIT / 2 - 1); \ + temp.E1(j + ofs * (2 * i + 1)) = do_ssrarns_ ## E1(Vd->E2(j + ofs * i), \ + imm, BIT / 2 - 1); \ + } \ + } \ + *Vd = temp; \ +} + +static void do_vssrarni_d_q(VReg *Vd, VReg *Vj, + uint64_t imm, int idx, Int128 mask1, Int128 mask2) +{ + Int128 shft_res1, shft_res2, r1, r2; + + if (imm == 0) { + shft_res1 = Vj->Q(idx); + shft_res2 = Vd->Q(idx); + } else { + r1 = int128_and(int128_rshift(Vj->Q(idx), (imm - 1)), int128_one()); + r2 = int128_and(int128_rshift(Vd->Q(idx), (imm - 1)), int128_one()); + shft_res1 = int128_add(int128_rshift(Vj->Q(idx), imm), r1); + shft_res2 = int128_add(int128_rshift(Vd->Q(idx), imm), r2); + } + if (int128_gt(shft_res1, mask1)) { + Vd->D(idx * 2) = int128_getlo(mask1); + } else if (int128_lt(shft_res1, int128_neg(mask2))) { + Vd->D(idx * 2) = int128_getlo(mask2); + } else { + Vd->D(idx * 2) = int128_getlo(shft_res1); + } + + if (int128_gt(shft_res2, mask1)) { + Vd->D(idx * 2 + 1) = int128_getlo(mask1); + } else if (int128_lt(shft_res2, int128_neg(mask2))) { + Vd->D(idx * 2 + 1) = int128_getlo(mask2); + } else { + Vd->D(idx * 2 + 1) = int128_getlo(shft_res2); + } +} + +void HELPER(vssrarni_d_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + Int128 mask1, mask2; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + mask1 = int128_sub(int128_lshift(int128_one(), 63), int128_one()); + mask2 = int128_lshift(int128_one(), 63); + + for (i = 0; i < oprsz / 16; i++) { + do_vssrarni_d_q(Vd, Vj, imm, i, mask1, mask2); + } +} + +VSSRARNI(vssrarni_b_h, 16, B, H) +VSSRARNI(vssrarni_h_w, 32, H, W) +VSSRARNI(vssrarni_w_d, 64, W, D) + +#define VSSRLRNUI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_ssrlrnu_ ## E1(Vj->E2(j + ofs * i), \ + imm, BIT / 2); \ + temp.E1(j + ofs * (2 * i + 1)) = do_ssrlrnu_ ## E1(Vd->E2(j + ofs * i), \ + imm, BIT / 2); \ + } \ + } \ + *Vd = temp; \ +} + +void HELPER(vssrlrni_du_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + Int128 mask; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + mask = int128_sub(int128_lshift(int128_one(), 64), int128_one()); + + for (i = 0; i < oprsz / 16; i++) { + do_vssrlrni_q(Vd, Vj, imm, i, mask); + } +} + +VSSRLRNUI(vssrlrni_bu_h, 16, B, H) +VSSRLRNUI(vssrlrni_hu_w, 32, H, W) +VSSRLRNUI(vssrlrni_wu_d, 64, W, D) + +#define VSSRARNUI(NAME, BIT, E1, E2) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E1(j + ofs * 2 * i) = do_ssrarnu_ ## E1(Vj->E2(j + ofs * i), \ + imm, BIT / 2); \ + temp.E1(j + ofs * (2 * i + 1)) = do_ssrarnu_ ## E1(Vd->E2(j + ofs * i), \ + imm, BIT / 2); \ + } \ + } \ + *Vd = temp; \ +} + +static void do_vssrarni_du_q(VReg *Vd, VReg *Vj, + uint64_t imm, int idx, Int128 mask1, Int128 mask2) +{ + Int128 shft_res1, shft_res2, r1, r2; + + if (imm == 0) { + shft_res1 = Vj->Q(idx); + shft_res2 = Vd->Q(idx); + } else { + r1 = int128_and(int128_rshift(Vj->Q(idx), (imm - 1)), int128_one()); + r2 = int128_and(int128_rshift(Vd->Q(idx), (imm - 1)), int128_one()); + shft_res1 = int128_add(int128_rshift(Vj->Q(idx), imm), r1); + shft_res2 = int128_add(int128_rshift(Vd->Q(idx), imm), r2); + } + + if (int128_lt(Vj->Q(idx), int128_zero())) { + shft_res1 = int128_zero(); + } + if (int128_lt(Vd->Q(idx), int128_zero())) { + shft_res2 = int128_zero(); + } + + if (int128_gt(shft_res1, mask1)) { + Vd->D(idx * 2) = int128_getlo(mask1); + } else if (int128_lt(shft_res1, int128_neg(mask2))) { + Vd->D(idx * 2) = int128_getlo(mask2); + } else { + Vd->D(idx * 2) = int128_getlo(shft_res1); + } + + if (int128_gt(shft_res2, mask1)) { + Vd->D(idx * 2 + 1) = int128_getlo(mask1); + } else if (int128_lt(shft_res2, int128_neg(mask2))) { + Vd->D(idx * 2 + 1) = int128_getlo(mask2); + } else { + Vd->D(idx * 2 + 1) = int128_getlo(shft_res2); + } +} + +void HELPER(vssrarni_du_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + Int128 mask1, mask2; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + mask1 = int128_sub(int128_lshift(int128_one(), 64), int128_one()); + mask2 = int128_lshift(int128_one(), 64); + + for (i = 0; i < oprsz / 16; i++) { + do_vssrarni_du_q(Vd, Vj, imm, i, mask1, mask2); + } +} + +VSSRARNUI(vssrarni_bu_h, 16, B, H) +VSSRARNUI(vssrarni_hu_w, 32, H, W) +VSSRARNUI(vssrarni_wu_d, 64, W, D) + +#define DO_2OP(NAME, BIT, E, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) \ + { \ + Vd->E(i) = DO_OP(Vj->E(i)); \ + } \ +} + +DO_2OP(vclo_b, 8, UB, DO_CLO_B) +DO_2OP(vclo_h, 16, UH, DO_CLO_H) +DO_2OP(vclo_w, 32, UW, DO_CLO_W) +DO_2OP(vclo_d, 64, UD, DO_CLO_D) +DO_2OP(vclz_b, 8, UB, DO_CLZ_B) +DO_2OP(vclz_h, 16, UH, DO_CLZ_H) +DO_2OP(vclz_w, 32, UW, DO_CLZ_W) +DO_2OP(vclz_d, 64, UD, DO_CLZ_D) + +#define VPCNT(NAME, BIT, E, FN) \ +void HELPER(NAME)(void *vd, void *vj, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) \ + { \ + Vd->E(i) = FN(Vj->E(i)); \ + } \ +} + +VPCNT(vpcnt_b, 8, UB, ctpop8) +VPCNT(vpcnt_h, 16, UH, ctpop16) +VPCNT(vpcnt_w, 32, UW, ctpop32) +VPCNT(vpcnt_d, 64, UD, ctpop64) + +#define DO_BIT(NAME, BIT, E, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = DO_OP(Vj->E(i), Vk->E(i)%BIT); \ + } \ +} + +DO_BIT(vbitclr_b, 8, UB, DO_BITCLR) +DO_BIT(vbitclr_h, 16, UH, DO_BITCLR) +DO_BIT(vbitclr_w, 32, UW, DO_BITCLR) +DO_BIT(vbitclr_d, 64, UD, DO_BITCLR) +DO_BIT(vbitset_b, 8, UB, DO_BITSET) +DO_BIT(vbitset_h, 16, UH, DO_BITSET) +DO_BIT(vbitset_w, 32, UW, DO_BITSET) +DO_BIT(vbitset_d, 64, UD, DO_BITSET) +DO_BIT(vbitrev_b, 8, UB, DO_BITREV) +DO_BIT(vbitrev_h, 16, UH, DO_BITREV) +DO_BIT(vbitrev_w, 32, UW, DO_BITREV) +DO_BIT(vbitrev_d, 64, UD, DO_BITREV) + +#define DO_BITI(NAME, BIT, E, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = DO_OP(Vj->E(i), imm); \ + } \ +} + +DO_BITI(vbitclri_b, 8, UB, DO_BITCLR) +DO_BITI(vbitclri_h, 16, UH, DO_BITCLR) +DO_BITI(vbitclri_w, 32, UW, DO_BITCLR) +DO_BITI(vbitclri_d, 64, UD, DO_BITCLR) +DO_BITI(vbitseti_b, 8, UB, DO_BITSET) +DO_BITI(vbitseti_h, 16, UH, DO_BITSET) +DO_BITI(vbitseti_w, 32, UW, DO_BITSET) +DO_BITI(vbitseti_d, 64, UD, DO_BITSET) +DO_BITI(vbitrevi_b, 8, UB, DO_BITREV) +DO_BITI(vbitrevi_h, 16, UH, DO_BITREV) +DO_BITI(vbitrevi_w, 32, UW, DO_BITREV) +DO_BITI(vbitrevi_d, 64, UD, DO_BITREV) + +#define VFRSTP(NAME, BIT, MASK, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, m, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + m = Vk->E(i * ofs) & MASK; \ + for (j = 0; j < ofs; j++) { \ + if (Vj->E(j + ofs * i) < 0) { \ + break; \ + } \ + } \ + Vd->E(m + i * ofs) = j; \ + } \ +} + +VFRSTP(vfrstp_b, 8, 0xf, B) +VFRSTP(vfrstp_h, 16, 0x7, H) + +#define VFRSTPI(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, m, ofs; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + m = imm % ofs; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + if (Vj->E(j + ofs * i) < 0) { \ + break; \ + } \ + } \ + Vd->E(m + i * ofs) = j; \ + } \ +} + +VFRSTPI(vfrstpi_b, 8, B) +VFRSTPI(vfrstpi_h, 16, H) + +static void vec_update_fcsr0_mask(CPULoongArchState *env, + uintptr_t pc, int mask) +{ + int flags = get_float_exception_flags(&env->fp_status); + + set_float_exception_flags(0, &env->fp_status); + + flags &= ~mask; + + if (flags) { + flags = ieee_ex_to_loongarch(flags); + UPDATE_FP_CAUSE(env->fcsr0, flags); + } + + if (GET_FP_ENABLES(env->fcsr0) & flags) { + do_raise_exception(env, EXCCODE_FPE, pc); + } else { + UPDATE_FP_FLAGS(env->fcsr0, flags); + } +} + +static void vec_update_fcsr0(CPULoongArchState *env, uintptr_t pc) +{ + vec_update_fcsr0_mask(env, pc, 0); +} + +static inline void vec_clear_cause(CPULoongArchState *env) +{ + SET_FP_CAUSE(env->fcsr0, 0); +} + +#define DO_3OP_F(NAME, BIT, E, FN) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, \ + CPULoongArchState *env, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + vec_clear_cause(env); \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = FN(Vj->E(i), Vk->E(i), &env->fp_status); \ + vec_update_fcsr0(env, GETPC()); \ + } \ +} + +DO_3OP_F(vfadd_s, 32, UW, float32_add) +DO_3OP_F(vfadd_d, 64, UD, float64_add) +DO_3OP_F(vfsub_s, 32, UW, float32_sub) +DO_3OP_F(vfsub_d, 64, UD, float64_sub) +DO_3OP_F(vfmul_s, 32, UW, float32_mul) +DO_3OP_F(vfmul_d, 64, UD, float64_mul) +DO_3OP_F(vfdiv_s, 32, UW, float32_div) +DO_3OP_F(vfdiv_d, 64, UD, float64_div) +DO_3OP_F(vfmax_s, 32, UW, float32_maxnum) +DO_3OP_F(vfmax_d, 64, UD, float64_maxnum) +DO_3OP_F(vfmin_s, 32, UW, float32_minnum) +DO_3OP_F(vfmin_d, 64, UD, float64_minnum) +DO_3OP_F(vfmaxa_s, 32, UW, float32_maxnummag) +DO_3OP_F(vfmaxa_d, 64, UD, float64_maxnummag) +DO_3OP_F(vfmina_s, 32, UW, float32_minnummag) +DO_3OP_F(vfmina_d, 64, UD, float64_minnummag) + +#define DO_4OP_F(NAME, BIT, E, FN, flags) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, void *va, \ + CPULoongArchState *env, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + VReg *Va = (VReg *)va; \ + int oprsz = simd_oprsz(desc); \ + \ + vec_clear_cause(env); \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = FN(Vj->E(i), Vk->E(i), Va->E(i), flags, &env->fp_status); \ + vec_update_fcsr0(env, GETPC()); \ + } \ +} + +DO_4OP_F(vfmadd_s, 32, UW, float32_muladd, 0) +DO_4OP_F(vfmadd_d, 64, UD, float64_muladd, 0) +DO_4OP_F(vfmsub_s, 32, UW, float32_muladd, float_muladd_negate_c) +DO_4OP_F(vfmsub_d, 64, UD, float64_muladd, float_muladd_negate_c) +DO_4OP_F(vfnmadd_s, 32, UW, float32_muladd, float_muladd_negate_result) +DO_4OP_F(vfnmadd_d, 64, UD, float64_muladd, float_muladd_negate_result) +DO_4OP_F(vfnmsub_s, 32, UW, float32_muladd, + float_muladd_negate_c | float_muladd_negate_result) +DO_4OP_F(vfnmsub_d, 64, UD, float64_muladd, + float_muladd_negate_c | float_muladd_negate_result) + +#define DO_2OP_F(NAME, BIT, E, FN) \ +void HELPER(NAME)(void *vd, void *vj, \ + CPULoongArchState *env, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + vec_clear_cause(env); \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = FN(env, Vj->E(i)); \ + } \ +} + +#define FLOGB(BIT, T) \ +static T do_flogb_## BIT(CPULoongArchState *env, T fj) \ +{ \ + T fp, fd; \ + float_status *status = &env->fp_status; \ + FloatRoundMode old_mode = get_float_rounding_mode(status); \ + \ + set_float_rounding_mode(float_round_down, status); \ + fp = float ## BIT ##_log2(fj, status); \ + fd = float ## BIT ##_round_to_int(fp, status); \ + set_float_rounding_mode(old_mode, status); \ + vec_update_fcsr0_mask(env, GETPC(), float_flag_inexact); \ + return fd; \ +} + +FLOGB(32, uint32_t) +FLOGB(64, uint64_t) + +#define FCLASS(NAME, BIT, E, FN) \ +void HELPER(NAME)(void *vd, void *vj, \ + CPULoongArchState *env, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = FN(env, Vj->E(i)); \ + } \ +} + +FCLASS(vfclass_s, 32, UW, helper_fclass_s) +FCLASS(vfclass_d, 64, UD, helper_fclass_d) + +#define FSQRT(BIT, T) \ +static T do_fsqrt_## BIT(CPULoongArchState *env, T fj) \ +{ \ + T fd; \ + fd = float ## BIT ##_sqrt(fj, &env->fp_status); \ + vec_update_fcsr0(env, GETPC()); \ + return fd; \ +} + +FSQRT(32, uint32_t) +FSQRT(64, uint64_t) + +#define FRECIP(BIT, T) \ +static T do_frecip_## BIT(CPULoongArchState *env, T fj) \ +{ \ + T fd; \ + fd = float ## BIT ##_div(float ## BIT ##_one, fj, &env->fp_status); \ + vec_update_fcsr0(env, GETPC()); \ + return fd; \ +} + +FRECIP(32, uint32_t) +FRECIP(64, uint64_t) + +#define FRSQRT(BIT, T) \ +static T do_frsqrt_## BIT(CPULoongArchState *env, T fj) \ +{ \ + T fd, fp; \ + fp = float ## BIT ##_sqrt(fj, &env->fp_status); \ + fd = float ## BIT ##_div(float ## BIT ##_one, fp, &env->fp_status); \ + vec_update_fcsr0(env, GETPC()); \ + return fd; \ +} + +FRSQRT(32, uint32_t) +FRSQRT(64, uint64_t) + +DO_2OP_F(vflogb_s, 32, UW, do_flogb_32) +DO_2OP_F(vflogb_d, 64, UD, do_flogb_64) +DO_2OP_F(vfsqrt_s, 32, UW, do_fsqrt_32) +DO_2OP_F(vfsqrt_d, 64, UD, do_fsqrt_64) +DO_2OP_F(vfrecip_s, 32, UW, do_frecip_32) +DO_2OP_F(vfrecip_d, 64, UD, do_frecip_64) +DO_2OP_F(vfrsqrt_s, 32, UW, do_frsqrt_32) +DO_2OP_F(vfrsqrt_d, 64, UD, do_frsqrt_64) + +static uint32_t float16_cvt_float32(uint16_t h, float_status *status) +{ + return float16_to_float32(h, true, status); +} +static uint64_t float32_cvt_float64(uint32_t s, float_status *status) +{ + return float32_to_float64(s, status); +} + +static uint16_t float32_cvt_float16(uint32_t s, float_status *status) +{ + return float32_to_float16(s, true, status); +} +static uint32_t float64_cvt_float32(uint64_t d, float_status *status) +{ + return float64_to_float32(d, status); +} + +void HELPER(vfcvtl_s_h)(void *vd, void *vj, + CPULoongArchState *env, uint32_t desc) +{ + int i, j, ofs; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + ofs = LSX_LEN / 32; + vec_clear_cause(env); + for (i = 0; i < oprsz / 16; i++) { + for (j = 0; j < ofs; j++) { + temp.UW(j + ofs * i) =float16_cvt_float32(Vj->UH(j + ofs * 2 * i), + &env->fp_status); + } + vec_update_fcsr0(env, GETPC()); + } + *Vd = temp; +} + +void HELPER(vfcvtl_d_s)(void *vd, void *vj, + CPULoongArchState *env, uint32_t desc) +{ + int i, j, ofs; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + ofs = LSX_LEN / 64; + vec_clear_cause(env); + for (i = 0; i < oprsz / 16; i++) { + for (j = 0; j < ofs; j++) { + temp.UD(j + ofs * i) = float32_cvt_float64(Vj->UW(j + ofs * 2 * i), + &env->fp_status); + } + vec_update_fcsr0(env, GETPC()); + } + *Vd = temp; +} + +void HELPER(vfcvth_s_h)(void *vd, void *vj, + CPULoongArchState *env, uint32_t desc) +{ + int i, j, ofs; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + ofs = LSX_LEN / 32; + vec_clear_cause(env); + for (i = 0; i < oprsz / 16; i++) { + for (j = 0; j < ofs; j++) { + temp.UW(j + ofs * i) = float16_cvt_float32(Vj->UH(j + ofs * (2 * i + 1)), + &env->fp_status); + } + vec_update_fcsr0(env, GETPC()); + } + *Vd = temp; +} + +void HELPER(vfcvth_d_s)(void *vd, void *vj, + CPULoongArchState *env, uint32_t desc) +{ + int i, j, ofs; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + ofs = LSX_LEN / 64; + vec_clear_cause(env); + for (i = 0; i < oprsz / 16; i++) { + for (j = 0; j < ofs; j++) { + temp.UD(j + ofs * i) = float32_cvt_float64(Vj->UW(j + ofs * (2 * i + 1)), + &env->fp_status); + } + vec_update_fcsr0(env, GETPC()); + } + *Vd = temp; +} + +void HELPER(vfcvt_h_s)(void *vd, void *vj, void *vk, + CPULoongArchState *env, uint32_t desc) +{ + int i, j, ofs; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + ofs = LSX_LEN / 32; + vec_clear_cause(env); + for(i = 0; i < oprsz / 16; i++) { + for (j = 0; j < ofs; j++) { + temp.UH(j + ofs * (2 * i + 1)) = float32_cvt_float16(Vj->UW(j + ofs * i), + &env->fp_status); + temp.UH(j + ofs * 2 * i) = float32_cvt_float16(Vk->UW(j + ofs * i), + &env->fp_status); + } + vec_update_fcsr0(env, GETPC()); + } + *Vd = temp; +} + +void HELPER(vfcvt_s_d)(void *vd, void *vj, void *vk, + CPULoongArchState *env, uint32_t desc) +{ + int i, j, ofs; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + ofs = LSX_LEN / 64; + vec_clear_cause(env); + for(i = 0; i < oprsz / 16; i++) { + for (j = 0; j < ofs; j++) { + temp.UW(j + ofs * (2 * i + 1)) = float64_cvt_float32(Vj->UD(j + ofs * i), + &env->fp_status); + temp.UW(j + ofs * 2 * i) = float64_cvt_float32(Vk->UD(j + ofs * i), + &env->fp_status); + } + vec_update_fcsr0(env, GETPC()); + } + *Vd = temp; +} + +void HELPER(vfrint_s)(void *vd, void *vj, + CPULoongArchState *env, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + vec_clear_cause(env); + for (i = 0; i < oprsz / 4; i++) { + Vd->W(i) = float32_round_to_int(Vj->UW(i), &env->fp_status); + vec_update_fcsr0(env, GETPC()); + } +} + +void HELPER(vfrint_d)(void *vd, void *vj, + CPULoongArchState *env, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + vec_clear_cause(env); + for (i = 0; i < oprsz / 8; i++) { + Vd->D(i) = float64_round_to_int(Vj->UD(i), &env->fp_status); + vec_update_fcsr0(env, GETPC()); + } +} + +#define FCVT_2OP(NAME, BIT, E, MODE) \ +void HELPER(NAME)(void *vd, void *vj, \ + CPULoongArchState *env, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + vec_clear_cause(env); \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + FloatRoundMode old_mode = get_float_rounding_mode(&env->fp_status); \ + set_float_rounding_mode(MODE, &env->fp_status); \ + Vd->E(i) = float## BIT ## _round_to_int(Vj->E(i), &env->fp_status); \ + set_float_rounding_mode(old_mode, &env->fp_status); \ + vec_update_fcsr0(env, GETPC()); \ + } \ +} + +FCVT_2OP(vfrintrne_s, 32, UW, float_round_nearest_even) +FCVT_2OP(vfrintrne_d, 64, UD, float_round_nearest_even) +FCVT_2OP(vfrintrz_s, 32, UW, float_round_to_zero) +FCVT_2OP(vfrintrz_d, 64, UD, float_round_to_zero) +FCVT_2OP(vfrintrp_s, 32, UW, float_round_up) +FCVT_2OP(vfrintrp_d, 64, UD, float_round_up) +FCVT_2OP(vfrintrm_s, 32, UW, float_round_down) +FCVT_2OP(vfrintrm_d, 64, UD, float_round_down) + +#define FTINT(NAME, FMT1, FMT2, T1, T2, MODE) \ +static T2 do_ftint ## NAME(CPULoongArchState *env, T1 fj) \ +{ \ + T2 fd; \ + FloatRoundMode old_mode = get_float_rounding_mode(&env->fp_status); \ + \ + set_float_rounding_mode(MODE, &env->fp_status); \ + fd = do_## FMT1 ##_to_## FMT2(env, fj); \ + set_float_rounding_mode(old_mode, &env->fp_status); \ + return fd; \ +} + +#define DO_FTINT(FMT1, FMT2, T1, T2) \ +static T2 do_## FMT1 ##_to_## FMT2(CPULoongArchState *env, T1 fj) \ +{ \ + T2 fd; \ + \ + fd = FMT1 ##_to_## FMT2(fj, &env->fp_status); \ + if (get_float_exception_flags(&env->fp_status) & (float_flag_invalid)) { \ + if (FMT1 ##_is_any_nan(fj)) { \ + fd = 0; \ + } \ + } \ + vec_update_fcsr0(env, GETPC()); \ + return fd; \ +} + +DO_FTINT(float32, int32, uint32_t, uint32_t) +DO_FTINT(float64, int64, uint64_t, uint64_t) +DO_FTINT(float32, uint32, uint32_t, uint32_t) +DO_FTINT(float64, uint64, uint64_t, uint64_t) +DO_FTINT(float64, int32, uint64_t, uint32_t) +DO_FTINT(float32, int64, uint32_t, uint64_t) + +FTINT(rne_w_s, float32, int32, uint32_t, uint32_t, float_round_nearest_even) +FTINT(rne_l_d, float64, int64, uint64_t, uint64_t, float_round_nearest_even) +FTINT(rp_w_s, float32, int32, uint32_t, uint32_t, float_round_up) +FTINT(rp_l_d, float64, int64, uint64_t, uint64_t, float_round_up) +FTINT(rz_w_s, float32, int32, uint32_t, uint32_t, float_round_to_zero) +FTINT(rz_l_d, float64, int64, uint64_t, uint64_t, float_round_to_zero) +FTINT(rm_w_s, float32, int32, uint32_t, uint32_t, float_round_down) +FTINT(rm_l_d, float64, int64, uint64_t, uint64_t, float_round_down) + +DO_2OP_F(vftintrne_w_s, 32, UW, do_ftintrne_w_s) +DO_2OP_F(vftintrne_l_d, 64, UD, do_ftintrne_l_d) +DO_2OP_F(vftintrp_w_s, 32, UW, do_ftintrp_w_s) +DO_2OP_F(vftintrp_l_d, 64, UD, do_ftintrp_l_d) +DO_2OP_F(vftintrz_w_s, 32, UW, do_ftintrz_w_s) +DO_2OP_F(vftintrz_l_d, 64, UD, do_ftintrz_l_d) +DO_2OP_F(vftintrm_w_s, 32, UW, do_ftintrm_w_s) +DO_2OP_F(vftintrm_l_d, 64, UD, do_ftintrm_l_d) +DO_2OP_F(vftint_w_s, 32, UW, do_float32_to_int32) +DO_2OP_F(vftint_l_d, 64, UD, do_float64_to_int64) + +FTINT(rz_wu_s, float32, uint32, uint32_t, uint32_t, float_round_to_zero) +FTINT(rz_lu_d, float64, uint64, uint64_t, uint64_t, float_round_to_zero) + +DO_2OP_F(vftintrz_wu_s, 32, UW, do_ftintrz_wu_s) +DO_2OP_F(vftintrz_lu_d, 64, UD, do_ftintrz_lu_d) +DO_2OP_F(vftint_wu_s, 32, UW, do_float32_to_uint32) +DO_2OP_F(vftint_lu_d, 64, UD, do_float64_to_uint64) + +FTINT(rm_w_d, float64, int32, uint64_t, uint32_t, float_round_down) +FTINT(rp_w_d, float64, int32, uint64_t, uint32_t, float_round_up) +FTINT(rz_w_d, float64, int32, uint64_t, uint32_t, float_round_to_zero) +FTINT(rne_w_d, float64, int32, uint64_t, uint32_t, float_round_nearest_even) + +#define FTINT_W_D(NAME, FN) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, \ + CPULoongArchState *env, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / 64; \ + vec_clear_cause(env); \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.W(j + ofs * (2 * i + 1)) = FN(env, Vj->UD(j + ofs * i)); \ + temp.W(j + ofs * 2 * i) = FN(env, Vk->UD(j + ofs * i)); \ + } \ + } \ + *Vd = temp; \ +} + +FTINT_W_D(vftint_w_d, do_float64_to_int32) +FTINT_W_D(vftintrm_w_d, do_ftintrm_w_d) +FTINT_W_D(vftintrp_w_d, do_ftintrp_w_d) +FTINT_W_D(vftintrz_w_d, do_ftintrz_w_d) +FTINT_W_D(vftintrne_w_d, do_ftintrne_w_d) + +FTINT(rml_l_s, float32, int64, uint32_t, uint64_t, float_round_down) +FTINT(rpl_l_s, float32, int64, uint32_t, uint64_t, float_round_up) +FTINT(rzl_l_s, float32, int64, uint32_t, uint64_t, float_round_to_zero) +FTINT(rnel_l_s, float32, int64, uint32_t, uint64_t, float_round_nearest_even) +FTINT(rmh_l_s, float32, int64, uint32_t, uint64_t, float_round_down) +FTINT(rph_l_s, float32, int64, uint32_t, uint64_t, float_round_up) +FTINT(rzh_l_s, float32, int64, uint32_t, uint64_t, float_round_to_zero) +FTINT(rneh_l_s, float32, int64, uint32_t, uint64_t, float_round_nearest_even) + +#define FTINTL_L_S(NAME, FN) \ +void HELPER(NAME)(void *vd, void *vj, \ + CPULoongArchState *env, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / 64; \ + vec_clear_cause(env); \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.D(j + ofs * i) = FN(env, Vj->UW(j + ofs * 2 * i)); \ + } \ + } \ + *Vd = temp; \ +} + +FTINTL_L_S(vftintl_l_s, do_float32_to_int64) +FTINTL_L_S(vftintrml_l_s, do_ftintrml_l_s) +FTINTL_L_S(vftintrpl_l_s, do_ftintrpl_l_s) +FTINTL_L_S(vftintrzl_l_s, do_ftintrzl_l_s) +FTINTL_L_S(vftintrnel_l_s, do_ftintrnel_l_s) + +#define FTINTH_L_S(NAME, FN) \ +void HELPER(NAME)(void *vd, void *vj, \ + CPULoongArchState *env, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / 64; \ + vec_clear_cause(env); \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.D(j + ofs * i) = FN(env, Vj->UW(j + ofs * (2 * i + 1))); \ + } \ + } \ + *Vd = temp; \ +} + +FTINTH_L_S(vftinth_l_s, do_float32_to_int64) +FTINTH_L_S(vftintrmh_l_s, do_ftintrmh_l_s) +FTINTH_L_S(vftintrph_l_s, do_ftintrph_l_s) +FTINTH_L_S(vftintrzh_l_s, do_ftintrzh_l_s) +FTINTH_L_S(vftintrneh_l_s, do_ftintrneh_l_s) + +#define FFINT(NAME, FMT1, FMT2, T1, T2) \ +static T2 do_ffint_ ## NAME(CPULoongArchState *env, T1 fj) \ +{ \ + T2 fd; \ + \ + fd = FMT1 ##_to_## FMT2(fj, &env->fp_status); \ + vec_update_fcsr0(env, GETPC()); \ + return fd; \ +} + +FFINT(s_w, int32, float32, int32_t, uint32_t) +FFINT(d_l, int64, float64, int64_t, uint64_t) +FFINT(s_wu, uint32, float32, uint32_t, uint32_t) +FFINT(d_lu, uint64, float64, uint64_t, uint64_t) + +DO_2OP_F(vffint_s_w, 32, W, do_ffint_s_w) +DO_2OP_F(vffint_d_l, 64, D, do_ffint_d_l) +DO_2OP_F(vffint_s_wu, 32, UW, do_ffint_s_wu) +DO_2OP_F(vffint_d_lu, 64, UD, do_ffint_d_lu) + +void HELPER(vffintl_d_w)(void *vd, void *vj, + CPULoongArchState *env, uint32_t desc) +{ + int i, j, ofs; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + ofs = LSX_LEN / 64; + vec_clear_cause(env); + for (i = 0; i < oprsz / 16; i++) { + for (j = 0; j < ofs; j++) { + temp.D(j + ofs * i) = int32_to_float64(Vj->W(j + ofs * 2 * i), + &env->fp_status); + } + vec_update_fcsr0(env, GETPC()); + } + *Vd = temp; +} + +void HELPER(vffinth_d_w)(void *vd, void *vj, + CPULoongArchState *env, uint32_t desc) +{ + int i, j, ofs; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + ofs = LSX_LEN / 64; + vec_clear_cause(env); + for (i = 0; i < oprsz /16; i++) { + for (j = 0; j < ofs; j++) { + temp.D(j + ofs * i) = int32_to_float64(Vj->W(j + ofs * (2 * i + 1)), + &env->fp_status); + } + vec_update_fcsr0(env, GETPC()); + } + *Vd = temp; +} + +void HELPER(vffint_s_l)(void *vd, void *vj, void *vk, + CPULoongArchState *env, uint32_t desc) +{ + int i, j, ofs; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + int oprsz = simd_oprsz(desc); + + ofs = LSX_LEN / 64; + vec_clear_cause(env); + for (i = 0; i < oprsz / 16; i++) { + for (j = 0; j < ofs; j++) { + temp.W(j + ofs * (2 * i + 1)) = int64_to_float32(Vj->D(j + ofs * i), + &env->fp_status); + temp.W(j + ofs * 2 * i) = int64_to_float32(Vk->D(j + ofs * i), + &env->fp_status); + } + vec_update_fcsr0(env, GETPC()); + } + *Vd = temp; +} + +#define VCMPI(NAME, BIT, E, DO_OP) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + typedef __typeof(Vd->E(0)) TD; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = DO_OP(Vj->E(i), (TD)imm); \ + } \ +} + +VCMPI(vseqi_b, 8, B, VSEQ) +VCMPI(vseqi_h, 16, H, VSEQ) +VCMPI(vseqi_w, 32, W, VSEQ) +VCMPI(vseqi_d, 64, D, VSEQ) +VCMPI(vslei_b, 8, B, VSLE) +VCMPI(vslei_h, 16, H, VSLE) +VCMPI(vslei_w, 32, W, VSLE) +VCMPI(vslei_d, 64, D, VSLE) +VCMPI(vslei_bu, 8, UB, VSLE) +VCMPI(vslei_hu, 16, UH, VSLE) +VCMPI(vslei_wu, 32, UW, VSLE) +VCMPI(vslei_du, 64, UD, VSLE) +VCMPI(vslti_b, 8, B, VSLT) +VCMPI(vslti_h, 16, H, VSLT) +VCMPI(vslti_w, 32, W, VSLT) +VCMPI(vslti_d, 64, D, VSLT) +VCMPI(vslti_bu, 8, UB, VSLT) +VCMPI(vslti_hu, 16, UH, VSLT) +VCMPI(vslti_wu, 32, UW, VSLT) +VCMPI(vslti_du, 64, UD, VSLT) + +static uint64_t vfcmp_common(CPULoongArchState *env, + FloatRelation cmp, uint32_t flags) +{ + uint64_t ret = 0; + + switch (cmp) { + case float_relation_less: + ret = (flags & FCMP_LT); + break; + case float_relation_equal: + ret = (flags & FCMP_EQ); + break; + case float_relation_greater: + ret = (flags & FCMP_GT); + break; + case float_relation_unordered: + ret = (flags & FCMP_UN); + break; + default: + g_assert_not_reached(); + } + + if (ret) { + ret = -1; + } + + return ret; +} + +#define VFCMP(NAME, BIT, E, FN) \ +void HELPER(NAME)(CPULoongArchState *env, uint32_t oprsz, \ + uint32_t vd, uint32_t vj, uint32_t vk, uint32_t flags) \ +{ \ + int i; \ + VReg t; \ + VReg *Vd = &(env->fpr[vd].vreg); \ + VReg *Vj = &(env->fpr[vj].vreg); \ + VReg *Vk = &(env->fpr[vk].vreg); \ + \ + vec_clear_cause(env); \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + FloatRelation cmp; \ + cmp = FN(Vj->E(i), Vk->E(i), &env->fp_status); \ + t.E(i) = vfcmp_common(env, cmp, flags); \ + vec_update_fcsr0(env, GETPC()); \ + } \ + *Vd = t; \ +} + +VFCMP(vfcmp_c_s, 32, UW, float32_compare_quiet) +VFCMP(vfcmp_s_s, 32, UW, float32_compare) +VFCMP(vfcmp_c_d, 64, UD, float64_compare_quiet) +VFCMP(vfcmp_s_d, 64, UD, float64_compare) + +void HELPER(vbitseli_b)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + + for (i = 0; i < simd_oprsz(desc); i++) { + Vd->B(i) = (~Vd->B(i) & Vj->B(i)) | (Vd->B(i) & imm); + } +} + +/* Copy from target/arm/tcg/sve_helper.c */ +static inline bool do_match2(uint64_t n, uint64_t m0, uint64_t m1, int esz) +{ + int bits = 8 << esz; + uint64_t ones = dup_const(esz, 1); + uint64_t signs = ones << (bits - 1); + uint64_t cmp0, cmp1; + + cmp1 = dup_const(esz, n); + cmp0 = cmp1 ^ m0; + cmp1 = cmp1 ^ m1; + cmp0 = (cmp0 - ones) & ~cmp0; + cmp1 = (cmp1 - ones) & ~cmp1; + return (cmp0 | cmp1) & signs; +} + +#define SETANYEQZ(NAME, MO) \ +void HELPER(NAME)(CPULoongArchState *env, \ + uint32_t oprsz, uint32_t cd, uint32_t vj) \ +{ \ + VReg *Vj = &(env->fpr[vj].vreg); \ + \ + env->cf[cd & 0x7] = do_match2(0, Vj->D(0), Vj->D(1), MO); \ + if (oprsz == 32) { \ + env->cf[cd & 0x7] = env->cf[cd & 0x7] || \ + do_match2(0, Vj->D(2), Vj->D(3), MO); \ + } \ +} + +SETANYEQZ(vsetanyeqz_b, MO_8) +SETANYEQZ(vsetanyeqz_h, MO_16) +SETANYEQZ(vsetanyeqz_w, MO_32) +SETANYEQZ(vsetanyeqz_d, MO_64) + +#define SETALLNEZ(NAME, MO) \ +void HELPER(NAME)(CPULoongArchState *env, \ + uint32_t oprsz, uint32_t cd, uint32_t vj) \ +{ \ + VReg *Vj = &(env->fpr[vj].vreg); \ + \ + env->cf[cd & 0x7]= !do_match2(0, Vj->D(0), Vj->D(1), MO); \ + if (oprsz == 32) { \ + env->cf[cd & 0x7] = env->cf[cd & 0x7] && \ + !do_match2(0, Vj->D(2), Vj->D(3), MO); \ + } \ +} + +SETALLNEZ(vsetallnez_b, MO_8) +SETALLNEZ(vsetallnez_h, MO_16) +SETALLNEZ(vsetallnez_w, MO_32) +SETALLNEZ(vsetallnez_d, MO_64) + +#define XVINSVE0(NAME, E, MASK) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + Vd->E(imm & MASK) = Vj->E(0); \ +} + +XVINSVE0(xvinsve0_w, W, 0x7) +XVINSVE0(xvinsve0_d, D, 0x3) + +#define XVPICKVE(NAME, E, BIT, MASK) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + Vd->E(0) = Vj->E(imm & MASK); \ + for (i = 1; i < oprsz / (BIT / 8); i++) { \ + Vd->E(i) = 0; \ + } \ +} + +XVPICKVE(xvpickve_w, W, 32, 0x7) +XVPICKVE(xvpickve_d, D, 64, 0x3) + +#define VPACKEV(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + temp.E(2 * i + 1) = Vj->E(2 * i); \ + temp.E(2 *i) = Vk->E(2 * i); \ + } \ + *Vd = temp; \ +} + +VPACKEV(vpackev_b, 16, B) +VPACKEV(vpackev_h, 32, H) +VPACKEV(vpackev_w, 64, W) +VPACKEV(vpackev_d, 128, D) + +#define VPACKOD(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + temp.E(2 * i + 1) = Vj->E(2 * i + 1); \ + temp.E(2 * i) = Vk->E(2 * i + 1); \ + } \ + *Vd = temp; \ +} + +VPACKOD(vpackod_b, 16, B) +VPACKOD(vpackod_h, 32, H) +VPACKOD(vpackod_w, 64, W) +VPACKOD(vpackod_d, 128, D) + +#define VPICKEV(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E(j + ofs * (2 * i + 1)) = Vj->E(2 * (j + ofs * i)); \ + temp.E(j + ofs * 2 * i) = Vk->E(2 * (j + ofs * i)); \ + } \ + } \ + *Vd = temp; \ +} + +VPICKEV(vpickev_b, 16, B) +VPICKEV(vpickev_h, 32, H) +VPICKEV(vpickev_w, 64, W) +VPICKEV(vpickev_d, 128, D) + +#define VPICKOD(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E(j + ofs * (2 * i + 1)) = Vj->E(2 * (j + ofs * i) + 1); \ + temp.E(j + ofs * 2 * i) = Vk->E(2 * (j + ofs * i) + 1); \ + } \ + } \ + *Vd = temp; \ +} + +VPICKOD(vpickod_b, 16, B) +VPICKOD(vpickod_h, 32, H) +VPICKOD(vpickod_w, 64, W) +VPICKOD(vpickod_d, 128, D) + +#define VILVL(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E(2 * (j + ofs * i) + 1) = Vj->E(j + ofs * 2 * i); \ + temp.E(2 * (j + ofs * i)) = Vk->E(j + ofs * 2 * i); \ + } \ + } \ + *Vd = temp; \ +} + +VILVL(vilvl_b, 16, B) +VILVL(vilvl_h, 32, H) +VILVL(vilvl_w, 64, W) +VILVL(vilvl_d, 128, D) + +#define VILVH(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, ofs; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + ofs = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / 16; i++) { \ + for (j = 0; j < ofs; j++) { \ + temp.E(2 * (j + ofs * i) + 1) = Vj->E(j + ofs * (2 * i + 1)); \ + temp.E(2 * (j + ofs * i)) = Vk->E(j + ofs * (2 * i + 1)); \ + } \ + } \ + *Vd = temp; \ +} + +VILVH(vilvh_b, 16, B) +VILVH(vilvh_h, 32, H) +VILVH(vilvh_w, 64, W) +VILVH(vilvh_d, 128, D) + +void HELPER(vshuf_b)(void *vd, void *vj, void *vk, void *va, uint32_t desc) +{ + int i, j, m; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + VReg *Va = (VReg *)va; + int oprsz = simd_oprsz(desc); + + m = LSX_LEN / 8; + for (i = 0; i < (oprsz / 16) * m; i++) { + j = i < m ? 0 : 1; + uint64_t k = (uint8_t)Va->B(i) % (2 * m); + temp.B(i) = k < m ? Vk->B(k + j * m): Vj->B(k + (j - 1) * m); + } + *Vd = temp; +} + +#define VSHUF(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, void *vk, uint32_t desc) \ +{ \ + int i, j, m; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + VReg *Vk = (VReg *)vk; \ + int oprsz = simd_oprsz(desc); \ + \ + m = LSX_LEN / BIT; \ + for (i = 0; i < (oprsz / 16) * m; i++) { \ + j = i < m ? 0 : 1; \ + uint64_t k = ((uint8_t)Vd->E(i)) % (2 * m); \ + temp.E(i) = k < m ? Vk->E(k + j * m) : Vj->E(k + (j - 1) * m); \ + } \ + *Vd = temp; \ +} + +VSHUF(vshuf_h, 16, H) +VSHUF(vshuf_w, 32, W) +VSHUF(vshuf_d, 64, D) + +#define VSHUF4I(NAME, BIT, E) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, j, max; \ + VReg temp = {}; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + max = LSX_LEN / BIT; \ + for (i = 0; i < oprsz / (BIT / 8); i++) { \ + j = i < max ? 1 : 2; \ + temp.E(i) = Vj->E(SHF_POS(i - ((j -1)* max), imm) + (j - 1) * max); \ + } \ + *Vd = temp; \ +} + +VSHUF4I(vshuf4i_b, 8, B) +VSHUF4I(vshuf4i_h, 16, H) +VSHUF4I(vshuf4i_w, 32, W) + +void HELPER(vshuf4i_d)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + temp.D(2 * i) = (imm & 2 ? Vj : Vd)->D((imm & 1) + 2 * i); + temp.D(2 * i + 1) = (imm & 8 ? Vj : Vd)->D(((imm >> 2) & 1) + 2 * i); + } + *Vd = temp; +} + +void HELPER(vperm_w)(void *vd, void *vj, void *vk, uint32_t desc) +{ + int i, m; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + VReg *Vk = (VReg *)vk; + + m = LASX_LEN / 32; + for (i = 0; i < m ; i++) { + uint64_t k = (uint8_t)Vk->W(i) % 8; + temp.W(i) = Vj->W(k); + } + *Vd = temp; +} + +void HELPER(vpermi_w)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + int oprsz = simd_oprsz(desc); + + for (i = 0; i < oprsz / 16; i++) { + temp.W(4 * i) = Vj->W((imm & 0x3) + 4 * i); + temp.W(4 * i + 1) = Vj->W(((imm >> 2) & 0x3) + 4 * i); + temp.W(4 * i + 2) = Vd->W(((imm >> 4) & 0x3) + 4 * i); + temp.W(4 * i + 3) = Vd->W(((imm >> 6) & 0x3) + 4 * i); + } + *Vd = temp; +} + +void HELPER(vpermi_d)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + VReg temp = {}; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + + temp.D(0) = Vj->D(imm & 0x3); + temp.D(1) = Vj->D((imm >> 2) & 0x3); + temp.D(2) = Vj->D((imm >> 4) & 0x3); + temp.D(3) = Vj->D((imm >> 6) & 0x3); + *Vd = temp; +} + +void HELPER(vpermi_q)(void *vd, void *vj, uint64_t imm, uint32_t desc) +{ + int i; + VReg temp; + VReg *Vd = (VReg *)vd; + VReg *Vj = (VReg *)vj; + + for (i = 0; i < 2; i++, imm >>= 4) { + temp.Q(i) = (imm & 2 ? Vd: Vj)->Q(imm & 1); + } + *Vd = temp; +} + +#define VEXTRINS(NAME, BIT, E, MASK) \ +void HELPER(NAME)(void *vd, void *vj, uint64_t imm, uint32_t desc) \ +{ \ + int i, ins, extr, max; \ + VReg *Vd = (VReg *)vd; \ + VReg *Vj = (VReg *)vj; \ + int oprsz = simd_oprsz(desc); \ + \ + max = LSX_LEN / BIT; \ + ins = (imm >> 4) & MASK; \ + extr = imm & MASK; \ + for (i = 0; i < oprsz / 16; i++) { \ + Vd->E(ins + i * max) = Vj->E(extr + i * max); \ + } \ +} + +VEXTRINS(vextrins_b, 8, B, 0xf) +VEXTRINS(vextrins_h, 16, H, 0x7) +VEXTRINS(vextrins_w, 32, W, 0x3) +VEXTRINS(vextrins_d, 64, D, 0x1) diff --git a/target/m68k/m68k-semi.c b/target/m68k/m68k-semi.c index 239f6e44e9..80cd8d70db 100644 --- a/target/m68k/m68k-semi.c +++ b/target/m68k/m68k-semi.c @@ -15,6 +15,10 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, see <http://www.gnu.org/licenses/>. + * + * The semihosting protocol implemented here is described in the + * libgloss sources: + * https://sourceware.org/git/?p=newlib-cygwin.git;a=blob;f=libgloss/m68k/m68k-semi.txt;hb=HEAD */ #include "qemu/osdep.h" diff --git a/target/ppc/cpu-models.h b/target/ppc/cpu-models.h index 572b5e553a..0229ef3a9a 100644 --- a/target/ppc/cpu-models.h +++ b/target/ppc/cpu-models.h @@ -44,7 +44,7 @@ enum { /* PowerPC 405 cores */ CPU_POWERPC_405D2 = 0x20010000, CPU_POWERPC_405D4 = 0x41810000, - /* PowerPC 405 microcontrolers */ + /* PowerPC 405 microcontrollers */ /* XXX: missing 0x200108a0 */ CPU_POWERPC_405CRa = 0x40110041, CPU_POWERPC_405CRb = 0x401100C5, @@ -74,7 +74,7 @@ enum { #define CPU_POWERPC_440 CPU_POWERPC_440GXf /* PowerPC 440 cores */ CPU_POWERPC_440_XILINX = 0x7ff21910, - /* PowerPC 440 microcontrolers */ + /* PowerPC 440 microcontrollers */ CPU_POWERPC_440EPa = 0x42221850, CPU_POWERPC_440EPb = 0x422218D3, CPU_POWERPC_440GPb = 0x40120440, diff --git a/target/ppc/cpu.h b/target/ppc/cpu.h index 173e4c351a..d703a5f3c6 100644 --- a/target/ppc/cpu.h +++ b/target/ppc/cpu.h @@ -428,7 +428,7 @@ FIELD(MSR, LE, MSR_LE, 1) /* PMU bits */ #define MMCR0_FC PPC_BIT(32) /* Freeze Counters */ -#define MMCR0_PMAO PPC_BIT(56) /* Perf Monitor Alert Ocurred */ +#define MMCR0_PMAO PPC_BIT(56) /* Perf Monitor Alert Occurred */ #define MMCR0_PMAE PPC_BIT(37) /* Perf Monitor Alert Enable */ #define MMCR0_EBE PPC_BIT(43) /* Perf Monitor EBB Enable */ #define MMCR0_FCECE PPC_BIT(38) /* FC on Enabled Cond or Event */ diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c index 7ab5ee92d9..c62bf0e437 100644 --- a/target/ppc/cpu_init.c +++ b/target/ppc/cpu_init.c @@ -5347,7 +5347,7 @@ static void register_970_lpar_sprs(CPUPPCState *env) static void register_power5p_lpar_sprs(CPUPPCState *env) { #if !defined(CONFIG_USER_ONLY) - /* Logical partitionning */ + /* Logical partitioning */ spr_register_kvm_hv(env, SPR_LPCR, "LPCR", SPR_NOACCESS, SPR_NOACCESS, SPR_NOACCESS, SPR_NOACCESS, @@ -5760,7 +5760,7 @@ static void register_power9_mmu_sprs(CPUPPCState *env) static void register_power10_hash_sprs(CPUPPCState *env) { /* - * it's the OS responsability to generate a random value for the registers + * it's the OS responsibility to generate a random value for the registers * in each process' context. So, initialize it with 0 here. */ uint64_t hashkeyr_initial_value = 0, hashpkeyr_initial_value = 0; diff --git a/target/ppc/excp_helper.c b/target/ppc/excp_helper.c index 72ec2be92e..99099cb1f6 100644 --- a/target/ppc/excp_helper.c +++ b/target/ppc/excp_helper.c @@ -455,7 +455,7 @@ static void powerpc_excp_40x(PowerPCCPU *cpu, int excp) /* * new interrupt handler msr preserves existing ME unless - * explicitly overriden. + * explicitly overridden. */ new_msr = env->msr & (((target_ulong)1 << MSR_ME)); @@ -578,7 +578,7 @@ static void powerpc_excp_6xx(PowerPCCPU *cpu, int excp) /* * new interrupt handler msr preserves existing ME unless - * explicitly overriden + * explicitly overridden */ new_msr = env->msr & ((target_ulong)1 << MSR_ME); @@ -739,7 +739,7 @@ static void powerpc_excp_7xx(PowerPCCPU *cpu, int excp) /* * new interrupt handler msr preserves existing ME unless - * explicitly overriden + * explicitly overridden */ new_msr = env->msr & ((target_ulong)1 << MSR_ME); @@ -911,7 +911,7 @@ static void powerpc_excp_74xx(PowerPCCPU *cpu, int excp) /* * new interrupt handler msr preserves existing ME unless - * explicitly overriden + * explicitly overridden */ new_msr = env->msr & ((target_ulong)1 << MSR_ME); @@ -1075,7 +1075,7 @@ static void powerpc_excp_booke(PowerPCCPU *cpu, int excp) /* * new interrupt handler msr preserves existing ME unless - * explicitly overriden + * explicitly overridden */ new_msr = env->msr & ((target_ulong)1 << MSR_ME); @@ -1288,7 +1288,7 @@ static bool books_vhyp_handles_hcall(PowerPCCPU *cpu) /* * When running a nested KVM HV guest under vhyp, HV exceptions are not * delivered to the guest (because there is no concept of HV support), but - * rather they are sent tothe vhyp to exit from the L2 back to the L1 and + * rather they are sent to the vhyp to exit from the L2 back to the L1 and * return from the H_ENTER_NESTED hypercall. */ static bool books_vhyp_handles_hv_excp(PowerPCCPU *cpu) @@ -1377,7 +1377,7 @@ static void powerpc_excp_books(PowerPCCPU *cpu, int excp) /* * new interrupt handler msr preserves existing HV and ME unless - * explicitly overriden + * explicitly overridden */ new_msr = env->msr & (((target_ulong)1 << MSR_ME) | MSR_HVB); diff --git a/target/ppc/int_helper.c b/target/ppc/int_helper.c index 96cdb3c7e3..6fd00684a5 100644 --- a/target/ppc/int_helper.c +++ b/target/ppc/int_helper.c @@ -25,6 +25,7 @@ #include "exec/helper-proto.h" #include "crypto/aes.h" #include "crypto/aes-round.h" +#include "crypto/clmul.h" #include "fpu/softfloat.h" #include "qapi/error.h" #include "qemu/guest-random.h" @@ -1424,46 +1425,39 @@ void helper_vbpermq(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b) #undef VBPERMQ_INDEX #undef VBPERMQ_DW -#define PMSUM(name, srcfld, trgfld, trgtyp) \ -void helper_##name(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b) \ -{ \ - int i, j; \ - trgtyp prod[sizeof(ppc_avr_t) / sizeof(a->srcfld[0])]; \ - \ - VECTOR_FOR_INORDER_I(i, srcfld) { \ - prod[i] = 0; \ - for (j = 0; j < sizeof(a->srcfld[0]) * 8; j++) { \ - if (a->srcfld[i] & (1ull << j)) { \ - prod[i] ^= ((trgtyp)b->srcfld[i] << j); \ - } \ - } \ - } \ - \ - VECTOR_FOR_INORDER_I(i, trgfld) { \ - r->trgfld[i] = prod[2 * i] ^ prod[2 * i + 1]; \ - } \ -} - -PMSUM(vpmsumb, u8, u16, uint16_t) -PMSUM(vpmsumh, u16, u32, uint32_t) -PMSUM(vpmsumw, u32, u64, uint64_t) +/* + * There is no carry across the two doublewords, so their order does + * not matter. Nor is there partial overlap between registers. + */ +void helper_vpmsumb(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b) +{ + for (int i = 0; i < 2; ++i) { + uint64_t aa = a->u64[i], bb = b->u64[i]; + r->u64[i] = clmul_8x4_even(aa, bb) ^ clmul_8x4_odd(aa, bb); + } +} -void helper_VPMSUMD(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b) +void helper_vpmsumh(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b) { - int i, j; - Int128 tmp, prod[2] = {int128_zero(), int128_zero()}; - - for (j = 0; j < 64; j++) { - for (i = 0; i < ARRAY_SIZE(r->u64); i++) { - if (a->VsrD(i) & (1ull << j)) { - tmp = int128_make64(b->VsrD(i)); - tmp = int128_lshift(tmp, j); - prod[i] = int128_xor(prod[i], tmp); - } - } + for (int i = 0; i < 2; ++i) { + uint64_t aa = a->u64[i], bb = b->u64[i]; + r->u64[i] = clmul_16x2_even(aa, bb) ^ clmul_16x2_odd(aa, bb); + } +} + +void helper_vpmsumw(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b) +{ + for (int i = 0; i < 2; ++i) { + uint64_t aa = a->u64[i], bb = b->u64[i]; + r->u64[i] = clmul_32(aa, bb) ^ clmul_32(aa >> 32, bb >> 32); } +} - r->s128 = int128_xor(prod[0], prod[1]); +void helper_VPMSUMD(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b) +{ + Int128 e = clmul_64(a->u64[0], b->u64[0]); + Int128 o = clmul_64(a->u64[1], b->u64[1]); + r->s128 = int128_xor(e, o); } #if HOST_BIG_ENDIAN diff --git a/target/ppc/power8-pmu-regs.c.inc b/target/ppc/power8-pmu-regs.c.inc index c82feedaff..75513db894 100644 --- a/target/ppc/power8-pmu-regs.c.inc +++ b/target/ppc/power8-pmu-regs.c.inc @@ -16,7 +16,7 @@ * Checks whether the Group A SPR (MMCR0, MMCR2, MMCRA, and the * PMCs) has problem state read access. * - * Read acccess is granted for all PMCC values but 0b01, where a + * Read access is granted for all PMCC values but 0b01, where a * Facility Unavailable Interrupt will occur. */ static bool spr_groupA_read_allowed(DisasContext *ctx) @@ -33,7 +33,7 @@ static bool spr_groupA_read_allowed(DisasContext *ctx) * Checks whether the Group A SPR (MMCR0, MMCR2, MMCRA, and the * PMCs) has problem state write access. * - * Write acccess is granted for PMCC values 0b10 and 0b11. Userspace + * Write access is granted for PMCC values 0b10 and 0b11. Userspace * writing with PMCC 0b00 will generate a Hypervisor Emulation * Assistance Interrupt. Userspace writing with PMCC 0b01 will * generate a Facility Unavailable Interrupt. diff --git a/target/ppc/translate/vmx-impl.c.inc b/target/ppc/translate/vmx-impl.c.inc index 6d7669aabd..5cdf53a9df 100644 --- a/target/ppc/translate/vmx-impl.c.inc +++ b/target/ppc/translate/vmx-impl.c.inc @@ -119,7 +119,7 @@ static void gen_stve##name(DisasContext *ctx) \ } GEN_VR_LDX(lvx, 0x07, 0x03); -/* As we don't emulate the cache, lvxl is stricly equivalent to lvx */ +/* As we don't emulate the cache, lvxl is strictly equivalent to lvx */ GEN_VR_LDX(lvxl, 0x07, 0x0B); GEN_VR_LVE(bx, 0x07, 0x00, 1); @@ -127,7 +127,7 @@ GEN_VR_LVE(hx, 0x07, 0x01, 2); GEN_VR_LVE(wx, 0x07, 0x02, 4); GEN_VR_STX(svx, 0x07, 0x07); -/* As we don't emulate the cache, stvxl is stricly equivalent to stvx */ +/* As we don't emulate the cache, stvxl is strictly equivalent to stvx */ GEN_VR_STX(svxl, 0x07, 0x0F); GEN_VR_STVE(bx, 0x07, 0x04, 1); @@ -1526,7 +1526,7 @@ static void gen_vprtyb_vec(unsigned vece, TCGv_vec t, TCGv_vec b) { int i; TCGv_vec tmp = tcg_temp_new_vec_matching(b); - /* MO_32 is 2, so 2 iteractions for MO_32 and 3 for MO_64 */ + /* MO_32 is 2, so 2 iterations for MO_32 and 3 for MO_64 */ for (i = 0; i < vece; i++) { tcg_gen_shri_vec(vece, tmp, b, (4 << (vece - i))); tcg_gen_xor_vec(vece, b, tmp, b); diff --git a/target/s390x/tcg/vec_int_helper.c b/target/s390x/tcg/vec_int_helper.c index 53ab5c5eb3..b18d8a6d16 100644 --- a/target/s390x/tcg/vec_int_helper.c +++ b/target/s390x/tcg/vec_int_helper.c @@ -14,19 +14,13 @@ #include "vec.h" #include "exec/helper-proto.h" #include "tcg/tcg-gvec-desc.h" +#include "crypto/clmul.h" static bool s390_vec_is_zero(const S390Vector *v) { return !v->doubleword[0] && !v->doubleword[1]; } -static void s390_vec_xor(S390Vector *res, const S390Vector *a, - const S390Vector *b) -{ - res->doubleword[0] = a->doubleword[0] ^ b->doubleword[0]; - res->doubleword[1] = a->doubleword[1] ^ b->doubleword[1]; -} - static void s390_vec_and(S390Vector *res, const S390Vector *a, const S390Vector *b) { @@ -164,117 +158,105 @@ DEF_VCTZ(8) DEF_VCTZ(16) /* like binary multiplication, but XOR instead of addition */ -#define DEF_GALOIS_MULTIPLY(BITS, TBITS) \ -static uint##TBITS##_t galois_multiply##BITS(uint##TBITS##_t a, \ - uint##TBITS##_t b) \ -{ \ - uint##TBITS##_t res = 0; \ - \ - while (b) { \ - if (b & 0x1) { \ - res = res ^ a; \ - } \ - a = a << 1; \ - b = b >> 1; \ - } \ - return res; \ + +/* + * There is no carry across the two doublewords, so their order does + * not matter. Nor is there partial overlap between registers. + */ +static inline uint64_t do_gfma8(uint64_t n, uint64_t m, uint64_t a) +{ + return clmul_8x4_even(n, m) ^ clmul_8x4_odd(n, m) ^ a; } -DEF_GALOIS_MULTIPLY(8, 16) -DEF_GALOIS_MULTIPLY(16, 32) -DEF_GALOIS_MULTIPLY(32, 64) -static S390Vector galois_multiply64(uint64_t a, uint64_t b) +void HELPER(gvec_vgfm8)(void *v1, const void *v2, const void *v3, uint32_t d) { - S390Vector res = {}; - S390Vector va = { - .doubleword[1] = a, - }; - S390Vector vb = { - .doubleword[1] = b, - }; - - while (!s390_vec_is_zero(&vb)) { - if (vb.doubleword[1] & 0x1) { - s390_vec_xor(&res, &res, &va); - } - s390_vec_shl(&va, &va, 1); - s390_vec_shr(&vb, &vb, 1); - } - return res; + uint64_t *q1 = v1; + const uint64_t *q2 = v2, *q3 = v3; + + q1[0] = do_gfma8(q2[0], q3[0], 0); + q1[1] = do_gfma8(q2[1], q3[1], 0); } -#define DEF_VGFM(BITS, TBITS) \ -void HELPER(gvec_vgfm##BITS)(void *v1, const void *v2, const void *v3, \ - uint32_t desc) \ -{ \ - int i; \ - \ - for (i = 0; i < (128 / TBITS); i++) { \ - uint##BITS##_t a = s390_vec_read_element##BITS(v2, i * 2); \ - uint##BITS##_t b = s390_vec_read_element##BITS(v3, i * 2); \ - uint##TBITS##_t d = galois_multiply##BITS(a, b); \ - \ - a = s390_vec_read_element##BITS(v2, i * 2 + 1); \ - b = s390_vec_read_element##BITS(v3, i * 2 + 1); \ - d = d ^ galois_multiply32(a, b); \ - s390_vec_write_element##TBITS(v1, i, d); \ - } \ +void HELPER(gvec_vgfma8)(void *v1, const void *v2, const void *v3, + const void *v4, uint32_t desc) +{ + uint64_t *q1 = v1; + const uint64_t *q2 = v2, *q3 = v3, *q4 = v4; + + q1[0] = do_gfma8(q2[0], q3[0], q4[0]); + q1[1] = do_gfma8(q2[1], q3[1], q4[1]); } -DEF_VGFM(8, 16) -DEF_VGFM(16, 32) -DEF_VGFM(32, 64) -void HELPER(gvec_vgfm64)(void *v1, const void *v2, const void *v3, - uint32_t desc) +static inline uint64_t do_gfma16(uint64_t n, uint64_t m, uint64_t a) +{ + return clmul_16x2_even(n, m) ^ clmul_16x2_odd(n, m) ^ a; +} + +void HELPER(gvec_vgfm16)(void *v1, const void *v2, const void *v3, uint32_t d) { - S390Vector tmp1, tmp2; - uint64_t a, b; + uint64_t *q1 = v1; + const uint64_t *q2 = v2, *q3 = v3; - a = s390_vec_read_element64(v2, 0); - b = s390_vec_read_element64(v3, 0); - tmp1 = galois_multiply64(a, b); - a = s390_vec_read_element64(v2, 1); - b = s390_vec_read_element64(v3, 1); - tmp2 = galois_multiply64(a, b); - s390_vec_xor(v1, &tmp1, &tmp2); + q1[0] = do_gfma16(q2[0], q3[0], 0); + q1[1] = do_gfma16(q2[1], q3[1], 0); } -#define DEF_VGFMA(BITS, TBITS) \ -void HELPER(gvec_vgfma##BITS)(void *v1, const void *v2, const void *v3, \ - const void *v4, uint32_t desc) \ -{ \ - int i; \ - \ - for (i = 0; i < (128 / TBITS); i++) { \ - uint##BITS##_t a = s390_vec_read_element##BITS(v2, i * 2); \ - uint##BITS##_t b = s390_vec_read_element##BITS(v3, i * 2); \ - uint##TBITS##_t d = galois_multiply##BITS(a, b); \ - \ - a = s390_vec_read_element##BITS(v2, i * 2 + 1); \ - b = s390_vec_read_element##BITS(v3, i * 2 + 1); \ - d = d ^ galois_multiply32(a, b); \ - d = d ^ s390_vec_read_element##TBITS(v4, i); \ - s390_vec_write_element##TBITS(v1, i, d); \ - } \ +void HELPER(gvec_vgfma16)(void *v1, const void *v2, const void *v3, + const void *v4, uint32_t d) +{ + uint64_t *q1 = v1; + const uint64_t *q2 = v2, *q3 = v3, *q4 = v4; + + q1[0] = do_gfma16(q2[0], q3[0], q4[0]); + q1[1] = do_gfma16(q2[1], q3[1], q4[1]); +} + +static inline uint64_t do_gfma32(uint64_t n, uint64_t m, uint64_t a) +{ + return clmul_32(n, m) ^ clmul_32(n >> 32, m >> 32) ^ a; +} + +void HELPER(gvec_vgfm32)(void *v1, const void *v2, const void *v3, uint32_t d) +{ + uint64_t *q1 = v1; + const uint64_t *q2 = v2, *q3 = v3; + + q1[0] = do_gfma32(q2[0], q3[0], 0); + q1[1] = do_gfma32(q2[1], q3[1], 0); +} + +void HELPER(gvec_vgfma32)(void *v1, const void *v2, const void *v3, + const void *v4, uint32_t d) +{ + uint64_t *q1 = v1; + const uint64_t *q2 = v2, *q3 = v3, *q4 = v4; + + q1[0] = do_gfma32(q2[0], q3[0], q4[0]); + q1[1] = do_gfma32(q2[1], q3[1], q4[1]); +} + +void HELPER(gvec_vgfm64)(void *v1, const void *v2, const void *v3, + uint32_t desc) +{ + uint64_t *q1 = v1; + const uint64_t *q2 = v2, *q3 = v3; + Int128 r; + + r = int128_xor(clmul_64(q2[0], q3[0]), clmul_64(q2[1], q3[1])); + q1[0] = int128_gethi(r); + q1[1] = int128_getlo(r); } -DEF_VGFMA(8, 16) -DEF_VGFMA(16, 32) -DEF_VGFMA(32, 64) void HELPER(gvec_vgfma64)(void *v1, const void *v2, const void *v3, const void *v4, uint32_t desc) { - S390Vector tmp1, tmp2; - uint64_t a, b; - - a = s390_vec_read_element64(v2, 0); - b = s390_vec_read_element64(v3, 0); - tmp1 = galois_multiply64(a, b); - a = s390_vec_read_element64(v2, 1); - b = s390_vec_read_element64(v3, 1); - tmp2 = galois_multiply64(a, b); - s390_vec_xor(&tmp1, &tmp1, &tmp2); - s390_vec_xor(v1, &tmp1, v4); + uint64_t *q1 = v1; + const uint64_t *q2 = v2, *q3 = v3, *q4 = v4; + Int128 r; + + r = int128_xor(clmul_64(q2[0], q3[0]), clmul_64(q2[1], q3[1])); + q1[0] = q4[0] ^ int128_gethi(r); + q1[1] = q4[1] ^ int128_getlo(r); } #define DEF_VMAL(BITS) \ diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index 0931a69448..06ea3c7652 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -272,7 +272,7 @@ static bool is_shimm1632(uint32_t v32, int *cmode, int *imm8) } } -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -602,6 +602,10 @@ typedef enum { DMB_ISH = 0xd50338bf, DMB_LD = 0x00000100, DMB_ST = 0x00000200, + + BTI_C = 0xd503245f, + BTI_J = 0xd503249f, + BTI_JC = 0xd50324df, } AArch64Insn; static inline uint32_t tcg_in32(TCGContext *s) @@ -843,6 +847,17 @@ static void tcg_out_insn_3313(TCGContext *s, AArch64Insn insn, | rn << 5 | (rd & 0x1f)); } +static void tcg_out_bti(TCGContext *s, AArch64Insn insn) +{ + /* + * While BTI insns are nops on hosts without FEAT_BTI, + * there is no point in emitting them in that case either. + */ + if (cpuinfo & CPUINFO_BTI) { + tcg_out32(s, insn); + } +} + /* Register to register move using ORR (shifted register with no shift). */ static void tcg_out_movr(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rm) { @@ -1351,18 +1366,6 @@ static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) tcg_out_insn(s, 3206, B, offset); } -static void tcg_out_goto_long(TCGContext *s, const tcg_insn_unit *target) -{ - ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2; - if (offset == sextract64(offset, 0, 26)) { - tcg_out_insn(s, 3206, B, offset); - } else { - /* Choose X9 as a call-clobbered non-LR temporary. */ - tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X9, (intptr_t)target); - tcg_out_insn(s, 3207, BR, TCG_REG_X9); - } -} - static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *target) { ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2; @@ -1947,12 +1950,28 @@ static const tcg_insn_unit *tb_ret_addr; static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) { + const tcg_insn_unit *target; + ptrdiff_t offset; + /* Reuse the zeroing that exists for goto_ptr. */ if (a0 == 0) { - tcg_out_goto_long(s, tcg_code_gen_epilogue); + target = tcg_code_gen_epilogue; } else { tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0); - tcg_out_goto_long(s, tb_ret_addr); + target = tb_ret_addr; + } + + offset = tcg_pcrel_diff(s, target) >> 2; + if (offset == sextract64(offset, 0, 26)) { + tcg_out_insn(s, 3206, B, offset); + } else { + /* + * Only x16/x17 generate BTI type Jump (2), + * other registers generate BTI type Jump|Call (3). + */ + QEMU_BUILD_BUG_ON(TCG_REG_TMP0 != TCG_REG_X16); + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP0, (intptr_t)target); + tcg_out_insn(s, 3207, BR, TCG_REG_TMP0); } } @@ -1970,6 +1989,7 @@ static void tcg_out_goto_tb(TCGContext *s, int which) tcg_out32(s, I3206_B); tcg_out_insn(s, 3207, BR, TCG_REG_TMP0); set_jmp_reset_offset(s, which); + tcg_out_bti(s, BTI_J); } void tb_target_set_jmp_target(const TranslationBlock *tb, int n, @@ -3074,6 +3094,8 @@ static void tcg_target_qemu_prologue(TCGContext *s) { TCGReg r; + tcg_out_bti(s, BTI_C); + /* Push (FP, LR) and allocate space for all saved registers. */ tcg_out_insn(s, 3314, STP, TCG_REG_FP, TCG_REG_LR, TCG_REG_SP, -PUSH_SIZE, 1, 1); @@ -3114,10 +3136,12 @@ static void tcg_target_qemu_prologue(TCGContext *s) * and fall through to the rest of the epilogue. */ tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); + tcg_out_bti(s, BTI_J); tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_X0, 0); /* TB epilogue */ tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); + tcg_out_bti(s, BTI_J); /* Remove TCG locals stack space. */ tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP, @@ -3135,6 +3159,11 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_insn(s, 3207, RET, TCG_REG_LR); } +static void tcg_out_tb_start(TCGContext *s) +{ + tcg_out_bti(s, BTI_J); +} + static void tcg_out_nop_fill(tcg_insn_unit *p, int count) { int i; diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index acb5f23b54..b1d56362a7 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -509,7 +509,7 @@ static bool is_shimm1632(uint32_t v32, int *cmode, int *imm8) * mov operand2: values represented with x << (2 * y), x < 0x100 * add, sub, eor...: ditto */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -2962,6 +2962,11 @@ static void tcg_out_epilogue(TCGContext *s) (1 << TCG_REG_R10) | (1 << TCG_REG_R11) | (1 << TCG_REG_PC)); } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + typedef struct { DebugFrameHeader h; uint8_t fde_def_cfa[4]; diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index 0c3d1e4cef..4e47151241 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -198,7 +198,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, } /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -4191,6 +4191,11 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc(s, OPC_RET, 0, 0, 0); } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + static void tcg_out_nop_fill(tcg_insn_unit *p, int count) { memset(p, 0x90, count); diff --git a/tcg/loongarch64/tcg-insn-defs.c.inc b/tcg/loongarch64/tcg-insn-defs.c.inc index b5bb0c5e73..ee3b483b02 100644 --- a/tcg/loongarch64/tcg-insn-defs.c.inc +++ b/tcg/loongarch64/tcg-insn-defs.c.inc @@ -4,7 +4,7 @@ * * This file is auto-generated by genqemutcgdefs from * https://github.com/loongson-community/loongarch-opcodes, - * from commit 25ca7effe9d88101c1cf96c4005423643386d81f. + * from commit 8027da9a8157a8b47fc48ff1def292e09c5668bd. * DO NOT EDIT. */ @@ -74,6 +74,60 @@ typedef enum { OPC_ANDI = 0x03400000, OPC_ORI = 0x03800000, OPC_XORI = 0x03c00000, + OPC_VFMADD_S = 0x09100000, + OPC_VFMADD_D = 0x09200000, + OPC_VFMSUB_S = 0x09500000, + OPC_VFMSUB_D = 0x09600000, + OPC_VFNMADD_S = 0x09900000, + OPC_VFNMADD_D = 0x09a00000, + OPC_VFNMSUB_S = 0x09d00000, + OPC_VFNMSUB_D = 0x09e00000, + OPC_VFCMP_CAF_S = 0x0c500000, + OPC_VFCMP_SAF_S = 0x0c508000, + OPC_VFCMP_CLT_S = 0x0c510000, + OPC_VFCMP_SLT_S = 0x0c518000, + OPC_VFCMP_CEQ_S = 0x0c520000, + OPC_VFCMP_SEQ_S = 0x0c528000, + OPC_VFCMP_CLE_S = 0x0c530000, + OPC_VFCMP_SLE_S = 0x0c538000, + OPC_VFCMP_CUN_S = 0x0c540000, + OPC_VFCMP_SUN_S = 0x0c548000, + OPC_VFCMP_CULT_S = 0x0c550000, + OPC_VFCMP_SULT_S = 0x0c558000, + OPC_VFCMP_CUEQ_S = 0x0c560000, + OPC_VFCMP_SUEQ_S = 0x0c568000, + OPC_VFCMP_CULE_S = 0x0c570000, + OPC_VFCMP_SULE_S = 0x0c578000, + OPC_VFCMP_CNE_S = 0x0c580000, + OPC_VFCMP_SNE_S = 0x0c588000, + OPC_VFCMP_COR_S = 0x0c5a0000, + OPC_VFCMP_SOR_S = 0x0c5a8000, + OPC_VFCMP_CUNE_S = 0x0c5c0000, + OPC_VFCMP_SUNE_S = 0x0c5c8000, + OPC_VFCMP_CAF_D = 0x0c600000, + OPC_VFCMP_SAF_D = 0x0c608000, + OPC_VFCMP_CLT_D = 0x0c610000, + OPC_VFCMP_SLT_D = 0x0c618000, + OPC_VFCMP_CEQ_D = 0x0c620000, + OPC_VFCMP_SEQ_D = 0x0c628000, + OPC_VFCMP_CLE_D = 0x0c630000, + OPC_VFCMP_SLE_D = 0x0c638000, + OPC_VFCMP_CUN_D = 0x0c640000, + OPC_VFCMP_SUN_D = 0x0c648000, + OPC_VFCMP_CULT_D = 0x0c650000, + OPC_VFCMP_SULT_D = 0x0c658000, + OPC_VFCMP_CUEQ_D = 0x0c660000, + OPC_VFCMP_SUEQ_D = 0x0c668000, + OPC_VFCMP_CULE_D = 0x0c670000, + OPC_VFCMP_SULE_D = 0x0c678000, + OPC_VFCMP_CNE_D = 0x0c680000, + OPC_VFCMP_SNE_D = 0x0c688000, + OPC_VFCMP_COR_D = 0x0c6a0000, + OPC_VFCMP_SOR_D = 0x0c6a8000, + OPC_VFCMP_CUNE_D = 0x0c6c0000, + OPC_VFCMP_SUNE_D = 0x0c6c8000, + OPC_VBITSEL_V = 0x0d100000, + OPC_VSHUF_B = 0x0d500000, OPC_ADDU16I_D = 0x10000000, OPC_LU12I_W = 0x14000000, OPC_CU32I_D = 0x16000000, @@ -92,6 +146,16 @@ typedef enum { OPC_LD_BU = 0x2a000000, OPC_LD_HU = 0x2a400000, OPC_LD_WU = 0x2a800000, + OPC_VLD = 0x2c000000, + OPC_VST = 0x2c400000, + OPC_VLDREPL_D = 0x30100000, + OPC_VLDREPL_W = 0x30200000, + OPC_VLDREPL_H = 0x30400000, + OPC_VLDREPL_B = 0x30800000, + OPC_VSTELM_D = 0x31100000, + OPC_VSTELM_W = 0x31200000, + OPC_VSTELM_H = 0x31400000, + OPC_VSTELM_B = 0x31800000, OPC_LDX_B = 0x38000000, OPC_LDX_H = 0x38040000, OPC_LDX_W = 0x38080000, @@ -103,6 +167,8 @@ typedef enum { OPC_LDX_BU = 0x38200000, OPC_LDX_HU = 0x38240000, OPC_LDX_WU = 0x38280000, + OPC_VLDX = 0x38400000, + OPC_VSTX = 0x38440000, OPC_DBAR = 0x38720000, OPC_JIRL = 0x4c000000, OPC_B = 0x50000000, @@ -113,6 +179,652 @@ typedef enum { OPC_BLE = 0x64000000, OPC_BGTU = 0x68000000, OPC_BLEU = 0x6c000000, + OPC_VSEQ_B = 0x70000000, + OPC_VSEQ_H = 0x70008000, + OPC_VSEQ_W = 0x70010000, + OPC_VSEQ_D = 0x70018000, + OPC_VSLE_B = 0x70020000, + OPC_VSLE_H = 0x70028000, + OPC_VSLE_W = 0x70030000, + OPC_VSLE_D = 0x70038000, + OPC_VSLE_BU = 0x70040000, + OPC_VSLE_HU = 0x70048000, + OPC_VSLE_WU = 0x70050000, + OPC_VSLE_DU = 0x70058000, + OPC_VSLT_B = 0x70060000, + OPC_VSLT_H = 0x70068000, + OPC_VSLT_W = 0x70070000, + OPC_VSLT_D = 0x70078000, + OPC_VSLT_BU = 0x70080000, + OPC_VSLT_HU = 0x70088000, + OPC_VSLT_WU = 0x70090000, + OPC_VSLT_DU = 0x70098000, + OPC_VADD_B = 0x700a0000, + OPC_VADD_H = 0x700a8000, + OPC_VADD_W = 0x700b0000, + OPC_VADD_D = 0x700b8000, + OPC_VSUB_B = 0x700c0000, + OPC_VSUB_H = 0x700c8000, + OPC_VSUB_W = 0x700d0000, + OPC_VSUB_D = 0x700d8000, + OPC_VADDWEV_H_B = 0x701e0000, + OPC_VADDWEV_W_H = 0x701e8000, + OPC_VADDWEV_D_W = 0x701f0000, + OPC_VADDWEV_Q_D = 0x701f8000, + OPC_VSUBWEV_H_B = 0x70200000, + OPC_VSUBWEV_W_H = 0x70208000, + OPC_VSUBWEV_D_W = 0x70210000, + OPC_VSUBWEV_Q_D = 0x70218000, + OPC_VADDWOD_H_B = 0x70220000, + OPC_VADDWOD_W_H = 0x70228000, + OPC_VADDWOD_D_W = 0x70230000, + OPC_VADDWOD_Q_D = 0x70238000, + OPC_VSUBWOD_H_B = 0x70240000, + OPC_VSUBWOD_W_H = 0x70248000, + OPC_VSUBWOD_D_W = 0x70250000, + OPC_VSUBWOD_Q_D = 0x70258000, + OPC_VADDWEV_H_BU = 0x702e0000, + OPC_VADDWEV_W_HU = 0x702e8000, + OPC_VADDWEV_D_WU = 0x702f0000, + OPC_VADDWEV_Q_DU = 0x702f8000, + OPC_VSUBWEV_H_BU = 0x70300000, + OPC_VSUBWEV_W_HU = 0x70308000, + OPC_VSUBWEV_D_WU = 0x70310000, + OPC_VSUBWEV_Q_DU = 0x70318000, + OPC_VADDWOD_H_BU = 0x70320000, + OPC_VADDWOD_W_HU = 0x70328000, + OPC_VADDWOD_D_WU = 0x70330000, + OPC_VADDWOD_Q_DU = 0x70338000, + OPC_VSUBWOD_H_BU = 0x70340000, + OPC_VSUBWOD_W_HU = 0x70348000, + OPC_VSUBWOD_D_WU = 0x70350000, + OPC_VSUBWOD_Q_DU = 0x70358000, + OPC_VADDWEV_H_BU_B = 0x703e0000, + OPC_VADDWEV_W_HU_H = 0x703e8000, + OPC_VADDWEV_D_WU_W = 0x703f0000, + OPC_VADDWEV_Q_DU_D = 0x703f8000, + OPC_VADDWOD_H_BU_B = 0x70400000, + OPC_VADDWOD_W_HU_H = 0x70408000, + OPC_VADDWOD_D_WU_W = 0x70410000, + OPC_VADDWOD_Q_DU_D = 0x70418000, + OPC_VSADD_B = 0x70460000, + OPC_VSADD_H = 0x70468000, + OPC_VSADD_W = 0x70470000, + OPC_VSADD_D = 0x70478000, + OPC_VSSUB_B = 0x70480000, + OPC_VSSUB_H = 0x70488000, + OPC_VSSUB_W = 0x70490000, + OPC_VSSUB_D = 0x70498000, + OPC_VSADD_BU = 0x704a0000, + OPC_VSADD_HU = 0x704a8000, + OPC_VSADD_WU = 0x704b0000, + OPC_VSADD_DU = 0x704b8000, + OPC_VSSUB_BU = 0x704c0000, + OPC_VSSUB_HU = 0x704c8000, + OPC_VSSUB_WU = 0x704d0000, + OPC_VSSUB_DU = 0x704d8000, + OPC_VHADDW_H_B = 0x70540000, + OPC_VHADDW_W_H = 0x70548000, + OPC_VHADDW_D_W = 0x70550000, + OPC_VHADDW_Q_D = 0x70558000, + OPC_VHSUBW_H_B = 0x70560000, + OPC_VHSUBW_W_H = 0x70568000, + OPC_VHSUBW_D_W = 0x70570000, + OPC_VHSUBW_Q_D = 0x70578000, + OPC_VHADDW_HU_BU = 0x70580000, + OPC_VHADDW_WU_HU = 0x70588000, + OPC_VHADDW_DU_WU = 0x70590000, + OPC_VHADDW_QU_DU = 0x70598000, + OPC_VHSUBW_HU_BU = 0x705a0000, + OPC_VHSUBW_WU_HU = 0x705a8000, + OPC_VHSUBW_DU_WU = 0x705b0000, + OPC_VHSUBW_QU_DU = 0x705b8000, + OPC_VADDA_B = 0x705c0000, + OPC_VADDA_H = 0x705c8000, + OPC_VADDA_W = 0x705d0000, + OPC_VADDA_D = 0x705d8000, + OPC_VABSD_B = 0x70600000, + OPC_VABSD_H = 0x70608000, + OPC_VABSD_W = 0x70610000, + OPC_VABSD_D = 0x70618000, + OPC_VABSD_BU = 0x70620000, + OPC_VABSD_HU = 0x70628000, + OPC_VABSD_WU = 0x70630000, + OPC_VABSD_DU = 0x70638000, + OPC_VAVG_B = 0x70640000, + OPC_VAVG_H = 0x70648000, + OPC_VAVG_W = 0x70650000, + OPC_VAVG_D = 0x70658000, + OPC_VAVG_BU = 0x70660000, + OPC_VAVG_HU = 0x70668000, + OPC_VAVG_WU = 0x70670000, + OPC_VAVG_DU = 0x70678000, + OPC_VAVGR_B = 0x70680000, + OPC_VAVGR_H = 0x70688000, + OPC_VAVGR_W = 0x70690000, + OPC_VAVGR_D = 0x70698000, + OPC_VAVGR_BU = 0x706a0000, + OPC_VAVGR_HU = 0x706a8000, + OPC_VAVGR_WU = 0x706b0000, + OPC_VAVGR_DU = 0x706b8000, + OPC_VMAX_B = 0x70700000, + OPC_VMAX_H = 0x70708000, + OPC_VMAX_W = 0x70710000, + OPC_VMAX_D = 0x70718000, + OPC_VMIN_B = 0x70720000, + OPC_VMIN_H = 0x70728000, + OPC_VMIN_W = 0x70730000, + OPC_VMIN_D = 0x70738000, + OPC_VMAX_BU = 0x70740000, + OPC_VMAX_HU = 0x70748000, + OPC_VMAX_WU = 0x70750000, + OPC_VMAX_DU = 0x70758000, + OPC_VMIN_BU = 0x70760000, + OPC_VMIN_HU = 0x70768000, + OPC_VMIN_WU = 0x70770000, + OPC_VMIN_DU = 0x70778000, + OPC_VMUL_B = 0x70840000, + OPC_VMUL_H = 0x70848000, + OPC_VMUL_W = 0x70850000, + OPC_VMUL_D = 0x70858000, + OPC_VMUH_B = 0x70860000, + OPC_VMUH_H = 0x70868000, + OPC_VMUH_W = 0x70870000, + OPC_VMUH_D = 0x70878000, + OPC_VMUH_BU = 0x70880000, + OPC_VMUH_HU = 0x70888000, + OPC_VMUH_WU = 0x70890000, + OPC_VMUH_DU = 0x70898000, + OPC_VMULWEV_H_B = 0x70900000, + OPC_VMULWEV_W_H = 0x70908000, + OPC_VMULWEV_D_W = 0x70910000, + OPC_VMULWEV_Q_D = 0x70918000, + OPC_VMULWOD_H_B = 0x70920000, + OPC_VMULWOD_W_H = 0x70928000, + OPC_VMULWOD_D_W = 0x70930000, + OPC_VMULWOD_Q_D = 0x70938000, + OPC_VMULWEV_H_BU = 0x70980000, + OPC_VMULWEV_W_HU = 0x70988000, + OPC_VMULWEV_D_WU = 0x70990000, + OPC_VMULWEV_Q_DU = 0x70998000, + OPC_VMULWOD_H_BU = 0x709a0000, + OPC_VMULWOD_W_HU = 0x709a8000, + OPC_VMULWOD_D_WU = 0x709b0000, + OPC_VMULWOD_Q_DU = 0x709b8000, + OPC_VMULWEV_H_BU_B = 0x70a00000, + OPC_VMULWEV_W_HU_H = 0x70a08000, + OPC_VMULWEV_D_WU_W = 0x70a10000, + OPC_VMULWEV_Q_DU_D = 0x70a18000, + OPC_VMULWOD_H_BU_B = 0x70a20000, + OPC_VMULWOD_W_HU_H = 0x70a28000, + OPC_VMULWOD_D_WU_W = 0x70a30000, + OPC_VMULWOD_Q_DU_D = 0x70a38000, + OPC_VMADD_B = 0x70a80000, + OPC_VMADD_H = 0x70a88000, + OPC_VMADD_W = 0x70a90000, + OPC_VMADD_D = 0x70a98000, + OPC_VMSUB_B = 0x70aa0000, + OPC_VMSUB_H = 0x70aa8000, + OPC_VMSUB_W = 0x70ab0000, + OPC_VMSUB_D = 0x70ab8000, + OPC_VMADDWEV_H_B = 0x70ac0000, + OPC_VMADDWEV_W_H = 0x70ac8000, + OPC_VMADDWEV_D_W = 0x70ad0000, + OPC_VMADDWEV_Q_D = 0x70ad8000, + OPC_VMADDWOD_H_B = 0x70ae0000, + OPC_VMADDWOD_W_H = 0x70ae8000, + OPC_VMADDWOD_D_W = 0x70af0000, + OPC_VMADDWOD_Q_D = 0x70af8000, + OPC_VMADDWEV_H_BU = 0x70b40000, + OPC_VMADDWEV_W_HU = 0x70b48000, + OPC_VMADDWEV_D_WU = 0x70b50000, + OPC_VMADDWEV_Q_DU = 0x70b58000, + OPC_VMADDWOD_H_BU = 0x70b60000, + OPC_VMADDWOD_W_HU = 0x70b68000, + OPC_VMADDWOD_D_WU = 0x70b70000, + OPC_VMADDWOD_Q_DU = 0x70b78000, + OPC_VMADDWEV_H_BU_B = 0x70bc0000, + OPC_VMADDWEV_W_HU_H = 0x70bc8000, + OPC_VMADDWEV_D_WU_W = 0x70bd0000, + OPC_VMADDWEV_Q_DU_D = 0x70bd8000, + OPC_VMADDWOD_H_BU_B = 0x70be0000, + OPC_VMADDWOD_W_HU_H = 0x70be8000, + OPC_VMADDWOD_D_WU_W = 0x70bf0000, + OPC_VMADDWOD_Q_DU_D = 0x70bf8000, + OPC_VDIV_B = 0x70e00000, + OPC_VDIV_H = 0x70e08000, + OPC_VDIV_W = 0x70e10000, + OPC_VDIV_D = 0x70e18000, + OPC_VMOD_B = 0x70e20000, + OPC_VMOD_H = 0x70e28000, + OPC_VMOD_W = 0x70e30000, + OPC_VMOD_D = 0x70e38000, + OPC_VDIV_BU = 0x70e40000, + OPC_VDIV_HU = 0x70e48000, + OPC_VDIV_WU = 0x70e50000, + OPC_VDIV_DU = 0x70e58000, + OPC_VMOD_BU = 0x70e60000, + OPC_VMOD_HU = 0x70e68000, + OPC_VMOD_WU = 0x70e70000, + OPC_VMOD_DU = 0x70e78000, + OPC_VSLL_B = 0x70e80000, + OPC_VSLL_H = 0x70e88000, + OPC_VSLL_W = 0x70e90000, + OPC_VSLL_D = 0x70e98000, + OPC_VSRL_B = 0x70ea0000, + OPC_VSRL_H = 0x70ea8000, + OPC_VSRL_W = 0x70eb0000, + OPC_VSRL_D = 0x70eb8000, + OPC_VSRA_B = 0x70ec0000, + OPC_VSRA_H = 0x70ec8000, + OPC_VSRA_W = 0x70ed0000, + OPC_VSRA_D = 0x70ed8000, + OPC_VROTR_B = 0x70ee0000, + OPC_VROTR_H = 0x70ee8000, + OPC_VROTR_W = 0x70ef0000, + OPC_VROTR_D = 0x70ef8000, + OPC_VSRLR_B = 0x70f00000, + OPC_VSRLR_H = 0x70f08000, + OPC_VSRLR_W = 0x70f10000, + OPC_VSRLR_D = 0x70f18000, + OPC_VSRAR_B = 0x70f20000, + OPC_VSRAR_H = 0x70f28000, + OPC_VSRAR_W = 0x70f30000, + OPC_VSRAR_D = 0x70f38000, + OPC_VSRLN_B_H = 0x70f48000, + OPC_VSRLN_H_W = 0x70f50000, + OPC_VSRLN_W_D = 0x70f58000, + OPC_VSRAN_B_H = 0x70f68000, + OPC_VSRAN_H_W = 0x70f70000, + OPC_VSRAN_W_D = 0x70f78000, + OPC_VSRLRN_B_H = 0x70f88000, + OPC_VSRLRN_H_W = 0x70f90000, + OPC_VSRLRN_W_D = 0x70f98000, + OPC_VSRARN_B_H = 0x70fa8000, + OPC_VSRARN_H_W = 0x70fb0000, + OPC_VSRARN_W_D = 0x70fb8000, + OPC_VSSRLN_B_H = 0x70fc8000, + OPC_VSSRLN_H_W = 0x70fd0000, + OPC_VSSRLN_W_D = 0x70fd8000, + OPC_VSSRAN_B_H = 0x70fe8000, + OPC_VSSRAN_H_W = 0x70ff0000, + OPC_VSSRAN_W_D = 0x70ff8000, + OPC_VSSRLRN_B_H = 0x71008000, + OPC_VSSRLRN_H_W = 0x71010000, + OPC_VSSRLRN_W_D = 0x71018000, + OPC_VSSRARN_B_H = 0x71028000, + OPC_VSSRARN_H_W = 0x71030000, + OPC_VSSRARN_W_D = 0x71038000, + OPC_VSSRLN_BU_H = 0x71048000, + OPC_VSSRLN_HU_W = 0x71050000, + OPC_VSSRLN_WU_D = 0x71058000, + OPC_VSSRAN_BU_H = 0x71068000, + OPC_VSSRAN_HU_W = 0x71070000, + OPC_VSSRAN_WU_D = 0x71078000, + OPC_VSSRLRN_BU_H = 0x71088000, + OPC_VSSRLRN_HU_W = 0x71090000, + OPC_VSSRLRN_WU_D = 0x71098000, + OPC_VSSRARN_BU_H = 0x710a8000, + OPC_VSSRARN_HU_W = 0x710b0000, + OPC_VSSRARN_WU_D = 0x710b8000, + OPC_VBITCLR_B = 0x710c0000, + OPC_VBITCLR_H = 0x710c8000, + OPC_VBITCLR_W = 0x710d0000, + OPC_VBITCLR_D = 0x710d8000, + OPC_VBITSET_B = 0x710e0000, + OPC_VBITSET_H = 0x710e8000, + OPC_VBITSET_W = 0x710f0000, + OPC_VBITSET_D = 0x710f8000, + OPC_VBITREV_B = 0x71100000, + OPC_VBITREV_H = 0x71108000, + OPC_VBITREV_W = 0x71110000, + OPC_VBITREV_D = 0x71118000, + OPC_VPACKEV_B = 0x71160000, + OPC_VPACKEV_H = 0x71168000, + OPC_VPACKEV_W = 0x71170000, + OPC_VPACKEV_D = 0x71178000, + OPC_VPACKOD_B = 0x71180000, + OPC_VPACKOD_H = 0x71188000, + OPC_VPACKOD_W = 0x71190000, + OPC_VPACKOD_D = 0x71198000, + OPC_VILVL_B = 0x711a0000, + OPC_VILVL_H = 0x711a8000, + OPC_VILVL_W = 0x711b0000, + OPC_VILVL_D = 0x711b8000, + OPC_VILVH_B = 0x711c0000, + OPC_VILVH_H = 0x711c8000, + OPC_VILVH_W = 0x711d0000, + OPC_VILVH_D = 0x711d8000, + OPC_VPICKEV_B = 0x711e0000, + OPC_VPICKEV_H = 0x711e8000, + OPC_VPICKEV_W = 0x711f0000, + OPC_VPICKEV_D = 0x711f8000, + OPC_VPICKOD_B = 0x71200000, + OPC_VPICKOD_H = 0x71208000, + OPC_VPICKOD_W = 0x71210000, + OPC_VPICKOD_D = 0x71218000, + OPC_VREPLVE_B = 0x71220000, + OPC_VREPLVE_H = 0x71228000, + OPC_VREPLVE_W = 0x71230000, + OPC_VREPLVE_D = 0x71238000, + OPC_VAND_V = 0x71260000, + OPC_VOR_V = 0x71268000, + OPC_VXOR_V = 0x71270000, + OPC_VNOR_V = 0x71278000, + OPC_VANDN_V = 0x71280000, + OPC_VORN_V = 0x71288000, + OPC_VFRSTP_B = 0x712b0000, + OPC_VFRSTP_H = 0x712b8000, + OPC_VADD_Q = 0x712d0000, + OPC_VSUB_Q = 0x712d8000, + OPC_VSIGNCOV_B = 0x712e0000, + OPC_VSIGNCOV_H = 0x712e8000, + OPC_VSIGNCOV_W = 0x712f0000, + OPC_VSIGNCOV_D = 0x712f8000, + OPC_VFADD_S = 0x71308000, + OPC_VFADD_D = 0x71310000, + OPC_VFSUB_S = 0x71328000, + OPC_VFSUB_D = 0x71330000, + OPC_VFMUL_S = 0x71388000, + OPC_VFMUL_D = 0x71390000, + OPC_VFDIV_S = 0x713a8000, + OPC_VFDIV_D = 0x713b0000, + OPC_VFMAX_S = 0x713c8000, + OPC_VFMAX_D = 0x713d0000, + OPC_VFMIN_S = 0x713e8000, + OPC_VFMIN_D = 0x713f0000, + OPC_VFMAXA_S = 0x71408000, + OPC_VFMAXA_D = 0x71410000, + OPC_VFMINA_S = 0x71428000, + OPC_VFMINA_D = 0x71430000, + OPC_VFCVT_H_S = 0x71460000, + OPC_VFCVT_S_D = 0x71468000, + OPC_VFFINT_S_L = 0x71480000, + OPC_VFTINT_W_D = 0x71498000, + OPC_VFTINTRM_W_D = 0x714a0000, + OPC_VFTINTRP_W_D = 0x714a8000, + OPC_VFTINTRZ_W_D = 0x714b0000, + OPC_VFTINTRNE_W_D = 0x714b8000, + OPC_VSHUF_H = 0x717a8000, + OPC_VSHUF_W = 0x717b0000, + OPC_VSHUF_D = 0x717b8000, + OPC_VSEQI_B = 0x72800000, + OPC_VSEQI_H = 0x72808000, + OPC_VSEQI_W = 0x72810000, + OPC_VSEQI_D = 0x72818000, + OPC_VSLEI_B = 0x72820000, + OPC_VSLEI_H = 0x72828000, + OPC_VSLEI_W = 0x72830000, + OPC_VSLEI_D = 0x72838000, + OPC_VSLEI_BU = 0x72840000, + OPC_VSLEI_HU = 0x72848000, + OPC_VSLEI_WU = 0x72850000, + OPC_VSLEI_DU = 0x72858000, + OPC_VSLTI_B = 0x72860000, + OPC_VSLTI_H = 0x72868000, + OPC_VSLTI_W = 0x72870000, + OPC_VSLTI_D = 0x72878000, + OPC_VSLTI_BU = 0x72880000, + OPC_VSLTI_HU = 0x72888000, + OPC_VSLTI_WU = 0x72890000, + OPC_VSLTI_DU = 0x72898000, + OPC_VADDI_BU = 0x728a0000, + OPC_VADDI_HU = 0x728a8000, + OPC_VADDI_WU = 0x728b0000, + OPC_VADDI_DU = 0x728b8000, + OPC_VSUBI_BU = 0x728c0000, + OPC_VSUBI_HU = 0x728c8000, + OPC_VSUBI_WU = 0x728d0000, + OPC_VSUBI_DU = 0x728d8000, + OPC_VBSLL_V = 0x728e0000, + OPC_VBSRL_V = 0x728e8000, + OPC_VMAXI_B = 0x72900000, + OPC_VMAXI_H = 0x72908000, + OPC_VMAXI_W = 0x72910000, + OPC_VMAXI_D = 0x72918000, + OPC_VMINI_B = 0x72920000, + OPC_VMINI_H = 0x72928000, + OPC_VMINI_W = 0x72930000, + OPC_VMINI_D = 0x72938000, + OPC_VMAXI_BU = 0x72940000, + OPC_VMAXI_HU = 0x72948000, + OPC_VMAXI_WU = 0x72950000, + OPC_VMAXI_DU = 0x72958000, + OPC_VMINI_BU = 0x72960000, + OPC_VMINI_HU = 0x72968000, + OPC_VMINI_WU = 0x72970000, + OPC_VMINI_DU = 0x72978000, + OPC_VFRSTPI_B = 0x729a0000, + OPC_VFRSTPI_H = 0x729a8000, + OPC_VCLO_B = 0x729c0000, + OPC_VCLO_H = 0x729c0400, + OPC_VCLO_W = 0x729c0800, + OPC_VCLO_D = 0x729c0c00, + OPC_VCLZ_B = 0x729c1000, + OPC_VCLZ_H = 0x729c1400, + OPC_VCLZ_W = 0x729c1800, + OPC_VCLZ_D = 0x729c1c00, + OPC_VPCNT_B = 0x729c2000, + OPC_VPCNT_H = 0x729c2400, + OPC_VPCNT_W = 0x729c2800, + OPC_VPCNT_D = 0x729c2c00, + OPC_VNEG_B = 0x729c3000, + OPC_VNEG_H = 0x729c3400, + OPC_VNEG_W = 0x729c3800, + OPC_VNEG_D = 0x729c3c00, + OPC_VMSKLTZ_B = 0x729c4000, + OPC_VMSKLTZ_H = 0x729c4400, + OPC_VMSKLTZ_W = 0x729c4800, + OPC_VMSKLTZ_D = 0x729c4c00, + OPC_VMSKGEZ_B = 0x729c5000, + OPC_VMSKNZ_B = 0x729c6000, + OPC_VSETEQZ_V = 0x729c9800, + OPC_VSETNEZ_V = 0x729c9c00, + OPC_VSETANYEQZ_B = 0x729ca000, + OPC_VSETANYEQZ_H = 0x729ca400, + OPC_VSETANYEQZ_W = 0x729ca800, + OPC_VSETANYEQZ_D = 0x729cac00, + OPC_VSETALLNEZ_B = 0x729cb000, + OPC_VSETALLNEZ_H = 0x729cb400, + OPC_VSETALLNEZ_W = 0x729cb800, + OPC_VSETALLNEZ_D = 0x729cbc00, + OPC_VFLOGB_S = 0x729cc400, + OPC_VFLOGB_D = 0x729cc800, + OPC_VFCLASS_S = 0x729cd400, + OPC_VFCLASS_D = 0x729cd800, + OPC_VFSQRT_S = 0x729ce400, + OPC_VFSQRT_D = 0x729ce800, + OPC_VFRECIP_S = 0x729cf400, + OPC_VFRECIP_D = 0x729cf800, + OPC_VFRSQRT_S = 0x729d0400, + OPC_VFRSQRT_D = 0x729d0800, + OPC_VFRINT_S = 0x729d3400, + OPC_VFRINT_D = 0x729d3800, + OPC_VFRINTRM_S = 0x729d4400, + OPC_VFRINTRM_D = 0x729d4800, + OPC_VFRINTRP_S = 0x729d5400, + OPC_VFRINTRP_D = 0x729d5800, + OPC_VFRINTRZ_S = 0x729d6400, + OPC_VFRINTRZ_D = 0x729d6800, + OPC_VFRINTRNE_S = 0x729d7400, + OPC_VFRINTRNE_D = 0x729d7800, + OPC_VFCVTL_S_H = 0x729de800, + OPC_VFCVTH_S_H = 0x729dec00, + OPC_VFCVTL_D_S = 0x729df000, + OPC_VFCVTH_D_S = 0x729df400, + OPC_VFFINT_S_W = 0x729e0000, + OPC_VFFINT_S_WU = 0x729e0400, + OPC_VFFINT_D_L = 0x729e0800, + OPC_VFFINT_D_LU = 0x729e0c00, + OPC_VFFINTL_D_W = 0x729e1000, + OPC_VFFINTH_D_W = 0x729e1400, + OPC_VFTINT_W_S = 0x729e3000, + OPC_VFTINT_L_D = 0x729e3400, + OPC_VFTINTRM_W_S = 0x729e3800, + OPC_VFTINTRM_L_D = 0x729e3c00, + OPC_VFTINTRP_W_S = 0x729e4000, + OPC_VFTINTRP_L_D = 0x729e4400, + OPC_VFTINTRZ_W_S = 0x729e4800, + OPC_VFTINTRZ_L_D = 0x729e4c00, + OPC_VFTINTRNE_W_S = 0x729e5000, + OPC_VFTINTRNE_L_D = 0x729e5400, + OPC_VFTINT_WU_S = 0x729e5800, + OPC_VFTINT_LU_D = 0x729e5c00, + OPC_VFTINTRZ_WU_S = 0x729e7000, + OPC_VFTINTRZ_LU_D = 0x729e7400, + OPC_VFTINTL_L_S = 0x729e8000, + OPC_VFTINTH_L_S = 0x729e8400, + OPC_VFTINTRML_L_S = 0x729e8800, + OPC_VFTINTRMH_L_S = 0x729e8c00, + OPC_VFTINTRPL_L_S = 0x729e9000, + OPC_VFTINTRPH_L_S = 0x729e9400, + OPC_VFTINTRZL_L_S = 0x729e9800, + OPC_VFTINTRZH_L_S = 0x729e9c00, + OPC_VFTINTRNEL_L_S = 0x729ea000, + OPC_VFTINTRNEH_L_S = 0x729ea400, + OPC_VEXTH_H_B = 0x729ee000, + OPC_VEXTH_W_H = 0x729ee400, + OPC_VEXTH_D_W = 0x729ee800, + OPC_VEXTH_Q_D = 0x729eec00, + OPC_VEXTH_HU_BU = 0x729ef000, + OPC_VEXTH_WU_HU = 0x729ef400, + OPC_VEXTH_DU_WU = 0x729ef800, + OPC_VEXTH_QU_DU = 0x729efc00, + OPC_VREPLGR2VR_B = 0x729f0000, + OPC_VREPLGR2VR_H = 0x729f0400, + OPC_VREPLGR2VR_W = 0x729f0800, + OPC_VREPLGR2VR_D = 0x729f0c00, + OPC_VROTRI_B = 0x72a02000, + OPC_VROTRI_H = 0x72a04000, + OPC_VROTRI_W = 0x72a08000, + OPC_VROTRI_D = 0x72a10000, + OPC_VSRLRI_B = 0x72a42000, + OPC_VSRLRI_H = 0x72a44000, + OPC_VSRLRI_W = 0x72a48000, + OPC_VSRLRI_D = 0x72a50000, + OPC_VSRARI_B = 0x72a82000, + OPC_VSRARI_H = 0x72a84000, + OPC_VSRARI_W = 0x72a88000, + OPC_VSRARI_D = 0x72a90000, + OPC_VINSGR2VR_B = 0x72eb8000, + OPC_VINSGR2VR_H = 0x72ebc000, + OPC_VINSGR2VR_W = 0x72ebe000, + OPC_VINSGR2VR_D = 0x72ebf000, + OPC_VPICKVE2GR_B = 0x72ef8000, + OPC_VPICKVE2GR_H = 0x72efc000, + OPC_VPICKVE2GR_W = 0x72efe000, + OPC_VPICKVE2GR_D = 0x72eff000, + OPC_VPICKVE2GR_BU = 0x72f38000, + OPC_VPICKVE2GR_HU = 0x72f3c000, + OPC_VPICKVE2GR_WU = 0x72f3e000, + OPC_VPICKVE2GR_DU = 0x72f3f000, + OPC_VREPLVEI_B = 0x72f78000, + OPC_VREPLVEI_H = 0x72f7c000, + OPC_VREPLVEI_W = 0x72f7e000, + OPC_VREPLVEI_D = 0x72f7f000, + OPC_VSLLWIL_H_B = 0x73082000, + OPC_VSLLWIL_W_H = 0x73084000, + OPC_VSLLWIL_D_W = 0x73088000, + OPC_VEXTL_Q_D = 0x73090000, + OPC_VSLLWIL_HU_BU = 0x730c2000, + OPC_VSLLWIL_WU_HU = 0x730c4000, + OPC_VSLLWIL_DU_WU = 0x730c8000, + OPC_VEXTL_QU_DU = 0x730d0000, + OPC_VBITCLRI_B = 0x73102000, + OPC_VBITCLRI_H = 0x73104000, + OPC_VBITCLRI_W = 0x73108000, + OPC_VBITCLRI_D = 0x73110000, + OPC_VBITSETI_B = 0x73142000, + OPC_VBITSETI_H = 0x73144000, + OPC_VBITSETI_W = 0x73148000, + OPC_VBITSETI_D = 0x73150000, + OPC_VBITREVI_B = 0x73182000, + OPC_VBITREVI_H = 0x73184000, + OPC_VBITREVI_W = 0x73188000, + OPC_VBITREVI_D = 0x73190000, + OPC_VSAT_B = 0x73242000, + OPC_VSAT_H = 0x73244000, + OPC_VSAT_W = 0x73248000, + OPC_VSAT_D = 0x73250000, + OPC_VSAT_BU = 0x73282000, + OPC_VSAT_HU = 0x73284000, + OPC_VSAT_WU = 0x73288000, + OPC_VSAT_DU = 0x73290000, + OPC_VSLLI_B = 0x732c2000, + OPC_VSLLI_H = 0x732c4000, + OPC_VSLLI_W = 0x732c8000, + OPC_VSLLI_D = 0x732d0000, + OPC_VSRLI_B = 0x73302000, + OPC_VSRLI_H = 0x73304000, + OPC_VSRLI_W = 0x73308000, + OPC_VSRLI_D = 0x73310000, + OPC_VSRAI_B = 0x73342000, + OPC_VSRAI_H = 0x73344000, + OPC_VSRAI_W = 0x73348000, + OPC_VSRAI_D = 0x73350000, + OPC_VSRLNI_B_H = 0x73404000, + OPC_VSRLNI_H_W = 0x73408000, + OPC_VSRLNI_W_D = 0x73410000, + OPC_VSRLNI_D_Q = 0x73420000, + OPC_VSRLRNI_B_H = 0x73444000, + OPC_VSRLRNI_H_W = 0x73448000, + OPC_VSRLRNI_W_D = 0x73450000, + OPC_VSRLRNI_D_Q = 0x73460000, + OPC_VSSRLNI_B_H = 0x73484000, + OPC_VSSRLNI_H_W = 0x73488000, + OPC_VSSRLNI_W_D = 0x73490000, + OPC_VSSRLNI_D_Q = 0x734a0000, + OPC_VSSRLNI_BU_H = 0x734c4000, + OPC_VSSRLNI_HU_W = 0x734c8000, + OPC_VSSRLNI_WU_D = 0x734d0000, + OPC_VSSRLNI_DU_Q = 0x734e0000, + OPC_VSSRLRNI_B_H = 0x73504000, + OPC_VSSRLRNI_H_W = 0x73508000, + OPC_VSSRLRNI_W_D = 0x73510000, + OPC_VSSRLRNI_D_Q = 0x73520000, + OPC_VSSRLRNI_BU_H = 0x73544000, + OPC_VSSRLRNI_HU_W = 0x73548000, + OPC_VSSRLRNI_WU_D = 0x73550000, + OPC_VSSRLRNI_DU_Q = 0x73560000, + OPC_VSRANI_B_H = 0x73584000, + OPC_VSRANI_H_W = 0x73588000, + OPC_VSRANI_W_D = 0x73590000, + OPC_VSRANI_D_Q = 0x735a0000, + OPC_VSRARNI_B_H = 0x735c4000, + OPC_VSRARNI_H_W = 0x735c8000, + OPC_VSRARNI_W_D = 0x735d0000, + OPC_VSRARNI_D_Q = 0x735e0000, + OPC_VSSRANI_B_H = 0x73604000, + OPC_VSSRANI_H_W = 0x73608000, + OPC_VSSRANI_W_D = 0x73610000, + OPC_VSSRANI_D_Q = 0x73620000, + OPC_VSSRANI_BU_H = 0x73644000, + OPC_VSSRANI_HU_W = 0x73648000, + OPC_VSSRANI_WU_D = 0x73650000, + OPC_VSSRANI_DU_Q = 0x73660000, + OPC_VSSRARNI_B_H = 0x73684000, + OPC_VSSRARNI_H_W = 0x73688000, + OPC_VSSRARNI_W_D = 0x73690000, + OPC_VSSRARNI_D_Q = 0x736a0000, + OPC_VSSRARNI_BU_H = 0x736c4000, + OPC_VSSRARNI_HU_W = 0x736c8000, + OPC_VSSRARNI_WU_D = 0x736d0000, + OPC_VSSRARNI_DU_Q = 0x736e0000, + OPC_VEXTRINS_D = 0x73800000, + OPC_VEXTRINS_W = 0x73840000, + OPC_VEXTRINS_H = 0x73880000, + OPC_VEXTRINS_B = 0x738c0000, + OPC_VSHUF4I_B = 0x73900000, + OPC_VSHUF4I_H = 0x73940000, + OPC_VSHUF4I_W = 0x73980000, + OPC_VSHUF4I_D = 0x739c0000, + OPC_VBITSELI_B = 0x73c40000, + OPC_VANDI_B = 0x73d00000, + OPC_VORI_B = 0x73d40000, + OPC_VXORI_B = 0x73d80000, + OPC_VNORI_B = 0x73dc0000, + OPC_VLDI = 0x73e00000, + OPC_VPERMI_W = 0x73e40000, } LoongArchInsn; static int32_t __attribute__((unused)) @@ -134,6 +846,13 @@ encode_djk_slots(LoongArchInsn opc, uint32_t d, uint32_t j, uint32_t k) } static int32_t __attribute__((unused)) +encode_djka_slots(LoongArchInsn opc, uint32_t d, uint32_t j, uint32_t k, + uint32_t a) +{ + return opc | d | j << 5 | k << 10 | a << 15; +} + +static int32_t __attribute__((unused)) encode_djkm_slots(LoongArchInsn opc, uint32_t d, uint32_t j, uint32_t k, uint32_t m) { @@ -141,12 +860,27 @@ encode_djkm_slots(LoongArchInsn opc, uint32_t d, uint32_t j, uint32_t k, } static int32_t __attribute__((unused)) +encode_djkn_slots(LoongArchInsn opc, uint32_t d, uint32_t j, uint32_t k, + uint32_t n) +{ + return opc | d | j << 5 | k << 10 | n << 18; +} + +static int32_t __attribute__((unused)) encode_dk_slots(LoongArchInsn opc, uint32_t d, uint32_t k) { return opc | d | k << 10; } static int32_t __attribute__((unused)) +encode_cdvj_insn(LoongArchInsn opc, TCGReg cd, TCGReg vj) +{ + tcg_debug_assert(cd >= 0 && cd <= 0x7); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + return encode_dj_slots(opc, cd, vj & 0x1f); +} + +static int32_t __attribute__((unused)) encode_dj_insn(LoongArchInsn opc, TCGReg d, TCGReg j) { tcg_debug_assert(d >= 0 && d <= 0x1f); @@ -239,6 +973,42 @@ encode_dsj20_insn(LoongArchInsn opc, TCGReg d, int32_t sj20) } static int32_t __attribute__((unused)) +encode_dvjuk1_insn(LoongArchInsn opc, TCGReg d, TCGReg vj, uint32_t uk1) +{ + tcg_debug_assert(d >= 0 && d <= 0x1f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk1 <= 0x1); + return encode_djk_slots(opc, d, vj & 0x1f, uk1); +} + +static int32_t __attribute__((unused)) +encode_dvjuk2_insn(LoongArchInsn opc, TCGReg d, TCGReg vj, uint32_t uk2) +{ + tcg_debug_assert(d >= 0 && d <= 0x1f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk2 <= 0x3); + return encode_djk_slots(opc, d, vj & 0x1f, uk2); +} + +static int32_t __attribute__((unused)) +encode_dvjuk3_insn(LoongArchInsn opc, TCGReg d, TCGReg vj, uint32_t uk3) +{ + tcg_debug_assert(d >= 0 && d <= 0x1f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk3 <= 0x7); + return encode_djk_slots(opc, d, vj & 0x1f, uk3); +} + +static int32_t __attribute__((unused)) +encode_dvjuk4_insn(LoongArchInsn opc, TCGReg d, TCGReg vj, uint32_t uk4) +{ + tcg_debug_assert(d >= 0 && d <= 0x1f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk4 <= 0xf); + return encode_djk_slots(opc, d, vj & 0x1f, uk4); +} + +static int32_t __attribute__((unused)) encode_sd10k16_insn(LoongArchInsn opc, int32_t sd10k16) { tcg_debug_assert(sd10k16 >= -0x2000000 && sd10k16 <= 0x1ffffff); @@ -252,6 +1022,265 @@ encode_ud15_insn(LoongArchInsn opc, uint32_t ud15) return encode_d_slot(opc, ud15); } +static int32_t __attribute__((unused)) +encode_vdj_insn(LoongArchInsn opc, TCGReg vd, TCGReg j) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + return encode_dj_slots(opc, vd & 0x1f, j); +} + +static int32_t __attribute__((unused)) +encode_vdjk_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, TCGReg k) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(k >= 0 && k <= 0x1f); + return encode_djk_slots(opc, vd & 0x1f, j, k); +} + +static int32_t __attribute__((unused)) +encode_vdjsk10_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, int32_t sk10) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(sk10 >= -0x200 && sk10 <= 0x1ff); + return encode_djk_slots(opc, vd & 0x1f, j, sk10 & 0x3ff); +} + +static int32_t __attribute__((unused)) +encode_vdjsk11_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, int32_t sk11) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(sk11 >= -0x400 && sk11 <= 0x3ff); + return encode_djk_slots(opc, vd & 0x1f, j, sk11 & 0x7ff); +} + +static int32_t __attribute__((unused)) +encode_vdjsk12_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, int32_t sk12) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(sk12 >= -0x800 && sk12 <= 0x7ff); + return encode_djk_slots(opc, vd & 0x1f, j, sk12 & 0xfff); +} + +static int32_t __attribute__((unused)) +encode_vdjsk8un1_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, int32_t sk8, + uint32_t un1) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(sk8 >= -0x80 && sk8 <= 0x7f); + tcg_debug_assert(un1 <= 0x1); + return encode_djkn_slots(opc, vd & 0x1f, j, sk8 & 0xff, un1); +} + +static int32_t __attribute__((unused)) +encode_vdjsk8un2_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, int32_t sk8, + uint32_t un2) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(sk8 >= -0x80 && sk8 <= 0x7f); + tcg_debug_assert(un2 <= 0x3); + return encode_djkn_slots(opc, vd & 0x1f, j, sk8 & 0xff, un2); +} + +static int32_t __attribute__((unused)) +encode_vdjsk8un3_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, int32_t sk8, + uint32_t un3) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(sk8 >= -0x80 && sk8 <= 0x7f); + tcg_debug_assert(un3 <= 0x7); + return encode_djkn_slots(opc, vd & 0x1f, j, sk8 & 0xff, un3); +} + +static int32_t __attribute__((unused)) +encode_vdjsk8un4_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, int32_t sk8, + uint32_t un4) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(sk8 >= -0x80 && sk8 <= 0x7f); + tcg_debug_assert(un4 <= 0xf); + return encode_djkn_slots(opc, vd & 0x1f, j, sk8 & 0xff, un4); +} + +static int32_t __attribute__((unused)) +encode_vdjsk9_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, int32_t sk9) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(sk9 >= -0x100 && sk9 <= 0xff); + return encode_djk_slots(opc, vd & 0x1f, j, sk9 & 0x1ff); +} + +static int32_t __attribute__((unused)) +encode_vdjuk1_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, uint32_t uk1) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(uk1 <= 0x1); + return encode_djk_slots(opc, vd & 0x1f, j, uk1); +} + +static int32_t __attribute__((unused)) +encode_vdjuk2_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, uint32_t uk2) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(uk2 <= 0x3); + return encode_djk_slots(opc, vd & 0x1f, j, uk2); +} + +static int32_t __attribute__((unused)) +encode_vdjuk3_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, uint32_t uk3) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(uk3 <= 0x7); + return encode_djk_slots(opc, vd & 0x1f, j, uk3); +} + +static int32_t __attribute__((unused)) +encode_vdjuk4_insn(LoongArchInsn opc, TCGReg vd, TCGReg j, uint32_t uk4) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(j >= 0 && j <= 0x1f); + tcg_debug_assert(uk4 <= 0xf); + return encode_djk_slots(opc, vd & 0x1f, j, uk4); +} + +static int32_t __attribute__((unused)) +encode_vdsj13_insn(LoongArchInsn opc, TCGReg vd, int32_t sj13) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(sj13 >= -0x1000 && sj13 <= 0xfff); + return encode_dj_slots(opc, vd & 0x1f, sj13 & 0x1fff); +} + +static int32_t __attribute__((unused)) +encode_vdvj_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + return encode_dj_slots(opc, vd & 0x1f, vj & 0x1f); +} + +static int32_t __attribute__((unused)) +encode_vdvjk_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, TCGReg k) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(k >= 0 && k <= 0x1f); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, k); +} + +static int32_t __attribute__((unused)) +encode_vdvjsk5_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(sk5 >= -0x10 && sk5 <= 0xf); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, sk5 & 0x1f); +} + +static int32_t __attribute__((unused)) +encode_vdvjuk1_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, uint32_t uk1) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk1 <= 0x1); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, uk1); +} + +static int32_t __attribute__((unused)) +encode_vdvjuk2_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, uint32_t uk2) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk2 <= 0x3); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, uk2); +} + +static int32_t __attribute__((unused)) +encode_vdvjuk3_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk3 <= 0x7); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, uk3); +} + +static int32_t __attribute__((unused)) +encode_vdvjuk4_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk4 <= 0xf); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, uk4); +} + +static int32_t __attribute__((unused)) +encode_vdvjuk5_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk5 <= 0x1f); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, uk5); +} + +static int32_t __attribute__((unused)) +encode_vdvjuk6_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk6 <= 0x3f); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, uk6); +} + +static int32_t __attribute__((unused)) +encode_vdvjuk7_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk7 <= 0x7f); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, uk7); +} + +static int32_t __attribute__((unused)) +encode_vdvjuk8_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(uk8 <= 0xff); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, uk8); +} + +static int32_t __attribute__((unused)) +encode_vdvjvk_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(vk >= 0x20 && vk <= 0x3f); + return encode_djk_slots(opc, vd & 0x1f, vj & 0x1f, vk & 0x1f); +} + +static int32_t __attribute__((unused)) +encode_vdvjvkva_insn(LoongArchInsn opc, TCGReg vd, TCGReg vj, TCGReg vk, + TCGReg va) +{ + tcg_debug_assert(vd >= 0x20 && vd <= 0x3f); + tcg_debug_assert(vj >= 0x20 && vj <= 0x3f); + tcg_debug_assert(vk >= 0x20 && vk <= 0x3f); + tcg_debug_assert(va >= 0x20 && va <= 0x3f); + return encode_djka_slots(opc, vd & 0x1f, vj & 0x1f, vk & 0x1f, va & 0x1f); +} + /* Emits the `clz.w d, j` instruction. */ static void __attribute__((unused)) tcg_out_opc_clz_w(TCGContext *s, TCGReg d, TCGReg j) @@ -711,6 +1740,384 @@ tcg_out_opc_xori(TCGContext *s, TCGReg d, TCGReg j, uint32_t uk12) tcg_out32(s, encode_djuk12_insn(OPC_XORI, d, j, uk12)); } +/* Emits the `vfmadd.s vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmadd_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VFMADD_S, vd, vj, vk, va)); +} + +/* Emits the `vfmadd.d vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmadd_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VFMADD_D, vd, vj, vk, va)); +} + +/* Emits the `vfmsub.s vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmsub_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VFMSUB_S, vd, vj, vk, va)); +} + +/* Emits the `vfmsub.d vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmsub_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VFMSUB_D, vd, vj, vk, va)); +} + +/* Emits the `vfnmadd.s vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfnmadd_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VFNMADD_S, vd, vj, vk, va)); +} + +/* Emits the `vfnmadd.d vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfnmadd_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VFNMADD_D, vd, vj, vk, va)); +} + +/* Emits the `vfnmsub.s vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfnmsub_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VFNMSUB_S, vd, vj, vk, va)); +} + +/* Emits the `vfnmsub.d vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfnmsub_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VFNMSUB_D, vd, vj, vk, va)); +} + +/* Emits the `vfcmp.caf.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_caf_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CAF_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.saf.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_saf_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SAF_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.clt.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_clt_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CLT_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.slt.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_slt_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SLT_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.ceq.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_ceq_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CEQ_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.seq.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_seq_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SEQ_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.cle.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cle_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CLE_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.sle.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sle_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SLE_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.cun.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cun_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CUN_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.sun.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sun_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SUN_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.cult.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cult_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CULT_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.sult.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sult_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SULT_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.cueq.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cueq_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CUEQ_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.sueq.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sueq_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SUEQ_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.cule.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cule_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CULE_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.sule.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sule_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SULE_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.cne.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cne_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CNE_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.sne.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sne_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SNE_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.cor.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cor_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_COR_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.sor.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sor_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SOR_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.cune.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cune_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CUNE_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.sune.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sune_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SUNE_S, vd, vj, vk)); +} + +/* Emits the `vfcmp.caf.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_caf_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CAF_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.saf.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_saf_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SAF_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.clt.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_clt_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CLT_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.slt.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_slt_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SLT_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.ceq.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_ceq_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CEQ_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.seq.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_seq_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SEQ_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.cle.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cle_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CLE_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.sle.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sle_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SLE_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.cun.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cun_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CUN_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.sun.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sun_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SUN_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.cult.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cult_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CULT_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.sult.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sult_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SULT_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.cueq.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cueq_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CUEQ_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.sueq.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sueq_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SUEQ_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.cule.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cule_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CULE_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.sule.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sule_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SULE_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.cne.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cne_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CNE_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.sne.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sne_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SNE_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.cor.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cor_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_COR_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.sor.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sor_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SOR_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.cune.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_cune_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_CUNE_D, vd, vj, vk)); +} + +/* Emits the `vfcmp.sune.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcmp_sune_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCMP_SUNE_D, vd, vj, vk)); +} + +/* Emits the `vbitsel.v vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitsel_v(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VBITSEL_V, vd, vj, vk, va)); +} + +/* Emits the `vshuf.b vd, vj, vk, va` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vshuf_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk, TCGReg va) +{ + tcg_out32(s, encode_vdvjvkva_insn(OPC_VSHUF_B, vd, vj, vk, va)); +} + /* Emits the `addu16i.d d, j, sk16` instruction. */ static void __attribute__((unused)) tcg_out_opc_addu16i_d(TCGContext *s, TCGReg d, TCGReg j, int32_t sk16) @@ -837,6 +2244,80 @@ tcg_out_opc_ld_wu(TCGContext *s, TCGReg d, TCGReg j, int32_t sk12) tcg_out32(s, encode_djsk12_insn(OPC_LD_WU, d, j, sk12)); } +/* Emits the `vld vd, j, sk12` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vld(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk12) +{ + tcg_out32(s, encode_vdjsk12_insn(OPC_VLD, vd, j, sk12)); +} + +/* Emits the `vst vd, j, sk12` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vst(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk12) +{ + tcg_out32(s, encode_vdjsk12_insn(OPC_VST, vd, j, sk12)); +} + +/* Emits the `vldrepl.d vd, j, sk9` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vldrepl_d(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk9) +{ + tcg_out32(s, encode_vdjsk9_insn(OPC_VLDREPL_D, vd, j, sk9)); +} + +/* Emits the `vldrepl.w vd, j, sk10` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vldrepl_w(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk10) +{ + tcg_out32(s, encode_vdjsk10_insn(OPC_VLDREPL_W, vd, j, sk10)); +} + +/* Emits the `vldrepl.h vd, j, sk11` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vldrepl_h(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk11) +{ + tcg_out32(s, encode_vdjsk11_insn(OPC_VLDREPL_H, vd, j, sk11)); +} + +/* Emits the `vldrepl.b vd, j, sk12` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vldrepl_b(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk12) +{ + tcg_out32(s, encode_vdjsk12_insn(OPC_VLDREPL_B, vd, j, sk12)); +} + +/* Emits the `vstelm.d vd, j, sk8, un1` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vstelm_d(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk8, + uint32_t un1) +{ + tcg_out32(s, encode_vdjsk8un1_insn(OPC_VSTELM_D, vd, j, sk8, un1)); +} + +/* Emits the `vstelm.w vd, j, sk8, un2` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vstelm_w(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk8, + uint32_t un2) +{ + tcg_out32(s, encode_vdjsk8un2_insn(OPC_VSTELM_W, vd, j, sk8, un2)); +} + +/* Emits the `vstelm.h vd, j, sk8, un3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vstelm_h(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk8, + uint32_t un3) +{ + tcg_out32(s, encode_vdjsk8un3_insn(OPC_VSTELM_H, vd, j, sk8, un3)); +} + +/* Emits the `vstelm.b vd, j, sk8, un4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vstelm_b(TCGContext *s, TCGReg vd, TCGReg j, int32_t sk8, + uint32_t un4) +{ + tcg_out32(s, encode_vdjsk8un4_insn(OPC_VSTELM_B, vd, j, sk8, un4)); +} + /* Emits the `ldx.b d, j, k` instruction. */ static void __attribute__((unused)) tcg_out_opc_ldx_b(TCGContext *s, TCGReg d, TCGReg j, TCGReg k) @@ -914,6 +2395,20 @@ tcg_out_opc_ldx_wu(TCGContext *s, TCGReg d, TCGReg j, TCGReg k) tcg_out32(s, encode_djk_insn(OPC_LDX_WU, d, j, k)); } +/* Emits the `vldx vd, j, k` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vldx(TCGContext *s, TCGReg vd, TCGReg j, TCGReg k) +{ + tcg_out32(s, encode_vdjk_insn(OPC_VLDX, vd, j, k)); +} + +/* Emits the `vstx vd, j, k` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vstx(TCGContext *s, TCGReg vd, TCGReg j, TCGReg k) +{ + tcg_out32(s, encode_vdjk_insn(OPC_VSTX, vd, j, k)); +} + /* Emits the `dbar ud15` instruction. */ static void __attribute__((unused)) tcg_out_opc_dbar(TCGContext *s, uint32_t ud15) @@ -984,4 +2479,4526 @@ tcg_out_opc_bleu(TCGContext *s, TCGReg d, TCGReg j, int32_t sk16) tcg_out32(s, encode_djsk16_insn(OPC_BLEU, d, j, sk16)); } +/* Emits the `vseq.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vseq_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSEQ_B, vd, vj, vk)); +} + +/* Emits the `vseq.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vseq_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSEQ_H, vd, vj, vk)); +} + +/* Emits the `vseq.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vseq_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSEQ_W, vd, vj, vk)); +} + +/* Emits the `vseq.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vseq_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSEQ_D, vd, vj, vk)); +} + +/* Emits the `vsle.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsle_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLE_B, vd, vj, vk)); +} + +/* Emits the `vsle.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsle_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLE_H, vd, vj, vk)); +} + +/* Emits the `vsle.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsle_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLE_W, vd, vj, vk)); +} + +/* Emits the `vsle.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsle_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLE_D, vd, vj, vk)); +} + +/* Emits the `vsle.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsle_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLE_BU, vd, vj, vk)); +} + +/* Emits the `vsle.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsle_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLE_HU, vd, vj, vk)); +} + +/* Emits the `vsle.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsle_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLE_WU, vd, vj, vk)); +} + +/* Emits the `vsle.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsle_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLE_DU, vd, vj, vk)); +} + +/* Emits the `vslt.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslt_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLT_B, vd, vj, vk)); +} + +/* Emits the `vslt.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslt_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLT_H, vd, vj, vk)); +} + +/* Emits the `vslt.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslt_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLT_W, vd, vj, vk)); +} + +/* Emits the `vslt.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslt_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLT_D, vd, vj, vk)); +} + +/* Emits the `vslt.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslt_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLT_BU, vd, vj, vk)); +} + +/* Emits the `vslt.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslt_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLT_HU, vd, vj, vk)); +} + +/* Emits the `vslt.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslt_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLT_WU, vd, vj, vk)); +} + +/* Emits the `vslt.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslt_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLT_DU, vd, vj, vk)); +} + +/* Emits the `vadd.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vadd_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADD_B, vd, vj, vk)); +} + +/* Emits the `vadd.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vadd_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADD_H, vd, vj, vk)); +} + +/* Emits the `vadd.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vadd_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADD_W, vd, vj, vk)); +} + +/* Emits the `vadd.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vadd_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADD_D, vd, vj, vk)); +} + +/* Emits the `vsub.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsub_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUB_B, vd, vj, vk)); +} + +/* Emits the `vsub.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsub_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUB_H, vd, vj, vk)); +} + +/* Emits the `vsub.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsub_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUB_W, vd, vj, vk)); +} + +/* Emits the `vsub.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsub_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUB_D, vd, vj, vk)); +} + +/* Emits the `vaddwev.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_H_B, vd, vj, vk)); +} + +/* Emits the `vaddwev.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_W_H, vd, vj, vk)); +} + +/* Emits the `vaddwev.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_D_W, vd, vj, vk)); +} + +/* Emits the `vaddwev.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_Q_D, vd, vj, vk)); +} + +/* Emits the `vsubwev.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwev_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWEV_H_B, vd, vj, vk)); +} + +/* Emits the `vsubwev.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwev_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWEV_W_H, vd, vj, vk)); +} + +/* Emits the `vsubwev.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwev_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWEV_D_W, vd, vj, vk)); +} + +/* Emits the `vsubwev.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwev_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWEV_Q_D, vd, vj, vk)); +} + +/* Emits the `vaddwod.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_H_B, vd, vj, vk)); +} + +/* Emits the `vaddwod.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_W_H, vd, vj, vk)); +} + +/* Emits the `vaddwod.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_D_W, vd, vj, vk)); +} + +/* Emits the `vaddwod.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_Q_D, vd, vj, vk)); +} + +/* Emits the `vsubwod.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwod_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWOD_H_B, vd, vj, vk)); +} + +/* Emits the `vsubwod.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwod_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWOD_W_H, vd, vj, vk)); +} + +/* Emits the `vsubwod.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwod_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWOD_D_W, vd, vj, vk)); +} + +/* Emits the `vsubwod.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwod_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWOD_Q_D, vd, vj, vk)); +} + +/* Emits the `vaddwev.h.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_h_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_H_BU, vd, vj, vk)); +} + +/* Emits the `vaddwev.w.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_w_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_W_HU, vd, vj, vk)); +} + +/* Emits the `vaddwev.d.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_d_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_D_WU, vd, vj, vk)); +} + +/* Emits the `vaddwev.q.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_q_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_Q_DU, vd, vj, vk)); +} + +/* Emits the `vsubwev.h.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwev_h_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWEV_H_BU, vd, vj, vk)); +} + +/* Emits the `vsubwev.w.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwev_w_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWEV_W_HU, vd, vj, vk)); +} + +/* Emits the `vsubwev.d.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwev_d_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWEV_D_WU, vd, vj, vk)); +} + +/* Emits the `vsubwev.q.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwev_q_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWEV_Q_DU, vd, vj, vk)); +} + +/* Emits the `vaddwod.h.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_h_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_H_BU, vd, vj, vk)); +} + +/* Emits the `vaddwod.w.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_w_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_W_HU, vd, vj, vk)); +} + +/* Emits the `vaddwod.d.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_d_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_D_WU, vd, vj, vk)); +} + +/* Emits the `vaddwod.q.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_q_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_Q_DU, vd, vj, vk)); +} + +/* Emits the `vsubwod.h.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwod_h_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWOD_H_BU, vd, vj, vk)); +} + +/* Emits the `vsubwod.w.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwod_w_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWOD_W_HU, vd, vj, vk)); +} + +/* Emits the `vsubwod.d.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwod_d_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWOD_D_WU, vd, vj, vk)); +} + +/* Emits the `vsubwod.q.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubwod_q_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUBWOD_Q_DU, vd, vj, vk)); +} + +/* Emits the `vaddwev.h.bu.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_h_bu_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_H_BU_B, vd, vj, vk)); +} + +/* Emits the `vaddwev.w.hu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_w_hu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_W_HU_H, vd, vj, vk)); +} + +/* Emits the `vaddwev.d.wu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_d_wu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_D_WU_W, vd, vj, vk)); +} + +/* Emits the `vaddwev.q.du.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwev_q_du_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWEV_Q_DU_D, vd, vj, vk)); +} + +/* Emits the `vaddwod.h.bu.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_h_bu_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_H_BU_B, vd, vj, vk)); +} + +/* Emits the `vaddwod.w.hu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_w_hu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_W_HU_H, vd, vj, vk)); +} + +/* Emits the `vaddwod.d.wu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_d_wu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_D_WU_W, vd, vj, vk)); +} + +/* Emits the `vaddwod.q.du.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddwod_q_du_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDWOD_Q_DU_D, vd, vj, vk)); +} + +/* Emits the `vsadd.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsadd_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSADD_B, vd, vj, vk)); +} + +/* Emits the `vsadd.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsadd_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSADD_H, vd, vj, vk)); +} + +/* Emits the `vsadd.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsadd_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSADD_W, vd, vj, vk)); +} + +/* Emits the `vsadd.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsadd_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSADD_D, vd, vj, vk)); +} + +/* Emits the `vssub.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssub_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSUB_B, vd, vj, vk)); +} + +/* Emits the `vssub.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssub_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSUB_H, vd, vj, vk)); +} + +/* Emits the `vssub.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssub_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSUB_W, vd, vj, vk)); +} + +/* Emits the `vssub.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssub_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSUB_D, vd, vj, vk)); +} + +/* Emits the `vsadd.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsadd_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSADD_BU, vd, vj, vk)); +} + +/* Emits the `vsadd.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsadd_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSADD_HU, vd, vj, vk)); +} + +/* Emits the `vsadd.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsadd_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSADD_WU, vd, vj, vk)); +} + +/* Emits the `vsadd.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsadd_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSADD_DU, vd, vj, vk)); +} + +/* Emits the `vssub.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssub_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSUB_BU, vd, vj, vk)); +} + +/* Emits the `vssub.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssub_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSUB_HU, vd, vj, vk)); +} + +/* Emits the `vssub.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssub_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSUB_WU, vd, vj, vk)); +} + +/* Emits the `vssub.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssub_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSUB_DU, vd, vj, vk)); +} + +/* Emits the `vhaddw.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhaddw_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHADDW_H_B, vd, vj, vk)); +} + +/* Emits the `vhaddw.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhaddw_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHADDW_W_H, vd, vj, vk)); +} + +/* Emits the `vhaddw.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhaddw_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHADDW_D_W, vd, vj, vk)); +} + +/* Emits the `vhaddw.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhaddw_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHADDW_Q_D, vd, vj, vk)); +} + +/* Emits the `vhsubw.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhsubw_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHSUBW_H_B, vd, vj, vk)); +} + +/* Emits the `vhsubw.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhsubw_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHSUBW_W_H, vd, vj, vk)); +} + +/* Emits the `vhsubw.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhsubw_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHSUBW_D_W, vd, vj, vk)); +} + +/* Emits the `vhsubw.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhsubw_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHSUBW_Q_D, vd, vj, vk)); +} + +/* Emits the `vhaddw.hu.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhaddw_hu_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHADDW_HU_BU, vd, vj, vk)); +} + +/* Emits the `vhaddw.wu.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhaddw_wu_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHADDW_WU_HU, vd, vj, vk)); +} + +/* Emits the `vhaddw.du.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhaddw_du_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHADDW_DU_WU, vd, vj, vk)); +} + +/* Emits the `vhaddw.qu.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhaddw_qu_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHADDW_QU_DU, vd, vj, vk)); +} + +/* Emits the `vhsubw.hu.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhsubw_hu_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHSUBW_HU_BU, vd, vj, vk)); +} + +/* Emits the `vhsubw.wu.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhsubw_wu_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHSUBW_WU_HU, vd, vj, vk)); +} + +/* Emits the `vhsubw.du.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhsubw_du_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHSUBW_DU_WU, vd, vj, vk)); +} + +/* Emits the `vhsubw.qu.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vhsubw_qu_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VHSUBW_QU_DU, vd, vj, vk)); +} + +/* Emits the `vadda.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vadda_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDA_B, vd, vj, vk)); +} + +/* Emits the `vadda.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vadda_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDA_H, vd, vj, vk)); +} + +/* Emits the `vadda.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vadda_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDA_W, vd, vj, vk)); +} + +/* Emits the `vadda.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vadda_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADDA_D, vd, vj, vk)); +} + +/* Emits the `vabsd.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vabsd_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VABSD_B, vd, vj, vk)); +} + +/* Emits the `vabsd.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vabsd_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VABSD_H, vd, vj, vk)); +} + +/* Emits the `vabsd.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vabsd_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VABSD_W, vd, vj, vk)); +} + +/* Emits the `vabsd.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vabsd_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VABSD_D, vd, vj, vk)); +} + +/* Emits the `vabsd.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vabsd_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VABSD_BU, vd, vj, vk)); +} + +/* Emits the `vabsd.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vabsd_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VABSD_HU, vd, vj, vk)); +} + +/* Emits the `vabsd.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vabsd_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VABSD_WU, vd, vj, vk)); +} + +/* Emits the `vabsd.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vabsd_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VABSD_DU, vd, vj, vk)); +} + +/* Emits the `vavg.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavg_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVG_B, vd, vj, vk)); +} + +/* Emits the `vavg.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavg_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVG_H, vd, vj, vk)); +} + +/* Emits the `vavg.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavg_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVG_W, vd, vj, vk)); +} + +/* Emits the `vavg.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavg_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVG_D, vd, vj, vk)); +} + +/* Emits the `vavg.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavg_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVG_BU, vd, vj, vk)); +} + +/* Emits the `vavg.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavg_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVG_HU, vd, vj, vk)); +} + +/* Emits the `vavg.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavg_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVG_WU, vd, vj, vk)); +} + +/* Emits the `vavg.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavg_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVG_DU, vd, vj, vk)); +} + +/* Emits the `vavgr.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavgr_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVGR_B, vd, vj, vk)); +} + +/* Emits the `vavgr.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavgr_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVGR_H, vd, vj, vk)); +} + +/* Emits the `vavgr.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavgr_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVGR_W, vd, vj, vk)); +} + +/* Emits the `vavgr.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavgr_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVGR_D, vd, vj, vk)); +} + +/* Emits the `vavgr.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavgr_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVGR_BU, vd, vj, vk)); +} + +/* Emits the `vavgr.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavgr_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVGR_HU, vd, vj, vk)); +} + +/* Emits the `vavgr.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavgr_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVGR_WU, vd, vj, vk)); +} + +/* Emits the `vavgr.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vavgr_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAVGR_DU, vd, vj, vk)); +} + +/* Emits the `vmax.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmax_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMAX_B, vd, vj, vk)); +} + +/* Emits the `vmax.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmax_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMAX_H, vd, vj, vk)); +} + +/* Emits the `vmax.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmax_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMAX_W, vd, vj, vk)); +} + +/* Emits the `vmax.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmax_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMAX_D, vd, vj, vk)); +} + +/* Emits the `vmin.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmin_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMIN_B, vd, vj, vk)); +} + +/* Emits the `vmin.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmin_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMIN_H, vd, vj, vk)); +} + +/* Emits the `vmin.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmin_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMIN_W, vd, vj, vk)); +} + +/* Emits the `vmin.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmin_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMIN_D, vd, vj, vk)); +} + +/* Emits the `vmax.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmax_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMAX_BU, vd, vj, vk)); +} + +/* Emits the `vmax.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmax_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMAX_HU, vd, vj, vk)); +} + +/* Emits the `vmax.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmax_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMAX_WU, vd, vj, vk)); +} + +/* Emits the `vmax.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmax_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMAX_DU, vd, vj, vk)); +} + +/* Emits the `vmin.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmin_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMIN_BU, vd, vj, vk)); +} + +/* Emits the `vmin.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmin_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMIN_HU, vd, vj, vk)); +} + +/* Emits the `vmin.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmin_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMIN_WU, vd, vj, vk)); +} + +/* Emits the `vmin.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmin_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMIN_DU, vd, vj, vk)); +} + +/* Emits the `vmul.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmul_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUL_B, vd, vj, vk)); +} + +/* Emits the `vmul.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmul_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUL_H, vd, vj, vk)); +} + +/* Emits the `vmul.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmul_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUL_W, vd, vj, vk)); +} + +/* Emits the `vmul.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmul_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUL_D, vd, vj, vk)); +} + +/* Emits the `vmuh.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmuh_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUH_B, vd, vj, vk)); +} + +/* Emits the `vmuh.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmuh_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUH_H, vd, vj, vk)); +} + +/* Emits the `vmuh.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmuh_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUH_W, vd, vj, vk)); +} + +/* Emits the `vmuh.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmuh_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUH_D, vd, vj, vk)); +} + +/* Emits the `vmuh.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmuh_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUH_BU, vd, vj, vk)); +} + +/* Emits the `vmuh.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmuh_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUH_HU, vd, vj, vk)); +} + +/* Emits the `vmuh.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmuh_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUH_WU, vd, vj, vk)); +} + +/* Emits the `vmuh.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmuh_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMUH_DU, vd, vj, vk)); +} + +/* Emits the `vmulwev.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_H_B, vd, vj, vk)); +} + +/* Emits the `vmulwev.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_W_H, vd, vj, vk)); +} + +/* Emits the `vmulwev.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_D_W, vd, vj, vk)); +} + +/* Emits the `vmulwev.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_Q_D, vd, vj, vk)); +} + +/* Emits the `vmulwod.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_H_B, vd, vj, vk)); +} + +/* Emits the `vmulwod.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_W_H, vd, vj, vk)); +} + +/* Emits the `vmulwod.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_D_W, vd, vj, vk)); +} + +/* Emits the `vmulwod.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_Q_D, vd, vj, vk)); +} + +/* Emits the `vmulwev.h.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_h_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_H_BU, vd, vj, vk)); +} + +/* Emits the `vmulwev.w.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_w_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_W_HU, vd, vj, vk)); +} + +/* Emits the `vmulwev.d.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_d_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_D_WU, vd, vj, vk)); +} + +/* Emits the `vmulwev.q.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_q_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_Q_DU, vd, vj, vk)); +} + +/* Emits the `vmulwod.h.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_h_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_H_BU, vd, vj, vk)); +} + +/* Emits the `vmulwod.w.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_w_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_W_HU, vd, vj, vk)); +} + +/* Emits the `vmulwod.d.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_d_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_D_WU, vd, vj, vk)); +} + +/* Emits the `vmulwod.q.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_q_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_Q_DU, vd, vj, vk)); +} + +/* Emits the `vmulwev.h.bu.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_h_bu_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_H_BU_B, vd, vj, vk)); +} + +/* Emits the `vmulwev.w.hu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_w_hu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_W_HU_H, vd, vj, vk)); +} + +/* Emits the `vmulwev.d.wu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_d_wu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_D_WU_W, vd, vj, vk)); +} + +/* Emits the `vmulwev.q.du.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwev_q_du_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWEV_Q_DU_D, vd, vj, vk)); +} + +/* Emits the `vmulwod.h.bu.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_h_bu_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_H_BU_B, vd, vj, vk)); +} + +/* Emits the `vmulwod.w.hu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_w_hu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_W_HU_H, vd, vj, vk)); +} + +/* Emits the `vmulwod.d.wu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_d_wu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_D_WU_W, vd, vj, vk)); +} + +/* Emits the `vmulwod.q.du.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmulwod_q_du_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMULWOD_Q_DU_D, vd, vj, vk)); +} + +/* Emits the `vmadd.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmadd_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADD_B, vd, vj, vk)); +} + +/* Emits the `vmadd.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmadd_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADD_H, vd, vj, vk)); +} + +/* Emits the `vmadd.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmadd_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADD_W, vd, vj, vk)); +} + +/* Emits the `vmadd.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmadd_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADD_D, vd, vj, vk)); +} + +/* Emits the `vmsub.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmsub_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMSUB_B, vd, vj, vk)); +} + +/* Emits the `vmsub.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmsub_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMSUB_H, vd, vj, vk)); +} + +/* Emits the `vmsub.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmsub_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMSUB_W, vd, vj, vk)); +} + +/* Emits the `vmsub.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmsub_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMSUB_D, vd, vj, vk)); +} + +/* Emits the `vmaddwev.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_H_B, vd, vj, vk)); +} + +/* Emits the `vmaddwev.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_W_H, vd, vj, vk)); +} + +/* Emits the `vmaddwev.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_D_W, vd, vj, vk)); +} + +/* Emits the `vmaddwev.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_Q_D, vd, vj, vk)); +} + +/* Emits the `vmaddwod.h.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_h_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_H_B, vd, vj, vk)); +} + +/* Emits the `vmaddwod.w.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_w_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_W_H, vd, vj, vk)); +} + +/* Emits the `vmaddwod.d.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_d_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_D_W, vd, vj, vk)); +} + +/* Emits the `vmaddwod.q.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_q_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_Q_D, vd, vj, vk)); +} + +/* Emits the `vmaddwev.h.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_h_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_H_BU, vd, vj, vk)); +} + +/* Emits the `vmaddwev.w.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_w_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_W_HU, vd, vj, vk)); +} + +/* Emits the `vmaddwev.d.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_d_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_D_WU, vd, vj, vk)); +} + +/* Emits the `vmaddwev.q.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_q_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_Q_DU, vd, vj, vk)); +} + +/* Emits the `vmaddwod.h.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_h_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_H_BU, vd, vj, vk)); +} + +/* Emits the `vmaddwod.w.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_w_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_W_HU, vd, vj, vk)); +} + +/* Emits the `vmaddwod.d.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_d_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_D_WU, vd, vj, vk)); +} + +/* Emits the `vmaddwod.q.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_q_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_Q_DU, vd, vj, vk)); +} + +/* Emits the `vmaddwev.h.bu.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_h_bu_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_H_BU_B, vd, vj, vk)); +} + +/* Emits the `vmaddwev.w.hu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_w_hu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_W_HU_H, vd, vj, vk)); +} + +/* Emits the `vmaddwev.d.wu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_d_wu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_D_WU_W, vd, vj, vk)); +} + +/* Emits the `vmaddwev.q.du.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwev_q_du_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWEV_Q_DU_D, vd, vj, vk)); +} + +/* Emits the `vmaddwod.h.bu.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_h_bu_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_H_BU_B, vd, vj, vk)); +} + +/* Emits the `vmaddwod.w.hu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_w_hu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_W_HU_H, vd, vj, vk)); +} + +/* Emits the `vmaddwod.d.wu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_d_wu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_D_WU_W, vd, vj, vk)); +} + +/* Emits the `vmaddwod.q.du.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaddwod_q_du_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMADDWOD_Q_DU_D, vd, vj, vk)); +} + +/* Emits the `vdiv.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vdiv_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VDIV_B, vd, vj, vk)); +} + +/* Emits the `vdiv.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vdiv_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VDIV_H, vd, vj, vk)); +} + +/* Emits the `vdiv.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vdiv_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VDIV_W, vd, vj, vk)); +} + +/* Emits the `vdiv.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vdiv_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VDIV_D, vd, vj, vk)); +} + +/* Emits the `vmod.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmod_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMOD_B, vd, vj, vk)); +} + +/* Emits the `vmod.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmod_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMOD_H, vd, vj, vk)); +} + +/* Emits the `vmod.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmod_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMOD_W, vd, vj, vk)); +} + +/* Emits the `vmod.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmod_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMOD_D, vd, vj, vk)); +} + +/* Emits the `vdiv.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vdiv_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VDIV_BU, vd, vj, vk)); +} + +/* Emits the `vdiv.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vdiv_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VDIV_HU, vd, vj, vk)); +} + +/* Emits the `vdiv.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vdiv_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VDIV_WU, vd, vj, vk)); +} + +/* Emits the `vdiv.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vdiv_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VDIV_DU, vd, vj, vk)); +} + +/* Emits the `vmod.bu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmod_bu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMOD_BU, vd, vj, vk)); +} + +/* Emits the `vmod.hu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmod_hu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMOD_HU, vd, vj, vk)); +} + +/* Emits the `vmod.wu vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmod_wu(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMOD_WU, vd, vj, vk)); +} + +/* Emits the `vmod.du vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmod_du(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VMOD_DU, vd, vj, vk)); +} + +/* Emits the `vsll.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsll_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLL_B, vd, vj, vk)); +} + +/* Emits the `vsll.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsll_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLL_H, vd, vj, vk)); +} + +/* Emits the `vsll.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsll_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLL_W, vd, vj, vk)); +} + +/* Emits the `vsll.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsll_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSLL_D, vd, vj, vk)); +} + +/* Emits the `vsrl.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrl_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRL_B, vd, vj, vk)); +} + +/* Emits the `vsrl.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrl_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRL_H, vd, vj, vk)); +} + +/* Emits the `vsrl.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrl_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRL_W, vd, vj, vk)); +} + +/* Emits the `vsrl.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrl_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRL_D, vd, vj, vk)); +} + +/* Emits the `vsra.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsra_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRA_B, vd, vj, vk)); +} + +/* Emits the `vsra.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsra_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRA_H, vd, vj, vk)); +} + +/* Emits the `vsra.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsra_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRA_W, vd, vj, vk)); +} + +/* Emits the `vsra.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsra_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRA_D, vd, vj, vk)); +} + +/* Emits the `vrotr.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vrotr_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VROTR_B, vd, vj, vk)); +} + +/* Emits the `vrotr.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vrotr_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VROTR_H, vd, vj, vk)); +} + +/* Emits the `vrotr.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vrotr_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VROTR_W, vd, vj, vk)); +} + +/* Emits the `vrotr.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vrotr_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VROTR_D, vd, vj, vk)); +} + +/* Emits the `vsrlr.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlr_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLR_B, vd, vj, vk)); +} + +/* Emits the `vsrlr.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlr_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLR_H, vd, vj, vk)); +} + +/* Emits the `vsrlr.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlr_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLR_W, vd, vj, vk)); +} + +/* Emits the `vsrlr.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlr_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLR_D, vd, vj, vk)); +} + +/* Emits the `vsrar.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrar_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRAR_B, vd, vj, vk)); +} + +/* Emits the `vsrar.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrar_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRAR_H, vd, vj, vk)); +} + +/* Emits the `vsrar.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrar_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRAR_W, vd, vj, vk)); +} + +/* Emits the `vsrar.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrar_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRAR_D, vd, vj, vk)); +} + +/* Emits the `vsrln.b.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrln_b_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLN_B_H, vd, vj, vk)); +} + +/* Emits the `vsrln.h.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrln_h_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLN_H_W, vd, vj, vk)); +} + +/* Emits the `vsrln.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrln_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLN_W_D, vd, vj, vk)); +} + +/* Emits the `vsran.b.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsran_b_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRAN_B_H, vd, vj, vk)); +} + +/* Emits the `vsran.h.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsran_h_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRAN_H_W, vd, vj, vk)); +} + +/* Emits the `vsran.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsran_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRAN_W_D, vd, vj, vk)); +} + +/* Emits the `vsrlrn.b.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlrn_b_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLRN_B_H, vd, vj, vk)); +} + +/* Emits the `vsrlrn.h.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlrn_h_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLRN_H_W, vd, vj, vk)); +} + +/* Emits the `vsrlrn.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlrn_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRLRN_W_D, vd, vj, vk)); +} + +/* Emits the `vsrarn.b.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrarn_b_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRARN_B_H, vd, vj, vk)); +} + +/* Emits the `vsrarn.h.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrarn_h_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRARN_H_W, vd, vj, vk)); +} + +/* Emits the `vsrarn.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrarn_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSRARN_W_D, vd, vj, vk)); +} + +/* Emits the `vssrln.b.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrln_b_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLN_B_H, vd, vj, vk)); +} + +/* Emits the `vssrln.h.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrln_h_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLN_H_W, vd, vj, vk)); +} + +/* Emits the `vssrln.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrln_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLN_W_D, vd, vj, vk)); +} + +/* Emits the `vssran.b.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssran_b_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRAN_B_H, vd, vj, vk)); +} + +/* Emits the `vssran.h.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssran_h_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRAN_H_W, vd, vj, vk)); +} + +/* Emits the `vssran.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssran_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRAN_W_D, vd, vj, vk)); +} + +/* Emits the `vssrlrn.b.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrn_b_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLRN_B_H, vd, vj, vk)); +} + +/* Emits the `vssrlrn.h.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrn_h_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLRN_H_W, vd, vj, vk)); +} + +/* Emits the `vssrlrn.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrn_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLRN_W_D, vd, vj, vk)); +} + +/* Emits the `vssrarn.b.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarn_b_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRARN_B_H, vd, vj, vk)); +} + +/* Emits the `vssrarn.h.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarn_h_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRARN_H_W, vd, vj, vk)); +} + +/* Emits the `vssrarn.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarn_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRARN_W_D, vd, vj, vk)); +} + +/* Emits the `vssrln.bu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrln_bu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLN_BU_H, vd, vj, vk)); +} + +/* Emits the `vssrln.hu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrln_hu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLN_HU_W, vd, vj, vk)); +} + +/* Emits the `vssrln.wu.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrln_wu_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLN_WU_D, vd, vj, vk)); +} + +/* Emits the `vssran.bu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssran_bu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRAN_BU_H, vd, vj, vk)); +} + +/* Emits the `vssran.hu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssran_hu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRAN_HU_W, vd, vj, vk)); +} + +/* Emits the `vssran.wu.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssran_wu_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRAN_WU_D, vd, vj, vk)); +} + +/* Emits the `vssrlrn.bu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrn_bu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLRN_BU_H, vd, vj, vk)); +} + +/* Emits the `vssrlrn.hu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrn_hu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLRN_HU_W, vd, vj, vk)); +} + +/* Emits the `vssrlrn.wu.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrn_wu_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRLRN_WU_D, vd, vj, vk)); +} + +/* Emits the `vssrarn.bu.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarn_bu_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRARN_BU_H, vd, vj, vk)); +} + +/* Emits the `vssrarn.hu.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarn_hu_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRARN_HU_W, vd, vj, vk)); +} + +/* Emits the `vssrarn.wu.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarn_wu_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSSRARN_WU_D, vd, vj, vk)); +} + +/* Emits the `vbitclr.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitclr_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITCLR_B, vd, vj, vk)); +} + +/* Emits the `vbitclr.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitclr_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITCLR_H, vd, vj, vk)); +} + +/* Emits the `vbitclr.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitclr_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITCLR_W, vd, vj, vk)); +} + +/* Emits the `vbitclr.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitclr_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITCLR_D, vd, vj, vk)); +} + +/* Emits the `vbitset.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitset_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITSET_B, vd, vj, vk)); +} + +/* Emits the `vbitset.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitset_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITSET_H, vd, vj, vk)); +} + +/* Emits the `vbitset.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitset_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITSET_W, vd, vj, vk)); +} + +/* Emits the `vbitset.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitset_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITSET_D, vd, vj, vk)); +} + +/* Emits the `vbitrev.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitrev_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITREV_B, vd, vj, vk)); +} + +/* Emits the `vbitrev.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitrev_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITREV_H, vd, vj, vk)); +} + +/* Emits the `vbitrev.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitrev_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITREV_W, vd, vj, vk)); +} + +/* Emits the `vbitrev.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitrev_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VBITREV_D, vd, vj, vk)); +} + +/* Emits the `vpackev.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpackev_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPACKEV_B, vd, vj, vk)); +} + +/* Emits the `vpackev.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpackev_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPACKEV_H, vd, vj, vk)); +} + +/* Emits the `vpackev.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpackev_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPACKEV_W, vd, vj, vk)); +} + +/* Emits the `vpackev.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpackev_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPACKEV_D, vd, vj, vk)); +} + +/* Emits the `vpackod.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpackod_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPACKOD_B, vd, vj, vk)); +} + +/* Emits the `vpackod.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpackod_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPACKOD_H, vd, vj, vk)); +} + +/* Emits the `vpackod.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpackod_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPACKOD_W, vd, vj, vk)); +} + +/* Emits the `vpackod.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpackod_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPACKOD_D, vd, vj, vk)); +} + +/* Emits the `vilvl.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vilvl_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VILVL_B, vd, vj, vk)); +} + +/* Emits the `vilvl.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vilvl_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VILVL_H, vd, vj, vk)); +} + +/* Emits the `vilvl.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vilvl_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VILVL_W, vd, vj, vk)); +} + +/* Emits the `vilvl.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vilvl_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VILVL_D, vd, vj, vk)); +} + +/* Emits the `vilvh.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vilvh_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VILVH_B, vd, vj, vk)); +} + +/* Emits the `vilvh.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vilvh_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VILVH_H, vd, vj, vk)); +} + +/* Emits the `vilvh.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vilvh_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VILVH_W, vd, vj, vk)); +} + +/* Emits the `vilvh.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vilvh_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VILVH_D, vd, vj, vk)); +} + +/* Emits the `vpickev.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickev_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPICKEV_B, vd, vj, vk)); +} + +/* Emits the `vpickev.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickev_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPICKEV_H, vd, vj, vk)); +} + +/* Emits the `vpickev.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickev_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPICKEV_W, vd, vj, vk)); +} + +/* Emits the `vpickev.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickev_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPICKEV_D, vd, vj, vk)); +} + +/* Emits the `vpickod.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickod_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPICKOD_B, vd, vj, vk)); +} + +/* Emits the `vpickod.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickod_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPICKOD_H, vd, vj, vk)); +} + +/* Emits the `vpickod.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickod_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPICKOD_W, vd, vj, vk)); +} + +/* Emits the `vpickod.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickod_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VPICKOD_D, vd, vj, vk)); +} + +/* Emits the `vreplve.b vd, vj, k` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplve_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg k) +{ + tcg_out32(s, encode_vdvjk_insn(OPC_VREPLVE_B, vd, vj, k)); +} + +/* Emits the `vreplve.h vd, vj, k` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplve_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg k) +{ + tcg_out32(s, encode_vdvjk_insn(OPC_VREPLVE_H, vd, vj, k)); +} + +/* Emits the `vreplve.w vd, vj, k` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplve_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg k) +{ + tcg_out32(s, encode_vdvjk_insn(OPC_VREPLVE_W, vd, vj, k)); +} + +/* Emits the `vreplve.d vd, vj, k` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplve_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg k) +{ + tcg_out32(s, encode_vdvjk_insn(OPC_VREPLVE_D, vd, vj, k)); +} + +/* Emits the `vand.v vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vand_v(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VAND_V, vd, vj, vk)); +} + +/* Emits the `vor.v vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vor_v(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VOR_V, vd, vj, vk)); +} + +/* Emits the `vxor.v vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vxor_v(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VXOR_V, vd, vj, vk)); +} + +/* Emits the `vnor.v vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vnor_v(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VNOR_V, vd, vj, vk)); +} + +/* Emits the `vandn.v vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vandn_v(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VANDN_V, vd, vj, vk)); +} + +/* Emits the `vorn.v vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vorn_v(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VORN_V, vd, vj, vk)); +} + +/* Emits the `vfrstp.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrstp_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFRSTP_B, vd, vj, vk)); +} + +/* Emits the `vfrstp.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrstp_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFRSTP_H, vd, vj, vk)); +} + +/* Emits the `vadd.q vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vadd_q(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VADD_Q, vd, vj, vk)); +} + +/* Emits the `vsub.q vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsub_q(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSUB_Q, vd, vj, vk)); +} + +/* Emits the `vsigncov.b vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsigncov_b(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSIGNCOV_B, vd, vj, vk)); +} + +/* Emits the `vsigncov.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsigncov_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSIGNCOV_H, vd, vj, vk)); +} + +/* Emits the `vsigncov.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsigncov_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSIGNCOV_W, vd, vj, vk)); +} + +/* Emits the `vsigncov.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsigncov_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSIGNCOV_D, vd, vj, vk)); +} + +/* Emits the `vfadd.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfadd_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFADD_S, vd, vj, vk)); +} + +/* Emits the `vfadd.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfadd_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFADD_D, vd, vj, vk)); +} + +/* Emits the `vfsub.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfsub_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFSUB_S, vd, vj, vk)); +} + +/* Emits the `vfsub.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfsub_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFSUB_D, vd, vj, vk)); +} + +/* Emits the `vfmul.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmul_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMUL_S, vd, vj, vk)); +} + +/* Emits the `vfmul.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmul_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMUL_D, vd, vj, vk)); +} + +/* Emits the `vfdiv.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfdiv_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFDIV_S, vd, vj, vk)); +} + +/* Emits the `vfdiv.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfdiv_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFDIV_D, vd, vj, vk)); +} + +/* Emits the `vfmax.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmax_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMAX_S, vd, vj, vk)); +} + +/* Emits the `vfmax.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmax_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMAX_D, vd, vj, vk)); +} + +/* Emits the `vfmin.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmin_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMIN_S, vd, vj, vk)); +} + +/* Emits the `vfmin.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmin_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMIN_D, vd, vj, vk)); +} + +/* Emits the `vfmaxa.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmaxa_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMAXA_S, vd, vj, vk)); +} + +/* Emits the `vfmaxa.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmaxa_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMAXA_D, vd, vj, vk)); +} + +/* Emits the `vfmina.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmina_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMINA_S, vd, vj, vk)); +} + +/* Emits the `vfmina.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfmina_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFMINA_D, vd, vj, vk)); +} + +/* Emits the `vfcvt.h.s vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcvt_h_s(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCVT_H_S, vd, vj, vk)); +} + +/* Emits the `vfcvt.s.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcvt_s_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFCVT_S_D, vd, vj, vk)); +} + +/* Emits the `vffint.s.l vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vffint_s_l(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFFINT_S_L, vd, vj, vk)); +} + +/* Emits the `vftint.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftint_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFTINT_W_D, vd, vj, vk)); +} + +/* Emits the `vftintrm.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrm_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFTINTRM_W_D, vd, vj, vk)); +} + +/* Emits the `vftintrp.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrp_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFTINTRP_W_D, vd, vj, vk)); +} + +/* Emits the `vftintrz.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrz_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFTINTRZ_W_D, vd, vj, vk)); +} + +/* Emits the `vftintrne.w.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrne_w_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VFTINTRNE_W_D, vd, vj, vk)); +} + +/* Emits the `vshuf.h vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vshuf_h(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSHUF_H, vd, vj, vk)); +} + +/* Emits the `vshuf.w vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vshuf_w(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSHUF_W, vd, vj, vk)); +} + +/* Emits the `vshuf.d vd, vj, vk` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vshuf_d(TCGContext *s, TCGReg vd, TCGReg vj, TCGReg vk) +{ + tcg_out32(s, encode_vdvjvk_insn(OPC_VSHUF_D, vd, vj, vk)); +} + +/* Emits the `vseqi.b vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vseqi_b(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSEQI_B, vd, vj, sk5)); +} + +/* Emits the `vseqi.h vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vseqi_h(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSEQI_H, vd, vj, sk5)); +} + +/* Emits the `vseqi.w vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vseqi_w(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSEQI_W, vd, vj, sk5)); +} + +/* Emits the `vseqi.d vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vseqi_d(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSEQI_D, vd, vj, sk5)); +} + +/* Emits the `vslei.b vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslei_b(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSLEI_B, vd, vj, sk5)); +} + +/* Emits the `vslei.h vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslei_h(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSLEI_H, vd, vj, sk5)); +} + +/* Emits the `vslei.w vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslei_w(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSLEI_W, vd, vj, sk5)); +} + +/* Emits the `vslei.d vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslei_d(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSLEI_D, vd, vj, sk5)); +} + +/* Emits the `vslei.bu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslei_bu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLEI_BU, vd, vj, uk5)); +} + +/* Emits the `vslei.hu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslei_hu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLEI_HU, vd, vj, uk5)); +} + +/* Emits the `vslei.wu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslei_wu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLEI_WU, vd, vj, uk5)); +} + +/* Emits the `vslei.du vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslei_du(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLEI_DU, vd, vj, uk5)); +} + +/* Emits the `vslti.b vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslti_b(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSLTI_B, vd, vj, sk5)); +} + +/* Emits the `vslti.h vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslti_h(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSLTI_H, vd, vj, sk5)); +} + +/* Emits the `vslti.w vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslti_w(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSLTI_W, vd, vj, sk5)); +} + +/* Emits the `vslti.d vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslti_d(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VSLTI_D, vd, vj, sk5)); +} + +/* Emits the `vslti.bu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslti_bu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLTI_BU, vd, vj, uk5)); +} + +/* Emits the `vslti.hu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslti_hu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLTI_HU, vd, vj, uk5)); +} + +/* Emits the `vslti.wu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslti_wu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLTI_WU, vd, vj, uk5)); +} + +/* Emits the `vslti.du vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslti_du(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLTI_DU, vd, vj, uk5)); +} + +/* Emits the `vaddi.bu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddi_bu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VADDI_BU, vd, vj, uk5)); +} + +/* Emits the `vaddi.hu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddi_hu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VADDI_HU, vd, vj, uk5)); +} + +/* Emits the `vaddi.wu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddi_wu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VADDI_WU, vd, vj, uk5)); +} + +/* Emits the `vaddi.du vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vaddi_du(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VADDI_DU, vd, vj, uk5)); +} + +/* Emits the `vsubi.bu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubi_bu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSUBI_BU, vd, vj, uk5)); +} + +/* Emits the `vsubi.hu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubi_hu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSUBI_HU, vd, vj, uk5)); +} + +/* Emits the `vsubi.wu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubi_wu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSUBI_WU, vd, vj, uk5)); +} + +/* Emits the `vsubi.du vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsubi_du(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSUBI_DU, vd, vj, uk5)); +} + +/* Emits the `vbsll.v vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbsll_v(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VBSLL_V, vd, vj, uk5)); +} + +/* Emits the `vbsrl.v vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbsrl_v(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VBSRL_V, vd, vj, uk5)); +} + +/* Emits the `vmaxi.b vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaxi_b(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VMAXI_B, vd, vj, sk5)); +} + +/* Emits the `vmaxi.h vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaxi_h(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VMAXI_H, vd, vj, sk5)); +} + +/* Emits the `vmaxi.w vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaxi_w(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VMAXI_W, vd, vj, sk5)); +} + +/* Emits the `vmaxi.d vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaxi_d(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VMAXI_D, vd, vj, sk5)); +} + +/* Emits the `vmini.b vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmini_b(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VMINI_B, vd, vj, sk5)); +} + +/* Emits the `vmini.h vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmini_h(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VMINI_H, vd, vj, sk5)); +} + +/* Emits the `vmini.w vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmini_w(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VMINI_W, vd, vj, sk5)); +} + +/* Emits the `vmini.d vd, vj, sk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmini_d(TCGContext *s, TCGReg vd, TCGReg vj, int32_t sk5) +{ + tcg_out32(s, encode_vdvjsk5_insn(OPC_VMINI_D, vd, vj, sk5)); +} + +/* Emits the `vmaxi.bu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaxi_bu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VMAXI_BU, vd, vj, uk5)); +} + +/* Emits the `vmaxi.hu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaxi_hu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VMAXI_HU, vd, vj, uk5)); +} + +/* Emits the `vmaxi.wu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaxi_wu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VMAXI_WU, vd, vj, uk5)); +} + +/* Emits the `vmaxi.du vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmaxi_du(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VMAXI_DU, vd, vj, uk5)); +} + +/* Emits the `vmini.bu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmini_bu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VMINI_BU, vd, vj, uk5)); +} + +/* Emits the `vmini.hu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmini_hu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VMINI_HU, vd, vj, uk5)); +} + +/* Emits the `vmini.wu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmini_wu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VMINI_WU, vd, vj, uk5)); +} + +/* Emits the `vmini.du vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmini_du(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VMINI_DU, vd, vj, uk5)); +} + +/* Emits the `vfrstpi.b vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrstpi_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VFRSTPI_B, vd, vj, uk5)); +} + +/* Emits the `vfrstpi.h vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrstpi_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VFRSTPI_H, vd, vj, uk5)); +} + +/* Emits the `vclo.b vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vclo_b(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VCLO_B, vd, vj)); +} + +/* Emits the `vclo.h vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vclo_h(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VCLO_H, vd, vj)); +} + +/* Emits the `vclo.w vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vclo_w(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VCLO_W, vd, vj)); +} + +/* Emits the `vclo.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vclo_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VCLO_D, vd, vj)); +} + +/* Emits the `vclz.b vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vclz_b(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VCLZ_B, vd, vj)); +} + +/* Emits the `vclz.h vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vclz_h(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VCLZ_H, vd, vj)); +} + +/* Emits the `vclz.w vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vclz_w(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VCLZ_W, vd, vj)); +} + +/* Emits the `vclz.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vclz_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VCLZ_D, vd, vj)); +} + +/* Emits the `vpcnt.b vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpcnt_b(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VPCNT_B, vd, vj)); +} + +/* Emits the `vpcnt.h vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpcnt_h(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VPCNT_H, vd, vj)); +} + +/* Emits the `vpcnt.w vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpcnt_w(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VPCNT_W, vd, vj)); +} + +/* Emits the `vpcnt.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpcnt_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VPCNT_D, vd, vj)); +} + +/* Emits the `vneg.b vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vneg_b(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VNEG_B, vd, vj)); +} + +/* Emits the `vneg.h vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vneg_h(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VNEG_H, vd, vj)); +} + +/* Emits the `vneg.w vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vneg_w(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VNEG_W, vd, vj)); +} + +/* Emits the `vneg.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vneg_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VNEG_D, vd, vj)); +} + +/* Emits the `vmskltz.b vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmskltz_b(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VMSKLTZ_B, vd, vj)); +} + +/* Emits the `vmskltz.h vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmskltz_h(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VMSKLTZ_H, vd, vj)); +} + +/* Emits the `vmskltz.w vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmskltz_w(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VMSKLTZ_W, vd, vj)); +} + +/* Emits the `vmskltz.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmskltz_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VMSKLTZ_D, vd, vj)); +} + +/* Emits the `vmskgez.b vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmskgez_b(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VMSKGEZ_B, vd, vj)); +} + +/* Emits the `vmsknz.b vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vmsknz_b(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VMSKNZ_B, vd, vj)); +} + +/* Emits the `vseteqz.v cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vseteqz_v(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETEQZ_V, cd, vj)); +} + +/* Emits the `vsetnez.v cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsetnez_v(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETNEZ_V, cd, vj)); +} + +/* Emits the `vsetanyeqz.b cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsetanyeqz_b(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETANYEQZ_B, cd, vj)); +} + +/* Emits the `vsetanyeqz.h cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsetanyeqz_h(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETANYEQZ_H, cd, vj)); +} + +/* Emits the `vsetanyeqz.w cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsetanyeqz_w(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETANYEQZ_W, cd, vj)); +} + +/* Emits the `vsetanyeqz.d cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsetanyeqz_d(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETANYEQZ_D, cd, vj)); +} + +/* Emits the `vsetallnez.b cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsetallnez_b(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETALLNEZ_B, cd, vj)); +} + +/* Emits the `vsetallnez.h cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsetallnez_h(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETALLNEZ_H, cd, vj)); +} + +/* Emits the `vsetallnez.w cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsetallnez_w(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETALLNEZ_W, cd, vj)); +} + +/* Emits the `vsetallnez.d cd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsetallnez_d(TCGContext *s, TCGReg cd, TCGReg vj) +{ + tcg_out32(s, encode_cdvj_insn(OPC_VSETALLNEZ_D, cd, vj)); +} + +/* Emits the `vflogb.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vflogb_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFLOGB_S, vd, vj)); +} + +/* Emits the `vflogb.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vflogb_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFLOGB_D, vd, vj)); +} + +/* Emits the `vfclass.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfclass_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFCLASS_S, vd, vj)); +} + +/* Emits the `vfclass.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfclass_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFCLASS_D, vd, vj)); +} + +/* Emits the `vfsqrt.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfsqrt_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFSQRT_S, vd, vj)); +} + +/* Emits the `vfsqrt.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfsqrt_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFSQRT_D, vd, vj)); +} + +/* Emits the `vfrecip.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrecip_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRECIP_S, vd, vj)); +} + +/* Emits the `vfrecip.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrecip_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRECIP_D, vd, vj)); +} + +/* Emits the `vfrsqrt.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrsqrt_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRSQRT_S, vd, vj)); +} + +/* Emits the `vfrsqrt.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrsqrt_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRSQRT_D, vd, vj)); +} + +/* Emits the `vfrint.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrint_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINT_S, vd, vj)); +} + +/* Emits the `vfrint.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrint_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINT_D, vd, vj)); +} + +/* Emits the `vfrintrm.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrintrm_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINTRM_S, vd, vj)); +} + +/* Emits the `vfrintrm.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrintrm_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINTRM_D, vd, vj)); +} + +/* Emits the `vfrintrp.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrintrp_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINTRP_S, vd, vj)); +} + +/* Emits the `vfrintrp.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrintrp_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINTRP_D, vd, vj)); +} + +/* Emits the `vfrintrz.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrintrz_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINTRZ_S, vd, vj)); +} + +/* Emits the `vfrintrz.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrintrz_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINTRZ_D, vd, vj)); +} + +/* Emits the `vfrintrne.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrintrne_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINTRNE_S, vd, vj)); +} + +/* Emits the `vfrintrne.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfrintrne_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFRINTRNE_D, vd, vj)); +} + +/* Emits the `vfcvtl.s.h vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcvtl_s_h(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFCVTL_S_H, vd, vj)); +} + +/* Emits the `vfcvth.s.h vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcvth_s_h(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFCVTH_S_H, vd, vj)); +} + +/* Emits the `vfcvtl.d.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcvtl_d_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFCVTL_D_S, vd, vj)); +} + +/* Emits the `vfcvth.d.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vfcvth_d_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFCVTH_D_S, vd, vj)); +} + +/* Emits the `vffint.s.w vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vffint_s_w(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFFINT_S_W, vd, vj)); +} + +/* Emits the `vffint.s.wu vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vffint_s_wu(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFFINT_S_WU, vd, vj)); +} + +/* Emits the `vffint.d.l vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vffint_d_l(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFFINT_D_L, vd, vj)); +} + +/* Emits the `vffint.d.lu vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vffint_d_lu(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFFINT_D_LU, vd, vj)); +} + +/* Emits the `vffintl.d.w vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vffintl_d_w(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFFINTL_D_W, vd, vj)); +} + +/* Emits the `vffinth.d.w vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vffinth_d_w(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFFINTH_D_W, vd, vj)); +} + +/* Emits the `vftint.w.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftint_w_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINT_W_S, vd, vj)); +} + +/* Emits the `vftint.l.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftint_l_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINT_L_D, vd, vj)); +} + +/* Emits the `vftintrm.w.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrm_w_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRM_W_S, vd, vj)); +} + +/* Emits the `vftintrm.l.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrm_l_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRM_L_D, vd, vj)); +} + +/* Emits the `vftintrp.w.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrp_w_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRP_W_S, vd, vj)); +} + +/* Emits the `vftintrp.l.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrp_l_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRP_L_D, vd, vj)); +} + +/* Emits the `vftintrz.w.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrz_w_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRZ_W_S, vd, vj)); +} + +/* Emits the `vftintrz.l.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrz_l_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRZ_L_D, vd, vj)); +} + +/* Emits the `vftintrne.w.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrne_w_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRNE_W_S, vd, vj)); +} + +/* Emits the `vftintrne.l.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrne_l_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRNE_L_D, vd, vj)); +} + +/* Emits the `vftint.wu.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftint_wu_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINT_WU_S, vd, vj)); +} + +/* Emits the `vftint.lu.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftint_lu_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINT_LU_D, vd, vj)); +} + +/* Emits the `vftintrz.wu.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrz_wu_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRZ_WU_S, vd, vj)); +} + +/* Emits the `vftintrz.lu.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrz_lu_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRZ_LU_D, vd, vj)); +} + +/* Emits the `vftintl.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintl_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTL_L_S, vd, vj)); +} + +/* Emits the `vftinth.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftinth_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTH_L_S, vd, vj)); +} + +/* Emits the `vftintrml.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrml_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRML_L_S, vd, vj)); +} + +/* Emits the `vftintrmh.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrmh_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRMH_L_S, vd, vj)); +} + +/* Emits the `vftintrpl.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrpl_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRPL_L_S, vd, vj)); +} + +/* Emits the `vftintrph.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrph_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRPH_L_S, vd, vj)); +} + +/* Emits the `vftintrzl.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrzl_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRZL_L_S, vd, vj)); +} + +/* Emits the `vftintrzh.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrzh_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRZH_L_S, vd, vj)); +} + +/* Emits the `vftintrnel.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrnel_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRNEL_L_S, vd, vj)); +} + +/* Emits the `vftintrneh.l.s vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vftintrneh_l_s(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VFTINTRNEH_L_S, vd, vj)); +} + +/* Emits the `vexth.h.b vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vexth_h_b(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTH_H_B, vd, vj)); +} + +/* Emits the `vexth.w.h vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vexth_w_h(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTH_W_H, vd, vj)); +} + +/* Emits the `vexth.d.w vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vexth_d_w(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTH_D_W, vd, vj)); +} + +/* Emits the `vexth.q.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vexth_q_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTH_Q_D, vd, vj)); +} + +/* Emits the `vexth.hu.bu vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vexth_hu_bu(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTH_HU_BU, vd, vj)); +} + +/* Emits the `vexth.wu.hu vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vexth_wu_hu(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTH_WU_HU, vd, vj)); +} + +/* Emits the `vexth.du.wu vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vexth_du_wu(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTH_DU_WU, vd, vj)); +} + +/* Emits the `vexth.qu.du vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vexth_qu_du(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTH_QU_DU, vd, vj)); +} + +/* Emits the `vreplgr2vr.b vd, j` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplgr2vr_b(TCGContext *s, TCGReg vd, TCGReg j) +{ + tcg_out32(s, encode_vdj_insn(OPC_VREPLGR2VR_B, vd, j)); +} + +/* Emits the `vreplgr2vr.h vd, j` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplgr2vr_h(TCGContext *s, TCGReg vd, TCGReg j) +{ + tcg_out32(s, encode_vdj_insn(OPC_VREPLGR2VR_H, vd, j)); +} + +/* Emits the `vreplgr2vr.w vd, j` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplgr2vr_w(TCGContext *s, TCGReg vd, TCGReg j) +{ + tcg_out32(s, encode_vdj_insn(OPC_VREPLGR2VR_W, vd, j)); +} + +/* Emits the `vreplgr2vr.d vd, j` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplgr2vr_d(TCGContext *s, TCGReg vd, TCGReg j) +{ + tcg_out32(s, encode_vdj_insn(OPC_VREPLGR2VR_D, vd, j)); +} + +/* Emits the `vrotri.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vrotri_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VROTRI_B, vd, vj, uk3)); +} + +/* Emits the `vrotri.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vrotri_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VROTRI_H, vd, vj, uk4)); +} + +/* Emits the `vrotri.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vrotri_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VROTRI_W, vd, vj, uk5)); +} + +/* Emits the `vrotri.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vrotri_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VROTRI_D, vd, vj, uk6)); +} + +/* Emits the `vsrlri.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlri_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VSRLRI_B, vd, vj, uk3)); +} + +/* Emits the `vsrlri.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlri_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSRLRI_H, vd, vj, uk4)); +} + +/* Emits the `vsrlri.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlri_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSRLRI_W, vd, vj, uk5)); +} + +/* Emits the `vsrlri.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlri_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSRLRI_D, vd, vj, uk6)); +} + +/* Emits the `vsrari.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrari_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VSRARI_B, vd, vj, uk3)); +} + +/* Emits the `vsrari.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrari_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSRARI_H, vd, vj, uk4)); +} + +/* Emits the `vsrari.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrari_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSRARI_W, vd, vj, uk5)); +} + +/* Emits the `vsrari.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrari_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSRARI_D, vd, vj, uk6)); +} + +/* Emits the `vinsgr2vr.b vd, j, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vinsgr2vr_b(TCGContext *s, TCGReg vd, TCGReg j, uint32_t uk4) +{ + tcg_out32(s, encode_vdjuk4_insn(OPC_VINSGR2VR_B, vd, j, uk4)); +} + +/* Emits the `vinsgr2vr.h vd, j, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vinsgr2vr_h(TCGContext *s, TCGReg vd, TCGReg j, uint32_t uk3) +{ + tcg_out32(s, encode_vdjuk3_insn(OPC_VINSGR2VR_H, vd, j, uk3)); +} + +/* Emits the `vinsgr2vr.w vd, j, uk2` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vinsgr2vr_w(TCGContext *s, TCGReg vd, TCGReg j, uint32_t uk2) +{ + tcg_out32(s, encode_vdjuk2_insn(OPC_VINSGR2VR_W, vd, j, uk2)); +} + +/* Emits the `vinsgr2vr.d vd, j, uk1` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vinsgr2vr_d(TCGContext *s, TCGReg vd, TCGReg j, uint32_t uk1) +{ + tcg_out32(s, encode_vdjuk1_insn(OPC_VINSGR2VR_D, vd, j, uk1)); +} + +/* Emits the `vpickve2gr.b d, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickve2gr_b(TCGContext *s, TCGReg d, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_dvjuk4_insn(OPC_VPICKVE2GR_B, d, vj, uk4)); +} + +/* Emits the `vpickve2gr.h d, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickve2gr_h(TCGContext *s, TCGReg d, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_dvjuk3_insn(OPC_VPICKVE2GR_H, d, vj, uk3)); +} + +/* Emits the `vpickve2gr.w d, vj, uk2` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickve2gr_w(TCGContext *s, TCGReg d, TCGReg vj, uint32_t uk2) +{ + tcg_out32(s, encode_dvjuk2_insn(OPC_VPICKVE2GR_W, d, vj, uk2)); +} + +/* Emits the `vpickve2gr.d d, vj, uk1` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickve2gr_d(TCGContext *s, TCGReg d, TCGReg vj, uint32_t uk1) +{ + tcg_out32(s, encode_dvjuk1_insn(OPC_VPICKVE2GR_D, d, vj, uk1)); +} + +/* Emits the `vpickve2gr.bu d, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickve2gr_bu(TCGContext *s, TCGReg d, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_dvjuk4_insn(OPC_VPICKVE2GR_BU, d, vj, uk4)); +} + +/* Emits the `vpickve2gr.hu d, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickve2gr_hu(TCGContext *s, TCGReg d, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_dvjuk3_insn(OPC_VPICKVE2GR_HU, d, vj, uk3)); +} + +/* Emits the `vpickve2gr.wu d, vj, uk2` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickve2gr_wu(TCGContext *s, TCGReg d, TCGReg vj, uint32_t uk2) +{ + tcg_out32(s, encode_dvjuk2_insn(OPC_VPICKVE2GR_WU, d, vj, uk2)); +} + +/* Emits the `vpickve2gr.du d, vj, uk1` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpickve2gr_du(TCGContext *s, TCGReg d, TCGReg vj, uint32_t uk1) +{ + tcg_out32(s, encode_dvjuk1_insn(OPC_VPICKVE2GR_DU, d, vj, uk1)); +} + +/* Emits the `vreplvei.b vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplvei_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VREPLVEI_B, vd, vj, uk4)); +} + +/* Emits the `vreplvei.h vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplvei_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VREPLVEI_H, vd, vj, uk3)); +} + +/* Emits the `vreplvei.w vd, vj, uk2` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplvei_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk2) +{ + tcg_out32(s, encode_vdvjuk2_insn(OPC_VREPLVEI_W, vd, vj, uk2)); +} + +/* Emits the `vreplvei.d vd, vj, uk1` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vreplvei_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk1) +{ + tcg_out32(s, encode_vdvjuk1_insn(OPC_VREPLVEI_D, vd, vj, uk1)); +} + +/* Emits the `vsllwil.h.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsllwil_h_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VSLLWIL_H_B, vd, vj, uk3)); +} + +/* Emits the `vsllwil.w.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsllwil_w_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSLLWIL_W_H, vd, vj, uk4)); +} + +/* Emits the `vsllwil.d.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsllwil_d_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLLWIL_D_W, vd, vj, uk5)); +} + +/* Emits the `vextl.q.d vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vextl_q_d(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTL_Q_D, vd, vj)); +} + +/* Emits the `vsllwil.hu.bu vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsllwil_hu_bu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VSLLWIL_HU_BU, vd, vj, uk3)); +} + +/* Emits the `vsllwil.wu.hu vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsllwil_wu_hu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSLLWIL_WU_HU, vd, vj, uk4)); +} + +/* Emits the `vsllwil.du.wu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsllwil_du_wu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLLWIL_DU_WU, vd, vj, uk5)); +} + +/* Emits the `vextl.qu.du vd, vj` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vextl_qu_du(TCGContext *s, TCGReg vd, TCGReg vj) +{ + tcg_out32(s, encode_vdvj_insn(OPC_VEXTL_QU_DU, vd, vj)); +} + +/* Emits the `vbitclri.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitclri_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VBITCLRI_B, vd, vj, uk3)); +} + +/* Emits the `vbitclri.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitclri_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VBITCLRI_H, vd, vj, uk4)); +} + +/* Emits the `vbitclri.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitclri_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VBITCLRI_W, vd, vj, uk5)); +} + +/* Emits the `vbitclri.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitclri_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VBITCLRI_D, vd, vj, uk6)); +} + +/* Emits the `vbitseti.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitseti_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VBITSETI_B, vd, vj, uk3)); +} + +/* Emits the `vbitseti.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitseti_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VBITSETI_H, vd, vj, uk4)); +} + +/* Emits the `vbitseti.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitseti_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VBITSETI_W, vd, vj, uk5)); +} + +/* Emits the `vbitseti.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitseti_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VBITSETI_D, vd, vj, uk6)); +} + +/* Emits the `vbitrevi.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitrevi_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VBITREVI_B, vd, vj, uk3)); +} + +/* Emits the `vbitrevi.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitrevi_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VBITREVI_H, vd, vj, uk4)); +} + +/* Emits the `vbitrevi.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitrevi_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VBITREVI_W, vd, vj, uk5)); +} + +/* Emits the `vbitrevi.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitrevi_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VBITREVI_D, vd, vj, uk6)); +} + +/* Emits the `vsat.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsat_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VSAT_B, vd, vj, uk3)); +} + +/* Emits the `vsat.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsat_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSAT_H, vd, vj, uk4)); +} + +/* Emits the `vsat.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsat_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSAT_W, vd, vj, uk5)); +} + +/* Emits the `vsat.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsat_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSAT_D, vd, vj, uk6)); +} + +/* Emits the `vsat.bu vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsat_bu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VSAT_BU, vd, vj, uk3)); +} + +/* Emits the `vsat.hu vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsat_hu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSAT_HU, vd, vj, uk4)); +} + +/* Emits the `vsat.wu vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsat_wu(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSAT_WU, vd, vj, uk5)); +} + +/* Emits the `vsat.du vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsat_du(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSAT_DU, vd, vj, uk6)); +} + +/* Emits the `vslli.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslli_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VSLLI_B, vd, vj, uk3)); +} + +/* Emits the `vslli.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslli_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSLLI_H, vd, vj, uk4)); +} + +/* Emits the `vslli.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslli_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSLLI_W, vd, vj, uk5)); +} + +/* Emits the `vslli.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vslli_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSLLI_D, vd, vj, uk6)); +} + +/* Emits the `vsrli.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrli_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VSRLI_B, vd, vj, uk3)); +} + +/* Emits the `vsrli.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrli_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSRLI_H, vd, vj, uk4)); +} + +/* Emits the `vsrli.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrli_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSRLI_W, vd, vj, uk5)); +} + +/* Emits the `vsrli.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrli_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSRLI_D, vd, vj, uk6)); +} + +/* Emits the `vsrai.b vd, vj, uk3` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrai_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk3) +{ + tcg_out32(s, encode_vdvjuk3_insn(OPC_VSRAI_B, vd, vj, uk3)); +} + +/* Emits the `vsrai.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrai_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSRAI_H, vd, vj, uk4)); +} + +/* Emits the `vsrai.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrai_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSRAI_W, vd, vj, uk5)); +} + +/* Emits the `vsrai.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrai_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSRAI_D, vd, vj, uk6)); +} + +/* Emits the `vsrlni.b.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlni_b_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSRLNI_B_H, vd, vj, uk4)); +} + +/* Emits the `vsrlni.h.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlni_h_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSRLNI_H_W, vd, vj, uk5)); +} + +/* Emits the `vsrlni.w.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlni_w_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSRLNI_W_D, vd, vj, uk6)); +} + +/* Emits the `vsrlni.d.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlni_d_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSRLNI_D_Q, vd, vj, uk7)); +} + +/* Emits the `vsrlrni.b.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlrni_b_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSRLRNI_B_H, vd, vj, uk4)); +} + +/* Emits the `vsrlrni.h.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlrni_h_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSRLRNI_H_W, vd, vj, uk5)); +} + +/* Emits the `vsrlrni.w.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlrni_w_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSRLRNI_W_D, vd, vj, uk6)); +} + +/* Emits the `vsrlrni.d.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrlrni_d_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSRLRNI_D_Q, vd, vj, uk7)); +} + +/* Emits the `vssrlni.b.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlni_b_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSSRLNI_B_H, vd, vj, uk4)); +} + +/* Emits the `vssrlni.h.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlni_h_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSSRLNI_H_W, vd, vj, uk5)); +} + +/* Emits the `vssrlni.w.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlni_w_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSSRLNI_W_D, vd, vj, uk6)); +} + +/* Emits the `vssrlni.d.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlni_d_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSSRLNI_D_Q, vd, vj, uk7)); +} + +/* Emits the `vssrlni.bu.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlni_bu_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSSRLNI_BU_H, vd, vj, uk4)); +} + +/* Emits the `vssrlni.hu.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlni_hu_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSSRLNI_HU_W, vd, vj, uk5)); +} + +/* Emits the `vssrlni.wu.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlni_wu_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSSRLNI_WU_D, vd, vj, uk6)); +} + +/* Emits the `vssrlni.du.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlni_du_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSSRLNI_DU_Q, vd, vj, uk7)); +} + +/* Emits the `vssrlrni.b.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrni_b_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSSRLRNI_B_H, vd, vj, uk4)); +} + +/* Emits the `vssrlrni.h.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrni_h_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSSRLRNI_H_W, vd, vj, uk5)); +} + +/* Emits the `vssrlrni.w.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrni_w_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSSRLRNI_W_D, vd, vj, uk6)); +} + +/* Emits the `vssrlrni.d.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrni_d_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSSRLRNI_D_Q, vd, vj, uk7)); +} + +/* Emits the `vssrlrni.bu.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrni_bu_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSSRLRNI_BU_H, vd, vj, uk4)); +} + +/* Emits the `vssrlrni.hu.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrni_hu_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSSRLRNI_HU_W, vd, vj, uk5)); +} + +/* Emits the `vssrlrni.wu.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrni_wu_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSSRLRNI_WU_D, vd, vj, uk6)); +} + +/* Emits the `vssrlrni.du.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrlrni_du_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSSRLRNI_DU_Q, vd, vj, uk7)); +} + +/* Emits the `vsrani.b.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrani_b_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSRANI_B_H, vd, vj, uk4)); +} + +/* Emits the `vsrani.h.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrani_h_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSRANI_H_W, vd, vj, uk5)); +} + +/* Emits the `vsrani.w.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrani_w_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSRANI_W_D, vd, vj, uk6)); +} + +/* Emits the `vsrani.d.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrani_d_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSRANI_D_Q, vd, vj, uk7)); +} + +/* Emits the `vsrarni.b.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrarni_b_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSRARNI_B_H, vd, vj, uk4)); +} + +/* Emits the `vsrarni.h.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrarni_h_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSRARNI_H_W, vd, vj, uk5)); +} + +/* Emits the `vsrarni.w.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrarni_w_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSRARNI_W_D, vd, vj, uk6)); +} + +/* Emits the `vsrarni.d.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vsrarni_d_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSRARNI_D_Q, vd, vj, uk7)); +} + +/* Emits the `vssrani.b.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrani_b_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSSRANI_B_H, vd, vj, uk4)); +} + +/* Emits the `vssrani.h.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrani_h_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSSRANI_H_W, vd, vj, uk5)); +} + +/* Emits the `vssrani.w.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrani_w_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSSRANI_W_D, vd, vj, uk6)); +} + +/* Emits the `vssrani.d.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrani_d_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSSRANI_D_Q, vd, vj, uk7)); +} + +/* Emits the `vssrani.bu.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrani_bu_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSSRANI_BU_H, vd, vj, uk4)); +} + +/* Emits the `vssrani.hu.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrani_hu_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSSRANI_HU_W, vd, vj, uk5)); +} + +/* Emits the `vssrani.wu.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrani_wu_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSSRANI_WU_D, vd, vj, uk6)); +} + +/* Emits the `vssrani.du.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrani_du_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSSRANI_DU_Q, vd, vj, uk7)); +} + +/* Emits the `vssrarni.b.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarni_b_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSSRARNI_B_H, vd, vj, uk4)); +} + +/* Emits the `vssrarni.h.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarni_h_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSSRARNI_H_W, vd, vj, uk5)); +} + +/* Emits the `vssrarni.w.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarni_w_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSSRARNI_W_D, vd, vj, uk6)); +} + +/* Emits the `vssrarni.d.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarni_d_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSSRARNI_D_Q, vd, vj, uk7)); +} + +/* Emits the `vssrarni.bu.h vd, vj, uk4` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarni_bu_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk4) +{ + tcg_out32(s, encode_vdvjuk4_insn(OPC_VSSRARNI_BU_H, vd, vj, uk4)); +} + +/* Emits the `vssrarni.hu.w vd, vj, uk5` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarni_hu_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk5) +{ + tcg_out32(s, encode_vdvjuk5_insn(OPC_VSSRARNI_HU_W, vd, vj, uk5)); +} + +/* Emits the `vssrarni.wu.d vd, vj, uk6` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarni_wu_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk6) +{ + tcg_out32(s, encode_vdvjuk6_insn(OPC_VSSRARNI_WU_D, vd, vj, uk6)); +} + +/* Emits the `vssrarni.du.q vd, vj, uk7` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vssrarni_du_q(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk7) +{ + tcg_out32(s, encode_vdvjuk7_insn(OPC_VSSRARNI_DU_Q, vd, vj, uk7)); +} + +/* Emits the `vextrins.d vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vextrins_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VEXTRINS_D, vd, vj, uk8)); +} + +/* Emits the `vextrins.w vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vextrins_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VEXTRINS_W, vd, vj, uk8)); +} + +/* Emits the `vextrins.h vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vextrins_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VEXTRINS_H, vd, vj, uk8)); +} + +/* Emits the `vextrins.b vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vextrins_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VEXTRINS_B, vd, vj, uk8)); +} + +/* Emits the `vshuf4i.b vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vshuf4i_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VSHUF4I_B, vd, vj, uk8)); +} + +/* Emits the `vshuf4i.h vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vshuf4i_h(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VSHUF4I_H, vd, vj, uk8)); +} + +/* Emits the `vshuf4i.w vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vshuf4i_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VSHUF4I_W, vd, vj, uk8)); +} + +/* Emits the `vshuf4i.d vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vshuf4i_d(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VSHUF4I_D, vd, vj, uk8)); +} + +/* Emits the `vbitseli.b vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vbitseli_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VBITSELI_B, vd, vj, uk8)); +} + +/* Emits the `vandi.b vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vandi_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VANDI_B, vd, vj, uk8)); +} + +/* Emits the `vori.b vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vori_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VORI_B, vd, vj, uk8)); +} + +/* Emits the `vxori.b vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vxori_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VXORI_B, vd, vj, uk8)); +} + +/* Emits the `vnori.b vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vnori_b(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VNORI_B, vd, vj, uk8)); +} + +/* Emits the `vldi vd, sj13` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vldi(TCGContext *s, TCGReg vd, int32_t sj13) +{ + tcg_out32(s, encode_vdsj13_insn(OPC_VLDI, vd, sj13)); +} + +/* Emits the `vpermi.w vd, vj, uk8` instruction. */ +static void __attribute__((unused)) +tcg_out_opc_vpermi_w(TCGContext *s, TCGReg vd, TCGReg vj, uint32_t uk8) +{ + tcg_out32(s, encode_vdvjuk8_insn(OPC_VPERMI_W, vd, vj, uk8)); +} + /* End of generated code. */ diff --git a/tcg/loongarch64/tcg-target-con-set.h b/tcg/loongarch64/tcg-target-con-set.h index c2bde44613..77d62e38e7 100644 --- a/tcg/loongarch64/tcg-target-con-set.h +++ b/tcg/loongarch64/tcg-target-con-set.h @@ -17,7 +17,11 @@ C_O0_I1(r) C_O0_I2(rZ, r) C_O0_I2(rZ, rZ) +C_O0_I2(w, r) +C_O0_I3(r, r, r) C_O1_I1(r, r) +C_O1_I1(w, r) +C_O1_I1(w, w) C_O1_I2(r, r, rC) C_O1_I2(r, r, ri) C_O1_I2(r, r, rI) @@ -29,4 +33,9 @@ C_O1_I2(r, 0, rZ) C_O1_I2(r, rZ, ri) C_O1_I2(r, rZ, rJ) C_O1_I2(r, rZ, rZ) +C_O1_I2(w, w, w) +C_O1_I2(w, w, wM) +C_O1_I2(w, w, wA) +C_O1_I3(w, w, w, w) C_O1_I4(r, rZ, rJ, rZ, rZ) +C_O2_I1(r, r, r) diff --git a/tcg/loongarch64/tcg-target-con-str.h b/tcg/loongarch64/tcg-target-con-str.h index 6e9ccca3ad..2ba9c135ac 100644 --- a/tcg/loongarch64/tcg-target-con-str.h +++ b/tcg/loongarch64/tcg-target-con-str.h @@ -14,6 +14,7 @@ * REGS(letter, register_mask) */ REGS('r', ALL_GENERAL_REGS) +REGS('w', ALL_VECTOR_REGS) /* * Define constraint letters for constants: @@ -25,3 +26,5 @@ CONST('U', TCG_CT_CONST_U12) CONST('Z', TCG_CT_CONST_ZERO) CONST('C', TCG_CT_CONST_C12) CONST('W', TCG_CT_CONST_WSZ) +CONST('M', TCG_CT_CONST_VCMP) +CONST('A', TCG_CT_CONST_VADD) diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index baf5fc3819..b701df50db 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -32,6 +32,8 @@ #include "../tcg-ldst.c.inc" #include <asm/hwcap.h> +bool use_lsx_instructions; + #ifdef CONFIG_DEBUG_TCG static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { "zero", @@ -65,7 +67,39 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { "s5", "s6", "s7", - "s8" + "s8", + "vr0", + "vr1", + "vr2", + "vr3", + "vr4", + "vr5", + "vr6", + "vr7", + "vr8", + "vr9", + "vr10", + "vr11", + "vr12", + "vr13", + "vr14", + "vr15", + "vr16", + "vr17", + "vr18", + "vr19", + "vr20", + "vr21", + "vr22", + "vr23", + "vr24", + "vr25", + "vr26", + "vr27", + "vr28", + "vr29", + "vr30", + "vr31", }; #endif @@ -102,6 +136,15 @@ static const int tcg_target_reg_alloc_order[] = { TCG_REG_A2, TCG_REG_A1, TCG_REG_A0, + + /* Vector registers */ + TCG_REG_V0, TCG_REG_V1, TCG_REG_V2, TCG_REG_V3, + TCG_REG_V4, TCG_REG_V5, TCG_REG_V6, TCG_REG_V7, + TCG_REG_V8, TCG_REG_V9, TCG_REG_V10, TCG_REG_V11, + TCG_REG_V12, TCG_REG_V13, TCG_REG_V14, TCG_REG_V15, + TCG_REG_V16, TCG_REG_V17, TCG_REG_V18, TCG_REG_V19, + TCG_REG_V20, TCG_REG_V21, TCG_REG_V22, TCG_REG_V23, + /* V24 - V31 are caller-saved, and skipped. */ }; static const int tcg_target_call_iarg_regs[] = { @@ -133,8 +176,11 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define TCG_CT_CONST_U12 0x800 #define TCG_CT_CONST_C12 0x1000 #define TCG_CT_CONST_WSZ 0x2000 +#define TCG_CT_CONST_VCMP 0x4000 +#define TCG_CT_CONST_VADD 0x8000 #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) +#define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32) static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) { @@ -142,7 +188,7 @@ static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) } /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { if (ct & TCG_CT_CONST) { return true; @@ -165,6 +211,13 @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) { return true; } + int64_t vec_val = sextract64(val, 0, 8 << vece); + if ((ct & TCG_CT_CONST_VCMP) && -0x10 <= vec_val && vec_val <= 0x1f) { + return true; + } + if ((ct & TCG_CT_CONST_VADD) && -0x1f <= vec_val && vec_val <= 0x1f) { + return true; + } return false; } @@ -1028,6 +1081,48 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, } } +static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg data_lo, TCGReg data_hi, + TCGReg addr_reg, MemOpIdx oi, bool is_ld) +{ + TCGLabelQemuLdst *ldst; + HostAddress h; + + ldst = prepare_host_addr(s, &h, addr_reg, oi, is_ld); + + if (h.aa.atom == MO_128) { + /* + * Use VLDX/VSTX when 128-bit atomicity is required. + * If address is aligned to 16-bytes, the 128-bit load/store is atomic. + */ + if (is_ld) { + tcg_out_opc_vldx(s, TCG_VEC_TMP0, h.base, h.index); + tcg_out_opc_vpickve2gr_d(s, data_lo, TCG_VEC_TMP0, 0); + tcg_out_opc_vpickve2gr_d(s, data_hi, TCG_VEC_TMP0, 1); + } else { + tcg_out_opc_vinsgr2vr_d(s, TCG_VEC_TMP0, data_lo, 0); + tcg_out_opc_vinsgr2vr_d(s, TCG_VEC_TMP0, data_hi, 1); + tcg_out_opc_vstx(s, TCG_VEC_TMP0, h.base, h.index); + } + } else { + /* Otherwise use a pair of LD/ST. */ + tcg_out_opc_add_d(s, TCG_REG_TMP0, h.base, h.index); + if (is_ld) { + tcg_out_opc_ld_d(s, data_lo, TCG_REG_TMP0, 0); + tcg_out_opc_ld_d(s, data_hi, TCG_REG_TMP0, 8); + } else { + tcg_out_opc_st_d(s, data_lo, TCG_REG_TMP0, 0); + tcg_out_opc_st_d(s, data_hi, TCG_REG_TMP0, 8); + } + } + + if (ldst) { + ldst->type = TCG_TYPE_I128; + ldst->datalo_reg = data_lo; + ldst->datahi_reg = data_hi; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); + } +} + /* * Entry-points */ @@ -1092,6 +1187,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, TCGArg a0 = args[0]; TCGArg a1 = args[1]; TCGArg a2 = args[2]; + TCGArg a3 = args[3]; int c2 = const_args[2]; switch (opc) { @@ -1454,6 +1550,10 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_qemu_ld_a64_i64: tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; + case INDEX_op_qemu_ld_a32_i128: + case INDEX_op_qemu_ld_a64_i128: + tcg_out_qemu_ldst_i128(s, a0, a1, a2, a3, true); + break; case INDEX_op_qemu_st_a32_i32: case INDEX_op_qemu_st_a64_i32: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); @@ -1462,6 +1562,10 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_qemu_st_a64_i64: tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; + case INDEX_op_qemu_st_a32_i128: + case INDEX_op_qemu_st_a64_i128: + tcg_out_qemu_ldst_i128(s, a0, a1, a2, a3, false); + break; case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: @@ -1486,6 +1590,444 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } } +static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg rd, TCGReg rs) +{ + switch (vece) { + case MO_8: + tcg_out_opc_vreplgr2vr_b(s, rd, rs); + break; + case MO_16: + tcg_out_opc_vreplgr2vr_h(s, rd, rs); + break; + case MO_32: + tcg_out_opc_vreplgr2vr_w(s, rd, rs); + break; + case MO_64: + tcg_out_opc_vreplgr2vr_d(s, rd, rs); + break; + default: + g_assert_not_reached(); + } + return true; +} + +static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg r, TCGReg base, intptr_t offset) +{ + /* Handle imm overflow and division (vldrepl.d imm is divided by 8) */ + if (offset < -0x800 || offset > 0x7ff || \ + (offset & ((1 << vece) - 1)) != 0) { + tcg_out_addi(s, TCG_TYPE_I64, TCG_REG_TMP0, base, offset); + base = TCG_REG_TMP0; + offset = 0; + } + offset >>= vece; + + switch (vece) { + case MO_8: + tcg_out_opc_vldrepl_b(s, r, base, offset); + break; + case MO_16: + tcg_out_opc_vldrepl_h(s, r, base, offset); + break; + case MO_32: + tcg_out_opc_vldrepl_w(s, r, base, offset); + break; + case MO_64: + tcg_out_opc_vldrepl_d(s, r, base, offset); + break; + default: + g_assert_not_reached(); + } + return true; +} + +static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg rd, int64_t v64) +{ + /* Try vldi if imm can fit */ + int64_t value = sextract64(v64, 0, 8 << vece); + if (-0x200 <= value && value <= 0x1FF) { + uint32_t imm = (vece << 10) | ((uint32_t)v64 & 0x3FF); + tcg_out_opc_vldi(s, rd, imm); + return; + } + + /* TODO: vldi patterns when imm 12 is set */ + + /* Fallback to vreplgr2vr */ + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP0, value); + switch (vece) { + case MO_8: + tcg_out_opc_vreplgr2vr_b(s, rd, TCG_REG_TMP0); + break; + case MO_16: + tcg_out_opc_vreplgr2vr_h(s, rd, TCG_REG_TMP0); + break; + case MO_32: + tcg_out_opc_vreplgr2vr_w(s, rd, TCG_REG_TMP0); + break; + case MO_64: + tcg_out_opc_vreplgr2vr_d(s, rd, TCG_REG_TMP0); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_out_addsub_vec(TCGContext *s, unsigned vece, const TCGArg a0, + const TCGArg a1, const TCGArg a2, + bool a2_is_const, bool is_add) +{ + static const LoongArchInsn add_vec_insn[4] = { + OPC_VADD_B, OPC_VADD_H, OPC_VADD_W, OPC_VADD_D + }; + static const LoongArchInsn add_vec_imm_insn[4] = { + OPC_VADDI_BU, OPC_VADDI_HU, OPC_VADDI_WU, OPC_VADDI_DU + }; + static const LoongArchInsn sub_vec_insn[4] = { + OPC_VSUB_B, OPC_VSUB_H, OPC_VSUB_W, OPC_VSUB_D + }; + static const LoongArchInsn sub_vec_imm_insn[4] = { + OPC_VSUBI_BU, OPC_VSUBI_HU, OPC_VSUBI_WU, OPC_VSUBI_DU + }; + + if (a2_is_const) { + int64_t value = sextract64(a2, 0, 8 << vece); + if (!is_add) { + value = -value; + } + + /* Try vaddi/vsubi */ + if (0 <= value && value <= 0x1f) { + tcg_out32(s, encode_vdvjuk5_insn(add_vec_imm_insn[vece], a0, \ + a1, value)); + return; + } else if (-0x1f <= value && value < 0) { + tcg_out32(s, encode_vdvjuk5_insn(sub_vec_imm_insn[vece], a0, \ + a1, -value)); + return; + } + + /* constraint TCG_CT_CONST_VADD ensures unreachable */ + g_assert_not_reached(); + } + + if (is_add) { + tcg_out32(s, encode_vdvjvk_insn(add_vec_insn[vece], a0, a1, a2)); + } else { + tcg_out32(s, encode_vdvjvk_insn(sub_vec_insn[vece], a0, a1, a2)); + } +} + +static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, + unsigned vecl, unsigned vece, + const TCGArg args[TCG_MAX_OP_ARGS], + const int const_args[TCG_MAX_OP_ARGS]) +{ + TCGType type = vecl + TCG_TYPE_V64; + TCGArg a0, a1, a2, a3; + TCGReg temp = TCG_REG_TMP0; + TCGReg temp_vec = TCG_VEC_TMP0; + + static const LoongArchInsn cmp_vec_insn[16][4] = { + [TCG_COND_EQ] = {OPC_VSEQ_B, OPC_VSEQ_H, OPC_VSEQ_W, OPC_VSEQ_D}, + [TCG_COND_LE] = {OPC_VSLE_B, OPC_VSLE_H, OPC_VSLE_W, OPC_VSLE_D}, + [TCG_COND_LEU] = {OPC_VSLE_BU, OPC_VSLE_HU, OPC_VSLE_WU, OPC_VSLE_DU}, + [TCG_COND_LT] = {OPC_VSLT_B, OPC_VSLT_H, OPC_VSLT_W, OPC_VSLT_D}, + [TCG_COND_LTU] = {OPC_VSLT_BU, OPC_VSLT_HU, OPC_VSLT_WU, OPC_VSLT_DU}, + }; + static const LoongArchInsn cmp_vec_imm_insn[16][4] = { + [TCG_COND_EQ] = {OPC_VSEQI_B, OPC_VSEQI_H, OPC_VSEQI_W, OPC_VSEQI_D}, + [TCG_COND_LE] = {OPC_VSLEI_B, OPC_VSLEI_H, OPC_VSLEI_W, OPC_VSLEI_D}, + [TCG_COND_LEU] = {OPC_VSLEI_BU, OPC_VSLEI_HU, OPC_VSLEI_WU, OPC_VSLEI_DU}, + [TCG_COND_LT] = {OPC_VSLTI_B, OPC_VSLTI_H, OPC_VSLTI_W, OPC_VSLTI_D}, + [TCG_COND_LTU] = {OPC_VSLTI_BU, OPC_VSLTI_HU, OPC_VSLTI_WU, OPC_VSLTI_DU}, + }; + LoongArchInsn insn; + static const LoongArchInsn neg_vec_insn[4] = { + OPC_VNEG_B, OPC_VNEG_H, OPC_VNEG_W, OPC_VNEG_D + }; + static const LoongArchInsn mul_vec_insn[4] = { + OPC_VMUL_B, OPC_VMUL_H, OPC_VMUL_W, OPC_VMUL_D + }; + static const LoongArchInsn smin_vec_insn[4] = { + OPC_VMIN_B, OPC_VMIN_H, OPC_VMIN_W, OPC_VMIN_D + }; + static const LoongArchInsn umin_vec_insn[4] = { + OPC_VMIN_BU, OPC_VMIN_HU, OPC_VMIN_WU, OPC_VMIN_DU + }; + static const LoongArchInsn smax_vec_insn[4] = { + OPC_VMAX_B, OPC_VMAX_H, OPC_VMAX_W, OPC_VMAX_D + }; + static const LoongArchInsn umax_vec_insn[4] = { + OPC_VMAX_BU, OPC_VMAX_HU, OPC_VMAX_WU, OPC_VMAX_DU + }; + static const LoongArchInsn ssadd_vec_insn[4] = { + OPC_VSADD_B, OPC_VSADD_H, OPC_VSADD_W, OPC_VSADD_D + }; + static const LoongArchInsn usadd_vec_insn[4] = { + OPC_VSADD_BU, OPC_VSADD_HU, OPC_VSADD_WU, OPC_VSADD_DU + }; + static const LoongArchInsn sssub_vec_insn[4] = { + OPC_VSSUB_B, OPC_VSSUB_H, OPC_VSSUB_W, OPC_VSSUB_D + }; + static const LoongArchInsn ussub_vec_insn[4] = { + OPC_VSSUB_BU, OPC_VSSUB_HU, OPC_VSSUB_WU, OPC_VSSUB_DU + }; + static const LoongArchInsn shlv_vec_insn[4] = { + OPC_VSLL_B, OPC_VSLL_H, OPC_VSLL_W, OPC_VSLL_D + }; + static const LoongArchInsn shrv_vec_insn[4] = { + OPC_VSRL_B, OPC_VSRL_H, OPC_VSRL_W, OPC_VSRL_D + }; + static const LoongArchInsn sarv_vec_insn[4] = { + OPC_VSRA_B, OPC_VSRA_H, OPC_VSRA_W, OPC_VSRA_D + }; + static const LoongArchInsn shli_vec_insn[4] = { + OPC_VSLLI_B, OPC_VSLLI_H, OPC_VSLLI_W, OPC_VSLLI_D + }; + static const LoongArchInsn shri_vec_insn[4] = { + OPC_VSRLI_B, OPC_VSRLI_H, OPC_VSRLI_W, OPC_VSRLI_D + }; + static const LoongArchInsn sari_vec_insn[4] = { + OPC_VSRAI_B, OPC_VSRAI_H, OPC_VSRAI_W, OPC_VSRAI_D + }; + static const LoongArchInsn rotrv_vec_insn[4] = { + OPC_VROTR_B, OPC_VROTR_H, OPC_VROTR_W, OPC_VROTR_D + }; + + a0 = args[0]; + a1 = args[1]; + a2 = args[2]; + a3 = args[3]; + + /* Currently only supports V128 */ + tcg_debug_assert(type == TCG_TYPE_V128); + + switch (opc) { + case INDEX_op_st_vec: + /* Try to fit vst imm */ + if (-0x800 <= a2 && a2 <= 0x7ff) { + tcg_out_opc_vst(s, a0, a1, a2); + } else { + tcg_out_movi(s, TCG_TYPE_I64, temp, a2); + tcg_out_opc_vstx(s, a0, a1, temp); + } + break; + case INDEX_op_ld_vec: + /* Try to fit vld imm */ + if (-0x800 <= a2 && a2 <= 0x7ff) { + tcg_out_opc_vld(s, a0, a1, a2); + } else { + tcg_out_movi(s, TCG_TYPE_I64, temp, a2); + tcg_out_opc_vldx(s, a0, a1, temp); + } + break; + case INDEX_op_and_vec: + tcg_out_opc_vand_v(s, a0, a1, a2); + break; + case INDEX_op_andc_vec: + /* + * vandn vd, vj, vk: vd = vk & ~vj + * andc_vec vd, vj, vk: vd = vj & ~vk + * vk and vk are swapped + */ + tcg_out_opc_vandn_v(s, a0, a2, a1); + break; + case INDEX_op_or_vec: + tcg_out_opc_vor_v(s, a0, a1, a2); + break; + case INDEX_op_orc_vec: + tcg_out_opc_vorn_v(s, a0, a1, a2); + break; + case INDEX_op_xor_vec: + tcg_out_opc_vxor_v(s, a0, a1, a2); + break; + case INDEX_op_nor_vec: + tcg_out_opc_vnor_v(s, a0, a1, a2); + break; + case INDEX_op_not_vec: + tcg_out_opc_vnor_v(s, a0, a1, a1); + break; + case INDEX_op_cmp_vec: + TCGCond cond = args[3]; + if (const_args[2]) { + /* + * cmp_vec dest, src, value + * Try vseqi/vslei/vslti + */ + int64_t value = sextract64(a2, 0, 8 << vece); + if ((cond == TCG_COND_EQ || cond == TCG_COND_LE || \ + cond == TCG_COND_LT) && (-0x10 <= value && value <= 0x0f)) { + tcg_out32(s, encode_vdvjsk5_insn(cmp_vec_imm_insn[cond][vece], \ + a0, a1, value)); + break; + } else if ((cond == TCG_COND_LEU || cond == TCG_COND_LTU) && + (0x00 <= value && value <= 0x1f)) { + tcg_out32(s, encode_vdvjuk5_insn(cmp_vec_imm_insn[cond][vece], \ + a0, a1, value)); + break; + } + + /* + * Fallback to: + * dupi_vec temp, a2 + * cmp_vec a0, a1, temp, cond + */ + tcg_out_dupi_vec(s, type, vece, temp_vec, a2); + a2 = temp_vec; + } + + insn = cmp_vec_insn[cond][vece]; + if (insn == 0) { + TCGArg t; + t = a1, a1 = a2, a2 = t; + cond = tcg_swap_cond(cond); + insn = cmp_vec_insn[cond][vece]; + tcg_debug_assert(insn != 0); + } + tcg_out32(s, encode_vdvjvk_insn(insn, a0, a1, a2)); + break; + case INDEX_op_add_vec: + tcg_out_addsub_vec(s, vece, a0, a1, a2, const_args[2], true); + break; + case INDEX_op_sub_vec: + tcg_out_addsub_vec(s, vece, a0, a1, a2, const_args[2], false); + break; + case INDEX_op_neg_vec: + tcg_out32(s, encode_vdvj_insn(neg_vec_insn[vece], a0, a1)); + break; + case INDEX_op_mul_vec: + tcg_out32(s, encode_vdvjvk_insn(mul_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_smin_vec: + tcg_out32(s, encode_vdvjvk_insn(smin_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_smax_vec: + tcg_out32(s, encode_vdvjvk_insn(smax_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_umin_vec: + tcg_out32(s, encode_vdvjvk_insn(umin_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_umax_vec: + tcg_out32(s, encode_vdvjvk_insn(umax_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_ssadd_vec: + tcg_out32(s, encode_vdvjvk_insn(ssadd_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_usadd_vec: + tcg_out32(s, encode_vdvjvk_insn(usadd_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_sssub_vec: + tcg_out32(s, encode_vdvjvk_insn(sssub_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_ussub_vec: + tcg_out32(s, encode_vdvjvk_insn(ussub_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_shlv_vec: + tcg_out32(s, encode_vdvjvk_insn(shlv_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_shrv_vec: + tcg_out32(s, encode_vdvjvk_insn(shrv_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_sarv_vec: + tcg_out32(s, encode_vdvjvk_insn(sarv_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_shli_vec: + tcg_out32(s, encode_vdvjuk3_insn(shli_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_shri_vec: + tcg_out32(s, encode_vdvjuk3_insn(shri_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_sari_vec: + tcg_out32(s, encode_vdvjuk3_insn(sari_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_rotrv_vec: + tcg_out32(s, encode_vdvjvk_insn(rotrv_vec_insn[vece], a0, a1, a2)); + break; + case INDEX_op_rotlv_vec: + /* rotlv_vec a1, a2 = rotrv_vec a1, -a2 */ + tcg_out32(s, encode_vdvj_insn(neg_vec_insn[vece], temp_vec, a2)); + tcg_out32(s, encode_vdvjvk_insn(rotrv_vec_insn[vece], a0, a1, + temp_vec)); + break; + case INDEX_op_rotli_vec: + /* rotli_vec a1, a2 = rotri_vec a1, -a2 */ + a2 = extract32(-a2, 0, 3 + vece); + switch (vece) { + case MO_8: + tcg_out_opc_vrotri_b(s, a0, a1, a2); + break; + case MO_16: + tcg_out_opc_vrotri_h(s, a0, a1, a2); + break; + case MO_32: + tcg_out_opc_vrotri_w(s, a0, a1, a2); + break; + case MO_64: + tcg_out_opc_vrotri_d(s, a0, a1, a2); + break; + default: + g_assert_not_reached(); + } + break; + case INDEX_op_bitsel_vec: + /* vbitsel vd, vj, vk, va = bitsel_vec vd, va, vk, vj */ + tcg_out_opc_vbitsel_v(s, a0, a3, a2, a1); + break; + case INDEX_op_dupm_vec: + tcg_out_dupm_vec(s, type, vece, a0, a1, a2); + break; + default: + g_assert_not_reached(); + } +} + +int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) +{ + switch (opc) { + case INDEX_op_ld_vec: + case INDEX_op_st_vec: + case INDEX_op_dup_vec: + case INDEX_op_dupm_vec: + case INDEX_op_cmp_vec: + case INDEX_op_add_vec: + case INDEX_op_sub_vec: + case INDEX_op_and_vec: + case INDEX_op_andc_vec: + case INDEX_op_or_vec: + case INDEX_op_orc_vec: + case INDEX_op_xor_vec: + case INDEX_op_nor_vec: + case INDEX_op_not_vec: + case INDEX_op_neg_vec: + case INDEX_op_mul_vec: + case INDEX_op_smin_vec: + case INDEX_op_smax_vec: + case INDEX_op_umin_vec: + case INDEX_op_umax_vec: + case INDEX_op_ssadd_vec: + case INDEX_op_usadd_vec: + case INDEX_op_sssub_vec: + case INDEX_op_ussub_vec: + case INDEX_op_shlv_vec: + case INDEX_op_shrv_vec: + case INDEX_op_sarv_vec: + case INDEX_op_bitsel_vec: + return 1; + default: + return 0; + } +} + +void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, + TCGArg a0, ...) +{ + g_assert_not_reached(); +} + static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { switch (op) { @@ -1505,6 +2047,14 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_qemu_st_a64_i64: return C_O0_I2(rZ, r); + case INDEX_op_qemu_ld_a32_i128: + case INDEX_op_qemu_ld_a64_i128: + return C_O2_I1(r, r, r); + + case INDEX_op_qemu_st_a32_i128: + case INDEX_op_qemu_st_a64_i128: + return C_O0_I3(r, r, r); + case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: return C_O0_I2(rZ, rZ); @@ -1627,6 +2177,54 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_movcond_i64: return C_O1_I4(r, rZ, rJ, rZ, rZ); + case INDEX_op_ld_vec: + case INDEX_op_dupm_vec: + case INDEX_op_dup_vec: + return C_O1_I1(w, r); + + case INDEX_op_st_vec: + return C_O0_I2(w, r); + + case INDEX_op_cmp_vec: + return C_O1_I2(w, w, wM); + + case INDEX_op_add_vec: + case INDEX_op_sub_vec: + return C_O1_I2(w, w, wA); + + case INDEX_op_and_vec: + case INDEX_op_andc_vec: + case INDEX_op_or_vec: + case INDEX_op_orc_vec: + case INDEX_op_xor_vec: + case INDEX_op_nor_vec: + case INDEX_op_mul_vec: + case INDEX_op_smin_vec: + case INDEX_op_smax_vec: + case INDEX_op_umin_vec: + case INDEX_op_umax_vec: + case INDEX_op_ssadd_vec: + case INDEX_op_usadd_vec: + case INDEX_op_sssub_vec: + case INDEX_op_ussub_vec: + case INDEX_op_shlv_vec: + case INDEX_op_shrv_vec: + case INDEX_op_sarv_vec: + case INDEX_op_rotrv_vec: + case INDEX_op_rotlv_vec: + return C_O1_I2(w, w, w); + + case INDEX_op_not_vec: + case INDEX_op_neg_vec: + case INDEX_op_shli_vec: + case INDEX_op_shri_vec: + case INDEX_op_sari_vec: + case INDEX_op_rotli_vec: + return C_O1_I1(w, w); + + case INDEX_op_bitsel_vec: + return C_O1_I3(w, w, w, w); + default: g_assert_not_reached(); } @@ -1698,6 +2296,11 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_RA, 0); } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + static void tcg_target_init(TCGContext *s) { unsigned long hwcap = qemu_getauxval(AT_HWCAP); @@ -1708,6 +2311,10 @@ static void tcg_target_init(TCGContext *s) exit(EXIT_FAILURE); } + if (hwcap & HWCAP_LOONGARCH_LSX) { + use_lsx_instructions = 1; + } + tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS; @@ -1723,6 +2330,18 @@ static void tcg_target_init(TCGContext *s) tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S8); tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S9); + if (use_lsx_instructions) { + tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS; + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V24); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V25); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V26); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V27); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V28); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V29); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V30); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V31); + } + s->reserved_regs = 0; tcg_regset_set_reg(s->reserved_regs, TCG_REG_ZERO); tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP0); @@ -1731,6 +2350,7 @@ static void tcg_target_init(TCGContext *s) tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP); tcg_regset_set_reg(s->reserved_regs, TCG_REG_TP); tcg_regset_set_reg(s->reserved_regs, TCG_REG_RESERVED); + tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP0); } typedef struct { diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index 559be67186..03017672f6 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -30,7 +30,7 @@ #define LOONGARCH_TCG_TARGET_H #define TCG_TARGET_INSN_UNIT_SIZE 4 -#define TCG_TARGET_NB_REGS 32 +#define TCG_TARGET_NB_REGS 64 #define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) @@ -68,13 +68,25 @@ typedef enum { TCG_REG_S7, TCG_REG_S8, + TCG_REG_V0 = 32, TCG_REG_V1, TCG_REG_V2, TCG_REG_V3, + TCG_REG_V4, TCG_REG_V5, TCG_REG_V6, TCG_REG_V7, + TCG_REG_V8, TCG_REG_V9, TCG_REG_V10, TCG_REG_V11, + TCG_REG_V12, TCG_REG_V13, TCG_REG_V14, TCG_REG_V15, + TCG_REG_V16, TCG_REG_V17, TCG_REG_V18, TCG_REG_V19, + TCG_REG_V20, TCG_REG_V21, TCG_REG_V22, TCG_REG_V23, + TCG_REG_V24, TCG_REG_V25, TCG_REG_V26, TCG_REG_V27, + TCG_REG_V28, TCG_REG_V29, TCG_REG_V30, TCG_REG_V31, + /* aliases */ TCG_AREG0 = TCG_REG_S0, TCG_REG_TMP0 = TCG_REG_T8, TCG_REG_TMP1 = TCG_REG_T7, TCG_REG_TMP2 = TCG_REG_T6, + TCG_VEC_TMP0 = TCG_REG_V23, } TCGReg; +extern bool use_lsx_instructions; + /* used for function call generation */ #define TCG_REG_CALL_STACK TCG_REG_SP #define TCG_TARGET_STACK_ALIGN 16 @@ -159,7 +171,31 @@ typedef enum { #define TCG_TARGET_HAS_muluh_i64 1 #define TCG_TARGET_HAS_mulsh_i64 1 -#define TCG_TARGET_HAS_qemu_ldst_i128 0 +#define TCG_TARGET_HAS_qemu_ldst_i128 use_lsx_instructions + +#define TCG_TARGET_HAS_v64 0 +#define TCG_TARGET_HAS_v128 use_lsx_instructions +#define TCG_TARGET_HAS_v256 0 + +#define TCG_TARGET_HAS_not_vec 1 +#define TCG_TARGET_HAS_neg_vec 1 +#define TCG_TARGET_HAS_abs_vec 0 +#define TCG_TARGET_HAS_andc_vec 1 +#define TCG_TARGET_HAS_orc_vec 1 +#define TCG_TARGET_HAS_nand_vec 0 +#define TCG_TARGET_HAS_nor_vec 1 +#define TCG_TARGET_HAS_eqv_vec 0 +#define TCG_TARGET_HAS_mul_vec 1 +#define TCG_TARGET_HAS_shi_vec 1 +#define TCG_TARGET_HAS_shs_vec 0 +#define TCG_TARGET_HAS_shv_vec 1 +#define TCG_TARGET_HAS_roti_vec 1 +#define TCG_TARGET_HAS_rots_vec 0 +#define TCG_TARGET_HAS_rotv_vec 1 +#define TCG_TARGET_HAS_sat_vec 1 +#define TCG_TARGET_HAS_minmax_vec 1 +#define TCG_TARGET_HAS_bitsel_vec 1 +#define TCG_TARGET_HAS_cmpsel_vec 0 #define TCG_TARGET_DEFAULT_MO (0) diff --git a/tcg/loongarch64/tcg-target.opc.h b/tcg/loongarch64/tcg-target.opc.h new file mode 100644 index 0000000000..fd1a40b7fd --- /dev/null +++ b/tcg/loongarch64/tcg-target.opc.h @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2023 Jiajie Chen + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * (at your option) any later version. + * + * See the COPYING file in the top-level directory for details. + * + * Target-specific opcodes for host vector expansion. These will be + * emitted by tcg_expand_vec_op. For those familiar with GCC internals, + * consider these to be UNSPEC with names. + */ diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index 9faa8bdf0b..f52bda4828 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -190,7 +190,7 @@ static bool is_p2m1(tcg_target_long val) } /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -2628,6 +2628,11 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc_reg(s, OPC_OR, TCG_TMP3, TCG_TMP3, TCG_TMP1); } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + static void tcg_target_init(TCGContext *s) { tcg_target_detect_isa(); diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 090f11e71c..90d76c2c2c 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -261,7 +261,7 @@ static bool reloc_pc14(tcg_insn_unit *src_rw, const tcg_insn_unit *target) } /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -2527,6 +2527,11 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out32(s, BCLR | BO_ALWAYS); } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) { tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, arg); diff --git a/tcg/region.c b/tcg/region.c index 2b28ed3556..a078899096 100644 --- a/tcg/region.c +++ b/tcg/region.c @@ -33,8 +33,19 @@ #include "tcg/tcg.h" #include "exec/translation-block.h" #include "tcg-internal.h" +#include "host/cpuinfo.h" +/* + * Local source-level compatibility with Unix. + * Used by tcg_region_init below. + */ +#if defined(_WIN32) +#define PROT_READ 1 +#define PROT_WRITE 2 +#define PROT_EXEC 4 +#endif + struct tcg_region_tree { QemuMutex lock; QTree *tree; @@ -83,6 +94,18 @@ bool in_code_gen_buffer(const void *p) return (size_t)(p - region.start_aligned) <= region.total_size; } +#ifndef CONFIG_TCG_INTERPRETER +static int host_prot_read_exec(void) +{ +#if defined(CONFIG_LINUX) && defined(HOST_AARCH64) && defined(PROT_BTI) + if (cpuinfo & CPUINFO_BTI) { + return PROT_READ | PROT_EXEC | PROT_BTI; + } +#endif + return PROT_READ | PROT_EXEC; +} +#endif + #ifdef CONFIG_DEBUG_TCG const void *tcg_splitwx_to_rx(void *rw) { @@ -505,14 +528,6 @@ static int alloc_code_gen_buffer(size_t tb_size, int splitwx, Error **errp) return PROT_READ | PROT_WRITE; } #elif defined(_WIN32) -/* - * Local source-level compatibility with Unix. - * Used by tcg_region_init below. - */ -#define PROT_READ 1 -#define PROT_WRITE 2 -#define PROT_EXEC 4 - static int alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) { void *buf; @@ -567,7 +582,7 @@ static int alloc_code_gen_buffer_splitwx_memfd(size_t size, Error **errp) goto fail; } - buf_rx = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0); + buf_rx = mmap(NULL, size, host_prot_read_exec(), MAP_SHARED, fd, 0); if (buf_rx == MAP_FAILED) { goto fail_rx; } @@ -642,7 +657,7 @@ static int alloc_code_gen_buffer_splitwx_vmremap(size_t size, Error **errp) return -1; } - if (mprotect((void *)buf_rx, size, PROT_READ | PROT_EXEC) != 0) { + if (mprotect((void *)buf_rx, size, host_prot_read_exec()) != 0) { error_setg_errno(errp, errno, "mprotect for jit splitwx"); munmap((void *)buf_rx, size); munmap((void *)buf_rw, size); @@ -805,7 +820,7 @@ void tcg_region_init(size_t tb_size, int splitwx, unsigned max_cpus) need_prot = PROT_READ | PROT_WRITE; #ifndef CONFIG_TCG_INTERPRETER if (tcg_splitwx_diff == 0) { - need_prot |= PROT_EXEC; + need_prot |= host_prot_read_exec(); } #endif for (size_t i = 0, n = region.n; i < n; i++) { @@ -820,7 +835,11 @@ void tcg_region_init(size_t tb_size, int splitwx, unsigned max_cpus) } else if (need_prot == (PROT_READ | PROT_WRITE)) { rc = qemu_mprotect_rw(start, end - start); } else { +#ifdef CONFIG_POSIX + rc = mprotect(start, end - start, need_prot); +#else g_assert_not_reached(); +#endif } if (rc) { error_setg_errno(&error_fatal, errno, diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index 9be81c1b7b..c2bcdea33f 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -145,7 +145,7 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define sextreg sextract64 /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -2099,6 +2099,11 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_RA, 0); } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + static volatile sig_atomic_t got_sigill; static void sigill_handler(int signo, siginfo_t *si, void *data) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index ecd8aaf2a1..7552f63a05 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -540,7 +540,7 @@ static bool risbg_mask(uint64_t c) } /* Test if a constant matches the constraint. */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -3483,6 +3483,11 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_R14); } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + static void tcg_out_nop_fill(tcg_insn_unit *p, int count) { memset(p, 0x07, count * sizeof(tcg_insn_unit)); diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index 81a08bb6c5..01ac26c192 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -322,7 +322,7 @@ static bool patch_reloc(tcg_insn_unit *src_rw, int type, } /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -962,6 +962,11 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_movi_s13(s, TCG_REG_O0, 0); } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + static void tcg_out_nop_fill(tcg_insn_unit *p, int count) { int i; diff --git a/tcg/tcg-op-gvec.c b/tcg/tcg-op-gvec.c index e260a07c61..41b1ae18e4 100644 --- a/tcg/tcg-op-gvec.c +++ b/tcg/tcg-op-gvec.c @@ -3846,6 +3846,155 @@ void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs, } } +static void expand_cmps_vec(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t oprsz, uint32_t tysz, TCGType type, + TCGCond cond, TCGv_vec c) +{ + TCGv_vec t0 = tcg_temp_new_vec(type); + TCGv_vec t1 = tcg_temp_new_vec(type); + uint32_t i; + + for (i = 0; i < oprsz; i += tysz) { + tcg_gen_ld_vec(t1, cpu_env, aofs + i); + tcg_gen_cmp_vec(cond, vece, t0, t1, c); + tcg_gen_st_vec(t0, cpu_env, dofs + i); + } +} + +void tcg_gen_gvec_cmps(TCGCond cond, unsigned vece, uint32_t dofs, + uint32_t aofs, TCGv_i64 c, + uint32_t oprsz, uint32_t maxsz) +{ + static const TCGOpcode cmp_list[] = { INDEX_op_cmp_vec, 0 }; + static gen_helper_gvec_2i * const eq_fn[4] = { + gen_helper_gvec_eqs8, gen_helper_gvec_eqs16, + gen_helper_gvec_eqs32, gen_helper_gvec_eqs64 + }; + static gen_helper_gvec_2i * const lt_fn[4] = { + gen_helper_gvec_lts8, gen_helper_gvec_lts16, + gen_helper_gvec_lts32, gen_helper_gvec_lts64 + }; + static gen_helper_gvec_2i * const le_fn[4] = { + gen_helper_gvec_les8, gen_helper_gvec_les16, + gen_helper_gvec_les32, gen_helper_gvec_les64 + }; + static gen_helper_gvec_2i * const ltu_fn[4] = { + gen_helper_gvec_ltus8, gen_helper_gvec_ltus16, + gen_helper_gvec_ltus32, gen_helper_gvec_ltus64 + }; + static gen_helper_gvec_2i * const leu_fn[4] = { + gen_helper_gvec_leus8, gen_helper_gvec_leus16, + gen_helper_gvec_leus32, gen_helper_gvec_leus64 + }; + static gen_helper_gvec_2i * const * const fns[16] = { + [TCG_COND_EQ] = eq_fn, + [TCG_COND_LT] = lt_fn, + [TCG_COND_LE] = le_fn, + [TCG_COND_LTU] = ltu_fn, + [TCG_COND_LEU] = leu_fn, + }; + + TCGType type; + + check_size_align(oprsz, maxsz, dofs | aofs); + check_overlap_2(dofs, aofs, maxsz); + + if (cond == TCG_COND_NEVER || cond == TCG_COND_ALWAYS) { + do_dup(MO_8, dofs, oprsz, maxsz, + NULL, NULL, -(cond == TCG_COND_ALWAYS)); + return; + } + + /* + * Implement inline with a vector type, if possible. + * Prefer integer when 64-bit host and 64-bit comparison. + */ + type = choose_vector_type(cmp_list, vece, oprsz, + TCG_TARGET_REG_BITS == 64 && vece == MO_64); + if (type != 0) { + const TCGOpcode *hold_list = tcg_swap_vecop_list(cmp_list); + TCGv_vec t_vec = tcg_temp_new_vec(type); + uint32_t some; + + tcg_gen_dup_i64_vec(vece, t_vec, c); + switch (type) { + case TCG_TYPE_V256: + some = QEMU_ALIGN_DOWN(oprsz, 32); + expand_cmps_vec(vece, dofs, aofs, some, 32, + TCG_TYPE_V256, cond, t_vec); + aofs += some; + dofs += some; + oprsz -= some; + maxsz -= some; + /* fallthru */ + + case TCG_TYPE_V128: + some = QEMU_ALIGN_DOWN(oprsz, 16); + expand_cmps_vec(vece, dofs, aofs, some, 16, + TCG_TYPE_V128, cond, t_vec); + break; + + case TCG_TYPE_V64: + some = QEMU_ALIGN_DOWN(oprsz, 8); + expand_cmps_vec(vece, dofs, aofs, some, 8, + TCG_TYPE_V64, cond, t_vec); + break; + + default: + g_assert_not_reached(); + } + tcg_temp_free_vec(t_vec); + tcg_swap_vecop_list(hold_list); + } else if (vece == MO_64 && check_size_impl(oprsz, 8)) { + TCGv_i64 t0 = tcg_temp_ebb_new_i64(); + uint32_t i; + + for (i = 0; i < oprsz; i += 8) { + tcg_gen_ld_i64(t0, cpu_env, aofs + i); + tcg_gen_negsetcond_i64(cond, t0, t0, c); + tcg_gen_st_i64(t0, cpu_env, dofs + i); + } + tcg_temp_free_i64(t0); + } else if (vece == MO_32 && check_size_impl(oprsz, 4)) { + TCGv_i32 t0 = tcg_temp_ebb_new_i32(); + TCGv_i32 t1 = tcg_temp_ebb_new_i32(); + uint32_t i; + + tcg_gen_extrl_i64_i32(t1, c); + for (i = 0; i < oprsz; i += 8) { + tcg_gen_ld_i32(t0, cpu_env, aofs + i); + tcg_gen_negsetcond_i32(cond, t0, t0, t1); + tcg_gen_st_i32(t0, cpu_env, dofs + i); + } + tcg_temp_free_i32(t0); + tcg_temp_free_i32(t1); + } else { + gen_helper_gvec_2i * const *fn = fns[cond]; + bool inv = false; + + if (fn == NULL) { + cond = tcg_invert_cond(cond); + fn = fns[cond]; + assert(fn != NULL); + inv = true; + } + tcg_gen_gvec_2i_ool(dofs, aofs, c, oprsz, maxsz, inv, fn[vece]); + return; + } + + if (oprsz < maxsz) { + expand_clr(dofs + oprsz, maxsz - oprsz); + } +} + +void tcg_gen_gvec_cmpi(TCGCond cond, unsigned vece, uint32_t dofs, + uint32_t aofs, int64_t c, + uint32_t oprsz, uint32_t maxsz) +{ + TCGv_i64 tmp = tcg_constant_i64(c); + tcg_gen_gvec_cmps(cond, vece, dofs, aofs, tmp, oprsz, maxsz); +} + static void tcg_gen_bitsel_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 c) { TCGv_i64 t = tcg_temp_ebb_new_i64(); diff --git a/tcg/tcg.c b/tcg/tcg.c index 620dbe08da..604fa9bf3e 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -108,6 +108,7 @@ static void tcg_register_jit_int(const void *buf, size_t size, __attribute__((unused)); /* Forward declarations for functions declared and used in tcg-target.c.inc. */ +static void tcg_out_tb_start(TCGContext *s); static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, intptr_t arg2); static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); @@ -171,7 +172,7 @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, const TCGHelperInfo *info); static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot); -static bool tcg_target_const_match(int64_t val, TCGType type, int ct); +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece); #ifdef TCG_TARGET_NEED_LDST_LABELS static int tcg_out_ldst_finalize(TCGContext *s); #endif @@ -4689,7 +4690,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) ts = arg_temp(arg); if (ts->val_type == TEMP_VAL_CONST - && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { + && tcg_target_const_match(ts->val, ts->type, arg_ct->ct, TCGOP_VECE(op))) { /* constant is OK for instruction */ const_args[i] = 1; new_args[i] = ts->val; @@ -6014,6 +6015,8 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) s->gen_insn_data = tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * start_words); + tcg_out_tb_start(s); + num_insns = -1; QTAILQ_FOREACH(op, &s->ops, link) { TCGOpcode opc = op->opc; diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index 253f27f174..461f4b47ff 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -913,7 +913,7 @@ static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, } /* Test if a constant matches the constraint. */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece) { return ct & TCG_CT_CONST; } @@ -955,6 +955,11 @@ static inline void tcg_target_qemu_prologue(TCGContext *s) { } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + bool tcg_target_has_memory_bswap(MemOp memop) { return true; diff --git a/tests/avocado/avocado_qemu/__init__.py b/tests/avocado/avocado_qemu/__init__.py index 33090903f1..0172a359b7 100644 --- a/tests/avocado/avocado_qemu/__init__.py +++ b/tests/avocado/avocado_qemu/__init__.py @@ -137,7 +137,7 @@ def _console_interaction(test, success_message, failure_message, assert not keep_sending or send_string if vm is None: vm = test.vm - console = vm.console_socket.makefile(mode='rb', encoding='utf-8') + console = vm.console_file console_logger = logging.getLogger('console') while True: if send_string: diff --git a/tests/avocado/boot_linux_console.py b/tests/avocado/boot_linux_console.py index 6eab515718..01ee149812 100644 --- a/tests/avocado/boot_linux_console.py +++ b/tests/avocado/boot_linux_console.py @@ -116,6 +116,7 @@ class BootLinuxConsole(LinuxKernelTest): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta(self): """ :avocado: tags=arch:mips @@ -138,6 +139,7 @@ class BootLinuxConsole(LinuxKernelTest): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64el_malta(self): """ This test requires the ar tool to extract "data.tar.gz" from @@ -191,6 +193,7 @@ class BootLinuxConsole(LinuxKernelTest): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta_cpio(self): """ :avocado: tags=arch:mips @@ -232,6 +235,7 @@ class BootLinuxConsole(LinuxKernelTest): # Wait for VM to shut down gracefully self.vm.wait() + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') @skipUnless(os.getenv('AVOCADO_ALLOW_UNTRUSTED_CODE'), 'untrusted code') def test_mips64el_malta_5KEc_cpio(self): """ @@ -292,6 +296,7 @@ class BootLinuxConsole(LinuxKernelTest): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_4k(self): """ :avocado: tags=arch:mipsel @@ -305,6 +310,7 @@ class BootLinuxConsole(LinuxKernelTest): kernel_hash = '477456aafd2a0f1ddc9482727f20fe9575565dd6' self.do_test_mips_malta32el_nanomips(kernel_url, kernel_hash) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_16k_up(self): """ :avocado: tags=arch:mipsel @@ -318,6 +324,7 @@ class BootLinuxConsole(LinuxKernelTest): kernel_hash = 'e882868f944c71c816e832e2303b7874d044a7bc' self.do_test_mips_malta32el_nanomips(kernel_url, kernel_hash) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_64k_dbg(self): """ :avocado: tags=arch:mipsel diff --git a/tests/avocado/machine_mips_malta.py b/tests/avocado/machine_mips_malta.py index 92233451c5..3620266589 100644 --- a/tests/avocado/machine_mips_malta.py +++ b/tests/avocado/machine_mips_malta.py @@ -11,6 +11,7 @@ import os import gzip import logging +from avocado import skip from avocado import skipIf from avocado import skipUnless from avocado.utils import archive @@ -93,6 +94,7 @@ class MaltaMachineFramebuffer(QemuSystemTest): cv2.imwrite(debug_png, screendump_bgr) self.assertGreaterEqual(tuxlogo_count, cpu_cores_count) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta_i6400_framebuffer_logo_1core(self): """ :avocado: tags=arch:mips64el @@ -101,6 +103,7 @@ class MaltaMachineFramebuffer(QemuSystemTest): """ self.do_test_i6400_framebuffer_logo(1) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_7cores(self): """ @@ -111,6 +114,7 @@ class MaltaMachineFramebuffer(QemuSystemTest): """ self.do_test_i6400_framebuffer_logo(7) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_8cores(self): """ @@ -142,6 +146,7 @@ class MaltaMachine(QemuSystemTest): wait_for_console_pattern(self, prompt) self.vm.shutdown() + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mipsel_malta_yamon(self): """ :avocado: tags=arch:mipsel @@ -150,6 +155,7 @@ class MaltaMachine(QemuSystemTest): """ self.do_test_yamon() + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64el_malta_yamon(self): """ :avocado: tags=arch:mips64el diff --git a/tests/avocado/replay_kernel.py b/tests/avocado/replay_kernel.py index a18610542e..f7ccfd2462 100644 --- a/tests/avocado/replay_kernel.py +++ b/tests/avocado/replay_kernel.py @@ -98,6 +98,7 @@ class ReplayKernelNormal(ReplayKernelBase): self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta(self): """ :avocado: tags=arch:mips @@ -116,6 +117,7 @@ class ReplayKernelNormal(ReplayKernelBase): self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64el_malta(self): """ This test requires the ar tool to extract "data.tar.gz" from @@ -431,6 +433,7 @@ class ReplayKernelSlow(ReplayKernelBase): # making it very slow. timeout = 180 + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta_cpio(self): """ :avocado: tags=arch:mips @@ -460,6 +463,7 @@ class ReplayKernelSlow(ReplayKernelBase): self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5, args=('-initrd', initrd_path)) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') @skipUnless(os.getenv('AVOCADO_ALLOW_UNTRUSTED_CODE'), 'untrusted code') def test_mips64el_malta_5KEc_cpio(self): """ @@ -502,6 +506,7 @@ class ReplayKernelSlow(ReplayKernelBase): console_pattern = 'Kernel command line: %s' % kernel_command_line self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_4k(self): """ :avocado: tags=arch:mipsel @@ -516,6 +521,7 @@ class ReplayKernelSlow(ReplayKernelBase): kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash) self.do_test_mips_malta32el_nanomips(kernel_path_xz) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_16k_up(self): """ :avocado: tags=arch:mipsel @@ -530,6 +536,7 @@ class ReplayKernelSlow(ReplayKernelBase): kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash) self.do_test_mips_malta32el_nanomips(kernel_path_xz) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_64k_dbg(self): """ :avocado: tags=arch:mipsel diff --git a/tests/avocado/tuxrun_baselines.py b/tests/avocado/tuxrun_baselines.py index e12250eabb..610b7e2bfa 100644 --- a/tests/avocado/tuxrun_baselines.py +++ b/tests/avocado/tuxrun_baselines.py @@ -352,6 +352,7 @@ class TuxRunBaselineTest(QemuSystemTest): self.common_tuxrun(csums=sums, drive="virtio-blk-pci") + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips32(self): """ :avocado: tags=arch:mips @@ -370,6 +371,7 @@ class TuxRunBaselineTest(QemuSystemTest): self.common_tuxrun(csums=sums, drive="driver=ide-hd,bus=ide.0,unit=0") + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips32el(self): """ :avocado: tags=arch:mipsel @@ -387,6 +389,7 @@ class TuxRunBaselineTest(QemuSystemTest): self.common_tuxrun(csums=sums, drive="driver=ide-hd,bus=ide.0,unit=0") + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64(self): """ :avocado: tags=arch:mips64 @@ -404,6 +407,7 @@ class TuxRunBaselineTest(QemuSystemTest): self.common_tuxrun(csums=sums, drive="driver=ide-hd,bus=ide.0,unit=0") + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64el(self): """ :avocado: tags=arch:mips64el diff --git a/tests/data/acpi/virt/SSDT.memhp b/tests/data/acpi/virt/SSDT.memhp index ef93c44464..31ff6ac469 100644 --- a/tests/data/acpi/virt/SSDT.memhp +++ b/tests/data/acpi/virt/SSDT.memhp Binary files differdiff --git a/tests/docker/dockerfiles/alpine.docker b/tests/docker/dockerfiles/alpine.docker index fa455f1474..d25649cb4f 100644 --- a/tests/docker/dockerfiles/alpine.docker +++ b/tests/docker/dockerfiles/alpine.docker @@ -59,6 +59,7 @@ RUN apk update && \ libtasn1-dev \ liburing-dev \ libusb-dev \ + libxdp-dev \ linux-pam-dev \ llvm \ lttng-ust-dev \ diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index fc1830966f..68bfe606f5 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -75,6 +75,7 @@ RUN dnf distro-sync -y && \ libubsan \ liburing-devel \ libusbx-devel \ + libxdp-devel \ libzstd-devel \ llvm \ lttng-ust-devel \ diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker index b66b9cc191..0991938595 100644 --- a/tests/docker/dockerfiles/debian-amd64-cross.docker +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch x86_64 debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch x86_64 debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -84,7 +80,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++-x86-64-linux-gnu \ gcc-x86-64-linux-gnu \ libaio-dev:amd64 \ - libasan5:amd64 \ + libasan6:amd64 \ libasound2-dev:amd64 \ libattr1-dev:amd64 \ libbpf-dev:amd64 \ @@ -146,6 +142,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:amd64 \ libvirglrenderer-dev:amd64 \ libvte-2.91-dev:amd64 \ + libxdp-dev:amd64 \ libxen-dev:amd64 \ libzstd-dev:amd64 \ nettle-dev:amd64 \ diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker index 02262bc70e..61dbc3ff24 100644 --- a/tests/docker/dockerfiles/debian-amd64.docker +++ b/tests/docker/dockerfiles/debian-amd64.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all debian-11 qemu +# $ lcitool dockerfile --layers all debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -32,7 +32,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ git \ hostname \ libaio-dev \ - libasan5 \ + libasan6 \ libasound2-dev \ libattr1-dev \ libbpf-dev \ @@ -97,6 +97,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev \ libvirglrenderer-dev \ libvte-2.91-dev \ + libxdp-dev \ libxen-dev \ libzstd-dev \ llvm \ @@ -115,11 +116,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -145,8 +144,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/g++ && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/gcc -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker index a0a968b8c6..74eabb274e 100644 --- a/tests/docker/dockerfiles/debian-arm64-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch aarch64 debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch aarch64 debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -84,7 +80,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++-aarch64-linux-gnu \ gcc-aarch64-linux-gnu \ libaio-dev:arm64 \ - libasan5:arm64 \ + libasan6:arm64 \ libasound2-dev:arm64 \ libattr1-dev:arm64 \ libbpf-dev:arm64 \ @@ -145,6 +141,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:arm64 \ libvirglrenderer-dev:arm64 \ libvte-2.91-dev:arm64 \ + libxdp-dev:arm64 \ libxen-dev:arm64 \ libzstd-dev:arm64 \ nettle-dev:arm64 \ diff --git a/tests/docker/dockerfiles/debian-armel-cross.docker b/tests/docker/dockerfiles/debian-armel-cross.docker index f1fc34a28a..75342c09b0 100644 --- a/tests/docker/dockerfiles/debian-armel-cross.docker +++ b/tests/docker/dockerfiles/debian-armel-cross.docker @@ -84,7 +84,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++-arm-linux-gnueabi \ gcc-arm-linux-gnueabi \ libaio-dev:armel \ - libasan5:armel \ + libasan6:armel \ libasound2-dev:armel \ libattr1-dev:armel \ libbpf-dev:armel \ diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker index a278578211..1ebd6ebd00 100644 --- a/tests/docker/dockerfiles/debian-armhf-cross.docker +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch armv7l debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch armv7l debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -84,7 +80,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++-arm-linux-gnueabihf \ gcc-arm-linux-gnueabihf \ libaio-dev:armhf \ - libasan5:armhf \ + libasan6:armhf \ libasound2-dev:armhf \ libattr1-dev:armhf \ libbpf-dev:armhf \ @@ -145,6 +141,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:armhf \ libvirglrenderer-dev:armhf \ libvte-2.91-dev:armhf \ + libxdp-dev:armhf \ libxen-dev:armhf \ libzstd-dev:armhf \ nettle-dev:armhf \ diff --git a/tests/docker/dockerfiles/debian-loongarch-cross.docker b/tests/docker/dockerfiles/debian-loongarch-cross.docker index 9d957547b5..b4bf265717 100644 --- a/tests/docker/dockerfiles/debian-loongarch-cross.docker +++ b/tests/docker/dockerfiles/debian-loongarch-cross.docker @@ -20,7 +20,7 @@ RUN apt-get update && \ git \ python3-minimal -RUN curl -#SL https://github.com/loongson/build-tools/releases/download/2022.05.29/loongarch64-clfs-5.0-cross-tools-gcc-glibc.tar.xz \ +RUN curl -#SL https://github.com/loongson/build-tools/releases/download/2023.08.08/CLFS-loongarch64-8.1-x86_64-cross-tools-gcc-glibc.tar.xz \ | tar -xJC /opt ENV PATH $PATH:/opt/cross-tools/bin diff --git a/tests/docker/dockerfiles/debian-ppc64el-cross.docker b/tests/docker/dockerfiles/debian-ppc64el-cross.docker index 30e5efa986..59091fed02 100644 --- a/tests/docker/dockerfiles/debian-ppc64el-cross.docker +++ b/tests/docker/dockerfiles/debian-ppc64el-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch ppc64le debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch ppc64le debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -84,7 +80,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++-powerpc64le-linux-gnu \ gcc-powerpc64le-linux-gnu \ libaio-dev:ppc64el \ - libasan5:ppc64el \ + libasan6:ppc64el \ libasound2-dev:ppc64el \ libattr1-dev:ppc64el \ libbpf-dev:ppc64el \ @@ -145,6 +141,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:ppc64el \ libvirglrenderer-dev:ppc64el \ libvte-2.91-dev:ppc64el \ + libxdp-dev:ppc64el \ libzstd-dev:ppc64el \ nettle-dev:ppc64el \ systemtap-sdt-dev:ppc64el \ diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker index ee6db7b526..48b2f28310 100644 --- a/tests/docker/dockerfiles/debian-s390x-cross.docker +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch s390x debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch s390x debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -84,7 +80,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++-s390x-linux-gnu \ gcc-s390x-linux-gnu \ libaio-dev:s390x \ - libasan5:s390x \ + libasan6:s390x \ libasound2-dev:s390x \ libattr1-dev:s390x \ libbpf-dev:s390x \ @@ -144,6 +140,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:s390x \ libvirglrenderer-dev:s390x \ libvte-2.91-dev:s390x \ + libxdp-dev:s390x \ libzstd-dev:s390x \ nettle-dev:s390x \ systemtap-sdt-dev:s390x \ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index c5b6c96943..f00e9e267c 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -82,6 +82,7 @@ exec "$@"\n' > /usr/bin/nosync && \ libubsan \ liburing-devel \ libusbx-devel \ + libxdp-devel \ libzstd-devel \ llvm \ lttng-ust-devel \ diff --git a/tests/docker/dockerfiles/opensuse-leap.docker b/tests/docker/dockerfiles/opensuse-leap.docker index fef8d5a2e4..ed04b4d6da 100644 --- a/tests/docker/dockerfiles/opensuse-leap.docker +++ b/tests/docker/dockerfiles/opensuse-leap.docker @@ -40,7 +40,7 @@ RUN zypper update -y && \ libSDL2-devel \ libSDL2_image-devel \ libaio-devel \ - libasan6 \ + libasan8 \ libattr-devel \ libbpf-devel \ libbz2-devel \ diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index 4180cd8674..d3e212060c 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -32,7 +32,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ git \ hostname \ libaio-dev \ - libasan5 \ + libasan6 \ libasound2-dev \ libattr1-dev \ libbrlapi-dev \ diff --git a/tests/docker/dockerfiles/ubuntu2204.docker b/tests/docker/dockerfiles/ubuntu2204.docker index 88493f00f6..94c2c16118 100644 --- a/tests/docker/dockerfiles/ubuntu2204.docker +++ b/tests/docker/dockerfiles/ubuntu2204.docker @@ -32,7 +32,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ git \ hostname \ libaio-dev \ - libasan5 \ + libasan6 \ libasound2-dev \ libattr1-dev \ libbpf-dev \ diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci -Subproject bbd55b4d18cce8f89b5167675e434a694131563 +Subproject e3ed1e5da101943e53d8d89424e17b22120743f diff --git a/tests/lcitool/projects/qemu.yml b/tests/lcitool/projects/qemu.yml index 584f78cb7f..6f0885170d 100644 --- a/tests/lcitool/projects/qemu.yml +++ b/tests/lcitool/projects/qemu.yml @@ -69,6 +69,7 @@ packages: - liburing - libusbx - libvdeplug + - libxdp - libzstd - llvm - lttng-ust diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index 4584870ea1..92e7d30982 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -93,7 +93,7 @@ def generate_pkglist(vm, target): # Netmap still needs to be manually built as it is yet to be packaged # into a distro. We also add cscope and gtags which are used in the CI # test -debian11_extras = [ +debian12_extras = [ "# netmap/cscope/global\n", "RUN DEBIAN_FRONTEND=noninteractive eatmydata \\\n", " apt install -y --no-install-recommends \\\n", @@ -123,8 +123,8 @@ try: # generate_dockerfile("alpine", "alpine-318") generate_dockerfile("centos8", "centos-stream-8") - generate_dockerfile("debian-amd64", "debian-11", - trailer="".join(debian11_extras)) + generate_dockerfile("debian-amd64", "debian-12", + trailer="".join(debian12_extras)) generate_dockerfile("fedora", "fedora-38") generate_dockerfile("opensuse-leap", "opensuse-leap-15") generate_dockerfile("ubuntu2004", "ubuntu-2004") @@ -133,24 +133,25 @@ try: # # Cross compiling builds # - generate_dockerfile("debian-amd64-cross", "debian-11", + generate_dockerfile("debian-amd64-cross", "debian-12", cross="x86_64", trailer=cross_build("x86_64-linux-gnu-", "x86_64-softmmu," "x86_64-linux-user," "i386-softmmu,i386-linux-user")) - generate_dockerfile("debian-arm64-cross", "debian-11", + generate_dockerfile("debian-arm64-cross", "debian-12", cross="aarch64", trailer=cross_build("aarch64-linux-gnu-", "aarch64-softmmu,aarch64-linux-user")) + # migration to bookworm stalled: https://lists.debian.org/debian-arm/2023/09/msg00006.html generate_dockerfile("debian-armel-cross", "debian-11", cross="armv6l", trailer=cross_build("arm-linux-gnueabi-", "arm-softmmu,arm-linux-user,armeb-linux-user")) - generate_dockerfile("debian-armhf-cross", "debian-11", + generate_dockerfile("debian-armhf-cross", "debian-12", cross="armv7l", trailer=cross_build("arm-linux-gnueabihf-", "arm-softmmu,arm-linux-user")) @@ -165,7 +166,7 @@ try: trailer=cross_build("mipsel-linux-gnu-", "mipsel-softmmu,mipsel-linux-user")) - generate_dockerfile("debian-ppc64el-cross", "debian-11", + generate_dockerfile("debian-ppc64el-cross", "debian-12", cross="ppc64le", trailer=cross_build("powerpc64le-linux-gnu-", "ppc64-softmmu,ppc64-linux-user")) @@ -176,7 +177,7 @@ try: trailer=cross_build("riscv64-linux-gnu-", "riscv64-softmmu,riscv64-linux-user")) - generate_dockerfile("debian-s390x-cross", "debian-11", + generate_dockerfile("debian-s390x-cross", "debian-12", cross="s390x", trailer=cross_build("s390x-linux-gnu-", "s390x-softmmu,s390x-linux-user")) diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out index 4d4af5a486..7e10c5fa1b 100644 --- a/tests/qemu-iotests/051.pc.out +++ b/tests/qemu-iotests/051.pc.out @@ -169,11 +169,11 @@ QEMU_PROG: -device scsi-hd,drive=disk: Device needs media, but drive is empty Testing: -drive file=TEST_DIR/t.qcow2,if=none,node-name=disk -object iothread,id=thread0 -device virtio-scsi,iothread=thread0,id=virtio-scsi0 -device scsi-hd,bus=virtio-scsi0.0,drive=disk,share-rw=on -device ide-hd,drive=disk,share-rw=on QEMU X.Y.Z monitor - type 'help' for more information -QEMU_PROG: -device ide-hd,drive=disk,share-rw=on: Cannot change iothread of active block backend +(qemu) QEMU_PROG: -device ide-hd,drive=disk,share-rw=on: Cannot change iothread of active block backend Testing: -drive file=TEST_DIR/t.qcow2,if=none,node-name=disk -object iothread,id=thread0 -device virtio-scsi,iothread=thread0,id=virtio-scsi0 -device scsi-hd,bus=virtio-scsi0.0,drive=disk,share-rw=on -device virtio-blk-pci,drive=disk,share-rw=on QEMU X.Y.Z monitor - type 'help' for more information -QEMU_PROG: -device virtio-blk-pci,drive=disk,share-rw=on: Cannot change iothread of active block backend +(qemu) QEMU_PROG: -device virtio-blk-pci,drive=disk,share-rw=on: Cannot change iothread of active block backend Testing: -drive file=TEST_DIR/t.qcow2,if=none,node-name=disk -object iothread,id=thread0 -device virtio-scsi,iothread=thread0,id=virtio-scsi0 -device scsi-hd,bus=virtio-scsi0.0,drive=disk,share-rw=on -device lsi53c895a,id=lsi0 -device scsi-hd,bus=lsi0.0,drive=disk,share-rw=on QEMU X.Y.Z monitor - type 'help' for more information @@ -185,7 +185,7 @@ QEMU X.Y.Z monitor - type 'help' for more information Testing: -drive file=TEST_DIR/t.qcow2,if=none,node-name=disk -object iothread,id=thread0 -device virtio-scsi,iothread=thread0,id=virtio-scsi0 -device scsi-hd,bus=virtio-scsi0.0,drive=disk,share-rw=on -device virtio-blk-pci,drive=disk,iothread=thread0,share-rw=on QEMU X.Y.Z monitor - type 'help' for more information -QEMU_PROG: -device virtio-blk-pci,drive=disk,iothread=thread0,share-rw=on: Cannot change iothread of active block backend +(qemu) QEMU_PROG: -device virtio-blk-pci,drive=disk,iothread=thread0,share-rw=on: Cannot change iothread of active block backend Testing: -drive file=TEST_DIR/t.qcow2,if=none,node-name=disk -object iothread,id=thread0 -device virtio-scsi,iothread=thread0,id=virtio-scsi0 -device scsi-hd,bus=virtio-scsi0.0,drive=disk,share-rw=on -device virtio-scsi,id=virtio-scsi1,iothread=thread0 -device scsi-hd,bus=virtio-scsi1.0,drive=disk,share-rw=on QEMU X.Y.Z monitor - type 'help' for more information diff --git a/tests/qemu-iotests/122.out b/tests/qemu-iotests/122.out index e18766e167..6a1aa3fe2b 100644 --- a/tests/qemu-iotests/122.out +++ b/tests/qemu-iotests/122.out @@ -67,12 +67,12 @@ read 65536/65536 bytes at offset 4194304 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 65536/65536 bytes at offset 8388608 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 65536, "length": 4128768, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 4194304, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 4259840, "length": 4128768, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 8388608, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 8454144, "length": 4128768, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 65536, "length": 4128768, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 4194304, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 4259840, "length": 4128768, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 8388608, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 8454144, "length": 4128768, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] read 65536/65536 bytes at offset 0 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 65536/65536 bytes at offset 4194304 @@ -94,12 +94,12 @@ wrote 1024/1024 bytes at offset 1046528 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 1024/1024 bytes at offset 0 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 65536, "length": 65536, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 131072, "length": 196608, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 327680, "length": 655360, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 983040, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 1048576, "length": 1046528, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 65536, "length": 65536, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 131072, "length": 196608, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 327680, "length": 655360, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 983040, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 1048576, "length": 1046528, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] read 16384/16384 bytes at offset 0 16 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 16384/16384 bytes at offset 16384 @@ -130,14 +130,14 @@ read 3145728/3145728 bytes at offset 0 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 63963136/63963136 bytes at offset 3145728 61 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] convert -c -S 0: read 3145728/3145728 bytes at offset 0 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 63963136/63963136 bytes at offset 3145728 61 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}] Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 wrote 33554432/33554432 bytes at offset 0 32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) @@ -152,7 +152,7 @@ read 30408704/30408704 bytes at offset 3145728 29 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 33554432/33554432 bytes at offset 33554432 32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] convert -c -S 0 with source backing file: read 3145728/3145728 bytes at offset 0 @@ -161,7 +161,7 @@ read 30408704/30408704 bytes at offset 3145728 29 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 33554432/33554432 bytes at offset 33554432 32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}] convert -S 0 -B ... read 3145728/3145728 bytes at offset 0 @@ -170,7 +170,7 @@ read 30408704/30408704 bytes at offset 3145728 29 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 33554432/33554432 bytes at offset 33554432 32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] convert -c -S 0 -B ... read 3145728/3145728 bytes at offset 0 @@ -179,7 +179,7 @@ read 30408704/30408704 bytes at offset 3145728 29 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 33554432/33554432 bytes at offset 33554432 32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}] === Non-zero -S === @@ -196,32 +196,32 @@ wrote 1024/1024 bytes at offset 66560 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) convert -S 4k -[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4096, "length": 4096, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 8192, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 12288, "length": 4096, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 16384, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 20480, "length": 67088384, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4096, "length": 4096, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 8192, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 12288, "length": 4096, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 16384, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 20480, "length": 67088384, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] convert -c -S 4k -[{ "start": 0, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 1024, "length": 7168, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 8192, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 9216, "length": 8192, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 17408, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 18432, "length": 67090432, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 1024, "length": 7168, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 8192, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 9216, "length": 8192, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 17408, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 18432, "length": 67090432, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] convert -S 8k -[{ "start": 0, "length": 24576, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 24576, "length": 67084288, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 24576, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 24576, "length": 67084288, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] convert -c -S 8k -[{ "start": 0, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 1024, "length": 7168, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 8192, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 9216, "length": 8192, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 17408, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true}, -{ "start": 18432, "length": 67090432, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 1024, "length": 7168, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 8192, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 9216, "length": 8192, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 17408, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "compressed": true}, +{ "start": 18432, "length": 67090432, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] === -n to a non-zero image === @@ -235,18 +235,18 @@ Images are identical. Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=67108864 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 -[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 -[{ "start": 0, "length": 67108864, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] === -n to an empty image with a backing file === Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=67108864 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT -[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT -[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "offset": 327680}] +[{ "start": 0, "length": 67108864, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 327680}] === -n -B to an image without a backing file === diff --git a/tests/qemu-iotests/131 b/tests/qemu-iotests/131 index 304bbb3f61..3119100e78 100755 --- a/tests/qemu-iotests/131 +++ b/tests/qemu-iotests/131 @@ -74,6 +74,58 @@ poke_file "$TEST_IMG" "$inuse_offset" "\x59\x6e\x6f\x74" echo "== read corrupted image with repairing ==" { $QEMU_IO -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +echo "== check discard ==" + +# Clear image +_make_test_img $size + +{ $QEMU_IO -c "write -P 0x11 0 $CLUSTER_DBL_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IMG map "$TEST_IMG"; } 2>&1 | _filter_qemu_img_map +{ $QEMU_IO -c "discard 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IMG map "$TEST_IMG"; } 2>&1 | _filter_qemu_img_map +{ $QEMU_IO -c "read -P 0 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== check simple allocation over the discarded hole ==" + +{ $QEMU_IO -c "write -P 0x11 $CLUSTER_DBL_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IMG map "$TEST_IMG"; } 2>&1 | _filter_qemu_img_map +{ $QEMU_IO -c "read -P 0x11 $CLUSTER_DBL_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== check more complex allocation over the discard hole ==" + +# Clear image +_make_test_img $size + +{ $QEMU_IO -c "write -P 0x11 $CLUSTER_DBL_SIZE $CLUSTER_DBL_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -c "discard $CLUSTER_DBL_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +# There is 1 cluster hole. Fill it fully and allocate 1 cluster at the end +{ $QEMU_IO -c "write -P 0x12 $CLUSTER_HALF_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IMG map "$TEST_IMG"; } 2>&1 | _filter_qemu_img_map +{ $QEMU_IO -c "read -P 0x12 $CLUSTER_HALF_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -c "read -P 0 0 $CLUSTER_HALF_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -c "read -P 0 $((CLUSTER_SIZE + CLUSTER_HALF_SIZE)) $CLUSTER_HALF_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== check write-zeroes ==" + +# Clear image +_make_test_img $size + +{ $QEMU_IO -c "write -P 0x11 0 $CLUSTER_DBL_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -c "write -z 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IMG map "$TEST_IMG"; } 2>&1 | _filter_qemu_img_map +{ $QEMU_IO -c "read -P 0 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== check cluster-partial write-zeroes ==" + +# Clear image +_make_test_img $size + +{ $QEMU_IO -c "write -P 0x11 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -c "write -z 0 $CLUSTER_HALF_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -c "read -P 0 0 $CLUSTER_HALF_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -c "read -P 0x11 $CLUSTER_HALF_SIZE $CLUSTER_HALF_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + echo "== allocate with backing ==" # Verify that allocating clusters works fine even when there is a backing image. # Regression test for a bug where we would pass a buffer read from the backing diff --git a/tests/qemu-iotests/131.out b/tests/qemu-iotests/131.out index d2904578df..86a2d2a49b 100644 --- a/tests/qemu-iotests/131.out +++ b/tests/qemu-iotests/131.out @@ -26,6 +26,66 @@ read 524288/524288 bytes at offset 0 Repairing image was not closed correctly read 1048576/1048576 bytes at offset 1048576 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check discard == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 +wrote 2097152/2097152 bytes at offset 0 +2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Offset Length File +0 0x200000 TEST_DIR/t.IMGFMT +discard 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Offset Length File +0x100000 0x100000 TEST_DIR/t.IMGFMT +read 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check simple allocation over the discarded hole == +wrote 1048576/1048576 bytes at offset 2097152 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Offset Length File +0x100000 0x100000 TEST_DIR/t.IMGFMT +0x200000 0x100000 TEST_DIR/t.IMGFMT +read 1048576/1048576 bytes at offset 2097152 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check more complex allocation over the discard hole == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 +wrote 2097152/2097152 bytes at offset 2097152 +2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +discard 1048576/1048576 bytes at offset 2097152 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +wrote 1048576/1048576 bytes at offset 524288 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Offset Length File +0 0x100000 TEST_DIR/t.IMGFMT +0x100000 0x100000 TEST_DIR/t.IMGFMT +0x300000 0x100000 TEST_DIR/t.IMGFMT +read 1048576/1048576 bytes at offset 524288 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +read 524288/524288 bytes at offset 0 +512 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +read 524288/524288 bytes at offset 1572864 +512 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check write-zeroes == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 +wrote 2097152/2097152 bytes at offset 0 +2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +wrote 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Offset Length File +0x100000 0x100000 TEST_DIR/t.IMGFMT +read 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +read 1048576/1048576 bytes at offset 1048576 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check cluster-partial write-zeroes == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 +wrote 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +wrote 524288/524288 bytes at offset 0 +512 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +read 524288/524288 bytes at offset 0 +512 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +read 524288/524288 bytes at offset 524288 +512 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) == allocate with backing == Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 diff --git a/tests/qemu-iotests/146.out b/tests/qemu-iotests/146.out index dfd6c77140..a48804154e 100644 --- a/tests/qemu-iotests/146.out +++ b/tests/qemu-iotests/146.out @@ -2,414 +2,414 @@ QA output created by 146 === Testing VPC Autodetect === -[{ "start": 0, "length": 136363130880, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 136363130880, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Testing VPC with current_size force === -[{ "start": 0, "length": 136365211648, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 136365211648, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Testing VPC with chs force === -[{ "start": 0, "length": 136363130880, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 136363130880, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Testing Hyper-V Autodetect === -[{ "start": 0, "length": 136365211648, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 136365211648, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Testing Hyper-V with current_size force === -[{ "start": 0, "length": 136365211648, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 136365211648, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Testing Hyper-V with chs force === -[{ "start": 0, "length": 136363130880, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 136363130880, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Testing d2v Autodetect === -[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4194304, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 8388608, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 12582912, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 16777216, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 33554432, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 35651584, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 37748736, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 39845888, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 41943040, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 44040192, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 46137344, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 48234496, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 50331648, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 52428800, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 54525952, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 56623104, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 58720256, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 60817408, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 62914560, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 65011712, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 67108864, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 69206016, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 71303168, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 73400320, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 75497472, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 77594624, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 79691776, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 81788928, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 83886080, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 85983232, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 88080384, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 90177536, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 92274688, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 94371840, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 96468992, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 98566144, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 100663296, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 102760448, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 104857600, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 106954752, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 109051904, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 111149056, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 113246208, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 115343360, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 117440512, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 119537664, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 121634816, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 123731968, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 125829120, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 127926272, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 130023424, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 132120576, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 134217728, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 136314880, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 138412032, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 140509184, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 142606336, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 144703488, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 146800640, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 148897792, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 150994944, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 153092096, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 155189248, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 157286400, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 159383552, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 161480704, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 163577856, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 165675008, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 167772160, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 169869312, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 171966464, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 174063616, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 176160768, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 178257920, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 180355072, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 182452224, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 184549376, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 186646528, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 188743680, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 190840832, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 192937984, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 195035136, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 197132288, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 199229440, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 201326592, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 203423744, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 205520896, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 207618048, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 209715200, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 211812352, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 213909504, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 216006656, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 218103808, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 220200960, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 222298112, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 224395264, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 226492416, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 228589568, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 230686720, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 232783872, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 234881024, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 236978176, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 239075328, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 241172480, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 243269632, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 245366784, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 247463936, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 249561088, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 251658240, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 253755392, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 255852544, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 257949696, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 260046848, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 262144000, "length": 1310720, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4194304, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 8388608, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 12582912, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 16777216, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 33554432, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 35651584, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 37748736, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 39845888, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 41943040, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 44040192, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 46137344, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 48234496, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 50331648, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 52428800, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 54525952, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 56623104, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 58720256, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 60817408, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 62914560, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 65011712, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 67108864, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 69206016, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 71303168, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 73400320, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 75497472, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 77594624, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 79691776, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 81788928, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 83886080, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 85983232, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 88080384, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 90177536, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 92274688, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 94371840, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 96468992, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 98566144, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 100663296, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 102760448, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 104857600, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 106954752, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 109051904, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 111149056, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 113246208, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 115343360, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 117440512, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 119537664, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 121634816, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 123731968, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 125829120, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 127926272, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 130023424, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 132120576, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 134217728, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 136314880, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 138412032, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 140509184, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 142606336, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 144703488, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 146800640, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 148897792, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 150994944, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 153092096, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 155189248, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 157286400, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 159383552, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 161480704, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 163577856, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 165675008, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 167772160, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 169869312, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 171966464, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 174063616, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 176160768, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 178257920, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 180355072, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 182452224, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 184549376, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 186646528, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 188743680, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 190840832, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 192937984, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 195035136, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 197132288, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 199229440, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 201326592, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 203423744, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 205520896, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 207618048, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 209715200, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 211812352, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 213909504, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 216006656, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 218103808, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 220200960, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 222298112, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 224395264, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 226492416, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 228589568, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 230686720, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 232783872, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 234881024, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 236978176, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 239075328, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 241172480, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 243269632, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 245366784, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 247463936, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 249561088, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 251658240, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 253755392, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 255852544, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 257949696, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 260046848, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 262144000, "length": 1310720, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] === Testing d2v with current_size force === -[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4194304, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 8388608, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 12582912, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 16777216, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 33554432, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 35651584, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 37748736, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 39845888, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 41943040, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 44040192, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 46137344, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 48234496, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 50331648, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 52428800, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 54525952, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 56623104, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 58720256, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 60817408, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 62914560, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 65011712, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 67108864, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 69206016, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 71303168, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 73400320, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 75497472, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 77594624, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 79691776, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 81788928, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 83886080, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 85983232, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 88080384, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 90177536, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 92274688, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 94371840, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 96468992, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 98566144, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 100663296, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 102760448, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 104857600, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 106954752, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 109051904, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 111149056, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 113246208, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 115343360, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 117440512, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 119537664, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 121634816, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 123731968, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 125829120, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 127926272, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 130023424, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 132120576, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 134217728, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 136314880, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 138412032, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 140509184, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 142606336, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 144703488, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 146800640, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 148897792, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 150994944, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 153092096, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 155189248, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 157286400, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 159383552, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 161480704, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 163577856, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 165675008, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 167772160, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 169869312, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 171966464, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 174063616, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 176160768, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 178257920, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 180355072, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 182452224, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 184549376, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 186646528, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 188743680, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 190840832, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 192937984, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 195035136, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 197132288, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 199229440, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 201326592, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 203423744, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 205520896, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 207618048, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 209715200, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 211812352, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 213909504, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 216006656, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 218103808, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 220200960, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 222298112, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 224395264, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 226492416, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 228589568, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 230686720, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 232783872, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 234881024, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 236978176, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 239075328, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 241172480, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 243269632, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 245366784, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 247463936, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 249561088, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 251658240, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 253755392, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 255852544, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 257949696, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 260046848, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 262144000, "length": 1310720, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4194304, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 8388608, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 12582912, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 16777216, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 33554432, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 35651584, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 37748736, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 39845888, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 41943040, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 44040192, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 46137344, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 48234496, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 50331648, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 52428800, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 54525952, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 56623104, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 58720256, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 60817408, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 62914560, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 65011712, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 67108864, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 69206016, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 71303168, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 73400320, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 75497472, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 77594624, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 79691776, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 81788928, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 83886080, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 85983232, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 88080384, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 90177536, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 92274688, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 94371840, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 96468992, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 98566144, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 100663296, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 102760448, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 104857600, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 106954752, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 109051904, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 111149056, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 113246208, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 115343360, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 117440512, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 119537664, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 121634816, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 123731968, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 125829120, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 127926272, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 130023424, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 132120576, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 134217728, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 136314880, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 138412032, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 140509184, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 142606336, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 144703488, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 146800640, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 148897792, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 150994944, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 153092096, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 155189248, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 157286400, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 159383552, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 161480704, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 163577856, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 165675008, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 167772160, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 169869312, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 171966464, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 174063616, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 176160768, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 178257920, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 180355072, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 182452224, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 184549376, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 186646528, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 188743680, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 190840832, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 192937984, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 195035136, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 197132288, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 199229440, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 201326592, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 203423744, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 205520896, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 207618048, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 209715200, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 211812352, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 213909504, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 216006656, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 218103808, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 220200960, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 222298112, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 224395264, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 226492416, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 228589568, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 230686720, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 232783872, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 234881024, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 236978176, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 239075328, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 241172480, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 243269632, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 245366784, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 247463936, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 249561088, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 251658240, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 253755392, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 255852544, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 257949696, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 260046848, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 262144000, "length": 1310720, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] === Testing d2v with chs force === -[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4194304, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 8388608, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 12582912, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 16777216, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 33554432, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 35651584, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 37748736, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 39845888, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 41943040, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 44040192, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 46137344, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 48234496, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 50331648, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 52428800, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 54525952, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 56623104, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 58720256, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 60817408, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 62914560, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 65011712, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 67108864, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 69206016, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 71303168, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 73400320, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 75497472, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 77594624, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 79691776, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 81788928, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 83886080, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 85983232, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 88080384, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 90177536, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 92274688, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 94371840, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 96468992, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 98566144, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 100663296, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 102760448, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 104857600, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 106954752, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 109051904, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 111149056, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 113246208, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 115343360, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 117440512, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 119537664, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 121634816, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 123731968, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 125829120, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 127926272, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 130023424, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 132120576, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 134217728, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 136314880, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 138412032, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 140509184, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 142606336, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 144703488, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 146800640, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 148897792, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 150994944, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 153092096, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 155189248, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 157286400, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 159383552, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 161480704, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 163577856, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 165675008, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 167772160, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 169869312, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 171966464, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 174063616, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 176160768, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 178257920, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 180355072, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 182452224, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 184549376, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 186646528, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 188743680, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 190840832, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 192937984, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 195035136, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 197132288, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 199229440, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 201326592, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 203423744, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 205520896, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 207618048, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 209715200, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 211812352, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 213909504, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 216006656, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 218103808, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 220200960, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 222298112, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 224395264, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 226492416, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 228589568, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 230686720, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 232783872, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 234881024, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 236978176, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 239075328, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 241172480, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 243269632, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 245366784, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 247463936, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 249561088, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 251658240, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 253755392, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 255852544, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 257949696, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 260046848, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 262144000, "length": 1310720, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4194304, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 8388608, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 12582912, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 16777216, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 33554432, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 35651584, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 37748736, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 39845888, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 41943040, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 44040192, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 46137344, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 48234496, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 50331648, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 52428800, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 54525952, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 56623104, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 58720256, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 60817408, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 62914560, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 65011712, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 67108864, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 69206016, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 71303168, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 73400320, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 75497472, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 77594624, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 79691776, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 81788928, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 83886080, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 85983232, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 88080384, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 90177536, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 92274688, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 94371840, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 96468992, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 98566144, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 100663296, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 102760448, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 104857600, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 106954752, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 109051904, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 111149056, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 113246208, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 115343360, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 117440512, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 119537664, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 121634816, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 123731968, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 125829120, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 127926272, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 130023424, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 132120576, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 134217728, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 136314880, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 138412032, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 140509184, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 142606336, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 144703488, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 146800640, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 148897792, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 150994944, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 153092096, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 155189248, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 157286400, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 159383552, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 161480704, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 163577856, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 165675008, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 167772160, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 169869312, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 171966464, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 174063616, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 176160768, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 178257920, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 180355072, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 182452224, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 184549376, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 186646528, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 188743680, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 190840832, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 192937984, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 195035136, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 197132288, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 199229440, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 201326592, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 203423744, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 205520896, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 207618048, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 209715200, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 211812352, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 213909504, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 216006656, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 218103808, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 220200960, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 222298112, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 224395264, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 226492416, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 228589568, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 230686720, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 232783872, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 234881024, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 236978176, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 239075328, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 241172480, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 243269632, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 245366784, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 247463936, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 249561088, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 251658240, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 253755392, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 255852544, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 257949696, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 260046848, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 262144000, "length": 1310720, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] === Testing Image create, default === @@ -417,15 +417,15 @@ Formatting 'TEST_DIR/IMGFMT-create-test.IMGFMT', fmt=IMGFMT size=4294967296 === Read created image, default opts ==== -[{ "start": 0, "length": 4295467008, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 4295467008, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Read created image, force_size_calc=chs ==== -[{ "start": 0, "length": 4295467008, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 4295467008, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Read created image, force_size_calc=current_size ==== -[{ "start": 0, "length": 4295467008, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 4295467008, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Testing Image create, force_size === @@ -433,13 +433,13 @@ Formatting 'TEST_DIR/IMGFMT-create-test.IMGFMT', fmt=IMGFMT size=4294967296 === Read created image, default opts ==== -[{ "start": 0, "length": 4294967296, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 4294967296, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Read created image, force_size_calc=chs ==== -[{ "start": 0, "length": 4294967296, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 4294967296, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === Read created image, force_size_calc=current_size ==== -[{ "start": 0, "length": 4294967296, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 4294967296, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] *** done diff --git a/tests/qemu-iotests/154.out b/tests/qemu-iotests/154.out index 1fa7ffc475..0199269add 100644 --- a/tests/qemu-iotests/154.out +++ b/tests/qemu-iotests/154.out @@ -11,14 +11,14 @@ wrote 2048/2048 bytes at offset 17408 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 2048/2048 bytes at offset 27648 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 4096, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 8192, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 12288, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 16384, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 20480, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 24576, "length": 8192, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 32768, "length": 134184960, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 4096, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 8192, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 12288, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 16384, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 20480, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 24576, "length": 8192, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 32768, "length": 134184960, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == backing file contains non-zero data before write_zeroes == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -41,11 +41,11 @@ read 1024/1024 bytes at offset 65536 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 2048/2048 bytes at offset 67584 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 32768, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 32768, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 36864, "length": 28672, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 69632, "length": 134148096, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 32768, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 32768, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 36864, "length": 28672, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 69632, "length": 134148096, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == backing file contains non-zero data after write_zeroes == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -68,11 +68,11 @@ read 1024/1024 bytes at offset 44032 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 3072/3072 bytes at offset 40960 3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 32768, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 32768, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 36864, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 40960, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 45056, "length": 134172672, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 32768, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 32768, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 36864, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 40960, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 45056, "length": 134172672, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == write_zeroes covers non-zero data == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -101,15 +101,15 @@ wrote 2048/2048 bytes at offset 29696 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 4096/4096 bytes at offset 28672 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 4096, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 8192, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 12288, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 16384, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 20480, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 24576, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 28672, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 32768, "length": 134184960, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 4096, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 8192, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 12288, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 16384, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 20480, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 24576, "length": 4096, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 28672, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 32768, "length": 134184960, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == spanning two clusters, non-zero before request == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -142,16 +142,16 @@ read 1024/1024 bytes at offset 67584 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 5120/5120 bytes at offset 68608 5 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 32768, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 32768, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 36864, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 40960, "length": 8192, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 49152, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 53248, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 57344, "length": 8192, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 69632, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 73728, "length": 134144000, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 32768, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 32768, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 36864, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 40960, "length": 8192, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 49152, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 53248, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 57344, "length": 8192, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 69632, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 73728, "length": 134144000, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == spanning two clusters, non-zero after request == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -184,16 +184,16 @@ read 7168/7168 bytes at offset 65536 7 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 1024/1024 bytes at offset 72704 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 32768, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 32768, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 36864, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 40960, "length": 8192, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 49152, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 53248, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 57344, "length": 8192, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 69632, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 73728, "length": 134144000, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 32768, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 32768, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 36864, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 40960, "length": 8192, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 49152, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 53248, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 57344, "length": 8192, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 69632, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 73728, "length": 134144000, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == spanning two clusters, partially overwriting backing file == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -212,8 +212,8 @@ read 1024/1024 bytes at offset 5120 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 2048/2048 bytes at offset 6144 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 8192, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 8192, "length": 134209536, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 8192, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 8192, "length": 134209536, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == spanning multiple clusters, non-zero in first cluster == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -226,10 +226,10 @@ read 2048/2048 bytes at offset 65536 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 10240/10240 bytes at offset 67584 10 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 69632, "length": 8192, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 77824, "length": 134139904, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 69632, "length": 8192, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 77824, "length": 134139904, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == spanning multiple clusters, non-zero in intermediate cluster == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -240,9 +240,9 @@ wrote 7168/7168 bytes at offset 67584 7 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 12288/12288 bytes at offset 65536 12 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 65536, "length": 12288, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 77824, "length": 134139904, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 65536, "length": 12288, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 77824, "length": 134139904, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == spanning multiple clusters, non-zero in final cluster == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -255,10 +255,10 @@ read 10240/10240 bytes at offset 65536 10 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 2048/2048 bytes at offset 75776 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 65536, "length": 8192, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 73728, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 77824, "length": 134139904, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 65536, "length": 8192, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 73728, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 77824, "length": 134139904, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == spanning multiple clusters, partially overwriting backing file == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 @@ -277,88 +277,88 @@ read 2048/2048 bytes at offset 74752 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 1024/1024 bytes at offset 76800 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 69632, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 73728, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 77824, "length": 134139904, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 65536, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 69632, "length": 4096, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 73728, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 77824, "length": 134139904, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] == unaligned image tail cluster, no allocation needed == Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 wrote 512/512 bytes at offset 134217728 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 wrote 512/512 bytes at offset 134219264 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 wrote 1024/1024 bytes at offset 134218240 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 wrote 2048/2048 bytes at offset 134217728 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134218752 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT wrote 512/512 bytes at offset 134217728 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT wrote 512/512 bytes at offset 134219264 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT wrote 1024/1024 bytes at offset 134218240 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT wrote 2048/2048 bytes at offset 134217728 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] wrote 512/512 bytes at offset 134217728 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT wrote 512/512 bytes at offset 134217728 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT wrote 512/512 bytes at offset 134219264 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT wrote 1024/1024 bytes at offset 134218240 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT wrote 2048/2048 bytes at offset 134217728 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 2048/2048 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134218752 wrote 1024/1024 bytes at offset 134217728 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) @@ -369,15 +369,15 @@ read 512/512 bytes at offset 134217728 read 512/512 bytes at offset 134218240 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 1024/1024 bytes allocated at offset 128 MiB -[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] wrote 1024/1024 bytes at offset 134217728 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 1024/1024 bytes allocated at offset 128 MiB read 1024/1024 bytes at offset 134217728 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 1024, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 134217728, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 1024, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] == unaligned image tail cluster, allocation required == Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134218752 @@ -390,8 +390,8 @@ read 512/512 bytes at offset 134217728 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 1536/1536 bytes at offset 134218240 1.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134218752 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134219776 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT wrote 512/512 bytes at offset 134218240 @@ -412,6 +412,6 @@ read 512/512 bytes at offset 134218240 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 1024/1024 bytes at offset 134218752 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 134217728, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 134217728, "length": 2048, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] *** done diff --git a/tests/qemu-iotests/179.out b/tests/qemu-iotests/179.out index 7cf22cd75f..65b909ebc2 100644 --- a/tests/qemu-iotests/179.out +++ b/tests/qemu-iotests/179.out @@ -13,11 +13,11 @@ wrote 2097152/2097152 bytes at offset 6291456 2 MiB (0x200000) bytes not allocated at offset 4 MiB (0x400000) 2 MiB (0x200000) bytes allocated at offset 6 MiB (0x600000) 56 MiB (0x3800000) bytes not allocated at offset 8 MiB (0x800000) -[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 8388608, "length": 58720256, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 8388608, "length": 58720256, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] wrote 2097150/2097150 bytes at offset 10485761 2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 2097150/2097150 bytes at offset 14680065 @@ -31,15 +31,15 @@ wrote 2097150/2097150 bytes at offset 14680065 2 MiB (0x200000) bytes not allocated at offset 12 MiB (0xc00000) 2 MiB (0x200000) bytes allocated at offset 14 MiB (0xe00000) 48 MiB (0x3000000) bytes not allocated at offset 16 MiB (0x1000000) -[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 8388608, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 12582912, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 16777216, "length": 50331648, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 8388608, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 12582912, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 16777216, "length": 50331648, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] wrote 14680064/14680064 bytes at offset 18874368 14 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 2097152/2097152 bytes at offset 20971520 @@ -57,21 +57,21 @@ wrote 6291456/6291456 bytes at offset 25165824 2 MiB (0x200000) bytes not allocated at offset 16 MiB (0x1000000) 14 MiB (0xe00000) bytes allocated at offset 18 MiB (0x1200000) 32 MiB (0x2000000) bytes not allocated at offset 32 MiB (0x2000000) -[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 8388608, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 12582912, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 16777216, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 25165824, "length": 6291456, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 33554432, "length": 33554432, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 8388608, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 12582912, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 16777216, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 25165824, "length": 6291456, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 33554432, "length": 33554432, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] wrote 2097152/2097152 bytes at offset 27262976 2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 2097152/2097152 bytes at offset 29360128 @@ -87,23 +87,23 @@ wrote 2097152/2097152 bytes at offset 29360128 2 MiB (0x200000) bytes not allocated at offset 16 MiB (0x1000000) 14 MiB (0xe00000) bytes allocated at offset 18 MiB (0x1200000) 32 MiB (0x2000000) bytes not allocated at offset 32 MiB (0x2000000) -[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 8388608, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 12582912, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 16777216, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 33554432, "length": 33554432, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 8388608, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 12582912, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 16777216, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 33554432, "length": 33554432, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] wrote 8388608/8388608 bytes at offset 33554432 8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 2097152/2097152 bytes at offset 35651584 @@ -121,24 +121,24 @@ wrote 2097152/2097152 bytes at offset 37748736 2 MiB (0x200000) bytes not allocated at offset 16 MiB (0x1000000) 22 MiB (0x1600000) bytes allocated at offset 18 MiB (0x1200000) 24 MiB (0x1800000) bytes not allocated at offset 40 MiB (0x2800000) -[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 8388608, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 12582912, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 16777216, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 33554432, "length": 8388608, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 41943040, "length": 25165824, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 4194304, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 6291456, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 8388608, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 10485760, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 12582912, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 14680064, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 16777216, "length": 2097152, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 18874368, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 20971520, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 23068672, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 25165824, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 27262976, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 29360128, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 31457280, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 33554432, "length": 8388608, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 41943040, "length": 25165824, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] wrote 8388608/8388608 bytes at offset 41943040 8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 8388608/8388608 bytes at offset 50331648 @@ -162,31 +162,31 @@ wrote 2097152/2097152 bytes at offset 62914560 4 MiB (0x400000) bytes not allocated at offset 54 MiB (0x3600000) 4 MiB (0x400000) bytes allocated at offset 58 MiB (0x3a00000) 2 MiB (0x200000) bytes not allocated at offset 62 MiB (0x3e00000) -[{ "start": 0, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 2097152, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false}, -{ "start": 4194304, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 6291456, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false}, -{ "start": 8388608, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 10485760, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false}, -{ "start": 12582912, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 14680064, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false}, -{ "start": 16777216, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 18874368, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 20971520, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false}, -{ "start": 23068672, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 25165824, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 27262976, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false}, -{ "start": 29360128, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 31457280, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 33554432, "length": 10485760, "depth": 1, "present": true, "zero": true, "data": false}, -{ "start": 44040192, "length": 4194304, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 48234496, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false}, -{ "start": 50331648, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 52428800, "length": 4194304, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 56623104, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 58720256, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 60817408, "length": 4194304, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 65011712, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 2097152, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 4194304, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 6291456, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 8388608, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 10485760, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 12582912, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 14680064, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 16777216, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 18874368, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 20971520, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 23068672, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 25165824, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 27262976, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 29360128, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 31457280, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 33554432, "length": 10485760, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 44040192, "length": 4194304, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 48234496, "length": 2097152, "depth": 1, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 50331648, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 52428800, "length": 4194304, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 56623104, "length": 2097152, "depth": 1, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 58720256, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 60817408, "length": 4194304, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 65011712, "length": 2097152, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] No errors were found on the image. No errors were found on the image. diff --git a/tests/qemu-iotests/209.out b/tests/qemu-iotests/209.out index 515906ac7a..79c4103a22 100644 --- a/tests/qemu-iotests/209.out +++ b/tests/qemu-iotests/209.out @@ -1,4 +1,4 @@ -[{ "start": 0, "length": 524288, "depth": 0, "present": true, "zero": false, "data": true, "offset": 0}, -{ "start": 524288, "length": 524288, "depth": 0, "present": true, "zero": true, "data": false, "offset": 524288}] +[{ "start": 0, "length": 524288, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 0}, +{ "start": 524288, "length": 524288, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 524288}] done. diff --git a/tests/qemu-iotests/221.out b/tests/qemu-iotests/221.out index 9cdd171a2d..df231c4e3d 100644 --- a/tests/qemu-iotests/221.out +++ b/tests/qemu-iotests/221.out @@ -5,14 +5,14 @@ QA output created by 221 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65537 discard 65537/65537 bytes at offset 0 64.001 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 66048, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] -[{ "start": 0, "length": 66048, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 66048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 66048, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] wrote 1/1 bytes at offset 65536 1 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 65536, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 65536, "length": 1, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 65537, "length": 511, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] -[{ "start": 0, "length": 65536, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 65536, "length": 1, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 65537, "length": 511, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 65536, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 65536, "length": 1, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 65537, "length": 511, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 65536, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 65536, "length": 1, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 65537, "length": 511, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] *** done diff --git a/tests/qemu-iotests/223.out b/tests/qemu-iotests/223.out index 26fb347c5d..86a37014d0 100644 --- a/tests/qemu-iotests/223.out +++ b/tests/qemu-iotests/223.out @@ -120,23 +120,23 @@ read 1048576/1048576 bytes at offset 1048576 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 2097152/2097152 bytes at offset 2097152 2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4096, "length": 1044480, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 1048576, "length": 3145728, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] -[{ "start": 0, "length": 65536, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 65536, "length": 2031616, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false}] +[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4096, "length": 1044480, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 1048576, "length": 3145728, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 65536, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 65536, "length": 2031616, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}] === Contrast to small granularity dirty-bitmap === -[{ "start": 0, "length": 512, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 512, "length": 512, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 1024, "length": 2096128, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false}] +[{ "start": 0, "length": 512, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 512, "length": 512, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 1024, "length": 2096128, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}] === Check bitmap taken from another node === -[{ "start": 0, "length": 4194304, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 4194304, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] === End qemu NBD server === @@ -239,23 +239,23 @@ read 1048576/1048576 bytes at offset 1048576 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 2097152/2097152 bytes at offset 2097152 2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4096, "length": 1044480, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 1048576, "length": 3145728, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] -[{ "start": 0, "length": 65536, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 65536, "length": 2031616, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false}] +[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4096, "length": 1044480, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 1048576, "length": 3145728, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 65536, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 65536, "length": 2031616, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}] === Contrast to small granularity dirty-bitmap === -[{ "start": 0, "length": 512, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 512, "length": 512, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 1024, "length": 2096128, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false}] +[{ "start": 0, "length": 512, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 512, "length": 512, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 1024, "length": 2096128, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}] === Check bitmap taken from another node === -[{ "start": 0, "length": 4194304, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 4194304, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] === End qemu NBD server === @@ -281,12 +281,12 @@ read 2097152/2097152 bytes at offset 2097152 === Use qemu-nbd as server === -[{ "start": 0, "length": 65536, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 65536, "length": 2031616, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false}] -[{ "start": 0, "length": 512, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 512, "length": 512, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 1024, "length": 11321, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] -[{ "start": 12345, "length": 2084807, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false}] +[{ "start": 0, "length": 65536, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 65536, "length": 2031616, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}] +[{ "start": 0, "length": 512, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 512, "length": 512, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 1024, "length": 11321, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] +[{ "start": 12345, "length": 2084807, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}] *** done diff --git a/tests/qemu-iotests/241.out b/tests/qemu-iotests/241.out index 88e8cfcd7e..7946c286d5 100644 --- a/tests/qemu-iotests/241.out +++ b/tests/qemu-iotests/241.out @@ -6,8 +6,8 @@ exports available: 1 export: '' size: 1024 min block: 1 -[{ "start": 0, "length": 1000, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 1000, "length": 24, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 1000, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 1000, "length": 24, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] 1 KiB (0x400) bytes allocated at offset 0 bytes (0x0) === Exporting unaligned raw image, forced server sector alignment === @@ -16,7 +16,7 @@ exports available: 1 export: '' size: 1024 min block: 512 -[{ "start": 0, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] 1 KiB (0x400) bytes allocated at offset 0 bytes (0x0) WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw. Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted. @@ -28,7 +28,7 @@ exports available: 1 export: '' size: 1024 min block: 1 -[{ "start": 0, "length": 1000, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 1000, "length": 24, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 1000, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 1000, "length": 24, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] 1 KiB (0x400) bytes allocated at offset 0 bytes (0x0) *** done diff --git a/tests/qemu-iotests/244.out b/tests/qemu-iotests/244.out index 4815a489b0..f46cfe93f1 100644 --- a/tests/qemu-iotests/244.out +++ b/tests/qemu-iotests/244.out @@ -57,12 +57,12 @@ wrote 3145728/3145728 bytes at offset 3145728 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) No errors were found on the image. -[{ "start": 0, "length": 1048576, "depth": 0, "present": false, "zero": true, "data": false}, -{ "start": 1048576, "length": 1048576, "depth": 0, "present": true, "zero": false, "data": true, "offset": 1048576}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 4194304, "length": 1048576, "depth": 0, "present": true, "zero": true, "data": false, "offset": 4194304}, -{ "start": 5242880, "length": 1048576, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 6291456, "length": 60817408, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 1048576, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 1048576, "length": 1048576, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 1048576}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 4194304, "length": 1048576, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 4194304}, +{ "start": 5242880, "length": 1048576, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 6291456, "length": 60817408, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] read 1048576/1048576 bytes at offset 0 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) @@ -94,10 +94,10 @@ wrote 3145728/3145728 bytes at offset 3145728 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) No errors were found on the image. -[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": 0}, -{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 4194304, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "offset": 4194304}, -{ "start": 6291456, "length": 60817408, "depth": 0, "present": true, "zero": false, "data": true, "offset": 6291456}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 0}, +{ "start": 2097152, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 4194304, "length": 2097152, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 4194304}, +{ "start": 6291456, "length": 60817408, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 6291456}] read 1048576/1048576 bytes at offset 0 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) @@ -123,8 +123,8 @@ read 1048576/1048576 bytes at offset 0 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) Offset Length Mapped to File 0 0x100000 0 TEST_DIR/t.qcow2.data -[{ "start": 0, "length": 1048576, "depth": 0, "present": true, "zero": false, "data": true, "offset": 0}, -{ "start": 1048576, "length": 66060288, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 1048576, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 0}, +{ "start": 1048576, "length": 66060288, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] === Copy offloading === diff --git a/tests/qemu-iotests/252.out b/tests/qemu-iotests/252.out index c578129c25..b1aa94cb05 100644 --- a/tests/qemu-iotests/252.out +++ b/tests/qemu-iotests/252.out @@ -23,8 +23,8 @@ read 131072/131072 bytes at offset 131072 read 131072/131072 bytes at offset 262144 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 262144, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 262144, "length": 131072, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 262144, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 262144, "length": 131072, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] read 131072/131072 bytes at offset 0 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) @@ -33,7 +33,7 @@ read 131072/131072 bytes at offset 131072 read 131072/131072 bytes at offset 262144 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 262144, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 262144, "length": 65536, "depth": 0, "present": true, "zero": true, "data": false}, -{ "start": 327680, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 262144, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 262144, "length": 65536, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}, +{ "start": 327680, "length": 65536, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] *** done diff --git a/tests/qemu-iotests/253.out b/tests/qemu-iotests/253.out index b3dca75a89..b458085adb 100644 --- a/tests/qemu-iotests/253.out +++ b/tests/qemu-iotests/253.out @@ -3,16 +3,16 @@ QA output created by 253 === Check mapping of unaligned raw image === Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048575 -[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4096, "length": 1044480, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] -[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4096, "length": 1044480, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4096, "length": 1044480, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4096, "length": 1044480, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] wrote 65535/65535 bytes at offset 983040 63.999 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4096, "length": 978944, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 983040, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] -[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 4096, "length": 978944, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}, -{ "start": 983040, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4096, "length": 978944, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 983040, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 4096, "length": 978944, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}, +{ "start": 983040, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] *** done diff --git a/tests/qemu-iotests/274.out b/tests/qemu-iotests/274.out index acd8b166a6..c2967335ca 100644 --- a/tests/qemu-iotests/274.out +++ b/tests/qemu-iotests/274.out @@ -20,18 +20,18 @@ read 1048576/1048576 bytes at offset 1048576 0/1048576 bytes allocated at offset 1 MiB === Checking map === -[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": 327680}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 327680}] Offset Length Mapped to File 0 0x200000 0x50000 TEST_DIR/PID-base -[{ "start": 0, "length": 1048576, "depth": 1, "present": true, "zero": false, "data": true, "offset": 327680}] +[{ "start": 0, "length": 1048576, "depth": 1, "present": true, "zero": false, "data": true, "compressed": false, "offset": 327680}] Offset Length Mapped to File 0 0x100000 0x50000 TEST_DIR/PID-base -[{ "start": 0, "length": 1048576, "depth": 2, "present": true, "zero": false, "data": true, "offset": 327680}, -{ "start": 1048576, "length": 1048576, "depth": 0, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 1048576, "depth": 2, "present": true, "zero": false, "data": true, "compressed": false, "offset": 327680}, +{ "start": 1048576, "length": 1048576, "depth": 0, "present": false, "zero": true, "data": false, "compressed": false}] Offset Length Mapped to File 0 0x100000 0x50000 TEST_DIR/PID-base @@ -186,8 +186,8 @@ read 65536/65536 bytes at offset 5368709120 1 GiB (0x40000000) bytes not allocated at offset 0 bytes (0x0) 7 GiB (0x1c0000000) bytes allocated at offset 1 GiB (0x40000000) -[{ "start": 0, "length": 1073741824, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 1073741824, "length": 7516192768, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 1073741824, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 1073741824, "length": 7516192768, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === preallocation=metadata === wrote 65536/65536 bytes at offset 33285996544 @@ -201,13 +201,13 @@ read 65536/65536 bytes at offset 33285996544 30 GiB (0x780000000) bytes not allocated at offset 0 bytes (0x0) 3 GiB (0xc0000000) bytes allocated at offset 30 GiB (0x780000000) -[{ "start": 0, "length": 32212254720, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 32212254720, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "offset": 327680}, -{ "start": 32749125632, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "offset": 537264128}, -{ "start": 33285996544, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "offset": 1074200576}, -{ "start": 33822867456, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "offset": 1611137024}, -{ "start": 34359738368, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "offset": 2148139008}, -{ "start": 34896609280, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "offset": 2685075456}] +[{ "start": 0, "length": 32212254720, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 32212254720, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 327680}, +{ "start": 32749125632, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 537264128}, +{ "start": 33285996544, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 1074200576}, +{ "start": 33822867456, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 1611137024}, +{ "start": 34359738368, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 2148139008}, +{ "start": 34896609280, "length": 536870912, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 2685075456}] === preallocation=falloc === wrote 65536/65536 bytes at offset 9437184 @@ -221,8 +221,8 @@ read 65536/65536 bytes at offset 9437184 5 MiB (0x500000) bytes not allocated at offset 0 bytes (0x0) 10 MiB (0xa00000) bytes allocated at offset 5 MiB (0x500000) -[{ "start": 0, "length": 5242880, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 5242880, "length": 10485760, "depth": 0, "present": true, "zero": false, "data": true, "offset": 327680}] +[{ "start": 0, "length": 5242880, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 5242880, "length": 10485760, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 327680}] === preallocation=full === wrote 65536/65536 bytes at offset 11534336 @@ -236,8 +236,8 @@ read 65536/65536 bytes at offset 11534336 8 MiB (0x800000) bytes not allocated at offset 0 bytes (0x0) 4 MiB (0x400000) bytes allocated at offset 8 MiB (0x800000) -[{ "start": 0, "length": 8388608, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 8388608, "length": 4194304, "depth": 0, "present": true, "zero": false, "data": true, "offset": 327680}] +[{ "start": 0, "length": 8388608, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 8388608, "length": 4194304, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 327680}] === preallocation=off === wrote 65536/65536 bytes at offset 259072 @@ -251,9 +251,9 @@ read 65536/65536 bytes at offset 259072 192 KiB (0x30000) bytes not allocated at offset 0 bytes (0x0) 320 KiB (0x50000) bytes allocated at offset 192 KiB (0x30000) -[{ "start": 0, "length": 196608, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 196608, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "offset": 327680}, -{ "start": 262144, "length": 262144, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 196608, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 196608, "length": 65536, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 327680}, +{ "start": 262144, "length": 262144, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === preallocation=off === wrote 65536/65536 bytes at offset 344064 @@ -267,8 +267,8 @@ read 65536/65536 bytes at offset 344064 256 KiB (0x40000) bytes not allocated at offset 0 bytes (0x0) 256 KiB (0x40000) bytes allocated at offset 256 KiB (0x40000) -[{ "start": 0, "length": 262144, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 262144, "length": 262144, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 262144, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 262144, "length": 262144, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] === preallocation=off === wrote 65536/65536 bytes at offset 446464 @@ -282,6 +282,6 @@ read 65536/65536 bytes at offset 446464 256 KiB (0x40000) bytes not allocated at offset 0 bytes (0x0) 244 KiB (0x3d000) bytes allocated at offset 256 KiB (0x40000) -[{ "start": 0, "length": 262144, "depth": 1, "present": false, "zero": true, "data": false}, -{ "start": 262144, "length": 249856, "depth": 0, "present": true, "zero": true, "data": false}] +[{ "start": 0, "length": 262144, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}, +{ "start": 262144, "length": 249856, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false}] diff --git a/tests/qemu-iotests/tests/file-io-error b/tests/qemu-iotests/tests/file-io-error new file mode 100755 index 0000000000..88ee5f670c --- /dev/null +++ b/tests/qemu-iotests/tests/file-io-error @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# group: rw +# +# Produce an I/O error in file-posix, and hope that it is not catastrophic. +# Regression test for: https://bugzilla.redhat.com/show_bug.cgi?id=2234374 +# +# Copyright (C) 2023 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +seq=$(basename "$0") +echo "QA output created by $seq" + +status=1 # failure is the default! + +_cleanup() +{ + _cleanup_qemu + rm -f "$TEST_DIR/fuse-export" +} +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# get standard environment, filters and checks +. ../common.rc +. ../common.filter +. ../common.qemu + +# Format-agnostic (we do not use any), but we do test the file protocol +_supported_proto file +_require_drivers blkdebug null-co + +if [ "$IMGOPTSSYNTAX" = "true" ]; then + # We need `$QEMU_IO -f file` to work; IMGOPTSSYNTAX uses --image-opts, + # breaking -f. + _unsupported_fmt $IMGFMT +fi + +# This is a regression test of a bug in which flie-posix would access zone +# information in case of an I/O error even when there is no zone information, +# resulting in a division by zero. +# To reproduce the problem, we need to trigger an I/O error inside of +# file-posix, which can be done (rootless) by providing a FUSE export that +# presents only errors when accessed. + +_launch_qemu +_send_qemu_cmd $QEMU_HANDLE \ + "{'execute': 'qmp_capabilities'}" \ + 'return' + +_send_qemu_cmd $QEMU_HANDLE \ + "{'execute': 'blockdev-add', + 'arguments': { + 'driver': 'blkdebug', + 'node-name': 'node0', + 'inject-error': [{'event': 'none'}], + 'image': { + 'driver': 'null-co' + } + }}" \ + 'return' + +# FUSE mountpoint must exist and be a regular file +touch "$TEST_DIR/fuse-export" + +# The grep -v to filter fusermount's (benign) error when /etc/fuse.conf does +# not contain user_allow_other and the subsequent check for missing FUSE support +# have both been taken from iotest 308. +output=$(_send_qemu_cmd $QEMU_HANDLE \ + "{'execute': 'block-export-add', + 'arguments': { + 'id': 'exp0', + 'type': 'fuse', + 'node-name': 'node0', + 'mountpoint': '$TEST_DIR/fuse-export', + 'writable': true + }}" \ + 'return' \ + | grep -v 'option allow_other only allowed if') + +if echo "$output" | grep -q "Parameter 'type' does not accept value 'fuse'"; then + _notrun 'No FUSE support' +fi +echo "$output" + +echo +# This should fail, but gracefully, i.e. just print an I/O error, not crash. +$QEMU_IO -f file -c 'write 0 64M' "$TEST_DIR/fuse-export" | _filter_qemu_io +echo + +_send_qemu_cmd $QEMU_HANDLE \ + "{'execute': 'block-export-del', + 'arguments': {'id': 'exp0'}}" \ + 'return' + +_send_qemu_cmd $QEMU_HANDLE \ + '' \ + 'BLOCK_EXPORT_DELETED' + +_send_qemu_cmd $QEMU_HANDLE \ + "{'execute': 'blockdev-del', + 'arguments': {'node-name': 'node0'}}" \ + 'return' + +# success, all done +echo "*** done" +rm -f $seq.full +status=0 diff --git a/tests/qemu-iotests/tests/file-io-error.out b/tests/qemu-iotests/tests/file-io-error.out new file mode 100644 index 0000000000..0f46455a94 --- /dev/null +++ b/tests/qemu-iotests/tests/file-io-error.out @@ -0,0 +1,33 @@ +QA output created by file-io-error +{'execute': 'qmp_capabilities'} +{"return": {}} +{'execute': 'blockdev-add', + 'arguments': { + 'driver': 'blkdebug', + 'node-name': 'node0', + 'inject-error': [{'event': 'none'}], + 'image': { + 'driver': 'null-co' + } + }} +{"return": {}} +{'execute': 'block-export-add', + 'arguments': { + 'id': 'exp0', + 'type': 'fuse', + 'node-name': 'node0', + 'mountpoint': 'TEST_DIR/fuse-export', + 'writable': true + }} +{"return": {}} + +write failed: Input/output error + +{'execute': 'block-export-del', + 'arguments': {'id': 'exp0'}} +{"return": {}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_EXPORT_DELETED", "data": {"id": "exp0"}} +{'execute': 'blockdev-del', + 'arguments': {'node-name': 'node0'}} +{"return": {}} +*** done diff --git a/tests/qemu-iotests/tests/nbd-qemu-allocation.out b/tests/qemu-iotests/tests/nbd-qemu-allocation.out index 9d938db24e..138eb09c6d 100644 --- a/tests/qemu-iotests/tests/nbd-qemu-allocation.out +++ b/tests/qemu-iotests/tests/nbd-qemu-allocation.out @@ -11,9 +11,9 @@ wrote 2097152/2097152 bytes at offset 1048576 === Check allocation over NBD === -[{ "start": 0, "length": 1048576, "depth": 1, "present": true, "zero": false, "data": true, "offset": 327680}, -{ "start": 1048576, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": 327680}, -{ "start": 3145728, "length": 1048576, "depth": 1, "present": false, "zero": true, "data": false}] +[{ "start": 0, "length": 1048576, "depth": 1, "present": true, "zero": false, "data": true, "compressed": false, "offset": 327680}, +{ "start": 1048576, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 327680}, +{ "start": 3145728, "length": 1048576, "depth": 1, "present": false, "zero": true, "data": false, "compressed": false}] exports available: 1 export: '' size: 4194304 @@ -24,9 +24,9 @@ exports available: 1 available meta contexts: 2 base:allocation qemu:allocation-depth -[{ "start": 0, "length": 3145728, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 3145728, "length": 1048576, "depth": 0, "present": true, "zero": true, "data": false, "offset": OFFSET}] -[{ "start": 0, "length": 1048576, "depth": 0, "present": true, "zero": true, "data": true, "offset": OFFSET}, -{ "start": 1048576, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 3145728, "length": 1048576, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 3145728, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 3145728, "length": 1048576, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 1048576, "depth": 0, "present": true, "zero": true, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 1048576, "length": 2097152, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 3145728, "length": 1048576, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] *** done diff --git a/tests/qemu-iotests/tests/parallels-checks b/tests/qemu-iotests/tests/parallels-checks index a7a1b357b5..b281246a42 100755 --- a/tests/qemu-iotests/tests/parallels-checks +++ b/tests/qemu-iotests/tests/parallels-checks @@ -91,7 +91,7 @@ file_size=`stat --printf="%s" "$TEST_IMG"` echo "file size: $file_size" echo "== check last cluster ==" -{ $QEMU_IO -c "read -P 0x11 $LAST_CLUSTER_OFF $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -r -c "read -P 0x11 $LAST_CLUSTER_OFF $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir # Clear image _make_test_img $SIZE @@ -105,25 +105,68 @@ echo "== write another pattern to second cluster ==" { $QEMU_IO -c "write -P 0x55 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir echo "== check second cluster ==" -{ $QEMU_IO -c "read -P 0x55 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -r -c "read -P 0x55 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + echo "== corrupt image ==" poke_file "$TEST_IMG" "$(($BAT_OFFSET + 4))" "\x01\x00\x00\x00" echo "== check second cluster ==" -{ $QEMU_IO -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -r -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir echo "== repair image ==" _check_test_img -r all +echo "== check the first cluster ==" +{ $QEMU_IO -r -c "read -P 0x11 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== check second cluster ==" +{ $QEMU_IO -r -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== write another pattern to the first clusters ==" +{ $QEMU_IO -c "write -P 0x66 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== check the first cluster ==" +{ $QEMU_IO -r -c "read -P 0x66 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== check the second cluster (deduplicated) ==" +{ $QEMU_IO -r -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +# Clear image +_make_test_img $SIZE + +echo "== TEST DUPLICATION SELF-CURE ==" + +echo "== write pattern to whole image ==" +{ $QEMU_IO -c "write -P 0x11 0 $SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== write another pattern to second cluster ==" +{ $QEMU_IO -c "write -P 0x55 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== check second cluster ==" +{ $QEMU_IO -r -c "read -P 0x55 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + + +echo "== corrupt image ==" +poke_file "$TEST_IMG" "$(($BAT_OFFSET + 4))" "\x01\x00\x00\x00" + +echo "== check second cluster ==" +{ $QEMU_IO -r -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== check the first cluster with self-repair ==" +{ $QEMU_IO -c "read -P 0x11 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + echo "== check second cluster ==" -{ $QEMU_IO -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +{ $QEMU_IO -r -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== write another pattern to the first clusters ==" +{ $QEMU_IO -c "write -P 0x66 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir -echo "== check first cluster on host ==" -printf "content: 0x%02x\n" `peek_file_le $TEST_IMG $(($CLUSTER_SIZE)) 1` +echo "== check the first cluster ==" +{ $QEMU_IO -r -c "read -P 0x66 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir -echo "== check second cluster on host ==" -printf "content: 0x%02x\n" `peek_file_le $TEST_IMG $(($CLUSTER_SIZE)) 1` +echo "== check the second cluster (deduplicated) ==" +{ $QEMU_IO -r -c "read -P 0x11 $CLUSTER_SIZE $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir # Clear image _make_test_img $SIZE @@ -139,6 +182,23 @@ poke_file "$TEST_IMG" "$DATA_OFF_OFFSET" "\xff\xff\xff\xff" echo "== check first cluster ==" { $QEMU_IO -c "read -P 0x55 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir +# Clear image +_make_test_img $SIZE + +echo "== TEST DATA_OFF THROUGH REPAIR ==" + +echo "== write pattern to first cluster ==" +{ $QEMU_IO -c "write -P 0x55 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + +echo "== spoil data_off field ==" +poke_file "$TEST_IMG" "$DATA_OFF_OFFSET" "\xff\xff\xff\xff" + +echo "== repair image ==" +_check_test_img -r all + +echo "== check first cluster ==" +{ $QEMU_IO -r -c "read -P 0x55 0 $CLUSTER_SIZE" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir + # success, all done echo "*** done" rm -f $seq.full diff --git a/tests/qemu-iotests/tests/parallels-checks.out b/tests/qemu-iotests/tests/parallels-checks.out index 98a3a7f55e..9793423111 100644 --- a/tests/qemu-iotests/tests/parallels-checks.out +++ b/tests/qemu-iotests/tests/parallels-checks.out @@ -55,13 +55,52 @@ The following inconsistencies were found and repaired: Double checking the fixed image now... No errors were found on the image. +== check the first cluster == +read 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) == check second cluster == read 1048576/1048576 bytes at offset 1048576 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -== check first cluster on host == -content: 0x11 -== check second cluster on host == -content: 0x11 +== write another pattern to the first clusters == +wrote 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check the first cluster == +read 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check the second cluster (deduplicated) == +read 1048576/1048576 bytes at offset 1048576 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4194304 +== TEST DUPLICATION SELF-CURE == +== write pattern to whole image == +wrote 4194304/4194304 bytes at offset 0 +4 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== write another pattern to second cluster == +wrote 1048576/1048576 bytes at offset 1048576 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check second cluster == +read 1048576/1048576 bytes at offset 1048576 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== corrupt image == +== check second cluster == +read 1048576/1048576 bytes at offset 1048576 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check the first cluster with self-repair == +Repairing duplicate offset in BAT entry 1 +read 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check second cluster == +read 1048576/1048576 bytes at offset 1048576 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== write another pattern to the first clusters == +wrote 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check the first cluster == +read 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== check the second cluster (deduplicated) == +read 1048576/1048576 bytes at offset 1048576 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4194304 == TEST DATA_OFF CHECK == == write pattern to first cluster == @@ -72,4 +111,22 @@ wrote 1048576/1048576 bytes at offset 0 Repairing data_off field has incorrect value read 1048576/1048576 bytes at offset 0 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4194304 +== TEST DATA_OFF THROUGH REPAIR == +== write pattern to first cluster == +wrote 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +== spoil data_off field == +== repair image == +Repairing data_off field has incorrect value +The following inconsistencies were found and repaired: + + 0 leaked clusters + 1 corruptions + +Double checking the fixed image now... +No errors were found on the image. +== check first cluster == +read 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) *** done diff --git a/tests/qemu-iotests/tests/qemu-img-bitmaps.out b/tests/qemu-iotests/tests/qemu-img-bitmaps.out index e851f0320e..74b81f703b 100644 --- a/tests/qemu-iotests/tests/qemu-img-bitmaps.out +++ b/tests/qemu-iotests/tests/qemu-img-bitmaps.out @@ -103,18 +103,18 @@ Format specific information: === Check bitmap contents === -[{ "start": 0, "length": 3145728, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 3145728, "length": 1048576, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 4194304, "length": 6291456, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] -[{ "start": 0, "length": 1048576, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 1048576, "length": 1048576, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 2097152, "length": 8388608, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] -[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 1048576, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 3145728, "length": 7340032, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] -[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}, -{ "start": 2097152, "length": 1048576, "depth": 0, "present": false, "zero": false, "data": false}, -{ "start": 3145728, "length": 7340032, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 0, "length": 3145728, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 3145728, "length": 1048576, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 4194304, "length": 6291456, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 1048576, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 1048576, "length": 1048576, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 2097152, "length": 8388608, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 1048576, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 3145728, "length": 7340032, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] +[{ "start": 0, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}, +{ "start": 2097152, "length": 1048576, "depth": 0, "present": false, "zero": false, "data": false, "compressed": false}, +{ "start": 3145728, "length": 7340032, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": OFFSET}] === Check handling of inconsistent bitmap === diff --git a/tests/qtest/libqos/igb.c b/tests/qtest/libqos/igb.c index a603468beb..f40c4ec4cd 100644 --- a/tests/qtest/libqos/igb.c +++ b/tests/qtest/libqos/igb.c @@ -109,6 +109,11 @@ static void igb_pci_start_hw(QOSGraphObject *obj) E1000_RAH_AV | E1000_RAH_POOL_1 | le16_to_cpu(*(uint16_t *)(address + 4))); + /* Set supported receive descriptor mode */ + e1000e_macreg_write(&d->e1000e, + E1000_SRRCTL(0), + E1000_SRRCTL_DESCTYPE_ADV_ONEBUF); + /* Enable receive */ e1000e_macreg_write(&d->e1000e, E1000_RFCTL, E1000_RFCTL_EXTEN); e1000e_macreg_write(&d->e1000e, E1000_RCTL, E1000_RCTL_EN); diff --git a/tests/qtest/libqtest.c b/tests/qtest/libqtest.c index 34b9c14b75..b1eba71ffe 100644 --- a/tests/qtest/libqtest.c +++ b/tests/qtest/libqtest.c @@ -24,6 +24,9 @@ #ifdef __linux__ #include <sys/prctl.h> #endif /* __linux__ */ +#ifdef __FreeBSD__ +#include <sys/procctl.h> +#endif /* __FreeBSD__ */ #include "libqtest.h" #include "libqmp.h" @@ -414,6 +417,10 @@ static QTestState *G_GNUC_PRINTF(1, 2) qtest_spawn_qemu(const char *fmt, ...) */ prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); #endif /* __linux__ */ +#ifdef __FreeBSD__ + int sig = SIGKILL; + procctl(P_PID, getpid(), PROC_PDEATHSIG_CTL, &sig); +#endif /* __FreeBSD__ */ if (!g_setenv("QEMU_AUDIO_DRV", "none", true)) { exit(1); } diff --git a/tests/qtest/microbit-test.c b/tests/qtest/microbit-test.c index 2abcad8e31..72190d38f7 100644 --- a/tests/qtest/microbit-test.c +++ b/tests/qtest/microbit-test.c @@ -434,6 +434,8 @@ static void test_nrf51_gpio_detect(void) g_assert_true(qtest_get_irq(qts, 0)); qtest_set_irq_in(qts, "/machine/nrf51", "unnamed-gpio-in", 3, 0); g_assert_true(qtest_get_irq(qts, 0)); + + qtest_quit(qts); } static void timer_task(QTestState *qts, hwaddr task) diff --git a/tests/tcg/aarch64/sysregs.c b/tests/tcg/aarch64/sysregs.c index d8eb06abcf..f7a055f1d5 100644 --- a/tests/tcg/aarch64/sysregs.c +++ b/tests/tcg/aarch64/sysregs.c @@ -126,7 +126,7 @@ int main(void) */ get_cpu_reg_check_mask(id_aa64isar0_el1, _m(f0ff,ffff,f0ff,fff0)); get_cpu_reg_check_mask(id_aa64isar1_el1, _m(00ff,f0ff,ffff,ffff)); - get_cpu_reg_check_mask(SYS_ID_AA64ISAR2_EL1, _m(0000,0000,0000,ffff)); + get_cpu_reg_check_mask(SYS_ID_AA64ISAR2_EL1, _m(00ff,0000,00ff,ffff)); /* TGran4 & TGran64 as pegged to -1 */ get_cpu_reg_check_mask(id_aa64mmfr0_el1, _m(f000,0000,ff00,0000)); get_cpu_reg_check_mask(id_aa64mmfr1_el1, _m(0000,f000,0000,0000)); @@ -138,7 +138,7 @@ int main(void) get_cpu_reg_check_mask(id_aa64dfr0_el1, _m(0000,0000,0000,0006)); get_cpu_reg_check_zero(id_aa64dfr1_el1); get_cpu_reg_check_mask(SYS_ID_AA64ZFR0_EL1, _m(0ff0,ff0f,00ff,00ff)); - get_cpu_reg_check_mask(SYS_ID_AA64SMFR0_EL1, _m(80f1,00fd,0000,0000)); + get_cpu_reg_check_mask(SYS_ID_AA64SMFR0_EL1, _m(8ff1,fcff,0000,0000)); get_cpu_reg_check_zero(id_aa64afr0_el1); get_cpu_reg_check_zero(id_aa64afr1_el1); diff --git a/tests/tcg/i386/system/boot.S b/tests/tcg/i386/system/boot.S index 794c2cb0ad..9e8920cbfe 100644 --- a/tests/tcg/i386/system/boot.S +++ b/tests/tcg/i386/system/boot.S @@ -71,7 +71,7 @@ _start: add $8,%esp /* - * Don't worry about stack frame, assume everthing + * Don't worry about stack frame, assume everything * is garbage when we return, we won't need it. */ call main diff --git a/tests/tcg/i386/x86.csv b/tests/tcg/i386/x86.csv index c43bf42dd3..5c0f628e35 100644 --- a/tests/tcg/i386/x86.csv +++ b/tests/tcg/i386/x86.csv @@ -19,7 +19,7 @@ # # 4. The instruction encoding. For example, "C1 /4 ib". # -# 5. The validity of the instruction in 32-bit (aka compatiblity, legacy) mode. +# 5. The validity of the instruction in 32-bit (aka compatibility, legacy) mode. # # 6. The validity of the instruction in 64-bit mode. # diff --git a/tests/tcg/m68k/Makefile.target b/tests/tcg/m68k/Makefile.target index 1163c7ef03..6ff214e60a 100644 --- a/tests/tcg/m68k/Makefile.target +++ b/tests/tcg/m68k/Makefile.target @@ -4,7 +4,7 @@ # VPATH += $(SRC_PATH)/tests/tcg/m68k -TESTS += trap +TESTS += trap denormal # On m68k Linux supports 4k and 8k pages (but 8k is currently broken) EXTRA_RUNS+=run-test-mmap-4096 # run-test-mmap-8192 diff --git a/tests/tcg/m68k/denormal.c b/tests/tcg/m68k/denormal.c new file mode 100644 index 0000000000..20bd8c7332 --- /dev/null +++ b/tests/tcg/m68k/denormal.c @@ -0,0 +1,53 @@ +/* + * Test m68k extended double denormals. + */ + +#include <stdio.h> +#include <stdint.h> + +#define TEST(X, Y) { X, Y, X * Y } + +static volatile long double test[][3] = { + TEST(0x1p+16383l, 0x1p-16446l), + TEST(0x1.1p-8223l, 0x1.1p-8224l), + TEST(1.0l, 0x1p-16383l), +}; + +#undef TEST + +static void dump_ld(const char *label, long double ld) +{ + union { + long double d; + struct { + uint32_t exp:16; + uint32_t space:16; + uint32_t h; + uint32_t l; + }; + } u; + + u.d = ld; + printf("%12s: % -27La 0x%04x 0x%08x 0x%08x\n", label, u.d, u.exp, u.h, u.l); +} + +int main(void) +{ + int i, n = sizeof(test) / sizeof(test[0]), err = 0; + + for (i = 0; i < n; ++i) { + long double x = test[i][0]; + long double y = test[i][1]; + long double build_mul = test[i][2]; + long double runtime_mul = x * y; + + if (runtime_mul != build_mul) { + dump_ld("x", x); + dump_ld("y", y); + dump_ld("build_mul", build_mul); + dump_ld("runtime_mul", runtime_mul); + err = 1; + } + } + return err; +} diff --git a/tests/unit/test-bdrv-drain.c b/tests/unit/test-bdrv-drain.c index ccc453c29e..0b603e7c57 100644 --- a/tests/unit/test-bdrv-drain.c +++ b/tests/unit/test-bdrv-drain.c @@ -512,6 +512,7 @@ static void test_iothread_main_thread_bh(void *opaque) * executed during drain, otherwise this would deadlock. */ aio_context_acquire(bdrv_get_aio_context(data->bs)); bdrv_flush(data->bs); + bdrv_dec_in_flight(data->bs); /* incremented by test_iothread_common() */ aio_context_release(bdrv_get_aio_context(data->bs)); } @@ -583,6 +584,13 @@ static void test_iothread_common(enum drain_type drain_type, int drain_thread) aio_context_acquire(ctx_a); } + /* + * Increment in_flight so that do_drain_begin() waits for + * test_iothread_main_thread_bh(). This prevents the race between + * test_iothread_main_thread_bh() in IOThread a and do_drain_begin() in + * this thread. test_iothread_main_thread_bh() decrements in_flight. + */ + bdrv_inc_in_flight(bs); aio_bh_schedule_oneshot(ctx_a, test_iothread_main_thread_bh, &data); /* The request is running on the IOThread a. Draining its block device @@ -967,9 +975,12 @@ typedef struct BDRVTestTopState { static void bdrv_test_top_close(BlockDriverState *bs) { BdrvChild *c, *next_c; + + bdrv_graph_wrlock(NULL); QLIST_FOREACH_SAFE(c, &bs->children, next, next_c) { bdrv_unref_child(bs, c); } + bdrv_graph_wrunlock(); } static int coroutine_fn GRAPH_RDLOCK @@ -1024,7 +1035,7 @@ static void coroutine_fn test_co_delete_by_drain(void *opaque) } else { BdrvChild *c, *next_c; QLIST_FOREACH_SAFE(c, &bs->children, next, next_c) { - bdrv_unref_child(bs, c); + bdrv_co_unref_child(bs, c); } } @@ -1055,8 +1066,10 @@ static void do_test_delete_by_drain(bool detach_instead_of_delete, null_bs = bdrv_open("null-co://", NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL, &error_abort); + bdrv_graph_wrlock(NULL); bdrv_attach_child(bs, null_bs, "null-child", &child_of_bds, BDRV_CHILD_DATA, &error_abort); + bdrv_graph_wrunlock(); /* This child will be the one to pass to requests through to, and * it will stall until a drain occurs */ @@ -1064,17 +1077,21 @@ static void do_test_delete_by_drain(bool detach_instead_of_delete, &error_abort); child_bs->total_sectors = 65536 >> BDRV_SECTOR_BITS; /* Takes our reference to child_bs */ + bdrv_graph_wrlock(NULL); tts->wait_child = bdrv_attach_child(bs, child_bs, "wait-child", &child_of_bds, BDRV_CHILD_DATA | BDRV_CHILD_PRIMARY, &error_abort); + bdrv_graph_wrunlock(); /* This child is just there to be deleted * (for detach_instead_of_delete == true) */ null_bs = bdrv_open("null-co://", NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL, &error_abort); + bdrv_graph_wrlock(NULL); bdrv_attach_child(bs, null_bs, "null-child", &child_of_bds, BDRV_CHILD_DATA, &error_abort); + bdrv_graph_wrunlock(); blk = blk_new(qemu_get_aio_context(), BLK_PERM_ALL, BLK_PERM_ALL); blk_insert_bs(blk, bs, &error_abort); @@ -1156,12 +1173,15 @@ static void detach_indirect_bh(void *opaque) struct detach_by_parent_data *data = opaque; bdrv_dec_in_flight(data->child_b->bs); + + bdrv_graph_wrlock(NULL); bdrv_unref_child(data->parent_b, data->child_b); bdrv_ref(data->c); data->child_c = bdrv_attach_child(data->parent_b, data->c, "PB-C", &child_of_bds, BDRV_CHILD_DATA, &error_abort); + bdrv_graph_wrunlock(); } static void detach_by_parent_aio_cb(void *opaque, int ret) @@ -1258,6 +1278,7 @@ static void test_detach_indirect(bool by_parent_cb) /* Set child relationships */ bdrv_ref(b); bdrv_ref(a); + bdrv_graph_wrlock(NULL); child_b = bdrv_attach_child(parent_b, b, "PB-B", &child_of_bds, BDRV_CHILD_DATA, &error_abort); child_a = bdrv_attach_child(parent_b, a, "PB-A", &child_of_bds, @@ -1267,6 +1288,7 @@ static void test_detach_indirect(bool by_parent_cb) bdrv_attach_child(parent_a, a, "PA-A", by_parent_cb ? &child_of_bds : &detach_by_driver_cb_class, BDRV_CHILD_DATA, &error_abort); + bdrv_graph_wrunlock(); g_assert_cmpint(parent_a->refcnt, ==, 1); g_assert_cmpint(parent_b->refcnt, ==, 1); @@ -1359,7 +1381,10 @@ static void test_append_to_drained(void) g_assert_cmpint(base_s->drain_count, ==, 1); g_assert_cmpint(base->in_flight, ==, 0); + aio_context_acquire(qemu_get_aio_context()); bdrv_append(overlay, base, &error_abort); + aio_context_release(qemu_get_aio_context()); + g_assert_cmpint(base->in_flight, ==, 0); g_assert_cmpint(overlay->in_flight, ==, 0); @@ -1682,6 +1707,7 @@ static void test_drop_intermediate_poll(void) * Establish the chain last, so the chain links are the first * elements in the BDS.parents lists */ + bdrv_graph_wrlock(NULL); for (i = 0; i < 3; i++) { if (i) { /* Takes the reference to chain[i - 1] */ @@ -1689,6 +1715,7 @@ static void test_drop_intermediate_poll(void) &chain_child_class, BDRV_CHILD_COW, &error_abort); } } + bdrv_graph_wrunlock(); job = block_job_create("job", &test_simple_job_driver, NULL, job_node, 0, BLK_PERM_ALL, 0, 0, NULL, NULL, &error_abort); @@ -1933,8 +1960,10 @@ static void do_test_replace_child_mid_drain(int old_drain_count, new_child_bs->total_sectors = 1; bdrv_ref(old_child_bs); + bdrv_graph_wrlock(NULL); bdrv_attach_child(parent_bs, old_child_bs, "child", &child_of_bds, BDRV_CHILD_COW, &error_abort); + bdrv_graph_wrunlock(); parent_s->setup_completed = true; for (i = 0; i < old_drain_count; i++) { diff --git a/tests/unit/test-bdrv-graph-mod.c b/tests/unit/test-bdrv-graph-mod.c index 36eed4b464..8609f7f42b 100644 --- a/tests/unit/test-bdrv-graph-mod.c +++ b/tests/unit/test-bdrv-graph-mod.c @@ -137,11 +137,15 @@ static void test_update_perm_tree(void) blk_insert_bs(root, bs, &error_abort); + bdrv_graph_wrlock(NULL); bdrv_attach_child(filter, bs, "child", &child_of_bds, BDRV_CHILD_DATA, &error_abort); + bdrv_graph_wrunlock(); + aio_context_acquire(qemu_get_aio_context()); ret = bdrv_append(filter, bs, NULL); g_assert_cmpint(ret, <, 0); + aio_context_release(qemu_get_aio_context()); bdrv_unref(filter); blk_unref(root); @@ -203,9 +207,13 @@ static void test_should_update_child(void) bdrv_set_backing_hd(target, bs, &error_abort); g_assert(target->backing->bs == bs); + bdrv_graph_wrlock(NULL); bdrv_attach_child(filter, target, "target", &child_of_bds, BDRV_CHILD_DATA, &error_abort); + bdrv_graph_wrunlock(); + aio_context_acquire(qemu_get_aio_context()); bdrv_append(filter, bs, &error_abort); + aio_context_release(qemu_get_aio_context()); g_assert(target->backing->bs == bs); bdrv_unref(filter); @@ -232,6 +240,7 @@ static void test_parallel_exclusive_write(void) */ bdrv_ref(base); + bdrv_graph_wrlock(NULL); bdrv_attach_child(top, fl1, "backing", &child_of_bds, BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY, &error_abort); @@ -241,6 +250,7 @@ static void test_parallel_exclusive_write(void) bdrv_attach_child(fl2, base, "backing", &child_of_bds, BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY, &error_abort); + bdrv_graph_wrunlock(); bdrv_replace_node(fl1, fl2, &error_abort); @@ -345,6 +355,7 @@ static void test_parallel_perm_update(void) */ bdrv_ref(base); + bdrv_graph_wrlock(NULL); bdrv_attach_child(top, ws, "file", &child_of_bds, BDRV_CHILD_DATA, &error_abort); c_fl1 = bdrv_attach_child(ws, fl1, "first", &child_of_bds, @@ -357,9 +368,13 @@ static void test_parallel_perm_update(void) bdrv_attach_child(fl2, base, "backing", &child_of_bds, BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY, &error_abort); + bdrv_graph_wrunlock(); /* Select fl1 as first child to be active */ s->selected = c_fl1; + + bdrv_graph_rdlock_main_loop(); + bdrv_child_refresh_perms(top, top->children.lh_first, &error_abort); assert(c_fl1->perm & BLK_PERM_WRITE); @@ -379,6 +394,7 @@ static void test_parallel_perm_update(void) assert(c_fl1->perm & BLK_PERM_WRITE); assert(!(c_fl2->perm & BLK_PERM_WRITE)); + bdrv_graph_rdunlock_main_loop(); bdrv_unref(top); } @@ -406,11 +422,15 @@ static void test_append_greedy_filter(void) BlockDriverState *base = no_perm_node("base"); BlockDriverState *fl = exclusive_writer_node("fl1"); + bdrv_graph_wrlock(NULL); bdrv_attach_child(top, base, "backing", &child_of_bds, BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY, &error_abort); + bdrv_graph_wrunlock(); + aio_context_acquire(qemu_get_aio_context()); bdrv_append(fl, base, &error_abort); + aio_context_release(qemu_get_aio_context()); bdrv_unref(fl); bdrv_unref(top); } diff --git a/tests/unit/test-block-iothread.c b/tests/unit/test-block-iothread.c index d727a5fee8..9155547313 100644 --- a/tests/unit/test-block-iothread.c +++ b/tests/unit/test-block-iothread.c @@ -756,11 +756,14 @@ static void test_propagate_mirror(void) &error_abort); /* Start a mirror job */ + aio_context_acquire(main_ctx); mirror_start("job0", src, target, NULL, JOB_DEFAULT, 0, 0, 0, MIRROR_SYNC_MODE_NONE, MIRROR_OPEN_BACKING_CHAIN, false, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT, false, "filter_node", MIRROR_COPY_MODE_BACKGROUND, &error_abort); + aio_context_release(main_ctx); + WITH_JOB_LOCK_GUARD() { job = job_get_locked("job0"); } diff --git a/tests/unit/test-throttle.c b/tests/unit/test-throttle.c index dc8739c1d6..cb587e33e7 100644 --- a/tests/unit/test-throttle.c +++ b/tests/unit/test-throttle.c @@ -169,8 +169,72 @@ static void test_init(void) /* check initialized fields */ g_assert(tt->clock_type == QEMU_CLOCK_VIRTUAL); - g_assert(tt->timers[0]); - g_assert(tt->timers[1]); + g_assert(tt->timers[THROTTLE_READ]); + g_assert(tt->timers[THROTTLE_WRITE]); + + /* check other fields where cleared */ + g_assert(!ts.previous_leak); + g_assert(!ts.cfg.op_size); + for (i = 0; i < BUCKETS_COUNT; i++) { + g_assert(!ts.cfg.buckets[i].avg); + g_assert(!ts.cfg.buckets[i].max); + g_assert(!ts.cfg.buckets[i].level); + } + + throttle_timers_destroy(tt); +} + +static void test_init_readonly(void) +{ + int i; + + tt = &tgm.throttle_timers; + + /* fill the structures with crap */ + memset(&ts, 1, sizeof(ts)); + memset(tt, 1, sizeof(*tt)); + + /* init structures */ + throttle_init(&ts); + throttle_timers_init(tt, ctx, QEMU_CLOCK_VIRTUAL, + read_timer_cb, NULL, &ts); + + /* check initialized fields */ + g_assert(tt->clock_type == QEMU_CLOCK_VIRTUAL); + g_assert(tt->timers[THROTTLE_READ]); + g_assert(!tt->timers[THROTTLE_WRITE]); + + /* check other fields where cleared */ + g_assert(!ts.previous_leak); + g_assert(!ts.cfg.op_size); + for (i = 0; i < BUCKETS_COUNT; i++) { + g_assert(!ts.cfg.buckets[i].avg); + g_assert(!ts.cfg.buckets[i].max); + g_assert(!ts.cfg.buckets[i].level); + } + + throttle_timers_destroy(tt); +} + +static void test_init_writeonly(void) +{ + int i; + + tt = &tgm.throttle_timers; + + /* fill the structures with crap */ + memset(&ts, 1, sizeof(ts)); + memset(tt, 1, sizeof(*tt)); + + /* init structures */ + throttle_init(&ts); + throttle_timers_init(tt, ctx, QEMU_CLOCK_VIRTUAL, + NULL, write_timer_cb, &ts); + + /* check initialized fields */ + g_assert(tt->clock_type == QEMU_CLOCK_VIRTUAL); + g_assert(!tt->timers[THROTTLE_READ]); + g_assert(tt->timers[THROTTLE_WRITE]); /* check other fields where cleared */ g_assert(!ts.previous_leak); @@ -191,7 +255,7 @@ static void test_destroy(void) throttle_timers_init(tt, ctx, QEMU_CLOCK_VIRTUAL, read_timer_cb, write_timer_cb, &ts); throttle_timers_destroy(tt); - for (i = 0; i < 2; i++) { + for (i = 0; i < THROTTLE_MAX; i++) { g_assert(!tt->timers[i]); } } @@ -573,9 +637,9 @@ static bool do_test_accounting(bool is_ops, /* are we testing bps or ops */ throttle_config(&ts, QEMU_CLOCK_VIRTUAL, &cfg); /* account a read */ - throttle_account(&ts, false, size); + throttle_account(&ts, THROTTLE_READ, size); /* account a write */ - throttle_account(&ts, true, size); + throttle_account(&ts, THROTTLE_WRITE, size); /* check total result */ index = to_test[is_ops][0]; @@ -752,6 +816,8 @@ int main(int argc, char **argv) g_test_add_func("/throttle/leak_bucket", test_leak_bucket); g_test_add_func("/throttle/compute_wait", test_compute_wait); g_test_add_func("/throttle/init", test_init); + g_test_add_func("/throttle/init_readonly", test_init_readonly); + g_test_add_func("/throttle/init_writeonly", test_init_writeonly); g_test_add_func("/throttle/destroy", test_destroy); g_test_add_func("/throttle/have_timer", test_have_timer); g_test_add_func("/throttle/detach_attach", test_detach_attach); diff --git a/util/cpuinfo-aarch64.c b/util/cpuinfo-aarch64.c index 7d39f47e3b..4c8a005715 100644 --- a/util/cpuinfo-aarch64.c +++ b/util/cpuinfo-aarch64.c @@ -13,6 +13,9 @@ # include <asm/hwcap.h> # include "elf.h" # endif +# ifndef HWCAP2_BTI +# define HWCAP2_BTI 0 /* added in glibc 2.32 */ +# endif #endif #ifdef CONFIG_DARWIN # include <sys/sysctl.h> @@ -56,12 +59,18 @@ unsigned __attribute__((constructor)) cpuinfo_init(void) unsigned long hwcap = qemu_getauxval(AT_HWCAP); info |= (hwcap & HWCAP_ATOMICS ? CPUINFO_LSE : 0); info |= (hwcap & HWCAP_USCAT ? CPUINFO_LSE2 : 0); - info |= (hwcap & HWCAP_AES ? CPUINFO_AES: 0); + info |= (hwcap & HWCAP_AES ? CPUINFO_AES : 0); + info |= (hwcap & HWCAP_PMULL ? CPUINFO_PMULL : 0); + + unsigned long hwcap2 = qemu_getauxval(AT_HWCAP2); + info |= (hwcap2 & HWCAP2_BTI ? CPUINFO_BTI : 0); #endif #ifdef CONFIG_DARWIN info |= sysctl_for_bool("hw.optional.arm.FEAT_LSE") * CPUINFO_LSE; info |= sysctl_for_bool("hw.optional.arm.FEAT_LSE2") * CPUINFO_LSE2; info |= sysctl_for_bool("hw.optional.arm.FEAT_AES") * CPUINFO_AES; + info |= sysctl_for_bool("hw.optional.arm.FEAT_PMULL") * CPUINFO_PMULL; + info |= sysctl_for_bool("hw.optional.arm.FEAT_BTI") * CPUINFO_BTI; #endif cpuinfo = info; diff --git a/util/cpuinfo-i386.c b/util/cpuinfo-i386.c index b2ed65bb10..9fddb18303 100644 --- a/util/cpuinfo-i386.c +++ b/util/cpuinfo-i386.c @@ -39,6 +39,7 @@ unsigned __attribute__((constructor)) cpuinfo_init(void) info |= (c & bit_SSE4_1 ? CPUINFO_SSE4 : 0); info |= (c & bit_MOVBE ? CPUINFO_MOVBE : 0); info |= (c & bit_POPCNT ? CPUINFO_POPCNT : 0); + info |= (c & bit_PCLMUL ? CPUINFO_PCLMUL : 0); /* Our AES support requires PSHUFB as well. */ info |= ((c & bit_AES) && (c & bit_SSSE3) ? CPUINFO_AES : 0); diff --git a/util/oslib-posix.c b/util/oslib-posix.c index 4d583da7ce..e86fd64e09 100644 --- a/util/oslib-posix.c +++ b/util/oslib-posix.c @@ -585,7 +585,7 @@ char *qemu_get_pid_name(pid_t pid) void *qemu_alloc_stack(size_t *sz) { - void *ptr, *guardpage; + void *ptr; int flags; #ifdef CONFIG_DEBUG_STACK_USAGE void *ptr2; @@ -618,17 +618,8 @@ void *qemu_alloc_stack(size_t *sz) abort(); } -#if defined(HOST_IA64) - /* separate register stack */ - guardpage = ptr + (((*sz - pagesz) / 2) & ~pagesz); -#elif defined(HOST_HPPA) - /* stack grows up */ - guardpage = ptr + *sz - pagesz; -#else - /* stack grows down */ - guardpage = ptr; -#endif - if (mprotect(guardpage, pagesz, PROT_NONE) != 0) { + /* Stack grows down -- guard page at the bottom. */ + if (mprotect(ptr, pagesz, PROT_NONE) != 0) { perror("failed to set up stack guard page"); abort(); } diff --git a/util/thread-pool.c b/util/thread-pool.c index e3d8292d14..22f9ba3286 100644 --- a/util/thread-pool.c +++ b/util/thread-pool.c @@ -228,17 +228,9 @@ static void thread_pool_cancel(BlockAIOCB *acb) } -static AioContext *thread_pool_get_aio_context(BlockAIOCB *acb) -{ - ThreadPoolElement *elem = (ThreadPoolElement *)acb; - ThreadPool *pool = elem->pool; - return pool->ctx; -} - static const AIOCBInfo thread_pool_aiocb_info = { .aiocb_size = sizeof(ThreadPoolElement), .cancel_async = thread_pool_cancel, - .get_aio_context = thread_pool_get_aio_context, }; BlockAIOCB *thread_pool_submit_aio(ThreadPoolFunc *func, void *arg, diff --git a/util/throttle.c b/util/throttle.c index 81f247a8d1..9582899da3 100644 --- a/util/throttle.c +++ b/util/throttle.c @@ -136,13 +136,14 @@ int64_t throttle_compute_wait(LeakyBucket *bkt) /* This function compute the time that must be waited while this IO * - * @is_write: true if the current IO is a write, false if it's a read + * @direction: throttle direction * @ret: time to wait */ static int64_t throttle_compute_wait_for(ThrottleState *ts, - bool is_write) + ThrottleDirection direction) { - BucketType to_check[2][4] = { {THROTTLE_BPS_TOTAL, + static const BucketType to_check[THROTTLE_MAX][4] = { + {THROTTLE_BPS_TOTAL, THROTTLE_OPS_TOTAL, THROTTLE_BPS_READ, THROTTLE_OPS_READ}, @@ -153,8 +154,8 @@ static int64_t throttle_compute_wait_for(ThrottleState *ts, int64_t wait, max_wait = 0; int i; - for (i = 0; i < 4; i++) { - BucketType index = to_check[is_write][i]; + for (i = 0; i < ARRAY_SIZE(to_check[THROTTLE_READ]); i++) { + BucketType index = to_check[direction][i]; wait = throttle_compute_wait(&ts->cfg.buckets[index]); if (wait > max_wait) { max_wait = wait; @@ -166,13 +167,13 @@ static int64_t throttle_compute_wait_for(ThrottleState *ts, /* compute the timer for this type of operation * - * @is_write: the type of operation + * @direction: throttle direction * @now: the current clock timestamp * @next_timestamp: the resulting timer * @ret: true if a timer must be set */ static bool throttle_compute_timer(ThrottleState *ts, - bool is_write, + ThrottleDirection direction, int64_t now, int64_t *next_timestamp) { @@ -182,7 +183,7 @@ static bool throttle_compute_timer(ThrottleState *ts, throttle_do_leak(ts, now); /* compute the wait time if any */ - wait = throttle_compute_wait_for(ts, is_write); + wait = throttle_compute_wait_for(ts, direction); /* if the code must wait compute when the next timer should fire */ if (wait) { @@ -199,10 +200,15 @@ static bool throttle_compute_timer(ThrottleState *ts, void throttle_timers_attach_aio_context(ThrottleTimers *tt, AioContext *new_context) { - tt->timers[0] = aio_timer_new(new_context, tt->clock_type, SCALE_NS, - tt->read_timer_cb, tt->timer_opaque); - tt->timers[1] = aio_timer_new(new_context, tt->clock_type, SCALE_NS, - tt->write_timer_cb, tt->timer_opaque); + ThrottleDirection dir; + + for (dir = THROTTLE_READ; dir < THROTTLE_MAX; dir++) { + if (tt->timer_cb[dir]) { + tt->timers[dir] = + aio_timer_new(new_context, tt->clock_type, SCALE_NS, + tt->timer_cb[dir], tt->timer_opaque); + } + } } /* @@ -233,11 +239,12 @@ void throttle_timers_init(ThrottleTimers *tt, QEMUTimerCB *write_timer_cb, void *timer_opaque) { + assert(read_timer_cb || write_timer_cb); memset(tt, 0, sizeof(ThrottleTimers)); tt->clock_type = clock_type; - tt->read_timer_cb = read_timer_cb; - tt->write_timer_cb = write_timer_cb; + tt->timer_cb[THROTTLE_READ] = read_timer_cb; + tt->timer_cb[THROTTLE_WRITE] = write_timer_cb; tt->timer_opaque = timer_opaque; throttle_timers_attach_aio_context(tt, aio_context); } @@ -245,7 +252,9 @@ void throttle_timers_init(ThrottleTimers *tt, /* destroy a timer */ static void throttle_timer_destroy(QEMUTimer **timer) { - assert(*timer != NULL); + if (*timer == NULL) { + return; + } timer_free(*timer); *timer = NULL; @@ -254,10 +263,10 @@ static void throttle_timer_destroy(QEMUTimer **timer) /* Remove timers from event loop */ void throttle_timers_detach_aio_context(ThrottleTimers *tt) { - int i; + ThrottleDirection dir; - for (i = 0; i < 2; i++) { - throttle_timer_destroy(&tt->timers[i]); + for (dir = THROTTLE_READ; dir < THROTTLE_MAX; dir++) { + throttle_timer_destroy(&tt->timers[dir]); } } @@ -270,8 +279,12 @@ void throttle_timers_destroy(ThrottleTimers *tt) /* is any throttling timer configured */ bool throttle_timers_are_initialized(ThrottleTimers *tt) { - if (tt->timers[0]) { - return true; + ThrottleDirection dir; + + for (dir = THROTTLE_READ; dir < THROTTLE_MAX; dir++) { + if (tt->timers[dir]) { + return true; + } } return false; @@ -413,19 +426,24 @@ void throttle_get_config(ThrottleState *ts, ThrottleConfig *cfg) * NOTE: this function is not unit tested due to it's usage of timer_mod * * @tt: the timers structure - * @is_write: the type of operation (read/write) + * @direction: throttle direction * @ret: true if the timer has been scheduled else false */ bool throttle_schedule_timer(ThrottleState *ts, ThrottleTimers *tt, - bool is_write) + ThrottleDirection direction) { int64_t now = qemu_clock_get_ns(tt->clock_type); int64_t next_timestamp; + QEMUTimer *timer; bool must_wait; + assert(direction < THROTTLE_MAX); + timer = tt->timers[direction]; + assert(timer); + must_wait = throttle_compute_timer(ts, - is_write, + direction, now, &next_timestamp); @@ -435,48 +453,50 @@ bool throttle_schedule_timer(ThrottleState *ts, } /* request throttled and timer pending -> do nothing */ - if (timer_pending(tt->timers[is_write])) { + if (timer_pending(timer)) { return true; } /* request throttled and timer not pending -> arm timer */ - timer_mod(tt->timers[is_write], next_timestamp); + timer_mod(timer, next_timestamp); return true; } /* do the accounting for this operation * - * @is_write: the type of operation (read/write) + * @direction: throttle direction * @size: the size of the operation */ -void throttle_account(ThrottleState *ts, bool is_write, uint64_t size) +void throttle_account(ThrottleState *ts, ThrottleDirection direction, + uint64_t size) { - const BucketType bucket_types_size[2][2] = { + static const BucketType bucket_types_size[THROTTLE_MAX][2] = { { THROTTLE_BPS_TOTAL, THROTTLE_BPS_READ }, { THROTTLE_BPS_TOTAL, THROTTLE_BPS_WRITE } }; - const BucketType bucket_types_units[2][2] = { + static const BucketType bucket_types_units[THROTTLE_MAX][2] = { { THROTTLE_OPS_TOTAL, THROTTLE_OPS_READ }, { THROTTLE_OPS_TOTAL, THROTTLE_OPS_WRITE } }; double units = 1.0; unsigned i; + assert(direction < THROTTLE_MAX); /* if cfg.op_size is defined and smaller than size we compute unit count */ if (ts->cfg.op_size && size > ts->cfg.op_size) { units = (double) size / ts->cfg.op_size; } - for (i = 0; i < 2; i++) { + for (i = 0; i < ARRAY_SIZE(bucket_types_size[THROTTLE_READ]); i++) { LeakyBucket *bkt; - bkt = &ts->cfg.buckets[bucket_types_size[is_write][i]]; + bkt = &ts->cfg.buckets[bucket_types_size[direction][i]]; bkt->level += size; if (bkt->burst_length > 1) { bkt->burst_level += size; } - bkt = &ts->cfg.buckets[bucket_types_units[is_write][i]]; + bkt = &ts->cfg.buckets[bucket_types_units[direction][i]]; bkt->level += units; if (bkt->burst_length > 1) { bkt->burst_level += units; |