diff options
99 files changed, 2124 insertions, 2911 deletions
diff --git a/.gitlab-ci.d/container-template.yml b/.gitlab-ci.d/container-template.yml index 519b8a9482..9ac4a0ee25 100644 --- a/.gitlab-ci.d/container-template.yml +++ b/.gitlab-ci.d/container-template.yml @@ -3,13 +3,13 @@ image: docker:stable stage: containers services: - - docker:dind + - docker:stable-dind before_script: - export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:latest" - export COMMON_TAG="$CI_REGISTRY/qemu-project/qemu/qemu/$NAME:latest" - apk add python3 - - docker info - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + - until docker info; do sleep 1; done script: - echo "TAG:$TAG" - echo "COMMON_TAG:$COMMON_TAG" diff --git a/.gitlab-ci.d/opensbi.yml b/.gitlab-ci.d/opensbi.yml index 9a651465d8..2e9d51764e 100644 --- a/.gitlab-ci.d/opensbi.yml +++ b/.gitlab-ci.d/opensbi.yml @@ -48,11 +48,9 @@ docker-opensbi: variables: GIT_DEPTH: 3 IMAGE_TAG: $CI_REGISTRY_IMAGE:opensbi-cross-build - # We don't use TLS - DOCKER_HOST: tcp://docker:2375 - DOCKER_TLS_CERTDIR: "" before_script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - until docker info; do sleep 1; done script: - docker pull $IMAGE_TAG || true - docker build --cache-from $IMAGE_TAG --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA diff --git a/.gitlab-ci.d/qemu-project.yml b/.gitlab-ci.d/qemu-project.yml index a7ed447fe4..4d914c4897 100644 --- a/.gitlab-ci.d/qemu-project.yml +++ b/.gitlab-ci.d/qemu-project.yml @@ -1,6 +1,13 @@ # This file contains the set of jobs run by the QEMU project: # https://gitlab.com/qemu-project/qemu/-/pipelines +variables: + RUNNER_TAG: "" + +default: + tags: + - $RUNNER_TAG + include: - local: '/.gitlab-ci.d/base.yml' - local: '/.gitlab-ci.d/stages.yml' diff --git a/MAINTAINERS b/MAINTAINERS index 1b6466496d..1c93ab0ee5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -157,6 +157,9 @@ F: include/exec/helper*.h F: include/sysemu/cpus.h F: include/sysemu/tcg.h F: include/hw/core/tcg-cpu-ops.h +F: host/include/*/host/cpuinfo.h +F: util/cpuinfo-*.c +F: include/tcg/ FPU emulation M: Aurelien Jarno <aurelien@aurel32.net> @@ -2237,6 +2240,7 @@ F: tests/qtest/fuzz-megasas-test.c Network packet abstractions M: Dmitry Fleytman <dmitry.fleytman@gmail.com> +R: Akihiko Odaki <akihiko.odaki@daynix.com> S: Maintained F: include/net/eth.h F: net/eth.c @@ -2279,7 +2283,7 @@ R: Sriram Yagnaraman <sriram.yagnaraman@est.tech> S: Maintained F: docs/system/devices/igb.rst F: hw/net/igb* -F: tests/avocado/igb.py +F: tests/avocado/netdev-ethtool.py F: tests/qtest/igb-test.c F: tests/qtest/libqos/igb.c diff --git a/accel/tcg/atomic_common.c.inc b/accel/tcg/atomic_common.c.inc index fe0eea018f..ee222fd7e7 100644 --- a/accel/tcg/atomic_common.c.inc +++ b/accel/tcg/atomic_common.c.inc @@ -19,20 +19,6 @@ static void atomic_trace_rmw_post(CPUArchState *env, uint64_t addr, qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_RW); } -#if HAVE_ATOMIC128 -static void atomic_trace_ld_post(CPUArchState *env, uint64_t addr, - MemOpIdx oi) -{ - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); -} - -static void atomic_trace_st_post(CPUArchState *env, uint64_t addr, - MemOpIdx oi) -{ - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); -} -#endif - /* * Atomic helpers callable from TCG. * These have a common interface and all defer to cpu_atomic_* @@ -62,36 +48,16 @@ CMPXCHG_HELPER(cmpxchgo_le, Int128) #undef CMPXCHG_HELPER -Int128 HELPER(nonatomic_cmpxchgo_be)(CPUArchState *env, uint64_t addr, - Int128 cmpv, Int128 newv, uint32_t oi) -{ -#if TCG_TARGET_REG_BITS == 32 - uintptr_t ra = GETPC(); - Int128 oldv; - - oldv = cpu_ld16_be_mmu(env, addr, oi, ra); - if (int128_eq(oldv, cmpv)) { - cpu_st16_be_mmu(env, addr, newv, oi, ra); - } else { - /* Even with comparison failure, still need a write cycle. */ - probe_write(env, addr, 16, get_mmuidx(oi), ra); - } - return oldv; -#else - g_assert_not_reached(); -#endif -} - -Int128 HELPER(nonatomic_cmpxchgo_le)(CPUArchState *env, uint64_t addr, - Int128 cmpv, Int128 newv, uint32_t oi) +Int128 HELPER(nonatomic_cmpxchgo)(CPUArchState *env, uint64_t addr, + Int128 cmpv, Int128 newv, uint32_t oi) { #if TCG_TARGET_REG_BITS == 32 uintptr_t ra = GETPC(); Int128 oldv; - oldv = cpu_ld16_le_mmu(env, addr, oi, ra); + oldv = cpu_ld16_mmu(env, addr, oi, ra); if (int128_eq(oldv, cmpv)) { - cpu_st16_le_mmu(env, addr, newv, oi, ra); + cpu_st16_mmu(env, addr, newv, oi, ra); } else { /* Even with comparison failure, still need a write cycle. */ probe_write(env, addr, 16, get_mmuidx(oi), ra); diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index 404a530f7c..e312acd16d 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -73,8 +73,7 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE cmpv, ABI_TYPE newv, MemOpIdx oi, uintptr_t retaddr) { - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, - PAGE_READ | PAGE_WRITE, retaddr); + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); DATA_TYPE ret; #if DATA_SIZE == 16 @@ -87,38 +86,11 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, return ret; } -#if DATA_SIZE >= 16 -#if HAVE_ATOMIC128 -ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) -{ - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, - PAGE_READ, retaddr); - DATA_TYPE val; - - val = atomic16_read(haddr); - ATOMIC_MMU_CLEANUP; - atomic_trace_ld_post(env, addr, oi); - return val; -} - -void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, - MemOpIdx oi, uintptr_t retaddr) -{ - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, - PAGE_WRITE, retaddr); - - atomic16_set(haddr, val); - ATOMIC_MMU_CLEANUP; - atomic_trace_st_post(env, addr, oi); -} -#endif -#else +#if DATA_SIZE < 16 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) { - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, - PAGE_READ | PAGE_WRITE, retaddr); + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); DATA_TYPE ret; ret = qatomic_xchg__nocheck(haddr, val); @@ -131,9 +103,8 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ { \ - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ - PAGE_READ | PAGE_WRITE, retaddr); \ - DATA_TYPE ret; \ + DATA_TYPE *haddr, ret; \ + haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); \ ret = qatomic_##X(haddr, val); \ ATOMIC_MMU_CLEANUP; \ atomic_trace_rmw_post(env, addr, oi); \ @@ -163,9 +134,8 @@ GEN_ATOMIC_HELPER(xor_fetch) ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ { \ - XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ - PAGE_READ | PAGE_WRITE, retaddr); \ - XDATA_TYPE cmp, old, new, val = xval; \ + XDATA_TYPE *haddr, cmp, old, new, val = xval; \ + haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); \ smp_mb(); \ cmp = qatomic_read__nocheck(haddr); \ do { \ @@ -188,7 +158,7 @@ GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) #undef GEN_ATOMIC_HELPER_FN -#endif /* DATA SIZE >= 16 */ +#endif /* DATA SIZE < 16 */ #undef END @@ -206,8 +176,7 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE cmpv, ABI_TYPE newv, MemOpIdx oi, uintptr_t retaddr) { - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, - PAGE_READ | PAGE_WRITE, retaddr); + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); DATA_TYPE ret; #if DATA_SIZE == 16 @@ -220,39 +189,11 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, return BSWAP(ret); } -#if DATA_SIZE >= 16 -#if HAVE_ATOMIC128 -ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr) -{ - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, - PAGE_READ, retaddr); - DATA_TYPE val; - - val = atomic16_read(haddr); - ATOMIC_MMU_CLEANUP; - atomic_trace_ld_post(env, addr, oi); - return BSWAP(val); -} - -void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, - MemOpIdx oi, uintptr_t retaddr) -{ - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, - PAGE_WRITE, retaddr); - - val = BSWAP(val); - atomic16_set(haddr, val); - ATOMIC_MMU_CLEANUP; - atomic_trace_st_post(env, addr, oi); -} -#endif -#else +#if DATA_SIZE < 16 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) { - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, - PAGE_READ | PAGE_WRITE, retaddr); + DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); ABI_TYPE ret; ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); @@ -265,9 +206,8 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ { \ - DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ - PAGE_READ | PAGE_WRITE, retaddr); \ - DATA_TYPE ret; \ + DATA_TYPE *haddr, ret; \ + haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); \ ret = qatomic_##X(haddr, BSWAP(val)); \ ATOMIC_MMU_CLEANUP; \ atomic_trace_rmw_post(env, addr, oi); \ @@ -294,9 +234,8 @@ GEN_ATOMIC_HELPER(xor_fetch) ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ { \ - XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ - PAGE_READ | PAGE_WRITE, retaddr); \ - XDATA_TYPE ldo, ldn, old, new, val = xval; \ + XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval; \ + haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, retaddr); \ smp_mb(); \ ldn = qatomic_read__nocheck(haddr); \ do { \ @@ -326,7 +265,7 @@ GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) #undef ADD #undef GEN_ATOMIC_HELPER_FN -#endif /* DATA_SIZE >= 16 */ +#endif /* DATA_SIZE < 16 */ #undef END #endif /* DATA_SIZE > 1 */ diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index bc0e1c3299..0e741960da 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -307,7 +307,6 @@ static void log_cpu_exec(target_ulong pc, CPUState *cpu, cpu->cpu_index, tb->tc.ptr, tb->cs_base, pc, tb->flags, tb->cflags, lookup_symbol(pc)); -#if defined(DEBUG_DISAS) if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) { FILE *logfile = qemu_log_trylock(); if (logfile) { @@ -323,7 +322,6 @@ static void log_cpu_exec(target_ulong pc, CPUState *cpu, qemu_log_unlock(logfile); } } -#endif /* DEBUG_DISAS */ } } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index ae0fbcdee2..90c72c9940 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -1896,12 +1896,9 @@ static bool mmu_lookup(CPUArchState *env, target_ulong addr, MemOpIdx oi, /* * Probe for an atomic operation. Do not allow unaligned operations, * or io operations to proceed. Return the host address. - * - * @prot may be PAGE_READ, PAGE_WRITE, or PAGE_READ|PAGE_WRITE. */ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, - MemOpIdx oi, int size, int prot, - uintptr_t retaddr) + MemOpIdx oi, int size, uintptr_t retaddr) { uintptr_t mmu_idx = get_mmuidx(oi); MemOp mop = get_memop(oi); @@ -1937,53 +1934,36 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, tlbe = tlb_entry(env, mmu_idx, addr); /* Check TLB entry and enforce page permissions. */ - if (prot & PAGE_WRITE) { - tlb_addr = tlb_addr_write(tlbe); - if (!tlb_hit(tlb_addr, addr)) { - if (!victim_tlb_hit(env, mmu_idx, index, MMU_DATA_STORE, - addr & TARGET_PAGE_MASK)) { - tlb_fill(env_cpu(env), addr, size, - MMU_DATA_STORE, mmu_idx, retaddr); - index = tlb_index(env, mmu_idx, addr); - tlbe = tlb_entry(env, mmu_idx, addr); - } - tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK; + tlb_addr = tlb_addr_write(tlbe); + if (!tlb_hit(tlb_addr, addr)) { + if (!victim_tlb_hit(env, mmu_idx, index, MMU_DATA_STORE, + addr & TARGET_PAGE_MASK)) { + tlb_fill(env_cpu(env), addr, size, + MMU_DATA_STORE, mmu_idx, retaddr); + index = tlb_index(env, mmu_idx, addr); + tlbe = tlb_entry(env, mmu_idx, addr); } + tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK; + } - if (prot & PAGE_READ) { - /* - * Let the guest notice RMW on a write-only page. - * We have just verified that the page is writable. - * Subpage lookups may have left TLB_INVALID_MASK set, - * but addr_read will only be -1 if PAGE_READ was unset. - */ - if (unlikely(tlbe->addr_read == -1)) { - tlb_fill(env_cpu(env), addr, size, - MMU_DATA_LOAD, mmu_idx, retaddr); - /* - * Since we don't support reads and writes to different - * addresses, and we do have the proper page loaded for - * write, this shouldn't ever return. But just in case, - * handle via stop-the-world. - */ - goto stop_the_world; - } - /* Collect TLB_WATCHPOINT for read. */ - tlb_addr |= tlbe->addr_read; - } - } else /* if (prot & PAGE_READ) */ { - tlb_addr = tlbe->addr_read; - if (!tlb_hit(tlb_addr, addr)) { - if (!victim_tlb_hit(env, mmu_idx, index, MMU_DATA_LOAD, - addr & TARGET_PAGE_MASK)) { - tlb_fill(env_cpu(env), addr, size, - MMU_DATA_LOAD, mmu_idx, retaddr); - index = tlb_index(env, mmu_idx, addr); - tlbe = tlb_entry(env, mmu_idx, addr); - } - tlb_addr = tlbe->addr_read & ~TLB_INVALID_MASK; - } + /* + * Let the guest notice RMW on a write-only page. + * We have just verified that the page is writable. + * Subpage lookups may have left TLB_INVALID_MASK set, + * but addr_read will only be -1 if PAGE_READ was unset. + */ + if (unlikely(tlbe->addr_read == -1)) { + tlb_fill(env_cpu(env), addr, size, MMU_DATA_LOAD, mmu_idx, retaddr); + /* + * Since we don't support reads and writes to different + * addresses, and we do have the proper page loaded for + * write, this shouldn't ever return. But just in case, + * handle via stop-the-world. + */ + goto stop_the_world; } + /* Collect TLB_WATCHPOINT for read. */ + tlb_addr |= tlbe->addr_read; /* Notice an IO access or a needs-MMU-lookup access */ if (unlikely(tlb_addr & (TLB_MMIO | TLB_DISCARD_WRITE))) { @@ -2000,11 +1980,8 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, } if (unlikely(tlb_addr & TLB_WATCHPOINT)) { - QEMU_BUILD_BUG_ON(PAGE_READ != BP_MEM_READ); - QEMU_BUILD_BUG_ON(PAGE_WRITE != BP_MEM_WRITE); - /* therefore prot == watchpoint bits */ - cpu_check_watchpoint(env_cpu(env), addr, size, - full->attrs, prot, retaddr); + cpu_check_watchpoint(env_cpu(env), addr, size, full->attrs, + BP_MEM_READ | BP_MEM_WRITE, retaddr); } return hostaddr; @@ -2575,89 +2552,45 @@ uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) return ret; } -uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) -{ - uint16_t ret; - - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUW); - ret = do_ld2_mmu(env, addr, oi, ra, MMU_DATA_LOAD); - plugin_load_cb(env, addr, oi); - return ret; -} - -uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) -{ - uint32_t ret; - - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUL); - ret = do_ld4_mmu(env, addr, oi, ra, MMU_DATA_LOAD); - plugin_load_cb(env, addr, oi); - return ret; -} - -uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) -{ - uint64_t ret; - - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUQ); - ret = do_ld8_mmu(env, addr, oi, ra, MMU_DATA_LOAD); - plugin_load_cb(env, addr, oi); - return ret; -} - -uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) { uint16_t ret; - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUW); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16); ret = do_ld2_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; } -uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) { uint32_t ret; - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUL); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32); ret = do_ld4_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; } -uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) { uint64_t ret; - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUQ); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64); ret = do_ld8_mmu(env, addr, oi, ra, MMU_DATA_LOAD); plugin_load_cb(env, addr, oi); return ret; } -Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) -{ - Int128 ret; - - tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_BE|MO_128)); - ret = do_ld16_mmu(env, addr, oi, ra); - plugin_load_cb(env, addr, oi); - return ret; -} - -Int128 cpu_ld16_le_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) { Int128 ret; - tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_LE|MO_128)); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128); ret = do_ld16_mmu(env, addr, oi, ra); plugin_load_cb(env, addr, oi); return ret; @@ -2779,7 +2712,7 @@ static uint64_t do_st16_leN(CPUArchState *env, MMULookupPageData *p, case MO_ATOM_WITHIN16_PAIR: /* Since size > 8, this is the half that must be atomic. */ - if (!HAVE_al16) { + if (!HAVE_ATOMIC128_RW) { cpu_loop_exit_atomic(env_cpu(env), ra); } return store_whole_le16(p->haddr, p->size, val_le); @@ -3045,66 +2978,34 @@ void cpu_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val, plugin_store_cb(env, addr, oi); } -void cpu_stw_be_mmu(CPUArchState *env, target_ulong addr, uint16_t val, - MemOpIdx oi, uintptr_t retaddr) -{ - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUW); - do_st2_mmu(env, addr, val, oi, retaddr); - plugin_store_cb(env, addr, oi); -} - -void cpu_stl_be_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - MemOpIdx oi, uintptr_t retaddr) -{ - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUL); - do_st4_mmu(env, addr, val, oi, retaddr); - plugin_store_cb(env, addr, oi); -} - -void cpu_stq_be_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - MemOpIdx oi, uintptr_t retaddr) -{ - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_BEUQ); - do_st8_mmu(env, addr, val, oi, retaddr); - plugin_store_cb(env, addr, oi); -} - -void cpu_stw_le_mmu(CPUArchState *env, target_ulong addr, uint16_t val, - MemOpIdx oi, uintptr_t retaddr) +void cpu_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, + MemOpIdx oi, uintptr_t retaddr) { - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUW); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16); do_st2_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } -void cpu_stl_le_mmu(CPUArchState *env, target_ulong addr, uint32_t val, +void cpu_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr) { - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUL); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32); do_st4_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } -void cpu_stq_le_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - MemOpIdx oi, uintptr_t retaddr) +void cpu_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, + MemOpIdx oi, uintptr_t retaddr) { - tcg_debug_assert((get_memop(oi) & (MO_BSWAP | MO_SIZE)) == MO_LEUQ); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64); do_st8_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } -void cpu_st16_be_mmu(CPUArchState *env, target_ulong addr, Int128 val, - MemOpIdx oi, uintptr_t retaddr) -{ - tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_BE|MO_128)); - do_st16_mmu(env, addr, val, oi, retaddr); - plugin_store_cb(env, addr, oi); -} - -void cpu_st16_le_mmu(CPUArchState *env, target_ulong addr, Int128 val, - MemOpIdx oi, uintptr_t retaddr) +void cpu_st16_mmu(CPUArchState *env, target_ulong addr, Int128 val, + MemOpIdx oi, uintptr_t retaddr) { - tcg_debug_assert((get_memop(oi) & (MO_BSWAP|MO_SIZE)) == (MO_LE|MO_128)); + tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128); do_st16_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, oi); } @@ -3137,7 +3038,7 @@ void cpu_st16_le_mmu(CPUArchState *env, target_ulong addr, Int128 val, #include "atomic_template.h" #endif -#if HAVE_CMPXCHG128 || HAVE_ATOMIC128 +#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128) #define DATA_SIZE 16 #include "atomic_template.h" #endif diff --git a/accel/tcg/ldst_atomicity.c.inc b/accel/tcg/ldst_atomicity.c.inc index ba5db7c366..0f6b3f8ab6 100644 --- a/accel/tcg/ldst_atomicity.c.inc +++ b/accel/tcg/ldst_atomicity.c.inc @@ -16,35 +16,6 @@ #endif #define HAVE_al8_fast (ATOMIC_REG_SIZE >= 8) -/* - * If __alignof(unsigned __int128) < 16, GCC may refuse to inline atomics - * that are supported by the host, e.g. s390x. We can force the pointer to - * have our known alignment with __builtin_assume_aligned, however prior to - * GCC 13 that was only reliable with optimization enabled. See - * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107389 - */ -#if defined(CONFIG_ATOMIC128_OPT) -# if !defined(__OPTIMIZE__) -# define ATTRIBUTE_ATOMIC128_OPT __attribute__((optimize("O1"))) -# endif -# define CONFIG_ATOMIC128 -#endif -#ifndef ATTRIBUTE_ATOMIC128_OPT -# define ATTRIBUTE_ATOMIC128_OPT -#endif - -#if defined(CONFIG_ATOMIC128) -# define HAVE_al16_fast true -#else -# define HAVE_al16_fast false -#endif -#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128) -# define HAVE_al16 true -#else -# define HAVE_al16 false -#endif - - /** * required_atomicity: * @@ -164,26 +135,6 @@ static inline uint64_t load_atomic8(void *pv) } /** - * load_atomic16: - * @pv: host address - * - * Atomically load 16 aligned bytes from @pv. - */ -static inline Int128 ATTRIBUTE_ATOMIC128_OPT -load_atomic16(void *pv) -{ -#ifdef CONFIG_ATOMIC128 - __uint128_t *p = __builtin_assume_aligned(pv, 16); - Int128Alias r; - - r.u = qatomic_read__nocheck(p); - return r.s; -#else - qemu_build_not_reached(); -#endif -} - -/** * load_atomic8_or_exit: * @env: cpu context * @ra: host unwind address @@ -228,8 +179,8 @@ static Int128 load_atomic16_or_exit(CPUArchState *env, uintptr_t ra, void *pv) { Int128 *p = __builtin_assume_aligned(pv, 16); - if (HAVE_al16_fast) { - return load_atomic16(p); + if (HAVE_ATOMIC128_RO) { + return atomic16_read_ro(p); } #ifdef CONFIG_USER_ONLY @@ -249,14 +200,9 @@ static Int128 load_atomic16_or_exit(CPUArchState *env, uintptr_t ra, void *pv) * In system mode all guest pages are writable, and for user-only * we have just checked writability. Try cmpxchg. */ -#if defined(CONFIG_CMPXCHG128) - /* Swap 0 with 0, with the side-effect of returning the old value. */ - { - Int128Alias r; - r.u = __sync_val_compare_and_swap_16((__uint128_t *)p, 0, 0); - return r.s; + if (HAVE_ATOMIC128_RW) { + return atomic16_read_rw(p); } -#endif /* Ultimate fallback: re-execute in serial context. */ cpu_loop_exit_atomic(env_cpu(env), ra); @@ -377,11 +323,10 @@ static uint64_t load_atom_extract_al16_or_exit(CPUArchState *env, uintptr_t ra, static inline uint64_t ATTRIBUTE_ATOMIC128_OPT load_atom_extract_al16_or_al8(void *pv, int s) { -#if defined(CONFIG_ATOMIC128) uintptr_t pi = (uintptr_t)pv; int o = pi & 7; int shr = (HOST_BIG_ENDIAN ? 16 - s - o : o) * 8; - __uint128_t r; + Int128 r; pv = (void *)(pi & ~7); if (pi & 8) { @@ -390,18 +335,14 @@ load_atom_extract_al16_or_al8(void *pv, int s) uint64_t b = qatomic_read__nocheck(p8 + 1); if (HOST_BIG_ENDIAN) { - r = ((__uint128_t)a << 64) | b; + r = int128_make128(b, a); } else { - r = ((__uint128_t)b << 64) | a; + r = int128_make128(a, b); } } else { - __uint128_t *p16 = __builtin_assume_aligned(pv, 16, 0); - r = qatomic_read__nocheck(p16); + r = atomic16_read_ro(pv); } - return r >> shr; -#else - qemu_build_not_reached(); -#endif + return int128_getlo(int128_urshift(r, shr)); } /** @@ -489,7 +430,7 @@ static uint16_t load_atom_2(CPUArchState *env, uintptr_t ra, if (likely((pi & 1) == 0)) { return load_atomic2(pv); } - if (HAVE_al16_fast) { + if (HAVE_ATOMIC128_RO) { return load_atom_extract_al16_or_al8(pv, 2); } @@ -528,7 +469,7 @@ static uint32_t load_atom_4(CPUArchState *env, uintptr_t ra, if (likely((pi & 3) == 0)) { return load_atomic4(pv); } - if (HAVE_al16_fast) { + if (HAVE_ATOMIC128_RO) { return load_atom_extract_al16_or_al8(pv, 4); } @@ -574,7 +515,7 @@ static uint64_t load_atom_8(CPUArchState *env, uintptr_t ra, if (HAVE_al8 && likely((pi & 7) == 0)) { return load_atomic8(pv); } - if (HAVE_al16_fast) { + if (HAVE_ATOMIC128_RO) { return load_atom_extract_al16_or_al8(pv, 8); } @@ -624,8 +565,8 @@ static Int128 load_atom_16(CPUArchState *env, uintptr_t ra, * If the host does not support 16-byte atomics, wait until we have * examined the atomicity parameters below. */ - if (HAVE_al16_fast && likely((pi & 15) == 0)) { - return load_atomic16(pv); + if (HAVE_ATOMIC128_RO && likely((pi & 15) == 0)) { + return atomic16_read_ro(pv); } atmax = required_atomicity(env, pi, memop); @@ -705,36 +646,6 @@ static inline void store_atomic8(void *pv, uint64_t val) } /** - * store_atomic16: - * @pv: host address - * @val: value to store - * - * Atomically store 16 aligned bytes to @pv. - */ -static inline void ATTRIBUTE_ATOMIC128_OPT -store_atomic16(void *pv, Int128Alias val) -{ -#if defined(CONFIG_ATOMIC128) - __uint128_t *pu = __builtin_assume_aligned(pv, 16); - qatomic_set__nocheck(pu, val.u); -#elif defined(CONFIG_CMPXCHG128) - __uint128_t *pu = __builtin_assume_aligned(pv, 16); - __uint128_t o; - - /* - * Without CONFIG_ATOMIC128, __atomic_compare_exchange_n will always - * defer to libatomic, so we must use __sync_*_compare_and_swap_16 - * and accept the sequential consistency that comes with it. - */ - do { - o = *pu; - } while (!__sync_bool_compare_and_swap_16(pu, o, val.u)); -#else - qemu_build_not_reached(); -#endif -} - -/** * store_atom_4x2 */ static inline void store_atom_4_by_2(void *pv, uint32_t val) @@ -974,7 +885,7 @@ static uint64_t store_whole_le16(void *pv, int size, Int128 val_le) int sh = o * 8; Int128 m, v; - qemu_build_assert(HAVE_al16); + qemu_build_assert(HAVE_ATOMIC128_RW); /* Like MAKE_64BIT_MASK(0, sz), but larger. */ if (sz <= 64) { @@ -1034,7 +945,7 @@ static void store_atom_2(CPUArchState *env, uintptr_t ra, return; } } else if ((pi & 15) == 7) { - if (HAVE_al16) { + if (HAVE_ATOMIC128_RW) { Int128 v = int128_lshift(int128_make64(val), 56); Int128 m = int128_lshift(int128_make64(0xffff), 56); store_atom_insert_al16(pv - 7, v, m); @@ -1103,7 +1014,7 @@ static void store_atom_4(CPUArchState *env, uintptr_t ra, return; } } else { - if (HAVE_al16) { + if (HAVE_ATOMIC128_RW) { store_whole_le16(pv, 4, int128_make64(cpu_to_le32(val))); return; } @@ -1168,7 +1079,7 @@ static void store_atom_8(CPUArchState *env, uintptr_t ra, } break; case MO_64: - if (HAVE_al16) { + if (HAVE_ATOMIC128_RW) { store_whole_le16(pv, 8, int128_make64(cpu_to_le64(val))); return; } @@ -1194,8 +1105,8 @@ static void store_atom_16(CPUArchState *env, uintptr_t ra, uint64_t a, b; int atmax; - if (HAVE_al16_fast && likely((pi & 15) == 0)) { - store_atomic16(pv, val); + if (HAVE_ATOMIC128_RW && likely((pi & 15) == 0)) { + atomic16_set(pv, val); return; } @@ -1223,7 +1134,7 @@ static void store_atom_16(CPUArchState *env, uintptr_t ra, } break; case -MO_64: - if (HAVE_al16) { + if (HAVE_ATOMIC128_RW) { uint64_t val_le; int s2 = pi & 15; int s1 = 16 - s2; @@ -1250,8 +1161,8 @@ static void store_atom_16(CPUArchState *env, uintptr_t ra, } break; case MO_128: - if (HAVE_al16) { - store_atomic16(pv, val); + if (HAVE_ATOMIC128_RW) { + atomic16_set(pv, val); return; } break; diff --git a/accel/tcg/ldst_common.c.inc b/accel/tcg/ldst_common.c.inc index 6ac8d871a3..5f8144b33a 100644 --- a/accel/tcg/ldst_common.c.inc +++ b/accel/tcg/ldst_common.c.inc @@ -26,7 +26,7 @@ uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx); - return cpu_ldw_be_mmu(env, addr, oi, ra); + return cpu_ldw_mmu(env, addr, oi, ra); } int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, @@ -39,21 +39,21 @@ uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx); - return cpu_ldl_be_mmu(env, addr, oi, ra); + return cpu_ldl_mmu(env, addr, oi, ra); } uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx); - return cpu_ldq_be_mmu(env, addr, oi, ra); + return cpu_ldq_mmu(env, addr, oi, ra); } uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx); - return cpu_ldw_le_mmu(env, addr, oi, ra); + return cpu_ldw_mmu(env, addr, oi, ra); } int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, @@ -66,14 +66,14 @@ uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx); - return cpu_ldl_le_mmu(env, addr, oi, ra); + return cpu_ldl_mmu(env, addr, oi, ra); } uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx); - return cpu_ldq_le_mmu(env, addr, oi, ra); + return cpu_ldq_mmu(env, addr, oi, ra); } void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, @@ -87,42 +87,42 @@ void cpu_stw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx); - cpu_stw_be_mmu(env, addr, val, oi, ra); + cpu_stw_mmu(env, addr, val, oi, ra); } void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx); - cpu_stl_be_mmu(env, addr, val, oi, ra); + cpu_stl_mmu(env, addr, val, oi, ra); } void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx); - cpu_stq_be_mmu(env, addr, val, oi, ra); + cpu_stq_mmu(env, addr, val, oi, ra); } void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx); - cpu_stw_le_mmu(env, addr, val, oi, ra); + cpu_stw_mmu(env, addr, val, oi, ra); } void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx); - cpu_stl_le_mmu(env, addr, val, oi, ra); + cpu_stl_mmu(env, addr, val, oi, ra); } void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val, int mmu_idx, uintptr_t ra) { MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx); - cpu_stq_le_mmu(env, addr, val, oi, ra); + cpu_stq_mmu(env, addr, val, oi, ra); } /*--------------------------*/ diff --git a/accel/tcg/tcg-runtime.h b/accel/tcg/tcg-runtime.h index 6f8c2061d0..39e68007f9 100644 --- a/accel/tcg/tcg-runtime.h +++ b/accel/tcg/tcg-runtime.h @@ -65,9 +65,7 @@ DEF_HELPER_FLAGS_5(atomic_cmpxchgo_le, TCG_CALL_NO_WG, i128, env, i64, i128, i128, i32) #endif -DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo_be, TCG_CALL_NO_WG, - i128, env, i64, i128, i128, i32) -DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo_le, TCG_CALL_NO_WG, +DEF_HELPER_FLAGS_5(nonatomic_cmpxchgo, TCG_CALL_NO_WG, i128, env, i64, i128, i128, i32) #ifdef CONFIG_ATOMIC64 diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 353849ca6d..c87648b99e 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -432,7 +432,6 @@ TranslationBlock *tb_gen_code(CPUState *cpu, qatomic_set(&prof->search_out_len, prof->search_out_len + search_size); #endif -#ifdef DEBUG_DISAS if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && qemu_log_in_addr_range(pc)) { FILE *logfile = qemu_log_trylock(); @@ -505,7 +504,6 @@ TranslationBlock *tb_gen_code(CPUState *cpu, qemu_log_unlock(logfile); } } -#endif qatomic_set(&tcg_ctx->code_gen_ptr, (void *) ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index 7bda43ff61..6120ef2a92 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -122,7 +122,6 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns, tb->size = db->pc_next - db->pc_first; tb->icount = db->num_insns; -#ifdef DEBUG_DISAS if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM) && qemu_log_in_addr_range(db->pc_first)) { FILE *logfile = qemu_log_trylock(); @@ -133,7 +132,6 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns, qemu_log_unlock(logfile); } } -#endif } static void *translator_access(CPUArchState *env, DisasContextBase *db, diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 36ad8284a5..dc8d6b5d40 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -940,8 +940,8 @@ uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, return ret; } -static uint16_t do_ld2_he_mmu(CPUArchState *env, abi_ptr addr, - MemOp mop, uintptr_t ra) +static uint16_t do_ld2_mmu(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra) { void *haddr; uint16_t ret; @@ -950,14 +950,6 @@ static uint16_t do_ld2_he_mmu(CPUArchState *env, abi_ptr addr, haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD); ret = load_atom_2(env, ra, haddr, mop); clear_helper_retaddr(); - return ret; -} - -tcg_target_ulong helper_lduw_mmu(CPUArchState *env, uint64_t addr, - MemOpIdx oi, uintptr_t ra) -{ - MemOp mop = get_memop(oi); - uint16_t ret = do_ld2_he_mmu(env, addr, mop, ra); if (mop & MO_BSWAP) { ret = bswap16(ret); @@ -965,44 +957,28 @@ tcg_target_ulong helper_lduw_mmu(CPUArchState *env, uint64_t addr, return ret; } -tcg_target_ulong helper_ldsw_mmu(CPUArchState *env, uint64_t addr, +tcg_target_ulong helper_lduw_mmu(CPUArchState *env, uint64_t addr, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - int16_t ret = do_ld2_he_mmu(env, addr, mop, ra); - - if (mop & MO_BSWAP) { - ret = bswap16(ret); - } - return ret; + return do_ld2_mmu(env, addr, get_memop(oi), ra); } -uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +tcg_target_ulong helper_ldsw_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - uint16_t ret; - - tcg_debug_assert((mop & MO_BSWAP) == MO_BE); - ret = do_ld2_he_mmu(env, addr, mop, ra); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - return cpu_to_be16(ret); + return (int16_t)do_ld2_mmu(env, addr, get_memop(oi), ra); } -uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - uint16_t ret; - - tcg_debug_assert((mop & MO_BSWAP) == MO_LE); - ret = do_ld2_he_mmu(env, addr, mop, ra); + uint16_t ret = do_ld2_mmu(env, addr, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - return cpu_to_le16(ret); + return ret; } -static uint32_t do_ld4_he_mmu(CPUArchState *env, abi_ptr addr, - MemOp mop, uintptr_t ra) +static uint32_t do_ld4_mmu(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra) { void *haddr; uint32_t ret; @@ -1011,14 +987,6 @@ static uint32_t do_ld4_he_mmu(CPUArchState *env, abi_ptr addr, haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD); ret = load_atom_4(env, ra, haddr, mop); clear_helper_retaddr(); - return ret; -} - -tcg_target_ulong helper_ldul_mmu(CPUArchState *env, uint64_t addr, - MemOpIdx oi, uintptr_t ra) -{ - MemOp mop = get_memop(oi); - uint32_t ret = do_ld4_he_mmu(env, addr, mop, ra); if (mop & MO_BSWAP) { ret = bswap32(ret); @@ -1026,44 +994,28 @@ tcg_target_ulong helper_ldul_mmu(CPUArchState *env, uint64_t addr, return ret; } -tcg_target_ulong helper_ldsl_mmu(CPUArchState *env, uint64_t addr, +tcg_target_ulong helper_ldul_mmu(CPUArchState *env, uint64_t addr, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - int32_t ret = do_ld4_he_mmu(env, addr, mop, ra); - - if (mop & MO_BSWAP) { - ret = bswap32(ret); - } - return ret; + return do_ld4_mmu(env, addr, get_memop(oi), ra); } -uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +tcg_target_ulong helper_ldsl_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - uint32_t ret; - - tcg_debug_assert((mop & MO_BSWAP) == MO_BE); - ret = do_ld4_he_mmu(env, addr, mop, ra); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - return cpu_to_be32(ret); + return (int32_t)do_ld4_mmu(env, addr, get_memop(oi), ra); } -uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - uint32_t ret; - - tcg_debug_assert((mop & MO_BSWAP) == MO_LE); - ret = do_ld4_he_mmu(env, addr, mop, ra); + uint32_t ret = do_ld4_mmu(env, addr, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - return cpu_to_le32(ret); + return ret; } -static uint64_t do_ld8_he_mmu(CPUArchState *env, abi_ptr addr, - MemOp mop, uintptr_t ra) +static uint64_t do_ld8_mmu(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra) { void *haddr; uint64_t ret; @@ -1072,14 +1024,6 @@ static uint64_t do_ld8_he_mmu(CPUArchState *env, abi_ptr addr, haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD); ret = load_atom_8(env, ra, haddr, mop); clear_helper_retaddr(); - return ret; -} - -uint64_t helper_ldq_mmu(CPUArchState *env, uint64_t addr, - MemOpIdx oi, uintptr_t ra) -{ - MemOp mop = get_memop(oi); - uint64_t ret = do_ld8_he_mmu(env, addr, mop, ra); if (mop & MO_BSWAP) { ret = bswap64(ret); @@ -1087,32 +1031,22 @@ uint64_t helper_ldq_mmu(CPUArchState *env, uint64_t addr, return ret; } -uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr, +uint64_t helper_ldq_mmu(CPUArchState *env, uint64_t addr, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - uint64_t ret; - - tcg_debug_assert((mop & MO_BSWAP) == MO_BE); - ret = do_ld8_he_mmu(env, addr, mop, ra); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - return cpu_to_be64(ret); + return do_ld8_mmu(env, addr, get_memop(oi), ra); } -uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - uint64_t ret; - - tcg_debug_assert((mop & MO_BSWAP) == MO_LE); - ret = do_ld8_he_mmu(env, addr, mop, ra); + uint64_t ret = do_ld8_mmu(env, addr, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - return cpu_to_le64(ret); + return ret; } -static Int128 do_ld16_he_mmu(CPUArchState *env, abi_ptr addr, - MemOp mop, uintptr_t ra) +static Int128 do_ld16_mmu(CPUArchState *env, abi_ptr addr, + MemOp mop, uintptr_t ra) { void *haddr; Int128 ret; @@ -1121,14 +1055,6 @@ static Int128 do_ld16_he_mmu(CPUArchState *env, abi_ptr addr, haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_LOAD); ret = load_atom_16(env, ra, haddr, mop); clear_helper_retaddr(); - return ret; -} - -Int128 helper_ld16_mmu(CPUArchState *env, uint64_t addr, - MemOpIdx oi, uintptr_t ra) -{ - MemOp mop = get_memop(oi); - Int128 ret = do_ld16_he_mmu(env, addr, mop, ra); if (mop & MO_BSWAP) { ret = bswap128(ret); @@ -1136,38 +1062,22 @@ Int128 helper_ld16_mmu(CPUArchState *env, uint64_t addr, return ret; } -Int128 helper_ld_i128(CPUArchState *env, uint64_t addr, MemOpIdx oi) +Int128 helper_ld16_mmu(CPUArchState *env, uint64_t addr, + MemOpIdx oi, uintptr_t ra) { - return helper_ld16_mmu(env, addr, oi, GETPC()); + return do_ld16_mmu(env, addr, get_memop(oi), ra); } -Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +Int128 helper_ld_i128(CPUArchState *env, uint64_t addr, MemOpIdx oi) { - MemOp mop = get_memop(oi); - Int128 ret; - - tcg_debug_assert((mop & MO_BSWAP) == MO_BE); - ret = do_ld16_he_mmu(env, addr, mop, ra); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - if (!HOST_BIG_ENDIAN) { - ret = bswap128(ret); - } - return ret; + return helper_ld16_mmu(env, addr, oi, GETPC()); } -Int128 cpu_ld16_le_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra) +Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr, + MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - Int128 ret; - - tcg_debug_assert((mop & MO_BSWAP) == MO_LE); - ret = do_ld16_he_mmu(env, addr, mop, ra); + Int128 ret = do_ld16_mmu(env, addr, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); - if (HOST_BIG_ENDIAN) { - ret = bswap128(ret); - } return ret; } @@ -1195,139 +1105,101 @@ void cpu_stb_mmu(CPUArchState *env, abi_ptr addr, uint8_t val, qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } -static void do_st2_he_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, - MemOp mop, uintptr_t ra) +static void do_st2_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, + MemOp mop, uintptr_t ra) { void *haddr; tcg_debug_assert((mop & MO_SIZE) == MO_16); haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE); - store_atom_2(env, ra, haddr, mop, val); - clear_helper_retaddr(); -} - -void helper_stw_mmu(CPUArchState *env, uint64_t addr, uint32_t val, - MemOpIdx oi, uintptr_t ra) -{ - MemOp mop = get_memop(oi); if (mop & MO_BSWAP) { val = bswap16(val); } - do_st2_he_mmu(env, addr, val, mop, ra); + store_atom_2(env, ra, haddr, mop, val); + clear_helper_retaddr(); } -void cpu_stw_be_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, +void helper_stw_mmu(CPUArchState *env, uint64_t addr, uint32_t val, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - - tcg_debug_assert((mop & MO_BSWAP) == MO_BE); - do_st2_he_mmu(env, addr, be16_to_cpu(val), mop, ra); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); + do_st2_mmu(env, addr, val, get_memop(oi), ra); } -void cpu_stw_le_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, +void cpu_stw_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - - tcg_debug_assert((mop & MO_BSWAP) == MO_LE); - do_st2_he_mmu(env, addr, le16_to_cpu(val), mop, ra); + do_st2_mmu(env, addr, val, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } -static void do_st4_he_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, - MemOp mop, uintptr_t ra) +static void do_st4_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, + MemOp mop, uintptr_t ra) { void *haddr; tcg_debug_assert((mop & MO_SIZE) == MO_32); haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE); - store_atom_4(env, ra, haddr, mop, val); - clear_helper_retaddr(); -} - -void helper_stl_mmu(CPUArchState *env, uint64_t addr, uint32_t val, - MemOpIdx oi, uintptr_t ra) -{ - MemOp mop = get_memop(oi); if (mop & MO_BSWAP) { val = bswap32(val); } - do_st4_he_mmu(env, addr, val, mop, ra); + store_atom_4(env, ra, haddr, mop, val); + clear_helper_retaddr(); } -void cpu_stl_be_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, +void helper_stl_mmu(CPUArchState *env, uint64_t addr, uint32_t val, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - - tcg_debug_assert((mop & MO_BSWAP) == MO_BE); - do_st4_he_mmu(env, addr, be32_to_cpu(val), mop, ra); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); + do_st4_mmu(env, addr, val, get_memop(oi), ra); } -void cpu_stl_le_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, - MemOpIdx oi, uintptr_t ra) +void cpu_stl_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, + MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - - tcg_debug_assert((mop & MO_BSWAP) == MO_LE); - do_st4_he_mmu(env, addr, le32_to_cpu(val), mop, ra); + do_st4_mmu(env, addr, val, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } -static void do_st8_he_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, - MemOp mop, uintptr_t ra) +static void do_st8_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, + MemOp mop, uintptr_t ra) { void *haddr; tcg_debug_assert((mop & MO_SIZE) == MO_64); haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE); - store_atom_8(env, ra, haddr, mop, val); - clear_helper_retaddr(); -} - -void helper_stq_mmu(CPUArchState *env, uint64_t addr, uint64_t val, - MemOpIdx oi, uintptr_t ra) -{ - MemOp mop = get_memop(oi); if (mop & MO_BSWAP) { val = bswap64(val); } - do_st8_he_mmu(env, addr, val, mop, ra); + store_atom_8(env, ra, haddr, mop, val); + clear_helper_retaddr(); } -void cpu_stq_be_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, +void helper_stq_mmu(CPUArchState *env, uint64_t addr, uint64_t val, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - - tcg_debug_assert((mop & MO_BSWAP) == MO_BE); - do_st8_he_mmu(env, addr, cpu_to_be64(val), mop, ra); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); + do_st8_mmu(env, addr, val, get_memop(oi), ra); } -void cpu_stq_le_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, +void cpu_stq_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - - tcg_debug_assert((mop & MO_BSWAP) == MO_LE); - do_st8_he_mmu(env, addr, cpu_to_le64(val), mop, ra); + do_st8_mmu(env, addr, val, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } -static void do_st16_he_mmu(CPUArchState *env, abi_ptr addr, Int128 val, - MemOp mop, uintptr_t ra) +static void do_st16_mmu(CPUArchState *env, abi_ptr addr, Int128 val, + MemOp mop, uintptr_t ra) { void *haddr; tcg_debug_assert((mop & MO_SIZE) == MO_128); haddr = cpu_mmu_lookup(env, addr, mop, ra, MMU_DATA_STORE); + + if (mop & MO_BSWAP) { + val = bswap128(val); + } store_atom_16(env, ra, haddr, mop, val); clear_helper_retaddr(); } @@ -1335,12 +1207,7 @@ static void do_st16_he_mmu(CPUArchState *env, abi_ptr addr, Int128 val, void helper_st16_mmu(CPUArchState *env, uint64_t addr, Int128 val, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - - if (mop & MO_BSWAP) { - val = bswap128(val); - } - do_st16_he_mmu(env, addr, val, mop, ra); + do_st16_mmu(env, addr, val, get_memop(oi), ra); } void helper_st_i128(CPUArchState *env, uint64_t addr, Int128 val, MemOpIdx oi) @@ -1348,29 +1215,10 @@ void helper_st_i128(CPUArchState *env, uint64_t addr, Int128 val, MemOpIdx oi) helper_st16_mmu(env, addr, val, oi, GETPC()); } -void cpu_st16_be_mmu(CPUArchState *env, abi_ptr addr, - Int128 val, MemOpIdx oi, uintptr_t ra) +void cpu_st16_mmu(CPUArchState *env, abi_ptr addr, + Int128 val, MemOpIdx oi, uintptr_t ra) { - MemOp mop = get_memop(oi); - - tcg_debug_assert((mop & MO_BSWAP) == MO_BE); - if (!HOST_BIG_ENDIAN) { - val = bswap128(val); - } - do_st16_he_mmu(env, addr, val, mop, ra); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); -} - -void cpu_st16_le_mmu(CPUArchState *env, abi_ptr addr, - Int128 val, MemOpIdx oi, uintptr_t ra) -{ - MemOp mop = get_memop(oi); - - tcg_debug_assert((mop & MO_BSWAP) == MO_LE); - if (HOST_BIG_ENDIAN) { - val = bswap128(val); - } - do_st16_he_mmu(env, addr, val, mop, ra); + do_st16_mmu(env, addr, val, get_memop(oi), ra); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } @@ -1475,12 +1323,9 @@ uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr, /* * Do not allow unaligned operations to proceed. Return the host address. - * - * @prot may be PAGE_READ, PAGE_WRITE, or PAGE_READ|PAGE_WRITE. */ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, - MemOpIdx oi, int size, int prot, - uintptr_t retaddr) + MemOpIdx oi, int size, uintptr_t retaddr) { MemOp mop = get_memop(oi); int a_bits = get_alignment_bits(mop); @@ -1488,8 +1333,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, /* Enforce guest required alignment. */ if (unlikely(addr & ((1 << a_bits) - 1))) { - MMUAccessType t = prot == PAGE_READ ? MMU_DATA_LOAD : MMU_DATA_STORE; - cpu_loop_exit_sigbus(env_cpu(env), addr, t, retaddr); + cpu_loop_exit_sigbus(env_cpu(env), addr, MMU_DATA_STORE, retaddr); } /* Enforce qemu required alignment. */ @@ -1527,7 +1371,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, #include "atomic_template.h" #endif -#if HAVE_ATOMIC128 || HAVE_CMPXCHG128 +#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128) #define DATA_SIZE 16 #include "atomic_template.h" #endif diff --git a/backends/hostmem-file.c b/backends/hostmem-file.c index 25141283c4..38ea65bec5 100644 --- a/backends/hostmem-file.c +++ b/backends/hostmem-file.c @@ -27,6 +27,7 @@ struct HostMemoryBackendFile { char *mem_path; uint64_t align; + uint64_t offset; bool discard_data; bool is_pmem; bool readonly; @@ -58,7 +59,8 @@ file_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) ram_flags |= fb->is_pmem ? RAM_PMEM : 0; memory_region_init_ram_from_file(&backend->mr, OBJECT(backend), name, backend->size, fb->align, ram_flags, - fb->mem_path, fb->readonly, errp); + fb->mem_path, fb->offset, fb->readonly, + errp); g_free(name); #endif } @@ -125,6 +127,36 @@ static void file_memory_backend_set_align(Object *o, Visitor *v, fb->align = val; } +static void file_memory_backend_get_offset(Object *o, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(o); + uint64_t val = fb->offset; + + visit_type_size(v, name, &val, errp); +} + +static void file_memory_backend_set_offset(Object *o, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + HostMemoryBackend *backend = MEMORY_BACKEND(o); + HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(o); + uint64_t val; + + if (host_memory_backend_mr_inited(backend)) { + error_setg(errp, "cannot change property '%s' of %s", name, + object_get_typename(o)); + return; + } + + if (!visit_type_size(v, name, &val, errp)) { + return; + } + fb->offset = val; +} + #ifdef CONFIG_LIBPMEM static bool file_memory_backend_get_pmem(Object *o, Error **errp) { @@ -197,6 +229,12 @@ file_backend_class_init(ObjectClass *oc, void *data) file_memory_backend_get_align, file_memory_backend_set_align, NULL, NULL); + object_class_property_add(oc, "offset", "int", + file_memory_backend_get_offset, + file_memory_backend_set_offset, + NULL, NULL); + object_class_property_set_description(oc, "offset", + "Offset into the target file (ex: 1G)"); #ifdef CONFIG_LIBPMEM object_class_property_add_bool(oc, "pmem", file_memory_backend_get_pmem, file_memory_backend_set_pmem); diff --git a/docs/system/devices/igb.rst b/docs/system/devices/igb.rst index 0bcdd85747..04e79dfe54 100644 --- a/docs/system/devices/igb.rst +++ b/docs/system/devices/igb.rst @@ -14,7 +14,8 @@ Limitations =========== This igb implementation was tested with Linux Test Project [2]_ and Windows HLK -[3]_ during the initial development. The command used when testing with LTP is: +[3]_ during the initial development. Later it was also tested with DPDK Test +Suite [4]_. The command used when testing with LTP is: .. code-block:: shell @@ -22,8 +23,8 @@ This igb implementation was tested with Linux Test Project [2]_ and Windows HLK Be aware that this implementation lacks many functionalities available with the actual hardware, and you may experience various failures if you try to use it -with a different operating system other than Linux and Windows or if you try -functionalities not covered by the tests. +with a different operating system other than DPDK, Linux, and Windows or if you +try functionalities not covered by the tests. Using igb ========= @@ -32,7 +33,7 @@ Using igb should be nothing different from using another network device. See :ref:`Network_emulation` in general. However, you may also need to perform additional steps to activate SR-IOV -feature on your guest. For Linux, refer to [4]_. +feature on your guest. For Linux, refer to [5]_. Developing igb ============== @@ -60,7 +61,7 @@ Avocado test and can be ran with the following command: .. code:: shell - make check-avocado AVOCADO_TESTS=tests/avocado/igb.py + make check-avocado AVOCADO_TESTS=tests/avocado/netdev-ethtool.py References ========== @@ -68,4 +69,5 @@ References .. [1] https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82576eb-gigabit-ethernet-controller-datasheet.pdf .. [2] https://github.com/linux-test-project/ltp .. [3] https://learn.microsoft.com/en-us/windows-hardware/test/hlk/ -.. [4] https://docs.kernel.org/PCI/pci-iov-howto.html +.. [4] https://doc.dpdk.org/dts/gsg/ +.. [5] https://docs.kernel.org/PCI/pci-iov-howto.html diff --git a/host/include/aarch64/host/atomic128-cas.h b/host/include/aarch64/host/atomic128-cas.h new file mode 100644 index 0000000000..58630107bc --- /dev/null +++ b/host/include/aarch64/host/atomic128-cas.h @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * Compare-and-swap for 128-bit atomic operations, AArch64 version. + * + * Copyright (C) 2018, 2023 Linaro, Ltd. + * + * See docs/devel/atomics.rst for discussion about the guarantees each + * atomic primitive is meant to provide. + */ + +#ifndef AARCH64_ATOMIC128_CAS_H +#define AARCH64_ATOMIC128_CAS_H + +/* Through gcc 10, aarch64 has no support for 128-bit atomics. */ +#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128) +#include "host/include/generic/host/atomic128-cas.h" +#else +static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) +{ + uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp); + uint64_t newl = int128_getlo(new), newh = int128_gethi(new); + uint64_t oldl, oldh; + uint32_t tmp; + + asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t" + "cmp %[oldl], %[cmpl]\n\t" + "ccmp %[oldh], %[cmph], #0, eq\n\t" + "b.ne 1f\n\t" + "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t" + "cbnz %w[tmp], 0b\n" + "1:" + : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), + [oldl] "=&r"(oldl), [oldh] "=&r"(oldh) + : [cmpl] "r"(cmpl), [cmph] "r"(cmph), + [newl] "r"(newl), [newh] "r"(newh) + : "memory", "cc"); + + return int128_make128(oldl, oldh); +} + +# define CONFIG_CMPXCHG128 1 +# define HAVE_CMPXCHG128 1 +#endif + +#endif /* AARCH64_ATOMIC128_CAS_H */ diff --git a/host/include/aarch64/host/atomic128-ldst.h b/host/include/aarch64/host/atomic128-ldst.h new file mode 100644 index 0000000000..a08f62c40a --- /dev/null +++ b/host/include/aarch64/host/atomic128-ldst.h @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * Load/store for 128-bit atomic operations, AArch64 version. + * + * Copyright (C) 2018, 2023 Linaro, Ltd. + * + * See docs/devel/atomics.rst for discussion about the guarantees each + * atomic primitive is meant to provide. + */ + +#ifndef AARCH64_ATOMIC128_LDST_H +#define AARCH64_ATOMIC128_LDST_H + +#include "host/cpuinfo.h" +#include "tcg/debug-assert.h" + +/* + * Through gcc 10, aarch64 has no support for 128-bit atomics. + * Through clang 16, without -march=armv8.4-a, __atomic_load_16 + * is incorrectly expanded to a read-write operation. + * + * Anyway, this method allows runtime detection of FEAT_LSE2. + */ + +#define HAVE_ATOMIC128_RO (cpuinfo & CPUINFO_LSE2) +#define HAVE_ATOMIC128_RW 1 + +static inline Int128 atomic16_read_ro(const Int128 *ptr) +{ + uint64_t l, h; + + tcg_debug_assert(HAVE_ATOMIC128_RO); + /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */ + asm("ldp %[l], %[h], %[mem]" + : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr)); + + return int128_make128(l, h); +} + +static inline Int128 atomic16_read_rw(Int128 *ptr) +{ + uint64_t l, h; + uint32_t tmp; + + if (cpuinfo & CPUINFO_LSE2) { + /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */ + asm("ldp %[l], %[h], %[mem]" + : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr)); + } else { + /* The load must be paired with the store to guarantee not tearing. */ + asm("0: ldxp %[l], %[h], %[mem]\n\t" + "stxp %w[tmp], %[l], %[h], %[mem]\n\t" + "cbnz %w[tmp], 0b" + : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), [l] "=&r"(l), [h] "=&r"(h)); + } + + return int128_make128(l, h); +} + +static inline void atomic16_set(Int128 *ptr, Int128 val) +{ + uint64_t l = int128_getlo(val), h = int128_gethi(val); + uint64_t t1, t2; + + if (cpuinfo & CPUINFO_LSE2) { + /* With FEAT_LSE2, 16-byte aligned STP is atomic. */ + asm("stp %[l], %[h], %[mem]" + : [mem] "=m"(*ptr) : [l] "r"(l), [h] "r"(h)); + } else { + /* Load into temporaries to acquire the exclusive access lock. */ + asm("0: ldxp %[t1], %[t2], %[mem]\n\t" + "stxp %w[t1], %[l], %[h], %[mem]\n\t" + "cbnz %w[t1], 0b" + : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2) + : [l] "r"(l), [h] "r"(h)); + } +} + +#endif /* AARCH64_ATOMIC128_LDST_H */ diff --git a/host/include/aarch64/host/cpuinfo.h b/host/include/aarch64/host/cpuinfo.h new file mode 100644 index 0000000000..82227890b4 --- /dev/null +++ b/host/include/aarch64/host/cpuinfo.h @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * Host specific cpu indentification for AArch64. + */ + +#ifndef HOST_CPUINFO_H +#define HOST_CPUINFO_H + +#define CPUINFO_ALWAYS (1u << 0) /* so cpuinfo is nonzero */ +#define CPUINFO_LSE (1u << 1) +#define CPUINFO_LSE2 (1u << 2) + +/* Initialized with a constructor. */ +extern unsigned cpuinfo; + +/* + * We cannot rely on constructor ordering, so other constructors must + * use the function interface rather than the variable above. + */ +unsigned cpuinfo_init(void); + +#endif /* HOST_CPUINFO_H */ diff --git a/host/include/generic/host/atomic128-cas.h b/host/include/generic/host/atomic128-cas.h new file mode 100644 index 0000000000..991d3da082 --- /dev/null +++ b/host/include/generic/host/atomic128-cas.h @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * Compare-and-swap for 128-bit atomic operations, generic version. + * + * Copyright (C) 2018, 2023 Linaro, Ltd. + * + * See docs/devel/atomics.rst for discussion about the guarantees each + * atomic primitive is meant to provide. + */ + +#ifndef HOST_ATOMIC128_CAS_H +#define HOST_ATOMIC128_CAS_H + +#if defined(CONFIG_ATOMIC128) +static inline Int128 ATTRIBUTE_ATOMIC128_OPT +atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) +{ + __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16); + Int128Alias r, c, n; + + c.s = cmp; + n.s = new; + r.i = qatomic_cmpxchg__nocheck(ptr_align, c.i, n.i); + return r.s; +} +# define HAVE_CMPXCHG128 1 +#elif defined(CONFIG_CMPXCHG128) +static inline Int128 ATTRIBUTE_ATOMIC128_OPT +atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) +{ + __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16); + Int128Alias r, c, n; + + c.s = cmp; + n.s = new; + r.i = __sync_val_compare_and_swap_16(ptr_align, c.i, n.i); + return r.s; +} +# define HAVE_CMPXCHG128 1 +#else +/* Fallback definition that must be optimized away, or error. */ +Int128 QEMU_ERROR("unsupported atomic") + atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new); +# define HAVE_CMPXCHG128 0 +#endif + +#endif /* HOST_ATOMIC128_CAS_H */ diff --git a/host/include/generic/host/atomic128-ldst.h b/host/include/generic/host/atomic128-ldst.h new file mode 100644 index 0000000000..80fff0643a --- /dev/null +++ b/host/include/generic/host/atomic128-ldst.h @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * Load/store for 128-bit atomic operations, generic version. + * + * Copyright (C) 2018, 2023 Linaro, Ltd. + * + * See docs/devel/atomics.rst for discussion about the guarantees each + * atomic primitive is meant to provide. + */ + +#ifndef HOST_ATOMIC128_LDST_H +#define HOST_ATOMIC128_LDST_H + +#if defined(CONFIG_ATOMIC128) +# define HAVE_ATOMIC128_RO 1 +# define HAVE_ATOMIC128_RW 1 + +static inline Int128 ATTRIBUTE_ATOMIC128_OPT +atomic16_read_ro(const Int128 *ptr) +{ + const __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16); + Int128Alias r; + + r.i = qatomic_read__nocheck(ptr_align); + return r.s; +} + +static inline Int128 ATTRIBUTE_ATOMIC128_OPT +atomic16_read_rw(Int128 *ptr) +{ + return atomic16_read_ro(ptr); +} + +static inline void ATTRIBUTE_ATOMIC128_OPT +atomic16_set(Int128 *ptr, Int128 val) +{ + __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16); + Int128Alias v; + + v.s = val; + qatomic_set__nocheck(ptr_align, v.i); +} + +#elif defined(CONFIG_CMPXCHG128) +# define HAVE_ATOMIC128_RO 0 +# define HAVE_ATOMIC128_RW 1 + +Int128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr); + +static inline Int128 ATTRIBUTE_ATOMIC128_OPT +atomic16_read_rw(Int128 *ptr) +{ + /* Maybe replace 0 with 0, returning the old value. */ + Int128 z = int128_make64(0); + return atomic16_cmpxchg(ptr, z, z); +} + +static inline void ATTRIBUTE_ATOMIC128_OPT +atomic16_set(Int128 *ptr, Int128 val) +{ + __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16); + __int128_t old; + Int128Alias new; + + new.s = val; + do { + old = *ptr_align; + } while (!__sync_bool_compare_and_swap_16(ptr_align, old, new.i)); +} + +#else +# define HAVE_ATOMIC128_RO 0 +# define HAVE_ATOMIC128_RW 0 + +/* Fallback definitions that must be optimized away, or error. */ +Int128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr); +Int128 QEMU_ERROR("unsupported atomic") atomic16_read_rw(Int128 *ptr); +void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val); +#endif + +#endif /* HOST_ATOMIC128_LDST_H */ diff --git a/host/include/generic/host/cpuinfo.h b/host/include/generic/host/cpuinfo.h new file mode 100644 index 0000000000..eca672064a --- /dev/null +++ b/host/include/generic/host/cpuinfo.h @@ -0,0 +1,4 @@ +/* + * No host specific cpu indentification. + * SPDX-License-Identifier: GPL-2.0-or-later + */ diff --git a/host/include/i386/host/cpuinfo.h b/host/include/i386/host/cpuinfo.h new file mode 100644 index 0000000000..a6537123cf --- /dev/null +++ b/host/include/i386/host/cpuinfo.h @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * Host specific cpu indentification for x86. + */ + +#ifndef HOST_CPUINFO_H +#define HOST_CPUINFO_H + +/* Digested version of <cpuid.h> */ + +#define CPUINFO_ALWAYS (1u << 0) /* so cpuinfo is nonzero */ +#define CPUINFO_CMOV (1u << 1) +#define CPUINFO_MOVBE (1u << 2) +#define CPUINFO_LZCNT (1u << 3) +#define CPUINFO_POPCNT (1u << 4) +#define CPUINFO_BMI1 (1u << 5) +#define CPUINFO_BMI2 (1u << 6) +#define CPUINFO_SSE2 (1u << 7) +#define CPUINFO_SSE4 (1u << 8) +#define CPUINFO_AVX1 (1u << 9) +#define CPUINFO_AVX2 (1u << 10) +#define CPUINFO_AVX512F (1u << 11) +#define CPUINFO_AVX512VL (1u << 12) +#define CPUINFO_AVX512BW (1u << 13) +#define CPUINFO_AVX512DQ (1u << 14) +#define CPUINFO_AVX512VBMI2 (1u << 15) +#define CPUINFO_ATOMIC_VMOVDQA (1u << 16) +#define CPUINFO_ATOMIC_VMOVDQU (1u << 17) + +/* Initialized with a constructor. */ +extern unsigned cpuinfo; + +/* + * We cannot rely on constructor ordering, so other constructors must + * use the function interface rather than the variable above. + */ +unsigned cpuinfo_init(void); + +#endif /* HOST_CPUINFO_H */ diff --git a/host/include/x86_64/host/cpuinfo.h b/host/include/x86_64/host/cpuinfo.h new file mode 100644 index 0000000000..67debab9a0 --- /dev/null +++ b/host/include/x86_64/host/cpuinfo.h @@ -0,0 +1 @@ +#include "host/include/i386/host/cpuinfo.h" diff --git a/hw/net/Kconfig b/hw/net/Kconfig index 18c7851efe..98e00be4f9 100644 --- a/hw/net/Kconfig +++ b/hw/net/Kconfig @@ -56,7 +56,7 @@ config RTL8139_PCI config VMXNET3_PCI bool - default y if PCI_DEVICES && PC_PCI + default y if PCI_DEVICES depends on PCI config SMC91C111 diff --git a/hw/net/e1000.c b/hw/net/e1000.c index 23d660619f..aae5f0bdc0 100644 --- a/hw/net/e1000.c +++ b/hw/net/e1000.c @@ -637,9 +637,8 @@ xmit_seg(E1000State *s) e1000x_inc_reg_if_not_full(s->mac_reg, TPT); e1000x_grow_8reg_if_not_full(s->mac_reg, TOTL, s->tx.size + 4); - s->mac_reg[GPTC] = s->mac_reg[TPT]; - s->mac_reg[GOTCL] = s->mac_reg[TOTL]; - s->mac_reg[GOTCH] = s->mac_reg[TOTH]; + e1000x_inc_reg_if_not_full(s->mac_reg, GPTC); + e1000x_grow_8reg_if_not_full(s->mac_reg, GOTCL, s->tx.size + 4); } static void @@ -805,38 +804,11 @@ start_xmit(E1000State *s) } static int -receive_filter(E1000State *s, const uint8_t *buf, int size) +receive_filter(E1000State *s, const void *buf) { - uint32_t rctl = s->mac_reg[RCTL]; - int isbcast = is_broadcast_ether_addr(buf); - int ismcast = is_multicast_ether_addr(buf); - - if (e1000x_is_vlan_packet(buf, le16_to_cpu(s->mac_reg[VET])) && - e1000x_vlan_rx_filter_enabled(s->mac_reg)) { - uint16_t vid = lduw_be_p(&PKT_GET_VLAN_HDR(buf)->h_tci); - uint32_t vfta = - ldl_le_p((uint32_t *)(s->mac_reg + VFTA) + - ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK)); - if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) { - return 0; - } - } - - if (!isbcast && !ismcast && (rctl & E1000_RCTL_UPE)) { /* promiscuous ucast */ - return 1; - } - - if (ismcast && (rctl & E1000_RCTL_MPE)) { /* promiscuous mcast */ - e1000x_inc_reg_if_not_full(s->mac_reg, MPRC); - return 1; - } - - if (isbcast && (rctl & E1000_RCTL_BAM)) { /* broadcast enabled */ - e1000x_inc_reg_if_not_full(s->mac_reg, BPRC); - return 1; - } - - return e1000x_rx_group_filter(s->mac_reg, buf); + return (!e1000x_is_vlan_packet(buf, s->mac_reg[VET]) || + e1000x_rx_vlan_filter(s->mac_reg, PKT_GET_VLAN_HDR(buf))) && + e1000x_rx_group_filter(s->mac_reg, buf); } static void @@ -923,6 +895,7 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt) size_t desc_offset; size_t desc_size; size_t total_size; + eth_pkt_types_e pkt_type; if (!e1000x_hw_rx_enabled(s->mac_reg)) { return -1; @@ -951,7 +924,7 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt) return size; } - if (!receive_filter(s, filter_buf, size)) { + if (!receive_filter(s, filter_buf)) { return size; } @@ -972,6 +945,7 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt) size -= 4; } + pkt_type = get_eth_packet_type(PKT_GET_ETH_HDR(filter_buf)); rdh_start = s->mac_reg[RDH]; desc_offset = 0; total_size = size + e1000x_fcs_len(s->mac_reg); @@ -1037,7 +1011,7 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt) } } while (desc_offset < total_size); - e1000x_update_rx_total_stats(s->mac_reg, size, total_size); + e1000x_update_rx_total_stats(s->mac_reg, pkt_type, size, total_size); n = E1000_ICS_RXT0; if ((rdt = s->mac_reg[RDT]) < s->mac_reg[RDH]) diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c index c0c09b6965..9f185d099c 100644 --- a/hw/net/e1000e_core.c +++ b/hw/net/e1000e_core.c @@ -165,14 +165,14 @@ e1000e_intrmgr_on_throttling_timer(void *opaque) timer->running = false; - if (msi_enabled(timer->core->owner)) { - trace_e1000e_irq_msi_notify_postponed(); - /* Clear msi_causes_pending to fire MSI eventually */ - timer->core->msi_causes_pending = 0; - e1000e_set_interrupt_cause(timer->core, 0); - } else { - trace_e1000e_irq_legacy_notify_postponed(); - e1000e_set_interrupt_cause(timer->core, 0); + if (timer->core->mac[IMS] & timer->core->mac[ICR]) { + if (msi_enabled(timer->core->owner)) { + trace_e1000e_irq_msi_notify_postponed(); + msi_notify(timer->core->owner, 0); + } else { + trace_e1000e_irq_legacy_notify_postponed(); + e1000e_raise_legacy_irq(timer->core); + } } } @@ -366,10 +366,6 @@ static void e1000e_intrmgr_fire_all_timers(E1000ECore *core) { int i; - uint32_t val = e1000e_intmgr_collect_delayed_causes(core); - - trace_e1000e_irq_adding_delayed_causes(val, core->mac[ICR]); - core->mac[ICR] |= val; if (core->itr.running) { timer_del(core->itr.timer); @@ -537,7 +533,7 @@ e1000e_rss_get_hash_type(E1000ECore *core, struct NetRxPkt *pkt) ip6info->rss_ex_dst_valid, ip6info->rss_ex_src_valid, core->mac[MRQC], - E1000_MRQC_EN_TCPIPV6(core->mac[MRQC]), + E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC]), E1000_MRQC_EN_IPV6EX(core->mac[MRQC]), E1000_MRQC_EN_IPV6(core->mac[MRQC])); @@ -546,8 +542,8 @@ e1000e_rss_get_hash_type(E1000ECore *core, struct NetRxPkt *pkt) ip6info->rss_ex_src_valid))) { if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && - E1000_MRQC_EN_TCPIPV6(core->mac[MRQC])) { - return E1000_MRQ_RSS_TYPE_IPV6TCP; + E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC])) { + return E1000_MRQ_RSS_TYPE_IPV6TCPEX; } if (E1000_MRQC_EN_IPV6EX(core->mac[MRQC])) { @@ -581,7 +577,7 @@ e1000e_rss_calc_hash(E1000ECore *core, case E1000_MRQ_RSS_TYPE_IPV4TCP: type = NetPktRssIpV4Tcp; break; - case E1000_MRQ_RSS_TYPE_IPV6TCP: + case E1000_MRQ_RSS_TYPE_IPV6TCPEX: type = NetPktRssIpV6TcpEx; break; case E1000_MRQ_RSS_TYPE_IPV6: @@ -711,9 +707,8 @@ e1000e_on_tx_done_update_stats(E1000ECore *core, struct NetTxPkt *tx_pkt) g_assert_not_reached(); } - core->mac[GPTC] = core->mac[TPT]; - core->mac[GOTCL] = core->mac[TOTL]; - core->mac[GOTCH] = core->mac[TOTH]; + e1000x_inc_reg_if_not_full(core->mac, GPTC); + e1000x_grow_8reg_if_not_full(core->mac, GOTCL, tot_len); } static void @@ -747,7 +742,8 @@ e1000e_process_tx_desc(E1000ECore *core, addr = le64_to_cpu(dp->buffer_addr); if (!tx->skip_cp) { - if (!net_tx_pkt_add_raw_fragment(tx->tx_pkt, addr, split_size)) { + if (!net_tx_pkt_add_raw_fragment_pci(tx->tx_pkt, core->owner, + addr, split_size)) { tx->skip_cp = true; } } @@ -765,7 +761,7 @@ e1000e_process_tx_desc(E1000ECore *core, } tx->skip_cp = false; - net_tx_pkt_reset(tx->tx_pkt, core->owner); + net_tx_pkt_reset(tx->tx_pkt, net_tx_pkt_unmap_frag_pci, core->owner); tx->sum_needed = 0; tx->cptse = 0; @@ -959,6 +955,8 @@ e1000e_start_xmit(E1000ECore *core, const E1000E_TxRing *txr) if (!ide || !e1000e_intrmgr_delay_tx_causes(core, &cause)) { e1000e_set_interrupt_cause(core, cause); } + + net_tx_pkt_reset(txr->tx->tx_pkt, net_tx_pkt_unmap_frag_pci, core->owner); } static bool @@ -1034,48 +1032,11 @@ e1000e_rx_l4_cso_enabled(E1000ECore *core) } static bool -e1000e_receive_filter(E1000ECore *core, const uint8_t *buf, int size) +e1000e_receive_filter(E1000ECore *core, const void *buf) { - uint32_t rctl = core->mac[RCTL]; - - if (e1000x_is_vlan_packet(buf, core->mac[VET]) && - e1000x_vlan_rx_filter_enabled(core->mac)) { - uint16_t vid = lduw_be_p(&PKT_GET_VLAN_HDR(buf)->h_tci); - uint32_t vfta = - ldl_le_p((uint32_t *)(core->mac + VFTA) + - ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK)); - if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) { - trace_e1000e_rx_flt_vlan_mismatch(vid); - return false; - } else { - trace_e1000e_rx_flt_vlan_match(vid); - } - } - - switch (net_rx_pkt_get_packet_type(core->rx_pkt)) { - case ETH_PKT_UCAST: - if (rctl & E1000_RCTL_UPE) { - return true; /* promiscuous ucast */ - } - break; - - case ETH_PKT_BCAST: - if (rctl & E1000_RCTL_BAM) { - return true; /* broadcast enabled */ - } - break; - - case ETH_PKT_MCAST: - if (rctl & E1000_RCTL_MPE) { - return true; /* promiscuous mcast */ - } - break; - - default: - g_assert_not_reached(); - } - - return e1000x_rx_group_filter(core->mac, buf); + return (!e1000x_is_vlan_packet(buf, core->mac[VET]) || + e1000x_rx_vlan_filter(core->mac, PKT_GET_VLAN_HDR(buf))) && + e1000x_rx_group_filter(core->mac, buf); } static inline void @@ -1149,6 +1110,11 @@ e1000e_verify_csum_in_sw(E1000ECore *core, return; } + if (l4hdr_proto != ETH_L4_HDR_PROTO_TCP && + l4hdr_proto != ETH_L4_HDR_PROTO_UDP) { + return; + } + if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) { trace_e1000e_rx_metadata_l4_csum_validation_failed(); return; @@ -1281,9 +1247,8 @@ e1000e_build_rx_metadata(E1000ECore *core, trace_e1000e_rx_metadata_l4_cso_disabled(); } - trace_e1000e_rx_metadata_status_flags(*status_flags); - func_exit: + trace_e1000e_rx_metadata_status_flags(*status_flags); *status_flags = cpu_to_le32(*status_flags); } @@ -1488,24 +1453,10 @@ e1000e_write_to_rx_buffers(E1000ECore *core, } static void -e1000e_update_rx_stats(E1000ECore *core, - size_t data_size, - size_t data_fcs_size) +e1000e_update_rx_stats(E1000ECore *core, size_t pkt_size, size_t pkt_fcs_size) { - e1000x_update_rx_total_stats(core->mac, data_size, data_fcs_size); - - switch (net_rx_pkt_get_packet_type(core->rx_pkt)) { - case ETH_PKT_BCAST: - e1000x_inc_reg_if_not_full(core->mac, BPRC); - break; - - case ETH_PKT_MCAST: - e1000x_inc_reg_if_not_full(core->mac, MPRC); - break; - - default: - break; - } + eth_pkt_types_e pkt_type = net_rx_pkt_get_packet_type(core->rx_pkt); + e1000x_update_rx_total_stats(core->mac, pkt_type, pkt_size, pkt_fcs_size); } static inline bool @@ -1700,12 +1651,9 @@ static ssize_t e1000e_receive_internal(E1000ECore *core, const struct iovec *iov, int iovcnt, bool has_vnet) { - static const int maximum_ethernet_hdr_len = (ETH_HLEN + 4); - - uint32_t n = 0; - uint8_t min_buf[ETH_ZLEN]; + uint32_t causes = 0; + uint8_t buf[ETH_ZLEN]; struct iovec min_iov; - uint8_t *filter_buf; size_t size, orig_size; size_t iov_ofs = 0; E1000E_RxRing rxr; @@ -1728,24 +1676,21 @@ e1000e_receive_internal(E1000ECore *core, const struct iovec *iov, int iovcnt, net_rx_pkt_unset_vhdr(core->rx_pkt); } - filter_buf = iov->iov_base + iov_ofs; orig_size = iov_size(iov, iovcnt); size = orig_size - iov_ofs; /* Pad to minimum Ethernet frame length */ - if (size < sizeof(min_buf)) { - iov_to_buf(iov, iovcnt, iov_ofs, min_buf, size); - memset(&min_buf[size], 0, sizeof(min_buf) - size); + if (size < sizeof(buf)) { + iov_to_buf(iov, iovcnt, iov_ofs, buf, size); + memset(&buf[size], 0, sizeof(buf) - size); e1000x_inc_reg_if_not_full(core->mac, RUC); - min_iov.iov_base = filter_buf = min_buf; - min_iov.iov_len = size = sizeof(min_buf); + min_iov.iov_base = buf; + min_iov.iov_len = size = sizeof(buf); iovcnt = 1; iov = &min_iov; iov_ofs = 0; - } else if (iov->iov_len < maximum_ethernet_hdr_len) { - /* This is very unlikely, but may happen. */ - iov_to_buf(iov, iovcnt, iov_ofs, min_buf, maximum_ethernet_hdr_len); - filter_buf = min_buf; + } else { + iov_to_buf(iov, iovcnt, iov_ofs, buf, ETH_HLEN + 4); } /* Discard oversized packets if !LPE and !SBP. */ @@ -1754,15 +1699,16 @@ e1000e_receive_internal(E1000ECore *core, const struct iovec *iov, int iovcnt, } net_rx_pkt_set_packet_type(core->rx_pkt, - get_eth_packet_type(PKT_GET_ETH_HDR(filter_buf))); + get_eth_packet_type(PKT_GET_ETH_HDR(buf))); - if (!e1000e_receive_filter(core, filter_buf, size)) { + if (!e1000e_receive_filter(core, buf)) { trace_e1000e_rx_flt_dropped(); return orig_size; } net_rx_pkt_attach_iovec_ex(core->rx_pkt, iov, iovcnt, iov_ofs, - e1000x_vlan_enabled(core->mac), core->mac[VET]); + e1000x_vlan_enabled(core->mac) ? 0 : -1, + core->mac[VET], 0); e1000e_rss_parse_packet(core, core->rx_pkt, &rss_info); e1000e_rx_ring_init(core, &rxr, rss_info.queue); @@ -1779,32 +1725,32 @@ e1000e_receive_internal(E1000ECore *core, const struct iovec *iov, int iovcnt, /* Perform small receive detection (RSRPD) */ if (total_size < core->mac[RSRPD]) { - n |= E1000_ICS_SRPD; + causes |= E1000_ICS_SRPD; } /* Perform ACK receive detection */ if (!(core->mac[RFCTL] & E1000_RFCTL_ACK_DIS) && (e1000e_is_tcp_ack(core, core->rx_pkt))) { - n |= E1000_ICS_ACK; + causes |= E1000_ICS_ACK; } /* Check if receive descriptor minimum threshold hit */ rdmts_hit = e1000e_rx_descr_threshold_hit(core, rxr.i); - n |= e1000e_rx_wb_interrupt_cause(core, rxr.i->idx, rdmts_hit); + causes |= e1000e_rx_wb_interrupt_cause(core, rxr.i->idx, rdmts_hit); trace_e1000e_rx_written_to_guest(rxr.i->idx); } else { - n |= E1000_ICS_RXO; + causes |= E1000_ICS_RXO; retval = 0; trace_e1000e_rx_not_written_to_guest(rxr.i->idx); } - if (!e1000e_intrmgr_delay_rx_causes(core, &n)) { - trace_e1000e_rx_interrupt_set(n); - e1000e_set_interrupt_cause(core, n); + if (!e1000e_intrmgr_delay_rx_causes(core, &causes)) { + trace_e1000e_rx_interrupt_set(causes); + e1000e_set_interrupt_cause(core, causes); } else { - trace_e1000e_rx_interrupt_delayed(n); + trace_e1000e_rx_interrupt_delayed(causes); } return retval; @@ -2024,13 +1970,6 @@ void(*e1000e_phyreg_writeops[E1000E_PHY_PAGES][E1000E_PHY_PAGE_SIZE]) } }; -static inline void -e1000e_clear_ims_bits(E1000ECore *core, uint32_t bits) -{ - trace_e1000e_irq_clear_ims(bits, core->mac[IMS], core->mac[IMS] & ~bits); - core->mac[IMS] &= ~bits; -} - static inline bool e1000e_postpone_interrupt(E1000IntrDelayTimer *timer) { @@ -2088,7 +2027,6 @@ e1000e_msix_notify_one(E1000ECore *core, uint32_t cause, uint32_t int_cfg) effective_eiac = core->mac[EIAC] & cause; core->mac[ICR] &= ~effective_eiac; - core->msi_causes_pending &= ~effective_eiac; if (!(core->mac[CTRL_EXT] & E1000_CTRL_EXT_IAME)) { core->mac[IMS] &= ~effective_eiac; @@ -2180,33 +2118,17 @@ e1000e_fix_icr_asserted(E1000ECore *core) trace_e1000e_irq_fix_icr_asserted(core->mac[ICR]); } -static void -e1000e_send_msi(E1000ECore *core, bool msix) +static void e1000e_raise_interrupts(E1000ECore *core, + size_t index, uint32_t causes) { - uint32_t causes = core->mac[ICR] & core->mac[IMS] & ~E1000_ICR_ASSERTED; - - core->msi_causes_pending &= causes; - causes ^= core->msi_causes_pending; - if (causes == 0) { - return; - } - core->msi_causes_pending |= causes; + bool is_msix = msix_enabled(core->owner); + uint32_t old_causes = core->mac[IMS] & core->mac[ICR]; + uint32_t raised_causes; - if (msix) { - e1000e_msix_notify(core, causes); - } else { - if (!e1000e_itr_should_postpone(core)) { - trace_e1000e_irq_msi_notify(causes); - msi_notify(core->owner, 0); - } - } -} + trace_e1000e_irq_set(index << 2, + core->mac[index], core->mac[index] | causes); -static void -e1000e_update_interrupt_state(E1000ECore *core) -{ - bool interrupts_pending; - bool is_msix = msix_enabled(core->owner); + core->mac[index] |= causes; /* Set ICR[OTHER] for MSI-X */ if (is_msix) { @@ -2228,40 +2150,58 @@ e1000e_update_interrupt_state(E1000ECore *core) */ core->mac[ICS] = core->mac[ICR]; - interrupts_pending = (core->mac[IMS] & core->mac[ICR]) ? true : false; - if (!interrupts_pending) { - core->msi_causes_pending = 0; - } - trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS], core->mac[ICR], core->mac[IMS]); - if (is_msix || msi_enabled(core->owner)) { - if (interrupts_pending) { - e1000e_send_msi(core, is_msix); - } - } else { - if (interrupts_pending) { - if (!e1000e_itr_should_postpone(core)) { - e1000e_raise_legacy_irq(core); - } + raised_causes = core->mac[IMS] & core->mac[ICR] & ~old_causes; + if (!raised_causes) { + return; + } + + if (is_msix) { + e1000e_msix_notify(core, raised_causes & ~E1000_ICR_ASSERTED); + } else if (!e1000e_itr_should_postpone(core)) { + if (msi_enabled(core->owner)) { + trace_e1000e_irq_msi_notify(raised_causes); + msi_notify(core->owner, 0); } else { - e1000e_lower_legacy_irq(core); + e1000e_raise_legacy_irq(core); } } } -static void -e1000e_set_interrupt_cause(E1000ECore *core, uint32_t val) +static void e1000e_lower_interrupts(E1000ECore *core, + size_t index, uint32_t causes) { - trace_e1000e_irq_set_cause_entry(val, core->mac[ICR]); + trace_e1000e_irq_clear(index << 2, + core->mac[index], core->mac[index] & ~causes); - val |= e1000e_intmgr_collect_delayed_causes(core); - core->mac[ICR] |= val; + core->mac[index] &= ~causes; + + /* + * Make sure ICR and ICS registers have the same value. + * The spec says that the ICS register is write-only. However in practice, + * on real hardware ICS is readable, and for reads it has the same value as + * ICR (except that ICS does not have the clear on read behaviour of ICR). + * + * The VxWorks PRO/1000 driver uses this behaviour. + */ + core->mac[ICS] = core->mac[ICR]; + + trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS], + core->mac[ICR], core->mac[IMS]); - trace_e1000e_irq_set_cause_exit(val, core->mac[ICR]); + if (!(core->mac[IMS] & core->mac[ICR]) && + !msix_enabled(core->owner) && !msi_enabled(core->owner)) { + e1000e_lower_legacy_irq(core); + } +} - e1000e_update_interrupt_state(core); +static void +e1000e_set_interrupt_cause(E1000ECore *core, uint32_t val) +{ + val |= e1000e_intmgr_collect_delayed_causes(core); + e1000e_raise_interrupts(core, ICR, val); } static inline void @@ -2527,30 +2467,27 @@ e1000e_set_ics(E1000ECore *core, int index, uint32_t val) static void e1000e_set_icr(E1000ECore *core, int index, uint32_t val) { - uint32_t icr = 0; if ((core->mac[ICR] & E1000_ICR_ASSERTED) && (core->mac[CTRL_EXT] & E1000_CTRL_EXT_IAME)) { trace_e1000e_irq_icr_process_iame(); - e1000e_clear_ims_bits(core, core->mac[IAM]); + e1000e_lower_interrupts(core, IMS, core->mac[IAM]); } - icr = core->mac[ICR] & ~val; /* * Windows driver expects that the "receive overrun" bit and other * ones to be cleared when the "Other" bit (#24) is cleared. */ - icr = (val & E1000_ICR_OTHER) ? (icr & ~E1000_ICR_OTHER_CAUSES) : icr; - trace_e1000e_irq_icr_write(val, core->mac[ICR], icr); - core->mac[ICR] = icr; - e1000e_update_interrupt_state(core); + if (val & E1000_ICR_OTHER) { + val |= E1000_ICR_OTHER_CAUSES; + } + e1000e_lower_interrupts(core, ICR, val); } static void e1000e_set_imc(E1000ECore *core, int index, uint32_t val) { trace_e1000e_irq_ims_clear_set_imc(val); - e1000e_clear_ims_bits(core, val); - e1000e_update_interrupt_state(core); + e1000e_lower_interrupts(core, IMS, val); } static void @@ -2571,9 +2508,6 @@ e1000e_set_ims(E1000ECore *core, int index, uint32_t val) uint32_t valid_val = val & ims_valid_mask; - trace_e1000e_irq_set_ims(val, core->mac[IMS], core->mac[IMS] | valid_val); - core->mac[IMS] |= valid_val; - if ((valid_val & ims_ext_mask) && (core->mac[CTRL_EXT] & E1000_CTRL_EXT_PBA_CLR) && msix_enabled(core->owner)) { @@ -2586,7 +2520,7 @@ e1000e_set_ims(E1000ECore *core, int index, uint32_t val) e1000e_intrmgr_fire_all_timers(core); } - e1000e_update_interrupt_state(core); + e1000e_raise_interrupts(core, IMS, valid_val); } static void @@ -2659,28 +2593,25 @@ static uint32_t e1000e_mac_icr_read(E1000ECore *core, int index) { uint32_t ret = core->mac[ICR]; - trace_e1000e_irq_icr_read_entry(ret); if (core->mac[IMS] == 0) { trace_e1000e_irq_icr_clear_zero_ims(); - core->mac[ICR] = 0; + e1000e_lower_interrupts(core, ICR, 0xffffffff); } if (!msix_enabled(core->owner)) { trace_e1000e_irq_icr_clear_nonmsix_icr_read(); - core->mac[ICR] = 0; + e1000e_lower_interrupts(core, ICR, 0xffffffff); } if ((core->mac[ICR] & E1000_ICR_ASSERTED) && (core->mac[CTRL_EXT] & E1000_CTRL_EXT_IAME)) { trace_e1000e_irq_icr_clear_iame(); - core->mac[ICR] = 0; + e1000e_lower_interrupts(core, ICR, 0xffffffff); trace_e1000e_irq_icr_process_iame(); - e1000e_clear_ims_bits(core, core->mac[IAM]); + e1000e_lower_interrupts(core, IMS, core->mac[IAM]); } - trace_e1000e_irq_icr_read_exit(core->mac[ICR]); - e1000e_update_interrupt_state(core); return ret; } @@ -3422,7 +3353,7 @@ e1000e_core_pci_realize(E1000ECore *core, qemu_add_vm_change_state_handler(e1000e_vm_state_change, core); for (i = 0; i < E1000E_NUM_QUEUES; i++) { - net_tx_pkt_init(&core->tx[i].tx_pkt, core->owner, E1000E_MAX_TX_FRAGS); + net_tx_pkt_init(&core->tx[i].tx_pkt, E1000E_MAX_TX_FRAGS); } net_rx_pkt_init(&core->rx_pkt); @@ -3447,7 +3378,6 @@ e1000e_core_pci_uninit(E1000ECore *core) qemu_del_vm_change_state_handler(core->vmstate); for (i = 0; i < E1000E_NUM_QUEUES; i++) { - net_tx_pkt_reset(core->tx[i].tx_pkt, core->owner); net_tx_pkt_uninit(core->tx[i].tx_pkt); } @@ -3572,7 +3502,6 @@ static void e1000e_reset(E1000ECore *core, bool sw) e1000x_reset_mac_addr(core->owner_nic, core->mac, core->permanent_mac); for (i = 0; i < ARRAY_SIZE(core->tx); i++) { - net_tx_pkt_reset(core->tx[i].tx_pkt, core->owner); memset(&core->tx[i].props, 0, sizeof(core->tx[i].props)); core->tx[i].skip_cp = false; } diff --git a/hw/net/e1000e_core.h b/hw/net/e1000e_core.h index 213a70530d..66b025cc43 100644 --- a/hw/net/e1000e_core.h +++ b/hw/net/e1000e_core.h @@ -111,8 +111,6 @@ struct E1000Core { PCIDevice *owner; void (*owner_start_recv)(PCIDevice *d); - uint32_t msi_causes_pending; - int64_t timadj; }; diff --git a/hw/net/e1000x_common.c b/hw/net/e1000x_common.c index b844af590a..212873fd77 100644 --- a/hw/net/e1000x_common.c +++ b/hw/net/e1000x_common.c @@ -58,33 +58,64 @@ bool e1000x_is_vlan_packet(const void *buf, uint16_t vet) return res; } -bool e1000x_rx_group_filter(uint32_t *mac, const uint8_t *buf) +bool e1000x_rx_vlan_filter(uint32_t *mac, const struct vlan_header *vhdr) +{ + if (e1000x_vlan_rx_filter_enabled(mac)) { + uint16_t vid = lduw_be_p(&vhdr->h_tci); + uint32_t vfta = + ldl_le_p((uint32_t *)(mac + VFTA) + + ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK)); + if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) { + trace_e1000x_rx_flt_vlan_mismatch(vid); + return false; + } + + trace_e1000x_rx_flt_vlan_match(vid); + } + + return true; +} + +bool e1000x_rx_group_filter(uint32_t *mac, const struct eth_header *ehdr) { static const int mta_shift[] = { 4, 3, 2, 0 }; uint32_t f, ra[2], *rp, rctl = mac[RCTL]; + if (is_broadcast_ether_addr(ehdr->h_dest)) { + if (rctl & E1000_RCTL_BAM) { + return true; + } + } else if (is_multicast_ether_addr(ehdr->h_dest)) { + if (rctl & E1000_RCTL_MPE) { + return true; + } + } else { + if (rctl & E1000_RCTL_UPE) { + return true; + } + } + for (rp = mac + RA; rp < mac + RA + 32; rp += 2) { if (!(rp[1] & E1000_RAH_AV)) { continue; } ra[0] = cpu_to_le32(rp[0]); ra[1] = cpu_to_le32(rp[1]); - if (!memcmp(buf, (uint8_t *)ra, ETH_ALEN)) { + if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) { trace_e1000x_rx_flt_ucast_match((int)(rp - mac - RA) / 2, - MAC_ARG(buf)); + MAC_ARG(ehdr->h_dest)); return true; } } - trace_e1000x_rx_flt_ucast_mismatch(MAC_ARG(buf)); + trace_e1000x_rx_flt_ucast_mismatch(MAC_ARG(ehdr->h_dest)); f = mta_shift[(rctl >> E1000_RCTL_MO_SHIFT) & 3]; - f = (((buf[5] << 8) | buf[4]) >> f) & 0xfff; + f = (((ehdr->h_dest[5] << 8) | ehdr->h_dest[4]) >> f) & 0xfff; if (mac[MTA + (f >> 5)] & (1 << (f & 0x1f))) { - e1000x_inc_reg_if_not_full(mac, MPRC); return true; } - trace_e1000x_rx_flt_inexact_mismatch(MAC_ARG(buf), + trace_e1000x_rx_flt_inexact_mismatch(MAC_ARG(ehdr->h_dest), (rctl >> E1000_RCTL_MO_SHIFT) & 3, f >> 5, mac[MTA + (f >> 5)]); @@ -109,16 +140,16 @@ bool e1000x_hw_rx_enabled(uint32_t *mac) bool e1000x_is_oversized(uint32_t *mac, size_t size) { + size_t header_size = sizeof(struct eth_header) + sizeof(struct vlan_header); /* this is the size past which hardware will drop packets when setting LPE=0 */ - static const int maximum_ethernet_vlan_size = 1522; + size_t maximum_short_size = header_size + ETH_MTU; /* this is the size past which hardware will drop packets when setting LPE=1 */ - static const int maximum_ethernet_lpe_size = 16 * KiB; + size_t maximum_large_size = 16 * KiB - ETH_FCS_LEN; - if ((size > maximum_ethernet_lpe_size || - (size > maximum_ethernet_vlan_size - && !(mac[RCTL] & E1000_RCTL_LPE))) + if ((size > maximum_large_size || + (size > maximum_short_size && !(mac[RCTL] & E1000_RCTL_LPE))) && !(mac[RCTL] & E1000_RCTL_SBP)) { e1000x_inc_reg_if_not_full(mac, ROC); trace_e1000x_rx_oversized(size); @@ -212,23 +243,36 @@ e1000x_rxbufsize(uint32_t rctl) void e1000x_update_rx_total_stats(uint32_t *mac, - size_t data_size, - size_t data_fcs_size) + eth_pkt_types_e pkt_type, + size_t pkt_size, + size_t pkt_fcs_size) { static const int PRCregs[6] = { PRC64, PRC127, PRC255, PRC511, PRC1023, PRC1522 }; - e1000x_increase_size_stats(mac, PRCregs, data_fcs_size); + e1000x_increase_size_stats(mac, PRCregs, pkt_fcs_size); e1000x_inc_reg_if_not_full(mac, TPR); - mac[GPRC] = mac[TPR]; + e1000x_inc_reg_if_not_full(mac, GPRC); /* TOR - Total Octets Received: * This register includes bytes received in a packet from the <Destination * Address> field through the <CRC> field, inclusively. * Always include FCS length (4) in size. */ - e1000x_grow_8reg_if_not_full(mac, TORL, data_size + 4); - mac[GORCL] = mac[TORL]; - mac[GORCH] = mac[TORH]; + e1000x_grow_8reg_if_not_full(mac, TORL, pkt_size + 4); + e1000x_grow_8reg_if_not_full(mac, GORCL, pkt_size + 4); + + switch (pkt_type) { + case ETH_PKT_BCAST: + e1000x_inc_reg_if_not_full(mac, BPRC); + break; + + case ETH_PKT_MCAST: + e1000x_inc_reg_if_not_full(mac, MPRC); + break; + + default: + break; + } } void diff --git a/hw/net/e1000x_common.h b/hw/net/e1000x_common.h index 911abd8a90..be291684de 100644 --- a/hw/net/e1000x_common.h +++ b/hw/net/e1000x_common.h @@ -91,8 +91,9 @@ e1000x_update_regs_on_link_up(uint32_t *mac, uint16_t *phy) } void e1000x_update_rx_total_stats(uint32_t *mac, - size_t data_size, - size_t data_fcs_size); + eth_pkt_types_e pkt_type, + size_t pkt_size, + size_t pkt_fcs_size); void e1000x_core_prepare_eeprom(uint16_t *eeprom, const uint16_t *templ, @@ -106,7 +107,9 @@ bool e1000x_rx_ready(PCIDevice *d, uint32_t *mac); bool e1000x_is_vlan_packet(const void *buf, uint16_t vet); -bool e1000x_rx_group_filter(uint32_t *mac, const uint8_t *buf); +bool e1000x_rx_vlan_filter(uint32_t *mac, const struct vlan_header *vhdr); + +bool e1000x_rx_group_filter(uint32_t *mac, const struct eth_header *ehdr); bool e1000x_hw_rx_enabled(uint32_t *mac); diff --git a/hw/net/e1000x_regs.h b/hw/net/e1000x_regs.h index 6d3c4c6d3a..13760c66d3 100644 --- a/hw/net/e1000x_regs.h +++ b/hw/net/e1000x_regs.h @@ -290,18 +290,18 @@ #define E1000_RETA_IDX(hash) ((hash) & (BIT(7) - 1)) #define E1000_RETA_VAL(reta, hash) (((uint8_t *)(reta))[E1000_RETA_IDX(hash)]) -#define E1000_MRQC_EN_TCPIPV4(mrqc) ((mrqc) & BIT(16)) -#define E1000_MRQC_EN_IPV4(mrqc) ((mrqc) & BIT(17)) -#define E1000_MRQC_EN_TCPIPV6(mrqc) ((mrqc) & BIT(18)) -#define E1000_MRQC_EN_IPV6EX(mrqc) ((mrqc) & BIT(19)) -#define E1000_MRQC_EN_IPV6(mrqc) ((mrqc) & BIT(20)) - -#define E1000_MRQ_RSS_TYPE_NONE (0) -#define E1000_MRQ_RSS_TYPE_IPV4TCP (1) -#define E1000_MRQ_RSS_TYPE_IPV4 (2) -#define E1000_MRQ_RSS_TYPE_IPV6TCP (3) -#define E1000_MRQ_RSS_TYPE_IPV6EX (4) -#define E1000_MRQ_RSS_TYPE_IPV6 (5) +#define E1000_MRQC_EN_TCPIPV4(mrqc) ((mrqc) & BIT(16)) +#define E1000_MRQC_EN_IPV4(mrqc) ((mrqc) & BIT(17)) +#define E1000_MRQC_EN_TCPIPV6EX(mrqc) ((mrqc) & BIT(18)) +#define E1000_MRQC_EN_IPV6EX(mrqc) ((mrqc) & BIT(19)) +#define E1000_MRQC_EN_IPV6(mrqc) ((mrqc) & BIT(20)) + +#define E1000_MRQ_RSS_TYPE_NONE (0) +#define E1000_MRQ_RSS_TYPE_IPV4TCP (1) +#define E1000_MRQ_RSS_TYPE_IPV4 (2) +#define E1000_MRQ_RSS_TYPE_IPV6TCPEX (3) +#define E1000_MRQ_RSS_TYPE_IPV6EX (4) +#define E1000_MRQ_RSS_TYPE_IPV6 (5) #define E1000_ICR_ASSERTED BIT(31) #define E1000_EIAC_MASK 0x01F00000 diff --git a/hw/net/igb.c b/hw/net/igb.c index 51a7e9133e..1c989d7677 100644 --- a/hw/net/igb.c +++ b/hw/net/igb.c @@ -433,16 +433,16 @@ static void igb_pci_realize(PCIDevice *pci_dev, Error **errp) pcie_ari_init(pci_dev, 0x150, 1); - pcie_sriov_pf_init(pci_dev, IGB_CAP_SRIOV_OFFSET, "igbvf", + pcie_sriov_pf_init(pci_dev, IGB_CAP_SRIOV_OFFSET, TYPE_IGBVF, IGB_82576_VF_DEV_ID, IGB_MAX_VF_FUNCTIONS, IGB_MAX_VF_FUNCTIONS, IGB_VF_OFFSET, IGB_VF_STRIDE); - pcie_sriov_pf_init_vf_bar(pci_dev, 0, + pcie_sriov_pf_init_vf_bar(pci_dev, IGBVF_MMIO_BAR_IDX, PCI_BASE_ADDRESS_MEM_TYPE_64 | PCI_BASE_ADDRESS_MEM_PREFETCH, - 16 * KiB); - pcie_sriov_pf_init_vf_bar(pci_dev, 3, + IGBVF_MMIO_SIZE); + pcie_sriov_pf_init_vf_bar(pci_dev, IGBVF_MSIX_BAR_IDX, PCI_BASE_ADDRESS_MEM_TYPE_64 | PCI_BASE_ADDRESS_MEM_PREFETCH, - 16 * KiB); + IGBVF_MSIX_SIZE); igb_init_net_peer(s, pci_dev, macaddr); diff --git a/hw/net/igb_common.h b/hw/net/igb_common.h index 69ac490f75..5c261ba9d3 100644 --- a/hw/net/igb_common.h +++ b/hw/net/igb_common.h @@ -28,6 +28,14 @@ #include "igb_regs.h" +#define TYPE_IGBVF "igbvf" + +#define IGBVF_MMIO_BAR_IDX (0) +#define IGBVF_MSIX_BAR_IDX (3) + +#define IGBVF_MMIO_SIZE (16 * 1024) +#define IGBVF_MSIX_SIZE (16 * 1024) + #define defreg(x) x = (E1000_##x >> 2) #define defreg_indexed(x, i) x##i = (E1000_##x(i) >> 2) #define defreg_indexeda(x, i) x##i##_A = (E1000_##x##_A(i) >> 2) @@ -43,7 +51,7 @@ defreg_indexeda(x, 0), defreg_indexeda(x, 1), \ defreg_indexeda(x, 2), defreg_indexeda(x, 3) -#define defregv(x) defreg_indexed(x, 0), defreg_indexed(x, 1), \ +#define defreg8(x) defreg_indexed(x, 0), defreg_indexed(x, 1), \ defreg_indexed(x, 2), defreg_indexed(x, 3), \ defreg_indexed(x, 4), defreg_indexed(x, 5), \ defreg_indexed(x, 6), defreg_indexed(x, 7) @@ -114,6 +122,8 @@ enum { defreg(EICS), defreg(EIMS), defreg(EIMC), defreg(EIAM), defreg(EICR), defreg(IVAR_MISC), defreg(GPIE), + defreg(TSYNCRXCFG), defreg8(ETQF), + defreg(RXPBS), defregd(RDBAL), defregd(RDBAH), defregd(RDLEN), defregd(SRRCTL), defregd(RDH), defregd(RDT), defregd(RXDCTL), defregd(RXCTL), defregd(RQDPC), defreg(RA2), @@ -125,15 +135,15 @@ enum { defreg(VT_CTL), - defregv(P2VMAILBOX), defregv(V2PMAILBOX), defreg(MBVFICR), defreg(MBVFIMR), + defreg8(P2VMAILBOX), defreg8(V2PMAILBOX), defreg(MBVFICR), defreg(MBVFIMR), defreg(VFLRE), defreg(VFRE), defreg(VFTE), defreg(WVBR), defreg(QDE), defreg(DTXSWC), defreg_indexed(VLVF, 0), - defregv(VMOLR), defreg(RPLOLR), defregv(VMBMEM), defregv(VMVIR), + defreg8(VMOLR), defreg(RPLOLR), defreg8(VMBMEM), defreg8(VMVIR), - defregv(PVTCTRL), defregv(PVTEICS), defregv(PVTEIMS), defregv(PVTEIMC), - defregv(PVTEIAC), defregv(PVTEIAM), defregv(PVTEICR), defregv(PVFGPRC), - defregv(PVFGPTC), defregv(PVFGORC), defregv(PVFGOTC), defregv(PVFMPRC), - defregv(PVFGPRLBC), defregv(PVFGPTLBC), defregv(PVFGORLBC), defregv(PVFGOTLBC), + defreg8(PVTCTRL), defreg8(PVTEICS), defreg8(PVTEIMS), defreg8(PVTEIMC), + defreg8(PVTEIAC), defreg8(PVTEIAM), defreg8(PVTEICR), defreg8(PVFGPRC), + defreg8(PVFGPTC), defreg8(PVFGORC), defreg8(PVFGOTC), defreg8(PVFMPRC), + defreg8(PVFGPRLBC), defreg8(PVFGPTLBC), defreg8(PVFGORLBC), defreg8(PVFGOTLBC), defreg(MTA_A), diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c index d733fed6cf..d00b1caa6a 100644 --- a/hw/net/igb_core.c +++ b/hw/net/igb_core.c @@ -67,14 +67,34 @@ typedef struct IGBTxPktVmdqCallbackContext { NetClientState *nc; } IGBTxPktVmdqCallbackContext; +typedef struct L2Header { + struct eth_header eth; + struct vlan_header vlan[2]; +} L2Header; + +typedef struct PTP2 { + uint8_t message_id_transport_specific; + uint8_t version_ptp; + uint16_t message_length; + uint8_t subdomain_number; + uint8_t reserved0; + uint16_t flags; + uint64_t correction; + uint8_t reserved1[5]; + uint8_t source_communication_technology; + uint32_t source_uuid_lo; + uint16_t source_uuid_hi; + uint16_t source_port_id; + uint16_t sequence_id; + uint8_t control; + uint8_t log_message_period; +} PTP2; + static ssize_t igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt, bool has_vnet, bool *external_tx); -static inline void -igb_set_interrupt_cause(IGBCore *core, uint32_t val); - -static void igb_update_interrupt_state(IGBCore *core); +static void igb_raise_interrupts(IGBCore *core, size_t index, uint32_t causes); static void igb_reset(IGBCore *core, bool sw); static inline void @@ -92,23 +112,31 @@ igb_lower_legacy_irq(IGBCore *core) pci_set_irq(core->owner, 0); } -static void igb_msix_notify(IGBCore *core, unsigned int vector) +static void igb_msix_notify(IGBCore *core, unsigned int cause) { PCIDevice *dev = core->owner; uint16_t vfn; + uint32_t effective_eiac; + unsigned int vector; - vfn = 8 - (vector + 2) / IGBVF_MSIX_VEC_NUM; + vfn = 8 - (cause + 2) / IGBVF_MSIX_VEC_NUM; if (vfn < pcie_sriov_num_vfs(core->owner)) { dev = pcie_sriov_get_vf_at_index(core->owner, vfn); assert(dev); - vector = (vector + 2) % IGBVF_MSIX_VEC_NUM; - } else if (vector >= IGB_MSIX_VEC_NUM) { + vector = (cause + 2) % IGBVF_MSIX_VEC_NUM; + } else if (cause >= IGB_MSIX_VEC_NUM) { qemu_log_mask(LOG_GUEST_ERROR, "igb: Tried to use vector unavailable for PF"); return; + } else { + vector = cause; } msix_notify(dev, vector); + + trace_e1000e_irq_icr_clear_eiac(core->mac[EICR], core->mac[EIAC]); + effective_eiac = core->mac[EIAC] & BIT(cause); + core->mac[EICR] &= ~effective_eiac; } static inline void @@ -274,6 +302,11 @@ igb_rss_get_hash_type(IGBCore *core, struct NetRxPkt *pkt) return E1000_MRQ_RSS_TYPE_IPV4TCP; } + if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP && + (core->mac[MRQC] & E1000_MRQC_RSS_FIELD_IPV4_UDP)) { + return E1000_MRQ_RSS_TYPE_IPV4UDP; + } + if (E1000_MRQC_EN_IPV4(core->mac[MRQC])) { return E1000_MRQ_RSS_TYPE_IPV4; } @@ -296,7 +329,7 @@ igb_rss_get_hash_type(IGBCore *core, struct NetRxPkt *pkt) ip6info->rss_ex_dst_valid, ip6info->rss_ex_src_valid, core->mac[MRQC], - E1000_MRQC_EN_TCPIPV6(core->mac[MRQC]), + E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC]), E1000_MRQC_EN_IPV6EX(core->mac[MRQC]), E1000_MRQC_EN_IPV6(core->mac[MRQC])); @@ -305,8 +338,13 @@ igb_rss_get_hash_type(IGBCore *core, struct NetRxPkt *pkt) ip6info->rss_ex_src_valid))) { if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && - E1000_MRQC_EN_TCPIPV6(core->mac[MRQC])) { - return E1000_MRQ_RSS_TYPE_IPV6TCP; + E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC])) { + return E1000_MRQ_RSS_TYPE_IPV6TCPEX; + } + + if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP && + (core->mac[MRQC] & E1000_MRQC_RSS_FIELD_IPV6_UDP)) { + return E1000_MRQ_RSS_TYPE_IPV6UDP; } if (E1000_MRQC_EN_IPV6EX(core->mac[MRQC])) { @@ -338,7 +376,7 @@ igb_rss_calc_hash(IGBCore *core, struct NetRxPkt *pkt, E1000E_RSSInfo *info) case E1000_MRQ_RSS_TYPE_IPV4TCP: type = NetPktRssIpV4Tcp; break; - case E1000_MRQ_RSS_TYPE_IPV6TCP: + case E1000_MRQ_RSS_TYPE_IPV6TCPEX: type = NetPktRssIpV6TcpEx; break; case E1000_MRQ_RSS_TYPE_IPV6: @@ -347,6 +385,12 @@ igb_rss_calc_hash(IGBCore *core, struct NetRxPkt *pkt, E1000E_RSSInfo *info) case E1000_MRQ_RSS_TYPE_IPV6EX: type = NetPktRssIpV6Ex; break; + case E1000_MRQ_RSS_TYPE_IPV4UDP: + type = NetPktRssIpV4Udp; + break; + case E1000_MRQ_RSS_TYPE_IPV6UDP: + type = NetPktRssIpV6Udp; + break; default: assert(false); return 0; @@ -402,7 +446,7 @@ igb_tx_insert_vlan(IGBCore *core, uint16_t qn, struct igb_tx *tx, } } - if (insert_vlan && e1000x_vlan_enabled(core->mac)) { + if (insert_vlan) { net_tx_pkt_setup_vlan_header_ex(tx->tx_pkt, vlan, core->mac[VET] & 0xffff); } @@ -411,9 +455,10 @@ igb_tx_insert_vlan(IGBCore *core, uint16_t qn, struct igb_tx *tx, static bool igb_setup_tx_offloads(IGBCore *core, struct igb_tx *tx) { + uint32_t idx = (tx->first_olinfo_status >> 4) & 1; + if (tx->first_cmd_type_len & E1000_ADVTXD_DCMD_TSE) { - uint32_t idx = (tx->first_olinfo_status >> 4) & 1; - uint32_t mss = tx->ctx[idx].mss_l4len_idx >> 16; + uint32_t mss = tx->ctx[idx].mss_l4len_idx >> E1000_ADVTXD_MSS_SHIFT; if (!net_tx_pkt_build_vheader(tx->tx_pkt, true, true, mss)) { return false; } @@ -423,10 +468,11 @@ igb_setup_tx_offloads(IGBCore *core, struct igb_tx *tx) return true; } - if (tx->first_olinfo_status & E1000_ADVTXD_POTS_TXSM) { - if (!net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0)) { - return false; - } + if ((tx->first_olinfo_status & E1000_ADVTXD_POTS_TXSM) && + !((tx->ctx[idx].type_tucmd_mlhl & E1000_ADVTXD_TUCMD_L4T_SCTP) ? + net_tx_pkt_update_sctp_checksum(tx->tx_pkt) : + net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0))) { + return false; } if (tx->first_olinfo_status & E1000_ADVTXD_POTS_IXSM) { @@ -538,9 +584,8 @@ igb_on_tx_done_update_stats(IGBCore *core, struct NetTxPkt *tx_pkt, int qn) g_assert_not_reached(); } - core->mac[GPTC] = core->mac[TPT]; - core->mac[GOTCL] = core->mac[TOTL]; - core->mac[GOTCH] = core->mac[TOTH]; + e1000x_inc_reg_if_not_full(core->mac, GPTC); + e1000x_grow_8reg_if_not_full(core->mac, GOTCL, tot_len); if (core->mac[MRQC] & 1) { uint16_t pool = qn % IGB_NUM_VM_POOLS; @@ -598,7 +643,8 @@ igb_process_tx_desc(IGBCore *core, length = cmd_type_len & 0xFFFF; if (!tx->skip_cp) { - if (!net_tx_pkt_add_raw_fragment(tx->tx_pkt, buffer_addr, length)) { + if (!net_tx_pkt_add_raw_fragment_pci(tx->tx_pkt, dev, + buffer_addr, length)) { tx->skip_cp = true; } } @@ -607,8 +653,15 @@ igb_process_tx_desc(IGBCore *core, if (!tx->skip_cp && net_tx_pkt_parse(tx->tx_pkt)) { idx = (tx->first_olinfo_status >> 4) & 1; igb_tx_insert_vlan(core, queue_index, tx, - tx->ctx[idx].vlan_macip_lens >> 16, - !!(cmd_type_len & E1000_TXD_CMD_VLE)); + tx->ctx[idx].vlan_macip_lens >> IGB_TX_FLAGS_VLAN_SHIFT, + !!(tx->first_cmd_type_len & E1000_TXD_CMD_VLE)); + + if ((tx->first_cmd_type_len & E1000_ADVTXD_MAC_TSTAMP) && + (core->mac[TSYNCTXCTL] & E1000_TSYNCTXCTL_ENABLED) && + !(core->mac[TSYNCTXCTL] & E1000_TSYNCTXCTL_VALID)) { + core->mac[TSYNCTXCTL] |= E1000_TSYNCTXCTL_VALID; + e1000x_timestamp(core->mac, core->timadj, TXSTMPL, TXSTMPH); + } if (igb_tx_pkt_send(core, tx, queue_index)) { igb_on_tx_done_update_stats(core, tx->tx_pkt, queue_index); @@ -617,7 +670,7 @@ igb_process_tx_desc(IGBCore *core, tx->first = true; tx->skip_cp = false; - net_tx_pkt_reset(tx->tx_pkt, dev); + net_tx_pkt_reset(tx->tx_pkt, net_tx_pkt_unmap_frag_pci, dev); } } @@ -843,8 +896,6 @@ igb_start_xmit(IGBCore *core, const IGB_TxRing *txr) d = core->owner; } - net_tx_pkt_reset(txr->tx->tx_pkt, d); - while (!igb_ring_empty(core, txi)) { base = igb_ring_head_descr(core, txi); @@ -859,9 +910,11 @@ igb_start_xmit(IGBCore *core, const IGB_TxRing *txr) } if (eic) { - core->mac[EICR] |= eic; - igb_set_interrupt_cause(core, E1000_ICR_TXDW); + igb_raise_interrupts(core, EICR, eic); + igb_raise_interrupts(core, ICR, E1000_ICR_TXDW); } + + net_tx_pkt_reset(txr->tx->tx_pkt, net_tx_pkt_unmap_frag_pci, d); } static uint32_t @@ -949,49 +1002,83 @@ igb_rx_l4_cso_enabled(IGBCore *core) return !!(core->mac[RXCSUM] & E1000_RXCSUM_TUOFLD); } -static bool -igb_rx_is_oversized(IGBCore *core, uint16_t qn, size_t size) +static bool igb_rx_is_oversized(IGBCore *core, const struct eth_header *ehdr, + size_t size, size_t vlan_num, + bool lpe, uint16_t rlpml) { - uint16_t pool = qn % IGB_NUM_VM_POOLS; - bool lpe = !!(core->mac[VMOLR0 + pool] & E1000_VMOLR_LPE); - int max_ethernet_lpe_size = - core->mac[VMOLR0 + pool] & E1000_VMOLR_RLPML_MASK; - int max_ethernet_vlan_size = 1522; - - return size > (lpe ? max_ethernet_lpe_size : max_ethernet_vlan_size); + size_t vlan_header_size = sizeof(struct vlan_header) * vlan_num; + size_t header_size = sizeof(struct eth_header) + vlan_header_size; + return lpe ? size + ETH_FCS_LEN > rlpml : size > header_size + ETH_MTU; } -static uint16_t igb_receive_assign(IGBCore *core, const struct eth_header *ehdr, - size_t size, E1000E_RSSInfo *rss_info, - bool *external_tx) +static uint16_t igb_receive_assign(IGBCore *core, const struct iovec *iov, + size_t iovcnt, size_t iov_ofs, + const L2Header *l2_header, size_t size, + E1000E_RSSInfo *rss_info, + uint16_t *etqf, bool *ts, bool *external_tx) { static const int ta_shift[] = { 4, 3, 2, 0 }; + const struct eth_header *ehdr = &l2_header->eth; uint32_t f, ra[2], *macp, rctl = core->mac[RCTL]; uint16_t queues = 0; uint16_t oversized = 0; - uint16_t vid = lduw_be_p(&PKT_GET_VLAN_HDR(ehdr)->h_tci) & VLAN_VID_MASK; - bool accepted = false; + size_t vlan_num = 0; + PTP2 ptp2; + bool lpe; + uint16_t rlpml; int i; memset(rss_info, 0, sizeof(E1000E_RSSInfo)); + *ts = false; if (external_tx) { *external_tx = true; } - if (e1000x_is_vlan_packet(ehdr, core->mac[VET] & 0xffff) && - e1000x_vlan_rx_filter_enabled(core->mac)) { - uint32_t vfta = - ldl_le_p((uint32_t *)(core->mac + VFTA) + - ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK)); - if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) { - trace_e1000e_rx_flt_vlan_mismatch(vid); - return queues; - } else { - trace_e1000e_rx_flt_vlan_match(vid); + if (core->mac[CTRL_EXT] & BIT(26)) { + if (be16_to_cpu(ehdr->h_proto) == core->mac[VET] >> 16 && + be16_to_cpu(l2_header->vlan[0].h_proto) == (core->mac[VET] & 0xffff)) { + vlan_num = 2; + } + } else { + if (be16_to_cpu(ehdr->h_proto) == (core->mac[VET] & 0xffff)) { + vlan_num = 1; } } + lpe = !!(core->mac[RCTL] & E1000_RCTL_LPE); + rlpml = core->mac[RLPML]; + if (!(core->mac[RCTL] & E1000_RCTL_SBP) && + igb_rx_is_oversized(core, ehdr, size, vlan_num, lpe, rlpml)) { + trace_e1000x_rx_oversized(size); + return queues; + } + + for (*etqf = 0; *etqf < 8; (*etqf)++) { + if ((core->mac[ETQF0 + *etqf] & E1000_ETQF_FILTER_ENABLE) && + be16_to_cpu(ehdr->h_proto) == (core->mac[ETQF0 + *etqf] & E1000_ETQF_ETYPE_MASK)) { + if ((core->mac[ETQF0 + *etqf] & E1000_ETQF_1588) && + (core->mac[TSYNCRXCTL] & E1000_TSYNCRXCTL_ENABLED) && + !(core->mac[TSYNCRXCTL] & E1000_TSYNCRXCTL_VALID) && + iov_to_buf(iov, iovcnt, iov_ofs + ETH_HLEN, &ptp2, sizeof(ptp2)) >= sizeof(ptp2) && + (ptp2.version_ptp & 15) == 2 && + ptp2.message_id_transport_specific == ((core->mac[TSYNCRXCFG] >> 8) & 255)) { + e1000x_timestamp(core->mac, core->timadj, RXSTMPL, RXSTMPH); + *ts = true; + core->mac[TSYNCRXCTL] |= E1000_TSYNCRXCTL_VALID; + core->mac[RXSATRL] = le32_to_cpu(ptp2.source_uuid_lo); + core->mac[RXSATRH] = le16_to_cpu(ptp2.source_uuid_hi) | + (le16_to_cpu(ptp2.sequence_id) << 16); + } + break; + } + } + + if (vlan_num && + !e1000x_rx_vlan_filter(core->mac, l2_header->vlan + vlan_num - 1)) { + return queues; + } + if (core->mac[MRQC] & 1) { if (is_broadcast_ether_addr(ehdr->h_dest)) { for (i = 0; i < IGB_NUM_VM_POOLS; i++) { @@ -1042,7 +1129,9 @@ static uint16_t igb_receive_assign(IGBCore *core, const struct eth_header *ehdr, if (e1000x_vlan_rx_filter_enabled(core->mac)) { uint16_t mask = 0; - if (e1000x_is_vlan_packet(ehdr, core->mac[VET] & 0xffff)) { + if (vlan_num) { + uint16_t vid = be16_to_cpu(l2_header->vlan[vlan_num - 1].h_tci) & VLAN_VID_MASK; + for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) { if ((core->mac[VLVF0 + i] & E1000_VLVF_VLANID_MASK) == vid && (core->mac[VLVF0 + i] & E1000_VLVF_VLANID_ENABLE)) { @@ -1070,7 +1159,11 @@ static uint16_t igb_receive_assign(IGBCore *core, const struct eth_header *ehdr, queues &= core->mac[VFRE]; if (queues) { for (i = 0; i < IGB_NUM_VM_POOLS; i++) { - if ((queues & BIT(i)) && igb_rx_is_oversized(core, i, size)) { + lpe = !!(core->mac[VMOLR0 + i] & E1000_VMOLR_LPE); + rlpml = core->mac[VMOLR0 + i] & E1000_VMOLR_RLPML_MASK; + if ((queues & BIT(i)) && + igb_rx_is_oversized(core, ehdr, size, vlan_num, + lpe, rlpml)) { oversized |= BIT(i); } } @@ -1097,33 +1190,7 @@ static uint16_t igb_receive_assign(IGBCore *core, const struct eth_header *ehdr, } } } else { - switch (net_rx_pkt_get_packet_type(core->rx_pkt)) { - case ETH_PKT_UCAST: - if (rctl & E1000_RCTL_UPE) { - accepted = true; /* promiscuous ucast */ - } - break; - - case ETH_PKT_BCAST: - if (rctl & E1000_RCTL_BAM) { - accepted = true; /* broadcast enabled */ - } - break; - - case ETH_PKT_MCAST: - if (rctl & E1000_RCTL_MPE) { - accepted = true; /* promiscuous mcast */ - } - break; - - default: - g_assert_not_reached(); - } - - if (!accepted) { - accepted = e1000x_rx_group_filter(core->mac, ehdr->h_dest); - } - + bool accepted = e1000x_rx_group_filter(core->mac, ehdr); if (!accepted) { for (macp = core->mac + RA2; macp < core->mac + RA2 + 16; macp += 2) { if (!(macp[1] & E1000_RAH_AV)) { @@ -1217,7 +1284,7 @@ static void igb_build_rx_metadata(IGBCore *core, struct NetRxPkt *pkt, bool is_eop, - const E1000E_RSSInfo *rss_info, + const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts, uint16_t *pkt_info, uint16_t *hdr_info, uint32_t *rss, uint32_t *status_flags, @@ -1225,9 +1292,8 @@ igb_build_rx_metadata(IGBCore *core, uint16_t *vlan_tag) { struct virtio_net_hdr *vhdr; - bool hasip4, hasip6; + bool hasip4, hasip6, csum_valid; EthL4HdrProto l4hdr_proto; - uint32_t pkt_type; *status_flags = E1000_RXD_STAT_DD; @@ -1266,34 +1332,47 @@ igb_build_rx_metadata(IGBCore *core, trace_e1000e_rx_metadata_ack(); } - if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) { - trace_e1000e_rx_metadata_ipv6_filtering_disabled(); - pkt_type = E1000_RXD_PKT_MAC; - } else if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP || - l4hdr_proto == ETH_L4_HDR_PROTO_UDP) { - pkt_type = hasip4 ? E1000_RXD_PKT_IP4_XDP : E1000_RXD_PKT_IP6_XDP; - } else if (hasip4 || hasip6) { - pkt_type = hasip4 ? E1000_RXD_PKT_IP4 : E1000_RXD_PKT_IP6; - } else { - pkt_type = E1000_RXD_PKT_MAC; - } + if (pkt_info) { + *pkt_info = rss_info->enabled ? rss_info->type : 0; - trace_e1000e_rx_metadata_pkt_type(pkt_type); + if (etqf < 8) { + *pkt_info |= (BIT(11) | etqf) << 4; + } else { + if (hasip4) { + *pkt_info |= E1000_ADVRXD_PKT_IP4; + } - if (pkt_info) { - if (rss_info->enabled) { - *pkt_info = rss_info->type; - } + if (hasip6) { + *pkt_info |= E1000_ADVRXD_PKT_IP6; + } - *pkt_info |= (pkt_type << 4); - } else { - *status_flags |= E1000_RXD_PKT_TYPE(pkt_type); + switch (l4hdr_proto) { + case ETH_L4_HDR_PROTO_TCP: + *pkt_info |= E1000_ADVRXD_PKT_TCP; + break; + + case ETH_L4_HDR_PROTO_UDP: + *pkt_info |= E1000_ADVRXD_PKT_UDP; + break; + + case ETH_L4_HDR_PROTO_SCTP: + *pkt_info |= E1000_ADVRXD_PKT_SCTP; + break; + + default: + break; + } + } } if (hdr_info) { *hdr_info = 0; } + if (ts) { + *status_flags |= BIT(16); + } + /* RX CSO information */ if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) { trace_e1000e_rx_metadata_ipv6_sum_disabled(); @@ -1317,6 +1396,15 @@ igb_build_rx_metadata(IGBCore *core, if (igb_rx_l4_cso_enabled(core)) { switch (l4hdr_proto) { + case ETH_L4_HDR_PROTO_SCTP: + if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) { + trace_e1000e_rx_metadata_l4_csum_validation_failed(); + goto func_exit; + } + if (!csum_valid) { + *status_flags |= E1000_RXDEXT_STATERR_TCPE; + } + /* fall through */ case ETH_L4_HDR_PROTO_TCP: *status_flags |= E1000_RXD_STAT_TCPCS; break; @@ -1326,22 +1414,21 @@ igb_build_rx_metadata(IGBCore *core, break; default: - goto func_exit; + break; } } else { trace_e1000e_rx_metadata_l4_cso_disabled(); } - trace_e1000e_rx_metadata_status_flags(*status_flags); - func_exit: + trace_e1000e_rx_metadata_status_flags(*status_flags); *status_flags = cpu_to_le32(*status_flags); } static inline void igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc, struct NetRxPkt *pkt, - const E1000E_RSSInfo *rss_info, + const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts, uint16_t length) { uint32_t status_flags, rss; @@ -1352,7 +1439,7 @@ igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc, desc->csum = 0; igb_build_rx_metadata(core, pkt, pkt != NULL, - rss_info, + rss_info, etqf, ts, NULL, NULL, &rss, &status_flags, &ip_id, &desc->special); @@ -1363,7 +1450,7 @@ igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc, static inline void igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc, struct NetRxPkt *pkt, - const E1000E_RSSInfo *rss_info, + const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts, uint16_t length) { memset(&desc->wb, 0, sizeof(desc->wb)); @@ -1371,7 +1458,7 @@ igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc, desc->wb.upper.length = cpu_to_le16(length); igb_build_rx_metadata(core, pkt, pkt != NULL, - rss_info, + rss_info, etqf, ts, &desc->wb.lower.lo_dword.pkt_info, &desc->wb.lower.lo_dword.hdr_info, &desc->wb.lower.hi_dword.rss, @@ -1382,12 +1469,15 @@ igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc, static inline void igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc, -struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, uint16_t length) + struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, + uint16_t etqf, bool ts, uint16_t length) { if (igb_rx_use_legacy_descriptor(core)) { - igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, length); + igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, + etqf, ts, length); } else { - igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info, length); + igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info, + etqf, ts, length); } } @@ -1438,29 +1528,17 @@ igb_write_to_rx_buffers(IGBCore *core, static void igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi, - size_t data_size, size_t data_fcs_size) + size_t pkt_size, size_t pkt_fcs_size) { - e1000x_update_rx_total_stats(core->mac, data_size, data_fcs_size); - - switch (net_rx_pkt_get_packet_type(core->rx_pkt)) { - case ETH_PKT_BCAST: - e1000x_inc_reg_if_not_full(core->mac, BPRC); - break; - - case ETH_PKT_MCAST: - e1000x_inc_reg_if_not_full(core->mac, MPRC); - break; - - default: - break; - } + eth_pkt_types_e pkt_type = net_rx_pkt_get_packet_type(core->rx_pkt); + e1000x_update_rx_total_stats(core->mac, pkt_type, pkt_size, pkt_fcs_size); if (core->mac[MRQC] & 1) { uint16_t pool = rxi->idx % IGB_NUM_VM_POOLS; - core->mac[PVFGORC0 + (pool * 64)] += data_size + 4; + core->mac[PVFGORC0 + (pool * 64)] += pkt_size + 4; core->mac[PVFGPRC0 + (pool * 64)]++; - if (net_rx_pkt_get_packet_type(core->rx_pkt) == ETH_PKT_MCAST) { + if (pkt_type == ETH_PKT_MCAST) { core->mac[PVFMPRC0 + (pool * 64)]++; } } @@ -1476,7 +1554,8 @@ igb_rx_descr_threshold_hit(IGBCore *core, const E1000E_RingInfo *rxi) static void igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt, const E1000E_RxRing *rxr, - const E1000E_RSSInfo *rss_info) + const E1000E_RSSInfo *rss_info, + uint16_t etqf, bool ts) { PCIDevice *d; dma_addr_t base; @@ -1558,7 +1637,7 @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt, } igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL, - rss_info, written); + rss_info, etqf, ts, written); igb_pci_dma_write_rx_desc(core, d, base, &desc, core->rx_desc_len); igb_ring_advance(core, rxi, core->rx_desc_len / E1000_MIN_RX_DESC_LEN); @@ -1602,19 +1681,22 @@ static ssize_t igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt, bool has_vnet, bool *external_tx) { - static const int maximum_ethernet_hdr_len = (ETH_HLEN + 4); - uint16_t queues = 0; - uint32_t n = 0; - uint8_t min_buf[ETH_ZLEN]; + uint32_t causes = 0; + uint32_t ecauses = 0; + union { + L2Header l2_header; + uint8_t octets[ETH_ZLEN]; + } buf; struct iovec min_iov; - struct eth_header *ehdr; - uint8_t *filter_buf; size_t size, orig_size; size_t iov_ofs = 0; E1000E_RxRing rxr; E1000E_RSSInfo rss_info; + uint16_t etqf; + bool ts; size_t total_size; + int strip_vlan_index; int i; trace_e1000e_rx_receive_iov(iovcnt); @@ -1635,36 +1717,30 @@ igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt, net_rx_pkt_unset_vhdr(core->rx_pkt); } - filter_buf = iov->iov_base + iov_ofs; orig_size = iov_size(iov, iovcnt); size = orig_size - iov_ofs; /* Pad to minimum Ethernet frame length */ - if (size < sizeof(min_buf)) { - iov_to_buf(iov, iovcnt, iov_ofs, min_buf, size); - memset(&min_buf[size], 0, sizeof(min_buf) - size); + if (size < sizeof(buf)) { + iov_to_buf(iov, iovcnt, iov_ofs, &buf, size); + memset(&buf.octets[size], 0, sizeof(buf) - size); e1000x_inc_reg_if_not_full(core->mac, RUC); - min_iov.iov_base = filter_buf = min_buf; - min_iov.iov_len = size = sizeof(min_buf); + min_iov.iov_base = &buf; + min_iov.iov_len = size = sizeof(buf); iovcnt = 1; iov = &min_iov; iov_ofs = 0; - } else if (iov->iov_len < maximum_ethernet_hdr_len) { - /* This is very unlikely, but may happen. */ - iov_to_buf(iov, iovcnt, iov_ofs, min_buf, maximum_ethernet_hdr_len); - filter_buf = min_buf; - } - - /* Discard oversized packets if !LPE and !SBP. */ - if (e1000x_is_oversized(core->mac, size)) { - return orig_size; + } else { + iov_to_buf(iov, iovcnt, iov_ofs, &buf, sizeof(buf.l2_header)); } - ehdr = PKT_GET_ETH_HDR(filter_buf); - net_rx_pkt_set_packet_type(core->rx_pkt, get_eth_packet_type(ehdr)); - net_rx_pkt_set_protocols(core->rx_pkt, filter_buf, size); + net_rx_pkt_set_packet_type(core->rx_pkt, + get_eth_packet_type(&buf.l2_header.eth)); + net_rx_pkt_set_protocols(core->rx_pkt, iov, iovcnt, iov_ofs); - queues = igb_receive_assign(core, ehdr, size, &rss_info, external_tx); + queues = igb_receive_assign(core, iov, iovcnt, iov_ofs, + &buf.l2_header, size, + &rss_info, &etqf, &ts, external_tx); if (!queues) { trace_e1000e_rx_flt_dropped(); return orig_size; @@ -1678,36 +1754,46 @@ igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt, igb_rx_ring_init(core, &rxr, i); + if (!igb_rx_strip_vlan(core, rxr.i)) { + strip_vlan_index = -1; + } else if (core->mac[CTRL_EXT] & BIT(26)) { + strip_vlan_index = 1; + } else { + strip_vlan_index = 0; + } + net_rx_pkt_attach_iovec_ex(core->rx_pkt, iov, iovcnt, iov_ofs, - igb_rx_strip_vlan(core, rxr.i), - core->mac[VET] & 0xffff); + strip_vlan_index, + core->mac[VET] & 0xffff, + core->mac[VET] >> 16); total_size = net_rx_pkt_get_total_len(core->rx_pkt) + e1000x_fcs_len(core->mac); if (!igb_has_rxbufs(core, rxr.i, total_size)) { - n |= E1000_ICS_RXO; + causes |= E1000_ICS_RXO; trace_e1000e_rx_not_written_to_guest(rxr.i->idx); continue; } - n |= E1000_ICR_RXDW; + causes |= E1000_ICR_RXDW; igb_rx_fix_l4_csum(core, core->rx_pkt); - igb_write_packet_to_guest(core, core->rx_pkt, &rxr, &rss_info); + igb_write_packet_to_guest(core, core->rx_pkt, &rxr, &rss_info, etqf, ts); /* Check if receive descriptor minimum threshold hit */ if (igb_rx_descr_threshold_hit(core, rxr.i)) { - n |= E1000_ICS_RXDMT0; + causes |= E1000_ICS_RXDMT0; } - core->mac[EICR] |= igb_rx_wb_eic(core, rxr.i->idx); + ecauses |= igb_rx_wb_eic(core, rxr.i->idx); trace_e1000e_rx_written_to_guest(rxr.i->idx); } - trace_e1000e_rx_interrupt_set(n); - igb_set_interrupt_cause(core, n); + trace_e1000e_rx_interrupt_set(causes); + igb_raise_interrupts(core, EICR, ecauses); + igb_raise_interrupts(core, ICR, causes); return orig_size; } @@ -1767,7 +1853,7 @@ void igb_core_set_link_status(IGBCore *core) } if (core->mac[STATUS] != old_status) { - igb_set_interrupt_cause(core, E1000_ICR_LSC); + igb_raise_interrupts(core, ICR, E1000_ICR_LSC); } } @@ -1847,13 +1933,6 @@ igb_set_rx_control(IGBCore *core, int index, uint32_t val) } } -static inline void -igb_clear_ims_bits(IGBCore *core, uint32_t bits) -{ - trace_e1000e_irq_clear_ims(bits, core->mac[IMS], core->mac[IMS] & ~bits); - core->mac[IMS] &= ~bits; -} - static inline bool igb_postpone_interrupt(IGBIntrDelayTimer *timer) { @@ -1876,10 +1955,8 @@ igb_eitr_should_postpone(IGBCore *core, int idx) return igb_postpone_interrupt(&core->eitr[idx]); } -static void igb_send_msix(IGBCore *core) +static void igb_send_msix(IGBCore *core, uint32_t causes) { - uint32_t causes = core->mac[EICR] & core->mac[EIMS]; - uint32_t effective_eiac; int vector; for (vector = 0; vector < IGB_INTR_NUM; ++vector) { @@ -1887,10 +1964,6 @@ static void igb_send_msix(IGBCore *core) trace_e1000e_irq_msix_notify_vec(vector); igb_msix_notify(core, vector); - - trace_e1000e_irq_icr_clear_eiac(core->mac[EICR], core->mac[EIAC]); - effective_eiac = core->mac[EIAC] & BIT(vector); - core->mac[EICR] &= ~effective_eiac; } } } @@ -1906,119 +1979,116 @@ igb_fix_icr_asserted(IGBCore *core) trace_e1000e_irq_fix_icr_asserted(core->mac[ICR]); } -static void -igb_update_interrupt_state(IGBCore *core) +static void igb_raise_interrupts(IGBCore *core, size_t index, uint32_t causes) { - uint32_t icr; - uint32_t causes; + uint32_t old_causes = core->mac[ICR] & core->mac[IMS]; + uint32_t old_ecauses = core->mac[EICR] & core->mac[EIMS]; + uint32_t raised_causes; + uint32_t raised_ecauses; uint32_t int_alloc; - icr = core->mac[ICR] & core->mac[IMS]; + trace_e1000e_irq_set(index << 2, + core->mac[index], core->mac[index] | causes); - if (msix_enabled(core->owner)) { - if (icr) { - causes = 0; - if (icr & E1000_ICR_DRSTA) { - int_alloc = core->mac[IVAR_MISC] & 0xff; - if (int_alloc & E1000_IVAR_VALID) { - causes |= BIT(int_alloc & 0x1f); - } + core->mac[index] |= causes; + + if (core->mac[GPIE] & E1000_GPIE_MSIX_MODE) { + raised_causes = core->mac[ICR] & core->mac[IMS] & ~old_causes; + + if (raised_causes & E1000_ICR_DRSTA) { + int_alloc = core->mac[IVAR_MISC] & 0xff; + if (int_alloc & E1000_IVAR_VALID) { + core->mac[EICR] |= BIT(int_alloc & 0x1f); } - /* Check if other bits (excluding the TCP Timer) are enabled. */ - if (icr & ~E1000_ICR_DRSTA) { - int_alloc = (core->mac[IVAR_MISC] >> 8) & 0xff; - if (int_alloc & E1000_IVAR_VALID) { - causes |= BIT(int_alloc & 0x1f); - } - trace_e1000e_irq_add_msi_other(core->mac[EICR]); + } + /* Check if other bits (excluding the TCP Timer) are enabled. */ + if (raised_causes & ~E1000_ICR_DRSTA) { + int_alloc = (core->mac[IVAR_MISC] >> 8) & 0xff; + if (int_alloc & E1000_IVAR_VALID) { + core->mac[EICR] |= BIT(int_alloc & 0x1f); } - core->mac[EICR] |= causes; } - if ((core->mac[EICR] & core->mac[EIMS])) { - igb_send_msix(core); + raised_ecauses = core->mac[EICR] & core->mac[EIMS] & ~old_ecauses; + if (!raised_ecauses) { + return; } + + igb_send_msix(core, raised_ecauses); } else { igb_fix_icr_asserted(core); - if (icr) { - core->mac[EICR] |= (icr & E1000_ICR_DRSTA) | E1000_EICR_OTHER; - } else { - core->mac[EICR] &= ~E1000_EICR_OTHER; + raised_causes = core->mac[ICR] & core->mac[IMS] & ~old_causes; + if (!raised_causes) { + return; } - trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS], - core->mac[ICR], core->mac[IMS]); + core->mac[EICR] |= (raised_causes & E1000_ICR_DRSTA) | E1000_EICR_OTHER; - if (msi_enabled(core->owner)) { - if (icr) { - msi_notify(core->owner, 0); - } + if (msix_enabled(core->owner)) { + trace_e1000e_irq_msix_notify_vec(0); + msix_notify(core->owner, 0); + } else if (msi_enabled(core->owner)) { + trace_e1000e_irq_msi_notify(raised_causes); + msi_notify(core->owner, 0); } else { - if (icr) { - igb_raise_legacy_irq(core); - } else { - igb_lower_legacy_irq(core); - } + igb_raise_legacy_irq(core); } } } -static void -igb_set_interrupt_cause(IGBCore *core, uint32_t val) +static void igb_lower_interrupts(IGBCore *core, size_t index, uint32_t causes) { - trace_e1000e_irq_set_cause_entry(val, core->mac[ICR]); + trace_e1000e_irq_clear(index << 2, + core->mac[index], core->mac[index] & ~causes); - core->mac[ICR] |= val; + core->mac[index] &= ~causes; - trace_e1000e_irq_set_cause_exit(val, core->mac[ICR]); + trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS], + core->mac[ICR], core->mac[IMS]); - igb_update_interrupt_state(core); + if (!(core->mac[ICR] & core->mac[IMS]) && + !(core->mac[GPIE] & E1000_GPIE_MSIX_MODE)) { + core->mac[EICR] &= ~E1000_EICR_OTHER; + + if (!msix_enabled(core->owner) && !msi_enabled(core->owner)) { + igb_lower_legacy_irq(core); + } + } } static void igb_set_eics(IGBCore *core, int index, uint32_t val) { bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE); + uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK; trace_igb_irq_write_eics(val, msix); - - core->mac[EICS] |= - val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK); - - /* - * TODO: Move to igb_update_interrupt_state if EICS is modified in other - * places. - */ - core->mac[EICR] = core->mac[EICS]; - - igb_update_interrupt_state(core); + igb_raise_interrupts(core, EICR, val & mask); } static void igb_set_eims(IGBCore *core, int index, uint32_t val) { bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE); + uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK; trace_igb_irq_write_eims(val, msix); - - core->mac[EIMS] |= - val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK); - - igb_update_interrupt_state(core); + igb_raise_interrupts(core, EIMS, val & mask); } static void mailbox_interrupt_to_vf(IGBCore *core, uint16_t vfn) { uint32_t ent = core->mac[VTIVAR_MISC + vfn]; + uint32_t causes; if ((ent & E1000_IVAR_VALID)) { - core->mac[EICR] |= (ent & 0x3) << (22 - vfn * IGBVF_MSIX_VEC_NUM); - igb_update_interrupt_state(core); + causes = (ent & 0x3) << (22 - vfn * IGBVF_MSIX_VEC_NUM); + igb_raise_interrupts(core, EICR, causes); } } static void mailbox_interrupt_to_pf(IGBCore *core) { - igb_set_interrupt_cause(core, E1000_ICR_VMMB); + igb_raise_interrupts(core, ICR, E1000_ICR_VMMB); } static void igb_set_pfmailbox(IGBCore *core, int index, uint32_t val) @@ -2109,13 +2179,12 @@ static void igb_w1c(IGBCore *core, int index, uint32_t val) static void igb_set_eimc(IGBCore *core, int index, uint32_t val) { bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE); + uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK; - /* Interrupts are disabled via a write to EIMC and reflected in EIMS. */ - core->mac[EIMS] &= - ~(val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK)); + trace_igb_irq_write_eimc(val, msix); - trace_igb_irq_write_eimc(val, core->mac[EIMS], msix); - igb_update_interrupt_state(core); + /* Interrupts are disabled via a write to EIMC and reflected in EIMS. */ + igb_lower_interrupts(core, EIMS, val & mask); } static void igb_set_eiac(IGBCore *core, int index, uint32_t val) @@ -2155,11 +2224,10 @@ static void igb_set_eicr(IGBCore *core, int index, uint32_t val) * TODO: In IOV mode, only bit zero of this vector is available for the PF * function. */ - core->mac[EICR] &= - ~(val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK)); + uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK; trace_igb_irq_write_eicr(val, msix); - igb_update_interrupt_state(core); + igb_lower_interrupts(core, EICR, val & mask); } static void igb_set_vtctrl(IGBCore *core, int index, uint32_t val) @@ -2259,7 +2327,7 @@ igb_autoneg_timer(void *opaque) igb_update_flowctl_status(core); /* signal link status change to the guest */ - igb_set_interrupt_cause(core, E1000_ICR_LSC); + igb_raise_interrupts(core, ICR, E1000_ICR_LSC); } } @@ -2332,7 +2400,7 @@ igb_set_mdic(IGBCore *core, int index, uint32_t val) core->mac[MDIC] = val | E1000_MDIC_READY; if (val & E1000_MDIC_INT_EN) { - igb_set_interrupt_cause(core, E1000_ICR_MDAC); + igb_raise_interrupts(core, ICR, E1000_ICR_MDAC); } } @@ -2440,49 +2508,39 @@ static void igb_set_ics(IGBCore *core, int index, uint32_t val) { trace_e1000e_irq_write_ics(val); - igb_set_interrupt_cause(core, val); + igb_raise_interrupts(core, ICR, val); } static void igb_set_imc(IGBCore *core, int index, uint32_t val) { trace_e1000e_irq_ims_clear_set_imc(val); - igb_clear_ims_bits(core, val); - igb_update_interrupt_state(core); + igb_lower_interrupts(core, IMS, val); } static void igb_set_ims(IGBCore *core, int index, uint32_t val) { - uint32_t valid_val = val & 0x77D4FBFD; - - trace_e1000e_irq_set_ims(val, core->mac[IMS], core->mac[IMS] | valid_val); - core->mac[IMS] |= valid_val; - igb_update_interrupt_state(core); + igb_raise_interrupts(core, IMS, val & 0x77D4FBFD); } -static void igb_commit_icr(IGBCore *core) +static void igb_nsicr(IGBCore *core) { /* - * If GPIE.NSICR = 0, then the copy of IAM to IMS will occur only if at + * If GPIE.NSICR = 0, then the clear of IMS will occur only if at * least one bit is set in the IMS and there is a true interrupt as * reflected in ICR.INTA. */ if ((core->mac[GPIE] & E1000_GPIE_NSICR) || (core->mac[IMS] && (core->mac[ICR] & E1000_ICR_INT_ASSERTED))) { - igb_set_ims(core, IMS, core->mac[IAM]); - } else { - igb_update_interrupt_state(core); + igb_lower_interrupts(core, IMS, core->mac[IAM]); } } static void igb_set_icr(IGBCore *core, int index, uint32_t val) { - uint32_t icr = core->mac[ICR] & ~val; - - trace_igb_irq_icr_write(val, core->mac[ICR], icr); - core->mac[ICR] = icr; - igb_commit_icr(core); + igb_nsicr(core); + igb_lower_interrupts(core, ICR, val); } static uint32_t @@ -2533,21 +2591,21 @@ static uint32_t igb_mac_icr_read(IGBCore *core, int index) { uint32_t ret = core->mac[ICR]; - trace_e1000e_irq_icr_read_entry(ret); if (core->mac[GPIE] & E1000_GPIE_NSICR) { trace_igb_irq_icr_clear_gpie_nsicr(); - core->mac[ICR] = 0; + igb_lower_interrupts(core, ICR, 0xffffffff); } else if (core->mac[IMS] == 0) { trace_e1000e_irq_icr_clear_zero_ims(); - core->mac[ICR] = 0; + igb_lower_interrupts(core, ICR, 0xffffffff); + } else if (core->mac[ICR] & E1000_ICR_INT_ASSERTED) { + igb_lower_interrupts(core, ICR, 0xffffffff); } else if (!msix_enabled(core->owner)) { trace_e1000e_irq_icr_clear_nonmsix_icr_read(); - core->mac[ICR] = 0; + igb_lower_interrupts(core, ICR, 0xffffffff); } - trace_e1000e_irq_icr_read_exit(core->mac[ICR]); - igb_commit_icr(core); + igb_nsicr(core); return ret; } @@ -3282,6 +3340,8 @@ static const readops igb_macreg_readops[] = { [EIAM] = igb_mac_readreg, [IVAR0 ... IVAR0 + 7] = igb_mac_readreg, igb_getreg(IVAR_MISC), + igb_getreg(TSYNCRXCFG), + [ETQF0 ... ETQF0 + 7] = igb_mac_readreg, igb_getreg(VT_CTL), [P2VMAILBOX0 ... P2VMAILBOX7] = igb_mac_readreg, [V2PMAILBOX0 ... V2PMAILBOX7] = igb_mac_vfmailbox_read, @@ -3689,6 +3749,8 @@ static const writeops igb_macreg_writeops[] = { [EIMS] = igb_set_eims, [IVAR0 ... IVAR0 + 7] = igb_mac_writereg, igb_putreg(IVAR_MISC), + igb_putreg(TSYNCRXCFG), + [ETQF0 ... ETQF0 + 7] = igb_mac_writereg, igb_putreg(VT_CTL), [P2VMAILBOX0 ... P2VMAILBOX7] = igb_set_pfmailbox, [V2PMAILBOX0 ... V2PMAILBOX7] = igb_set_vfmailbox, @@ -3955,7 +4017,7 @@ igb_core_pci_realize(IGBCore *core, core->vmstate = qemu_add_vm_change_state_handler(igb_vm_state_change, core); for (i = 0; i < IGB_NUM_QUEUES; i++) { - net_tx_pkt_init(&core->tx[i].tx_pkt, NULL, E1000E_MAX_TX_FRAGS); + net_tx_pkt_init(&core->tx[i].tx_pkt, E1000E_MAX_TX_FRAGS); } net_rx_pkt_init(&core->rx_pkt); @@ -3980,7 +4042,6 @@ igb_core_pci_uninit(IGBCore *core) qemu_del_vm_change_state_handler(core->vmstate); for (i = 0; i < IGB_NUM_QUEUES; i++) { - net_tx_pkt_reset(core->tx[i].tx_pkt, NULL); net_tx_pkt_uninit(core->tx[i].tx_pkt); } @@ -4073,54 +4134,54 @@ static const uint32_t igb_mac_reg_init[] = { [VMOLR0 ... VMOLR0 + 7] = 0x2600 | E1000_VMOLR_STRCRC, [RPLOLR] = E1000_RPLOLR_STRCRC, [RLPML] = 0x2600, - [TXCTL0] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL1] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL2] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL3] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL4] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL5] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL6] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL7] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL8] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL9] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL10] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL11] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL12] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL13] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL14] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, - [TXCTL15] = E1000_DCA_TXCTRL_DATA_RRO_EN | - E1000_DCA_TXCTRL_TX_WB_RO_EN | - E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL0] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL1] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL2] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL3] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL4] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL5] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL6] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL7] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL8] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL9] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL10] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL11] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL12] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL13] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL14] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL15] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, }; static void igb_reset(IGBCore *core, bool sw) @@ -4159,7 +4220,6 @@ static void igb_reset(IGBCore *core, bool sw) for (i = 0; i < ARRAY_SIZE(core->tx); i++) { tx = &core->tx[i]; - net_tx_pkt_reset(tx->tx_pkt, NULL); memset(tx->ctx, 0, sizeof(tx->ctx)); tx->first = true; tx->skip_cp = false; diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h index c5c5b3c3b8..82ff195dfc 100644 --- a/hw/net/igb_regs.h +++ b/hw/net/igb_regs.h @@ -42,11 +42,6 @@ union e1000_adv_tx_desc { } wb; }; -#define E1000_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Descriptor */ -#define E1000_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ -#define E1000_ADVTXD_DCMD_DEXT 0x20000000 /* Descriptor Extension (1=Adv) */ -#define E1000_ADVTXD_DCMD_TSE 0x80000000 /* TCP/UDP Segmentation Enable */ - #define E1000_ADVTXD_POTS_IXSM 0x00000100 /* Insert TCP/UDP Checksum */ #define E1000_ADVTXD_POTS_TXSM 0x00000200 /* Insert TCP/UDP Checksum */ @@ -151,6 +146,10 @@ union e1000_adv_rx_desc { #define IGB_82576_VF_DEV_ID 0x10CA #define IGB_I350_VF_DEV_ID 0x1520 +/* VLAN info */ +#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000 +#define IGB_TX_FLAGS_VLAN_SHIFT 16 + /* from igb/e1000_82575.h */ #define E1000_MRQC_ENABLE_RSS_MQ 0x00000002 @@ -160,6 +159,29 @@ union e1000_adv_rx_desc { #define E1000_MRQC_RSS_FIELD_IPV6_UDP 0x00800000 #define E1000_MRQC_RSS_FIELD_IPV6_UDP_EX 0x01000000 +/* Adv Transmit Descriptor Config Masks */ +#define E1000_ADVTXD_MAC_TSTAMP 0x00080000 /* IEEE1588 Timestamp packet */ +#define E1000_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Descriptor */ +#define E1000_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ +#define E1000_ADVTXD_DCMD_EOP 0x01000000 /* End of Packet */ +#define E1000_ADVTXD_DCMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ +#define E1000_ADVTXD_DCMD_RS 0x08000000 /* Report Status */ +#define E1000_ADVTXD_DCMD_DEXT 0x20000000 /* Descriptor extension (1=Adv) */ +#define E1000_ADVTXD_DCMD_VLE 0x40000000 /* VLAN pkt enable */ +#define E1000_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ +#define E1000_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ + +#define E1000_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ +#define E1000_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */ +#define E1000_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ +#define E1000_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */ +#define E1000_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 packet TYPE of SCTP */ +/* IPSec Encrypt Enable for ESP */ +#define E1000_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ +#define E1000_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ +/* Adv ctxt IPSec SA IDX mask */ +/* Adv ctxt IPSec ESP len mask */ + /* Additional Transmit Descriptor Control definitions */ #define E1000_TXDCTL_QUEUE_ENABLE 0x02000000 /* Enable specific Tx Queue */ @@ -188,6 +210,15 @@ union e1000_adv_rx_desc { #define E1000_DCA_TXCTRL_CPUID_SHIFT 24 /* Tx CPUID now in the last byte */ #define E1000_DCA_RXCTRL_CPUID_SHIFT 24 /* Rx CPUID now in the last byte */ +/* ETQF register bit definitions */ +#define E1000_ETQF_FILTER_ENABLE BIT(26) +#define E1000_ETQF_1588 BIT(30) +#define E1000_ETQF_IMM_INT BIT(29) +#define E1000_ETQF_QUEUE_ENABLE BIT(31) +#define E1000_ETQF_QUEUE_SHIFT 16 +#define E1000_ETQF_QUEUE_MASK 0x00070000 +#define E1000_ETQF_ETYPE_MASK 0x0000FFFF + #define E1000_DTXSWC_MAC_SPOOF_MASK 0x000000FF /* Per VF MAC spoof control */ #define E1000_DTXSWC_VLAN_SPOOF_MASK 0x0000FF00 /* Per VF VLAN spoof control */ #define E1000_DTXSWC_LLE_MASK 0x00FF0000 /* Per VF Local LB enables */ @@ -291,6 +322,9 @@ union e1000_adv_rx_desc { /* E1000_EITR_CNT_IGNR is only for 82576 and newer */ #define E1000_EITR_CNT_IGNR 0x80000000 /* Don't reset counters on write */ +#define E1000_TSYNCTXCTL_VALID 0x00000001 /* tx timestamp valid */ +#define E1000_TSYNCTXCTL_ENABLED 0x00000010 /* enable tx timestampping */ + /* PCI Express Control */ #define E1000_GCR_CMPL_TMOUT_MASK 0x0000F000 #define E1000_GCR_CMPL_TMOUT_10ms 0x00001000 @@ -362,6 +396,20 @@ union e1000_adv_rx_desc { #define E1000_FRTIMER 0x01048 /* Free Running Timer - RW */ #define E1000_FCRTV 0x02460 /* Flow Control Refresh Timer Value - RW */ +#define E1000_TSYNCRXCFG 0x05F50 /* Time Sync Rx Configuration - RW */ + +/* Filtering Registers */ +#define E1000_SAQF(_n) (0x5980 + 4 * (_n)) +#define E1000_DAQF(_n) (0x59A0 + 4 * (_n)) +#define E1000_SPQF(_n) (0x59C0 + 4 * (_n)) +#define E1000_FTQF(_n) (0x59E0 + 4 * (_n)) +#define E1000_SAQF0 E1000_SAQF(0) +#define E1000_DAQF0 E1000_DAQF(0) +#define E1000_SPQF0 E1000_SPQF(0) +#define E1000_FTQF0 E1000_FTQF(0) +#define E1000_SYNQF(_n) (0x055FC + (4 * (_n))) /* SYN Packet Queue Fltr */ +#define E1000_ETQF(_n) (0x05CB0 + (4 * (_n))) /* EType Queue Fltr */ + #define E1000_RQDPC(_n) (0x0C030 + ((_n) * 0x40)) #define E1000_RXPBS 0x02404 /* Rx Packet Buffer Size - RW */ @@ -637,10 +685,19 @@ union e1000_adv_rx_desc { #define E1000_RSS_QUEUE(reta, hash) (E1000_RETA_VAL(reta, hash) & 0x0F) +#define E1000_MRQ_RSS_TYPE_IPV4UDP 7 +#define E1000_MRQ_RSS_TYPE_IPV6UDP 8 + #define E1000_STATUS_IOV_MODE 0x00040000 #define E1000_STATUS_NUM_VFS_SHIFT 14 +#define E1000_ADVRXD_PKT_IP4 BIT(4) +#define E1000_ADVRXD_PKT_IP6 BIT(6) +#define E1000_ADVRXD_PKT_TCP BIT(8) +#define E1000_ADVRXD_PKT_UDP BIT(9) +#define E1000_ADVRXD_PKT_SCTP BIT(10) + static inline uint8_t igb_ivar_entry_rx(uint8_t i) { return i < 8 ? i * 4 : (i - 8) * 4 + 2; diff --git a/hw/net/igbvf.c b/hw/net/igbvf.c index 70beb7af50..284ea61184 100644 --- a/hw/net/igbvf.c +++ b/hw/net/igbvf.c @@ -50,15 +50,8 @@ #include "trace.h" #include "qapi/error.h" -#define TYPE_IGBVF "igbvf" OBJECT_DECLARE_SIMPLE_TYPE(IgbVfState, IGBVF) -#define IGBVF_MMIO_BAR_IDX (0) -#define IGBVF_MSIX_BAR_IDX (3) - -#define IGBVF_MMIO_SIZE (16 * 1024) -#define IGBVF_MSIX_SIZE (16 * 1024) - struct IgbVfState { PCIDevice parent_obj; diff --git a/hw/net/net_rx_pkt.c b/hw/net/net_rx_pkt.c index 39cdea06de..32e5f3f9cf 100644 --- a/hw/net/net_rx_pkt.c +++ b/hw/net/net_rx_pkt.c @@ -16,6 +16,7 @@ */ #include "qemu/osdep.h" +#include "qemu/crc32c.h" #include "trace.h" #include "net_rx_pkt.h" #include "net/checksum.h" @@ -23,7 +24,10 @@ struct NetRxPkt { struct virtio_net_hdr virt_hdr; - uint8_t ehdr_buf[sizeof(struct eth_header) + sizeof(struct vlan_header)]; + struct { + struct eth_header eth; + struct vlan_header vlan; + } ehdr_buf; struct iovec *vec; uint16_t vec_len_total; uint16_t vec_len; @@ -89,7 +93,7 @@ net_rx_pkt_pull_data(struct NetRxPkt *pkt, if (pkt->ehdr_buf_len) { net_rx_pkt_iovec_realloc(pkt, iovcnt + 1); - pkt->vec[0].iov_base = pkt->ehdr_buf; + pkt->vec[0].iov_base = &pkt->ehdr_buf; pkt->vec[0].iov_len = pkt->ehdr_buf_len; pkt->tot_len = pllen + pkt->ehdr_buf_len; @@ -103,7 +107,7 @@ net_rx_pkt_pull_data(struct NetRxPkt *pkt, iov, iovcnt, ploff, pkt->tot_len); } - eth_get_protocols(pkt->vec, pkt->vec_len, &pkt->hasip4, &pkt->hasip6, + eth_get_protocols(pkt->vec, pkt->vec_len, 0, &pkt->hasip4, &pkt->hasip6, &pkt->l3hdr_off, &pkt->l4hdr_off, &pkt->l5hdr_off, &pkt->ip6hdr_info, &pkt->ip4hdr_info, &pkt->l4hdr_info); @@ -120,7 +124,7 @@ void net_rx_pkt_attach_iovec(struct NetRxPkt *pkt, assert(pkt); if (strip_vlan) { - pkt->ehdr_buf_len = eth_strip_vlan(iov, iovcnt, iovoff, pkt->ehdr_buf, + pkt->ehdr_buf_len = eth_strip_vlan(iov, iovcnt, iovoff, &pkt->ehdr_buf, &ploff, &tci); } else { pkt->ehdr_buf_len = 0; @@ -133,20 +137,17 @@ void net_rx_pkt_attach_iovec(struct NetRxPkt *pkt, void net_rx_pkt_attach_iovec_ex(struct NetRxPkt *pkt, const struct iovec *iov, int iovcnt, - size_t iovoff, bool strip_vlan, - uint16_t vet) + size_t iovoff, int strip_vlan_index, + uint16_t vet, uint16_t vet_ext) { uint16_t tci = 0; uint16_t ploff = iovoff; assert(pkt); - if (strip_vlan) { - pkt->ehdr_buf_len = eth_strip_vlan_ex(iov, iovcnt, iovoff, vet, - pkt->ehdr_buf, - &ploff, &tci); - } else { - pkt->ehdr_buf_len = 0; - } + pkt->ehdr_buf_len = eth_strip_vlan_ex(iov, iovcnt, iovoff, + strip_vlan_index, vet, vet_ext, + &pkt->ehdr_buf, + &ploff, &tci); pkt->tci = tci; @@ -186,17 +187,13 @@ size_t net_rx_pkt_get_total_len(struct NetRxPkt *pkt) return pkt->tot_len; } -void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, const void *data, - size_t len) +void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, + const struct iovec *iov, size_t iovcnt, + size_t iovoff) { - const struct iovec iov = { - .iov_base = (void *)data, - .iov_len = len - }; - assert(pkt); - eth_get_protocols(&iov, 1, &pkt->hasip4, &pkt->hasip6, + eth_get_protocols(iov, iovcnt, iovoff, &pkt->hasip4, &pkt->hasip6, &pkt->l3hdr_off, &pkt->l4hdr_off, &pkt->l5hdr_off, &pkt->ip6hdr_info, &pkt->ip4hdr_info, &pkt->l4hdr_info); } @@ -240,11 +237,6 @@ eth_ip4_hdr_info *net_rx_pkt_get_ip4_info(struct NetRxPkt *pkt) return &pkt->ip4hdr_info; } -eth_l4_hdr_info *net_rx_pkt_get_l4_info(struct NetRxPkt *pkt) -{ - return &pkt->l4hdr_info; -} - static inline void _net_rx_rss_add_chunk(uint8_t *rss_input, size_t *bytes_written, void *ptr, size_t size) @@ -560,32 +552,73 @@ _net_rx_pkt_calc_l4_csum(struct NetRxPkt *pkt) return csum; } -bool net_rx_pkt_validate_l4_csum(struct NetRxPkt *pkt, bool *csum_valid) +static bool +_net_rx_pkt_validate_sctp_sum(struct NetRxPkt *pkt) { - uint16_t csum; + size_t csum_off; + size_t off = pkt->l4hdr_off; + size_t vec_len = pkt->vec_len; + struct iovec *vec; + uint32_t calculated = 0; + uint32_t original; + bool valid; - trace_net_rx_pkt_l4_csum_validate_entry(); + for (vec = pkt->vec; vec->iov_len < off; vec++) { + off -= vec->iov_len; + vec_len--; + } - if (pkt->l4hdr_info.proto != ETH_L4_HDR_PROTO_TCP && - pkt->l4hdr_info.proto != ETH_L4_HDR_PROTO_UDP) { - trace_net_rx_pkt_l4_csum_validate_not_xxp(); + csum_off = off + 8; + + if (!iov_to_buf(vec, vec_len, csum_off, &original, sizeof(original))) { return false; } - if (pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_UDP && - pkt->l4hdr_info.hdr.udp.uh_sum == 0) { - trace_net_rx_pkt_l4_csum_validate_udp_with_no_checksum(); + if (!iov_from_buf(vec, vec_len, csum_off, + &calculated, sizeof(calculated))) { return false; } + calculated = crc32c(0xffffffff, + (uint8_t *)vec->iov_base + off, vec->iov_len - off); + calculated = iov_crc32c(calculated ^ 0xffffffff, vec + 1, vec_len - 1); + valid = calculated == le32_to_cpu(original); + iov_from_buf(vec, vec_len, csum_off, &original, sizeof(original)); + + return valid; +} + +bool net_rx_pkt_validate_l4_csum(struct NetRxPkt *pkt, bool *csum_valid) +{ + uint32_t csum; + + trace_net_rx_pkt_l4_csum_validate_entry(); + if (pkt->hasip4 && pkt->ip4hdr_info.fragment) { trace_net_rx_pkt_l4_csum_validate_ip4_fragment(); return false; } - csum = _net_rx_pkt_calc_l4_csum(pkt); + switch (pkt->l4hdr_info.proto) { + case ETH_L4_HDR_PROTO_UDP: + if (pkt->l4hdr_info.hdr.udp.uh_sum == 0) { + trace_net_rx_pkt_l4_csum_validate_udp_with_no_checksum(); + return false; + } + /* fall through */ + case ETH_L4_HDR_PROTO_TCP: + csum = _net_rx_pkt_calc_l4_csum(pkt); + *csum_valid = ((csum == 0) || (csum == 0xFFFF)); + break; + + case ETH_L4_HDR_PROTO_SCTP: + *csum_valid = _net_rx_pkt_validate_sctp_sum(pkt); + break; - *csum_valid = ((csum == 0) || (csum == 0xFFFF)); + default: + trace_net_rx_pkt_l4_csum_validate_not_xxp(); + return false; + } trace_net_rx_pkt_l4_csum_validate_csum(*csum_valid); diff --git a/hw/net/net_rx_pkt.h b/hw/net/net_rx_pkt.h index d00b484900..55ec67a1a7 100644 --- a/hw/net/net_rx_pkt.h +++ b/hw/net/net_rx_pkt.h @@ -55,12 +55,14 @@ size_t net_rx_pkt_get_total_len(struct NetRxPkt *pkt); * parse and set packet analysis results * * @pkt: packet - * @data: pointer to the data buffer to be parsed - * @len: data length + * @iov: received data scatter-gather list + * @iovcnt: number of elements in iov + * @iovoff: data start offset in the iov * */ -void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, const void *data, - size_t len); +void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, + const struct iovec *iov, size_t iovcnt, + size_t iovoff); /** * fetches packet analysis results @@ -117,15 +119,6 @@ eth_ip6_hdr_info *net_rx_pkt_get_ip6_info(struct NetRxPkt *pkt); */ eth_ip4_hdr_info *net_rx_pkt_get_ip4_info(struct NetRxPkt *pkt); -/** - * fetches L4 header analysis results - * - * Return: pointer to analysis results structure which is stored in internal - * packet area. - * - */ -eth_l4_hdr_info *net_rx_pkt_get_l4_info(struct NetRxPkt *pkt); - typedef enum { NetPktRssIpV4, NetPktRssIpV4Tcp, @@ -230,18 +223,19 @@ void net_rx_pkt_attach_iovec(struct NetRxPkt *pkt, /** * attach scatter-gather data to rx packet * -* @pkt: packet -* @iov: received data scatter-gather list -* @iovcnt number of elements in iov -* @iovoff data start offset in the iov -* @strip_vlan: should the module strip vlan from data -* @vet: VLAN tag Ethernet type +* @pkt: packet +* @iov: received data scatter-gather list +* @iovcnt: number of elements in iov +* @iovoff: data start offset in the iov +* @strip_vlan_index: index of Q tag if it is to be stripped. negative otherwise. +* @vet: VLAN tag Ethernet type +* @vet_ext: outer VLAN tag Ethernet type * */ void net_rx_pkt_attach_iovec_ex(struct NetRxPkt *pkt, - const struct iovec *iov, int iovcnt, - size_t iovoff, bool strip_vlan, - uint16_t vet); + const struct iovec *iov, int iovcnt, + size_t iovoff, int strip_vlan_index, + uint16_t vet, uint16_t vet_ext); /** * attach data to rx packet diff --git a/hw/net/net_tx_pkt.c b/hw/net/net_tx_pkt.c index 8dc8568ba2..2e5f58b3c9 100644 --- a/hw/net/net_tx_pkt.c +++ b/hw/net/net_tx_pkt.c @@ -16,12 +16,13 @@ */ #include "qemu/osdep.h" -#include "net_tx_pkt.h" +#include "qemu/crc32c.h" #include "net/eth.h" #include "net/checksum.h" #include "net/tap.h" #include "net/net.h" #include "hw/pci/pci_device.h" +#include "net_tx_pkt.h" enum { NET_TX_PKT_VHDR_FRAG = 0, @@ -32,8 +33,6 @@ enum { /* TX packet private context */ struct NetTxPkt { - PCIDevice *pci_dev; - struct virtio_net_hdr virt_hdr; struct iovec *raw; @@ -42,7 +41,10 @@ struct NetTxPkt { struct iovec *vec; - uint8_t l2_hdr[ETH_MAX_L2_HDR_LEN]; + struct { + struct eth_header eth; + struct vlan_header vlan[3]; + } l2_hdr; union { struct ip_header ip; struct ip6_header ip6; @@ -59,13 +61,10 @@ struct NetTxPkt { uint8_t l4proto; }; -void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev, - uint32_t max_frags) +void net_tx_pkt_init(struct NetTxPkt **pkt, uint32_t max_frags) { struct NetTxPkt *p = g_malloc0(sizeof *p); - p->pci_dev = pci_dev; - p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG); p->raw = g_new(struct iovec, max_frags); @@ -137,6 +136,23 @@ void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt) pkt->virt_hdr.csum_offset, &csum, sizeof(csum)); } +bool net_tx_pkt_update_sctp_checksum(struct NetTxPkt *pkt) +{ + uint32_t csum = 0; + struct iovec *pl_start_frag = pkt->vec + NET_TX_PKT_PL_START_FRAG; + + if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) { + return false; + } + + csum = cpu_to_le32(iov_crc32c(0xffffffff, pl_start_frag, pkt->payload_frags)); + if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) { + return false; + } + + return true; +} + static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt) { pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len + @@ -370,24 +386,17 @@ bool net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt, uint16_t vlan, uint16_t vlan_ethtype) { - bool is_new; assert(pkt); - eth_setup_vlan_headers_ex(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base, - vlan, vlan_ethtype, &is_new); + eth_setup_vlan_headers(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base, + &pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len, + vlan, vlan_ethtype); - /* update l2hdrlen */ - if (is_new) { - pkt->hdr_len += sizeof(struct vlan_header); - pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len += - sizeof(struct vlan_header); - } + pkt->hdr_len += sizeof(struct vlan_header); } -bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa, - size_t len) +bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, void *base, size_t len) { - hwaddr mapped_len = 0; struct iovec *ventry; assert(pkt); @@ -395,23 +404,12 @@ bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa, return false; } - if (!len) { - return true; - } - ventry = &pkt->raw[pkt->raw_frags]; - mapped_len = len; - - ventry->iov_base = pci_dma_map(pkt->pci_dev, pa, - &mapped_len, DMA_DIRECTION_TO_DEVICE); + ventry->iov_base = base; + ventry->iov_len = len; + pkt->raw_frags++; - if ((ventry->iov_base != NULL) && (len == mapped_len)) { - ventry->iov_len = mapped_len; - pkt->raw_frags++; - return true; - } else { - return false; - } + return true; } bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt) @@ -445,7 +443,8 @@ void net_tx_pkt_dump(struct NetTxPkt *pkt) #endif } -void net_tx_pkt_reset(struct NetTxPkt *pkt, PCIDevice *pci_dev) +void net_tx_pkt_reset(struct NetTxPkt *pkt, + NetTxPktFreeFrag callback, void *context) { int i; @@ -465,17 +464,37 @@ void net_tx_pkt_reset(struct NetTxPkt *pkt, PCIDevice *pci_dev) assert(pkt->raw); for (i = 0; i < pkt->raw_frags; i++) { assert(pkt->raw[i].iov_base); - pci_dma_unmap(pkt->pci_dev, pkt->raw[i].iov_base, - pkt->raw[i].iov_len, DMA_DIRECTION_TO_DEVICE, 0); + callback(context, pkt->raw[i].iov_base, pkt->raw[i].iov_len); } } - pkt->pci_dev = pci_dev; pkt->raw_frags = 0; pkt->hdr_len = 0; pkt->l4proto = 0; } +void net_tx_pkt_unmap_frag_pci(void *context, void *base, size_t len) +{ + pci_dma_unmap(context, base, len, DMA_DIRECTION_TO_DEVICE, 0); +} + +bool net_tx_pkt_add_raw_fragment_pci(struct NetTxPkt *pkt, PCIDevice *pci_dev, + dma_addr_t pa, size_t len) +{ + dma_addr_t mapped_len = len; + void *base = pci_dma_map(pci_dev, pa, &mapped_len, DMA_DIRECTION_TO_DEVICE); + if (!base) { + return false; + } + + if (mapped_len != len || !net_tx_pkt_add_raw_fragment(pkt, base, len)) { + net_tx_pkt_unmap_frag_pci(pci_dev, base, mapped_len); + return false; + } + + return true; +} + static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt, struct iovec *iov, uint32_t iov_len, uint16_t csl) @@ -697,7 +716,7 @@ static void net_tx_pkt_udp_fragment_fix(struct NetTxPkt *pkt, } static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt, - NetTxPktCallback callback, + NetTxPktSend callback, void *context) { uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; @@ -794,7 +813,7 @@ bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc) } bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload, - NetTxPktCallback callback, void *context) + NetTxPktSend callback, void *context) { assert(pkt); diff --git a/hw/net/net_tx_pkt.h b/hw/net/net_tx_pkt.h index e5ce6f20bc..0a716e74a5 100644 --- a/hw/net/net_tx_pkt.h +++ b/hw/net/net_tx_pkt.h @@ -26,17 +26,16 @@ struct NetTxPkt; -typedef void (* NetTxPktCallback)(void *, const struct iovec *, int, const struct iovec *, int); +typedef void (*NetTxPktFreeFrag)(void *, void *, size_t); +typedef void (*NetTxPktSend)(void *, const struct iovec *, int, const struct iovec *, int); /** * Init function for tx packet functionality * * @pkt: packet pointer - * @pci_dev: PCI device processing this packet * @max_frags: max tx ip fragments */ -void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev, - uint32_t max_frags); +void net_tx_pkt_init(struct NetTxPkt **pkt, uint32_t max_frags); /** * Clean all tx packet resources. @@ -95,12 +94,11 @@ net_tx_pkt_setup_vlan_header(struct NetTxPkt *pkt, uint16_t vlan) * populate data fragment into pkt context. * * @pkt: packet - * @pa: physical address of fragment + * @base: pointer to fragment * @len: length of fragment * */ -bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa, - size_t len); +bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, void *base, size_t len); /** * Fix ip header fields and calculate IP header and pseudo header checksums. @@ -119,6 +117,14 @@ void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt); void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt); /** + * Calculate the SCTP checksum. + * + * @pkt: packet + * + */ +bool net_tx_pkt_update_sctp_checksum(struct NetTxPkt *pkt); + +/** * get length of all populated data. * * @pkt: packet @@ -148,10 +154,30 @@ void net_tx_pkt_dump(struct NetTxPkt *pkt); * reset tx packet private context (needed to be called between packets) * * @pkt: packet - * @dev: PCI device processing the next packet + * @callback: function to free the fragments + * @context: pointer to be passed to the callback + */ +void net_tx_pkt_reset(struct NetTxPkt *pkt, + NetTxPktFreeFrag callback, void *context); + +/** + * Unmap a fragment mapped from a PCI device. + * + * @context: PCI device owning fragment + * @base: pointer to fragment + * @len: length of fragment + */ +void net_tx_pkt_unmap_frag_pci(void *context, void *base, size_t len); + +/** + * map data fragment from PCI device and populate it into pkt context. * + * @pci_dev: PCI device owning fragment + * @pa: physical address of fragment + * @len: length of fragment */ -void net_tx_pkt_reset(struct NetTxPkt *pkt, PCIDevice *dev); +bool net_tx_pkt_add_raw_fragment_pci(struct NetTxPkt *pkt, PCIDevice *pci_dev, + dma_addr_t pa, size_t len); /** * Send packet to qemu. handles sw offloads if vhdr is not supported. @@ -173,7 +199,7 @@ bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc); * @ret: operation result */ bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload, - NetTxPktCallback callback, void *context); + NetTxPktSend callback, void *context); /** * parse raw packet data and analyze offload requirements. diff --git a/hw/net/rtl8139.c b/hw/net/rtl8139.c index 5a5aaf868d..5f1a4d359b 100644 --- a/hw/net/rtl8139.c +++ b/hw/net/rtl8139.c @@ -2154,6 +2154,9 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s) int large_send_mss = (txdw0 >> CP_TC_LGSEN_MSS_SHIFT) & CP_TC_LGSEN_MSS_MASK; + if (large_send_mss == 0) { + goto skip_offload; + } DPRINTF("+++ C+ mode offloaded task TSO IP data %d " "frame data %d specified MSS=%d\n", diff --git a/hw/net/trace-events b/hw/net/trace-events index d35554fce8..e4a98b2c7d 100644 --- a/hw/net/trace-events +++ b/hw/net/trace-events @@ -106,6 +106,8 @@ e1000_receiver_overrun(size_t s, uint32_t rdh, uint32_t rdt) "Receiver overrun: # e1000x_common.c e1000x_rx_can_recv_disabled(bool link_up, bool rx_enabled, bool pci_master) "link_up: %d, rx_enabled %d, pci_master %d" e1000x_vlan_is_vlan_pkt(bool is_vlan_pkt, uint16_t eth_proto, uint16_t vet) "Is VLAN packet: %d, ETH proto: 0x%X, VET: 0x%X" +e1000x_rx_flt_vlan_mismatch(uint16_t vid) "VID mismatch: 0x%X" +e1000x_rx_flt_vlan_match(uint16_t vid) "VID match: 0x%X" e1000x_rx_flt_ucast_match(uint32_t idx, uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "unicast match[%d]: %02x:%02x:%02x:%02x:%02x:%02x" e1000x_rx_flt_ucast_mismatch(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "unicast mismatch: %02x:%02x:%02x:%02x:%02x:%02x" e1000x_rx_flt_inexact_mismatch(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5, uint32_t mo, uint32_t mta, uint32_t mta_val) "inexact mismatch: %02x:%02x:%02x:%02x:%02x:%02x MO %d MTA[%d] 0x%x" @@ -154,8 +156,6 @@ e1000e_rx_can_recv_rings_full(void) "Cannot receive: all rings are full" e1000e_rx_can_recv(void) "Can receive" e1000e_rx_has_buffers(int ridx, uint32_t free_desc, size_t total_size, uint32_t desc_buf_size) "ring #%d: free descr: %u, packet size %zu, descr buffer size %u" e1000e_rx_null_descriptor(void) "Null RX descriptor!!" -e1000e_rx_flt_vlan_mismatch(uint16_t vid) "VID mismatch: 0x%X" -e1000e_rx_flt_vlan_match(uint16_t vid) "VID match: 0x%X" e1000e_rx_desc_ps_read(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3) "buffers: [0x%"PRIx64", 0x%"PRIx64", 0x%"PRIx64", 0x%"PRIx64"]" e1000e_rx_desc_ps_write(uint16_t a0, uint16_t a1, uint16_t a2, uint16_t a3) "bytes written: [%u, %u, %u, %u]" e1000e_rx_desc_buff_sizes(uint32_t b0, uint32_t b1, uint32_t b2, uint32_t b3) "buffer sizes: [%u, %u, %u, %u]" @@ -179,7 +179,7 @@ e1000e_rx_rss_disabled(void) "RSS is disabled" e1000e_rx_rss_type(uint32_t type) "RSS type is %u" e1000e_rx_rss_ip4(int l4hdr_proto, uint32_t mrqc, bool tcpipv4_enabled, bool ipv4_enabled) "RSS IPv4: L4 header protocol %d, mrqc 0x%X, tcpipv4 enabled %d, ipv4 enabled %d" e1000e_rx_rss_ip6_rfctl(uint32_t rfctl) "RSS IPv6: rfctl 0x%X" -e1000e_rx_rss_ip6(bool ex_dis, bool new_ex_dis, int l4hdr_proto, bool has_ext_headers, bool ex_dst_valid, bool ex_src_valid, uint32_t mrqc, bool tcpipv6_enabled, bool ipv6ex_enabled, bool ipv6_enabled) "RSS IPv6: ex_dis: %d, new_ex_dis: %d, L4 header protocol %d, has_ext_headers %d, ex_dst_valid %d, ex_src_valid %d, mrqc 0x%X, tcpipv6 enabled %d, ipv6ex enabled %d, ipv6 enabled %d" +e1000e_rx_rss_ip6(bool ex_dis, bool new_ex_dis, int l4hdr_proto, bool has_ext_headers, bool ex_dst_valid, bool ex_src_valid, uint32_t mrqc, bool tcpipv6ex_enabled, bool ipv6ex_enabled, bool ipv6_enabled) "RSS IPv6: ex_dis: %d, new_ex_dis: %d, L4 header protocol %d, has_ext_headers %d, ex_dst_valid %d, ex_src_valid %d, mrqc 0x%X, tcpipv6ex enabled %d, ipv6ex enabled %d, ipv6 enabled %d" e1000e_rx_metadata_protocols(bool hasip4, bool hasip6, int l4hdr_protocol) "protocols: ip4: %d, ip6: %d, l4hdr: %d" e1000e_rx_metadata_vlan(uint16_t vlan_tag) "VLAN tag is 0x%X" @@ -205,21 +205,16 @@ e1000e_irq_msix_notify_postponed_vec(int idx) "Sending MSI-X postponed by EITR[% e1000e_irq_legacy_notify(bool level) "IRQ line state: %d" e1000e_irq_msix_notify_vec(uint32_t vector) "MSI-X notify vector 0x%x" e1000e_irq_postponed_by_xitr(uint32_t reg) "Interrupt postponed by [E]ITR register 0x%x" -e1000e_irq_clear_ims(uint32_t bits, uint32_t old_ims, uint32_t new_ims) "Clearing IMS bits 0x%x: 0x%x --> 0x%x" -e1000e_irq_set_ims(uint32_t bits, uint32_t old_ims, uint32_t new_ims) "Setting IMS bits 0x%x: 0x%x --> 0x%x" +e1000e_irq_clear(uint32_t offset, uint32_t old, uint32_t new) "Clearing interrupt register 0x%x: 0x%x --> 0x%x" +e1000e_irq_set(uint32_t offset, uint32_t old, uint32_t new) "Setting interrupt register 0x%x: 0x%x --> 0x%x" e1000e_irq_fix_icr_asserted(uint32_t new_val) "ICR_ASSERTED bit fixed: 0x%x" e1000e_irq_add_msi_other(uint32_t new_val) "ICR_OTHER bit added: 0x%x" e1000e_irq_pending_interrupts(uint32_t pending, uint32_t icr, uint32_t ims) "ICR PENDING: 0x%x (ICR: 0x%x, IMS: 0x%x)" -e1000e_irq_set_cause_entry(uint32_t val, uint32_t icr) "Going to set IRQ cause 0x%x, ICR: 0x%x" -e1000e_irq_set_cause_exit(uint32_t val, uint32_t icr) "Set IRQ cause 0x%x, ICR: 0x%x" -e1000e_irq_icr_write(uint32_t bits, uint32_t old_icr, uint32_t new_icr) "Clearing ICR bits 0x%x: 0x%x --> 0x%x" e1000e_irq_write_ics(uint32_t val) "Adding ICR bits 0x%x" e1000e_irq_icr_process_iame(void) "Clearing IMS bits due to IAME" e1000e_irq_read_ics(uint32_t ics) "Current ICS: 0x%x" e1000e_irq_read_ims(uint32_t ims) "Current IMS: 0x%x" e1000e_irq_icr_clear_nonmsix_icr_read(void) "Clearing ICR on read due to non MSI-X int" -e1000e_irq_icr_read_entry(uint32_t icr) "Starting ICR read. Current ICR: 0x%x" -e1000e_irq_icr_read_exit(uint32_t icr) "Ending ICR read. Current ICR: 0x%x" e1000e_irq_icr_clear_zero_ims(void) "Clearing ICR on read due to zero IMS" e1000e_irq_icr_clear_iame(void) "Clearing ICR on read due to IAME" e1000e_irq_iam_clear_eiame(uint32_t iam, uint32_t cause) "Clearing IMS due to EIAME, IAM: 0x%X, cause: 0x%X" @@ -235,7 +230,6 @@ e1000e_irq_tidv_fpd_not_running(void) "FPD written while TIDV was not running" e1000e_irq_eitr_set(uint32_t eitr_num, uint32_t val) "EITR[%u] = %u" e1000e_irq_itr_set(uint32_t val) "ITR = %u" e1000e_irq_fire_all_timers(uint32_t val) "Firing all delay/throttling timers on all interrupts enable (0x%X written to IMS)" -e1000e_irq_adding_delayed_causes(uint32_t val, uint32_t icr) "Merging delayed causes 0x%X to ICR 0x%X" e1000e_irq_msix_pending_clearing(uint32_t cause, uint32_t int_cfg, uint32_t vec) "Clearing MSI-X pending bit for cause 0x%x, IVAR config 0x%x, vector %u" e1000e_wrn_msix_vec_wrong(uint32_t cause, uint32_t cfg) "Invalid configuration for cause 0x%x: 0x%x" @@ -288,12 +282,11 @@ igb_rx_desc_buff_write(uint64_t addr, uint16_t offset, const void* source, uint3 igb_rx_metadata_rss(uint32_t rss) "RSS data: 0x%X" igb_irq_icr_clear_gpie_nsicr(void) "Clearing ICR on read due to GPIE.NSICR enabled" -igb_irq_icr_write(uint32_t bits, uint32_t old_icr, uint32_t new_icr) "Clearing ICR bits 0x%x: 0x%x --> 0x%x" igb_irq_set_iam(uint32_t icr) "Update IAM: 0x%x" igb_irq_read_iam(uint32_t icr) "Current IAM: 0x%x" igb_irq_write_eics(uint32_t val, bool msix) "Update EICS: 0x%x MSI-X: %d" igb_irq_write_eims(uint32_t val, bool msix) "Update EIMS: 0x%x MSI-X: %d" -igb_irq_write_eimc(uint32_t val, uint32_t eims, bool msix) "Update EIMC: 0x%x EIMS: 0x%x MSI-X: %d" +igb_irq_write_eimc(uint32_t val, bool msix) "Update EIMC: 0x%x MSI-X: %d" igb_irq_write_eiac(uint32_t val) "Update EIAC: 0x%x" igb_irq_write_eiam(uint32_t val, bool msix) "Update EIAM: 0x%x MSI-X: %d" igb_irq_write_eicr(uint32_t val, bool msix) "Update EICR: 0x%x MSI-X: %d" diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c index 8ce239a34d..6df6b7329d 100644 --- a/hw/net/virtio-net.c +++ b/hw/net/virtio-net.c @@ -1834,9 +1834,12 @@ static int virtio_net_process_rss(NetClientState *nc, const uint8_t *buf, VIRTIO_NET_HASH_REPORT_UDPv6, VIRTIO_NET_HASH_REPORT_UDPv6_EX }; + struct iovec iov = { + .iov_base = (void *)buf, + .iov_len = size + }; - net_rx_pkt_set_protocols(pkt, buf + n->host_hdr_len, - size - n->host_hdr_len); + net_rx_pkt_set_protocols(pkt, &iov, 1, n->host_hdr_len); net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); net_hash_type = virtio_net_get_hash_type(hasip4, hasip6, l4hdr_proto, n->rss_data.hash_types); diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c index f7b874c139..18b9edfdb2 100644 --- a/hw/net/vmxnet3.c +++ b/hw/net/vmxnet3.c @@ -651,9 +651,8 @@ static void vmxnet3_process_tx_queue(VMXNET3State *s, int qidx) data_len = (txd.len > 0) ? txd.len : VMXNET3_MAX_TX_BUF_SIZE; data_pa = txd.addr; - if (!net_tx_pkt_add_raw_fragment(s->tx_pkt, - data_pa, - data_len)) { + if (!net_tx_pkt_add_raw_fragment_pci(s->tx_pkt, PCI_DEVICE(s), + data_pa, data_len)) { s->skip_current_tx_pkt = true; } } @@ -678,9 +677,12 @@ static void vmxnet3_process_tx_queue(VMXNET3State *s, int qidx) vmxnet3_complete_packet(s, qidx, txd_idx); s->tx_sop = true; s->skip_current_tx_pkt = false; - net_tx_pkt_reset(s->tx_pkt, PCI_DEVICE(s)); + net_tx_pkt_reset(s->tx_pkt, + net_tx_pkt_unmap_frag_pci, PCI_DEVICE(s)); } } + + net_tx_pkt_reset(s->tx_pkt, net_tx_pkt_unmap_frag_pci, PCI_DEVICE(s)); } static inline void @@ -1159,7 +1161,6 @@ static void vmxnet3_deactivate_device(VMXNET3State *s) { if (s->device_active) { VMW_CBPRN("Deactivating vmxnet3..."); - net_tx_pkt_reset(s->tx_pkt, PCI_DEVICE(s)); net_tx_pkt_uninit(s->tx_pkt); net_rx_pkt_uninit(s->rx_pkt); s->device_active = false; @@ -1519,7 +1520,7 @@ static void vmxnet3_activate_device(VMXNET3State *s) /* Preallocate TX packet wrapper */ VMW_CFPRN("Max TX fragments is %u", s->max_tx_frags); - net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s), s->max_tx_frags); + net_tx_pkt_init(&s->tx_pkt, s->max_tx_frags); net_rx_pkt_init(&s->rx_pkt); /* Read rings memory locations for RX queues */ @@ -2001,7 +2002,12 @@ vmxnet3_receive(NetClientState *nc, const uint8_t *buf, size_t size) get_eth_packet_type(PKT_GET_ETH_HDR(buf))); if (vmxnet3_rx_filter_may_indicate(s, buf, size)) { - net_rx_pkt_set_protocols(s->rx_pkt, buf, size); + struct iovec iov = { + .iov_base = (void *)buf, + .iov_len = size + }; + + net_rx_pkt_set_protocols(s->rx_pkt, &iov, 1, 0); vmxnet3_rx_need_csum_calculate(s->rx_pkt, buf, size); net_rx_pkt_attach_data(s->rx_pkt, buf, size, s->rx_vlan_stripping); bytes_indicated = vmxnet3_indicate_packet(s) ? size : -1; @@ -2399,7 +2405,7 @@ static int vmxnet3_post_load(void *opaque, int version_id) { VMXNET3State *s = opaque; - net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s), s->max_tx_frags); + net_tx_pkt_init(&s->tx_pkt, s->max_tx_frags); net_rx_pkt_init(&s->rx_pkt); if (s->msix_used) { diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index bf27a39905..73874a94de 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -2994,7 +2994,7 @@ static void vfio_realize(PCIDevice *pdev, Error **errp) qemu_uuid_unparse(&vdev->vf_token, uuid); name = g_strdup_printf("%s vf_token=%s", vbasedev->name, uuid); } else { - name = vbasedev->name; + name = g_strdup(vbasedev->name); } ret = vfio_get_device(group, name, vbasedev, errp); diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index e3ec8727da..74a2a28663 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -473,6 +473,7 @@ static MemoryRegion *vhost_user_get_mr_data(uint64_t addr, ram_addr_t *offset, assert((uintptr_t)addr == addr); mr = memory_region_from_host((void *)(uintptr_t)addr, offset); *fd = memory_region_get_fd(mr); + *offset += mr->ram_block->fd_offset; return mr; } diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index 7c867c94c3..5939688f69 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -207,43 +207,21 @@ void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint64_t val, int mmu_idx, uintptr_t ra); uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra); -uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr ptr, - MemOpIdx oi, uintptr_t ra); -uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr ptr, - MemOpIdx oi, uintptr_t ra); -uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr ptr, - MemOpIdx oi, uintptr_t ra); -uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr ptr, - MemOpIdx oi, uintptr_t ra); -uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr ptr, - MemOpIdx oi, uintptr_t ra); -uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr ptr, - MemOpIdx oi, uintptr_t ra); - -Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra); -Int128 cpu_ld16_le_mmu(CPUArchState *env, abi_ptr addr, - MemOpIdx oi, uintptr_t ra); +uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra); +uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra); +uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra); +Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra); void cpu_stb_mmu(CPUArchState *env, abi_ptr ptr, uint8_t val, MemOpIdx oi, uintptr_t ra); -void cpu_stw_be_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val, - MemOpIdx oi, uintptr_t ra); -void cpu_stl_be_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val, - MemOpIdx oi, uintptr_t ra); -void cpu_stq_be_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val, - MemOpIdx oi, uintptr_t ra); -void cpu_stw_le_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val, - MemOpIdx oi, uintptr_t ra); -void cpu_stl_le_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val, - MemOpIdx oi, uintptr_t ra); -void cpu_stq_le_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val, - MemOpIdx oi, uintptr_t ra); - -void cpu_st16_be_mmu(CPUArchState *env, abi_ptr addr, Int128 val, - MemOpIdx oi, uintptr_t ra); -void cpu_st16_le_mmu(CPUArchState *env, abi_ptr addr, Int128 val, - MemOpIdx oi, uintptr_t ra); +void cpu_stw_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val, + MemOpIdx oi, uintptr_t ra); +void cpu_stl_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val, + MemOpIdx oi, uintptr_t ra); +void cpu_stq_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val, + MemOpIdx oi, uintptr_t ra); +void cpu_st16_mmu(CPUArchState *env, abi_ptr addr, Int128 val, + MemOpIdx oi, uintptr_t ra); uint32_t cpu_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr, uint32_t cmpv, uint32_t newv, @@ -322,15 +300,6 @@ Int128 cpu_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr, Int128 cmpv, Int128 newv, MemOpIdx oi, uintptr_t retaddr); -Int128 cpu_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -Int128 cpu_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr, - MemOpIdx oi, uintptr_t retaddr); -void cpu_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val, - MemOpIdx oi, uintptr_t retaddr); -void cpu_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val, - MemOpIdx oi, uintptr_t retaddr); - #if defined(CONFIG_USER_ONLY) extern __thread uintptr_t helper_retaddr; @@ -416,9 +385,6 @@ static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx, # define cpu_ldsw_mmuidx_ra cpu_ldsw_be_mmuidx_ra # define cpu_ldl_mmuidx_ra cpu_ldl_be_mmuidx_ra # define cpu_ldq_mmuidx_ra cpu_ldq_be_mmuidx_ra -# define cpu_ldw_mmu cpu_ldw_be_mmu -# define cpu_ldl_mmu cpu_ldl_be_mmu -# define cpu_ldq_mmu cpu_ldq_be_mmu # define cpu_stw_data cpu_stw_be_data # define cpu_stl_data cpu_stl_be_data # define cpu_stq_data cpu_stq_be_data @@ -428,9 +394,6 @@ static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx, # define cpu_stw_mmuidx_ra cpu_stw_be_mmuidx_ra # define cpu_stl_mmuidx_ra cpu_stl_be_mmuidx_ra # define cpu_stq_mmuidx_ra cpu_stq_be_mmuidx_ra -# define cpu_stw_mmu cpu_stw_be_mmu -# define cpu_stl_mmu cpu_stl_be_mmu -# define cpu_stq_mmu cpu_stq_be_mmu #else # define cpu_lduw_data cpu_lduw_le_data # define cpu_ldsw_data cpu_ldsw_le_data @@ -444,9 +407,6 @@ static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx, # define cpu_ldsw_mmuidx_ra cpu_ldsw_le_mmuidx_ra # define cpu_ldl_mmuidx_ra cpu_ldl_le_mmuidx_ra # define cpu_ldq_mmuidx_ra cpu_ldq_le_mmuidx_ra -# define cpu_ldw_mmu cpu_ldw_le_mmu -# define cpu_ldl_mmu cpu_ldl_le_mmu -# define cpu_ldq_mmu cpu_ldq_le_mmu # define cpu_stw_data cpu_stw_le_data # define cpu_stl_data cpu_stl_le_data # define cpu_stq_data cpu_stq_le_data @@ -456,9 +416,6 @@ static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx, # define cpu_stw_mmuidx_ra cpu_stw_le_mmuidx_ra # define cpu_stl_mmuidx_ra cpu_stl_le_mmuidx_ra # define cpu_stq_mmuidx_ra cpu_stq_le_mmuidx_ra -# define cpu_stw_mmu cpu_stw_le_mmu -# define cpu_stl_mmu cpu_stl_le_mmu -# define cpu_stq_mmu cpu_stq_le_mmu #endif uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr, diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index ecded1f112..4d2b151986 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -27,9 +27,6 @@ #include "qemu/interval-tree.h" #include "qemu/clang-tsa.h" -/* allow to see translation results - the slowdown should be negligible, so we leave it */ -#define DEBUG_DISAS - /* Page tracking code uses ram addresses in system mode, and virtual addresses in userspace mode. Define tb_page_addr_t to be an appropriate type. */ diff --git a/include/exec/memory.h b/include/exec/memory.h index 6f24a3dd46..c3661b2276 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -1326,6 +1326,7 @@ void memory_region_init_resizeable_ram(MemoryRegion *mr, * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM, * RAM_NORESERVE, * @path: the path in which to allocate the RAM. + * @offset: offset within the file referenced by path * @readonly: true to open @path for reading, false for read/write. * @errp: pointer to Error*, to store an error if it happens. * @@ -1339,6 +1340,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, uint64_t align, uint32_t ram_flags, const char *path, + ram_addr_t offset, bool readonly, Error **errp); diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index f4fb6a2111..90a8269290 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -110,6 +110,7 @@ long qemu_maxrampagesize(void); * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM, * RAM_NORESERVE. * @mem_path or @fd: specify the backing file or device + * @offset: Offset into target file * @readonly: true to open @path for reading, false for read/write. * @errp: pointer to Error*, to store an error if it happens * @@ -119,7 +120,7 @@ long qemu_maxrampagesize(void); */ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, const char *mem_path, - bool readonly, Error **errp); + off_t offset, bool readonly, Error **errp); RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, int fd, off_t offset, bool readonly, Error **errp); diff --git a/include/exec/ramblock.h b/include/exec/ramblock.h index adc03df59c..69c6a53902 100644 --- a/include/exec/ramblock.h +++ b/include/exec/ramblock.h @@ -40,6 +40,7 @@ struct RAMBlock { QLIST_ENTRY(RAMBlock) next; QLIST_HEAD(, RAMBlockNotifier) ramblock_notifiers; int fd; + uint64_t fd_offset; size_t page_size; /* dirty bitmap used during migration */ unsigned long *bmap; diff --git a/include/net/eth.h b/include/net/eth.h index c5ae4493b4..3b80b6e07f 100644 --- a/include/net/eth.h +++ b/include/net/eth.h @@ -32,6 +32,8 @@ #define ETH_ALEN 6 #define ETH_HLEN 14 #define ETH_ZLEN 60 /* Min. octets in frame without FCS */ +#define ETH_FCS_LEN 4 +#define ETH_MTU 1500 struct eth_header { uint8_t h_dest[ETH_ALEN]; /* destination eth addr */ @@ -222,6 +224,7 @@ struct tcp_hdr { #define IP_HEADER_VERSION_6 (6) #define IP_PROTO_TCP (6) #define IP_PROTO_UDP (17) +#define IP_PROTO_SCTP (132) #define IPTOS_ECN_MASK 0x03 #define IPTOS_ECN(x) ((x) & IPTOS_ECN_MASK) #define IPTOS_ECN_CE 0x03 @@ -312,10 +315,10 @@ eth_get_l2_hdr_length(const void *p) } static inline uint32_t -eth_get_l2_hdr_length_iov(const struct iovec *iov, int iovcnt) +eth_get_l2_hdr_length_iov(const struct iovec *iov, size_t iovcnt, size_t iovoff) { uint8_t p[sizeof(struct eth_header) + sizeof(struct vlan_header)]; - size_t copied = iov_to_buf(iov, iovcnt, 0, p, ARRAY_SIZE(p)); + size_t copied = iov_to_buf(iov, iovcnt, iovoff, p, ARRAY_SIZE(p)); if (copied < ARRAY_SIZE(p)) { return copied; @@ -340,26 +343,19 @@ eth_get_pkt_tci(const void *p) size_t eth_strip_vlan(const struct iovec *iov, int iovcnt, size_t iovoff, - uint8_t *new_ehdr_buf, + void *new_ehdr_buf, uint16_t *payload_offset, uint16_t *tci); size_t -eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff, - uint16_t vet, uint8_t *new_ehdr_buf, +eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff, int index, + uint16_t vet, uint16_t vet_ext, void *new_ehdr_buf, uint16_t *payload_offset, uint16_t *tci); uint16_t eth_get_l3_proto(const struct iovec *l2hdr_iov, int iovcnt, size_t l2hdr_len); -void eth_setup_vlan_headers_ex(struct eth_header *ehdr, uint16_t vlan_tag, - uint16_t vlan_ethtype, bool *is_new); - -static inline void -eth_setup_vlan_headers(struct eth_header *ehdr, uint16_t vlan_tag, - bool *is_new) -{ - eth_setup_vlan_headers_ex(ehdr, vlan_tag, ETH_P_VLAN, is_new); -} +void eth_setup_vlan_headers(struct eth_header *ehdr, size_t *ehdr_size, + uint16_t vlan_tag, uint16_t vlan_ethtype); uint8_t eth_get_gso_type(uint16_t l3_proto, uint8_t *l3_hdr, uint8_t l4proto); @@ -384,7 +380,8 @@ typedef struct eth_ip4_hdr_info_st { typedef enum EthL4HdrProto { ETH_L4_HDR_PROTO_INVALID, ETH_L4_HDR_PROTO_TCP, - ETH_L4_HDR_PROTO_UDP + ETH_L4_HDR_PROTO_UDP, + ETH_L4_HDR_PROTO_SCTP } EthL4HdrProto; typedef struct eth_l4_hdr_info_st { @@ -397,7 +394,7 @@ typedef struct eth_l4_hdr_info_st { bool has_tcp_data; } eth_l4_hdr_info; -void eth_get_protocols(const struct iovec *iov, int iovcnt, +void eth_get_protocols(const struct iovec *iov, size_t iovcnt, size_t iovoff, bool *hasip4, bool *hasip6, size_t *l3hdr_off, size_t *l4hdr_off, diff --git a/include/qemu/atomic128.h b/include/qemu/atomic128.h index d0ba0b9c65..34554bf0ac 100644 --- a/include/qemu/atomic128.h +++ b/include/qemu/atomic128.h @@ -16,6 +16,23 @@ #include "qemu/int128.h" /* + * If __alignof(unsigned __int128) < 16, GCC may refuse to inline atomics + * that are supported by the host, e.g. s390x. We can force the pointer to + * have our known alignment with __builtin_assume_aligned, however prior to + * GCC 13 that was only reliable with optimization enabled. See + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107389 + */ +#if defined(CONFIG_ATOMIC128_OPT) +# if !defined(__OPTIMIZE__) +# define ATTRIBUTE_ATOMIC128_OPT __attribute__((optimize("O1"))) +# endif +# define CONFIG_ATOMIC128 +#endif +#ifndef ATTRIBUTE_ATOMIC128_OPT +# define ATTRIBUTE_ATOMIC128_OPT +#endif + +/* * GCC is a house divided about supporting large atomic operations. * * For hosts that only have large compare-and-swap, a legalistic reading @@ -41,132 +58,7 @@ * Therefore, special case each platform. */ -#if defined(CONFIG_ATOMIC128) -static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) -{ - Int128Alias r, c, n; - - c.s = cmp; - n.s = new; - r.i = qatomic_cmpxchg__nocheck((__int128_t *)ptr, c.i, n.i); - return r.s; -} -# define HAVE_CMPXCHG128 1 -#elif defined(CONFIG_CMPXCHG128) -static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) -{ - Int128Alias r, c, n; - - c.s = cmp; - n.s = new; - r.i = __sync_val_compare_and_swap_16((__int128_t *)ptr, c.i, n.i); - return r.s; -} -# define HAVE_CMPXCHG128 1 -#elif defined(__aarch64__) -/* Through gcc 8, aarch64 has no support for 128-bit at all. */ -static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) -{ - uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp); - uint64_t newl = int128_getlo(new), newh = int128_gethi(new); - uint64_t oldl, oldh; - uint32_t tmp; - - asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t" - "cmp %[oldl], %[cmpl]\n\t" - "ccmp %[oldh], %[cmph], #0, eq\n\t" - "b.ne 1f\n\t" - "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t" - "cbnz %w[tmp], 0b\n" - "1:" - : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), - [oldl] "=&r"(oldl), [oldh] "=&r"(oldh) - : [cmpl] "r"(cmpl), [cmph] "r"(cmph), - [newl] "r"(newl), [newh] "r"(newh) - : "memory", "cc"); - - return int128_make128(oldl, oldh); -} -# define HAVE_CMPXCHG128 1 -#else -/* Fallback definition that must be optimized away, or error. */ -Int128 QEMU_ERROR("unsupported atomic") - atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new); -# define HAVE_CMPXCHG128 0 -#endif /* Some definition for HAVE_CMPXCHG128 */ - - -#if defined(CONFIG_ATOMIC128) -static inline Int128 atomic16_read(Int128 *ptr) -{ - Int128Alias r; - - r.i = qatomic_read__nocheck((__int128_t *)ptr); - return r.s; -} - -static inline void atomic16_set(Int128 *ptr, Int128 val) -{ - Int128Alias v; - - v.s = val; - qatomic_set__nocheck((__int128_t *)ptr, v.i); -} - -# define HAVE_ATOMIC128 1 -#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__) -/* We can do better than cmpxchg for AArch64. */ -static inline Int128 atomic16_read(Int128 *ptr) -{ - uint64_t l, h; - uint32_t tmp; - - /* The load must be paired with the store to guarantee not tearing. */ - asm("0: ldxp %[l], %[h], %[mem]\n\t" - "stxp %w[tmp], %[l], %[h], %[mem]\n\t" - "cbnz %w[tmp], 0b" - : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h)); - - return int128_make128(l, h); -} - -static inline void atomic16_set(Int128 *ptr, Int128 val) -{ - uint64_t l = int128_getlo(val), h = int128_gethi(val); - uint64_t t1, t2; - - /* Load into temporaries to acquire the exclusive access lock. */ - asm("0: ldxp %[t1], %[t2], %[mem]\n\t" - "stxp %w[t1], %[l], %[h], %[mem]\n\t" - "cbnz %w[t1], 0b" - : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2) - : [l] "r"(l), [h] "r"(h)); -} - -# define HAVE_ATOMIC128 1 -#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128 -static inline Int128 atomic16_read(Int128 *ptr) -{ - /* Maybe replace 0 with 0, returning the old value. */ - Int128 z = int128_make64(0); - return atomic16_cmpxchg(ptr, z, z); -} - -static inline void atomic16_set(Int128 *ptr, Int128 val) -{ - Int128 old = *ptr, cmp; - do { - cmp = old; - old = atomic16_cmpxchg(ptr, cmp, val); - } while (int128_ne(old, cmp)); -} - -# define HAVE_ATOMIC128 1 -#else -/* Fallback definitions that must be optimized away, or error. */ -Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr); -void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val); -# define HAVE_ATOMIC128 0 -#endif /* Some definition for HAVE_ATOMIC128 */ +#include "host/atomic128-cas.h" +#include "host/atomic128-ldst.h" #endif /* QEMU_ATOMIC128_H */ diff --git a/include/qemu/crc32c.h b/include/qemu/crc32c.h index 5b78884c38..88b4d2b3b3 100644 --- a/include/qemu/crc32c.h +++ b/include/qemu/crc32c.h @@ -30,5 +30,6 @@ uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length); +uint32_t iov_crc32c(uint32_t crc, const struct iovec *iov, size_t iov_cnt); #endif diff --git a/include/tcg/debug-assert.h b/include/tcg/debug-assert.h new file mode 100644 index 0000000000..596765a3d2 --- /dev/null +++ b/include/tcg/debug-assert.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define tcg_debug_assert + * Copyright (c) 2008 Fabrice Bellard + */ + +#ifndef TCG_DEBUG_ASSERT_H +#define TCG_DEBUG_ASSERT_H + +#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS +# define tcg_debug_assert(X) do { assert(X); } while (0) +#else +# define tcg_debug_assert(X) \ + do { if (!(X)) { __builtin_unreachable(); } } while (0) +#endif + +#endif diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index cd6327b175..072c35f7f5 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -34,6 +34,7 @@ #include "tcg/tcg-mo.h" #include "tcg-target.h" #include "tcg/tcg-cond.h" +#include "tcg/debug-assert.h" /* XXX: make safe guess about sizes */ #define MAX_OP_PER_INSTR 266 @@ -222,14 +223,6 @@ typedef uint64_t tcg_insn_unit; /* The port better have done this. */ #endif - -#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS -# define tcg_debug_assert(X) do { assert(X); } while (0) -#else -# define tcg_debug_assert(X) \ - do { if (!(X)) { __builtin_unreachable(); } } while (0) -#endif - typedef struct TCGRelocation TCGRelocation; struct TCGRelocation { QSIMPLEQ_ENTRY(TCGRelocation) next; diff --git a/meson.build b/meson.build index 2e47608353..78890f0155 100644 --- a/meson.build +++ b/meson.build @@ -521,6 +521,16 @@ add_project_arguments('-iquote', '.', '-iquote', meson.current_source_dir() / 'include', language: all_languages) +# If a host-specific include directory exists, list that first... +host_include = meson.current_source_dir() / 'host/include/' +if fs.is_dir(host_include / host_arch) + add_project_arguments('-iquote', host_include / host_arch, + language: all_languages) +endif +# ... followed by the generic fallback. +add_project_arguments('-iquote', host_include / 'generic', + language: all_languages) + sparse = find_program('cgcc', required: get_option('sparse')) if sparse.found() run_target('sparse', @@ -2556,7 +2566,7 @@ if has_int128 # __alignof(unsigned __int128) for the host. atomic_test_128 = ''' int main(int ac, char **av) { - unsigned __int128 *p = __builtin_assume_aligned(av[ac - 1], sizeof(16)); + unsigned __int128 *p = __builtin_assume_aligned(av[ac - 1], 16); p[1] = __atomic_load_n(&p[0], __ATOMIC_RELAXED); __atomic_store_n(&p[2], p[3], __ATOMIC_RELAXED); __atomic_compare_exchange_n(&p[4], &p[5], p[6], 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); diff --git a/migration/meson.build b/migration/meson.build index a8e01e70ae..8ba6e420fe 100644 --- a/migration/meson.build +++ b/migration/meson.build @@ -8,7 +8,6 @@ migration_files = files( 'qemu-file.c', 'yank_functions.c', ) -softmmu_ss.add(migration_files) softmmu_ss.add(files( 'block-dirty-bitmap.c', diff --git a/migration/ram.c b/migration/ram.c index 9fb076fa58..88a6c82e63 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -90,34 +90,6 @@ #define RAM_SAVE_FLAG_MULTIFD_FLUSH 0x200 /* We can't use any flag that is bigger than 0x200 */ -int (*xbzrle_encode_buffer_func)(uint8_t *, uint8_t *, int, - uint8_t *, int) = xbzrle_encode_buffer; -#if defined(CONFIG_AVX512BW_OPT) -#include "qemu/cpuid.h" -static void __attribute__((constructor)) init_cpu_flag(void) -{ - unsigned max = __get_cpuid_max(0, NULL); - int a, b, c, d; - if (max >= 1) { - __cpuid(1, a, b, c, d); - /* We must check that AVX is not just available, but usable. */ - if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) { - int bv; - __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0)); - __cpuid_count(7, 0, a, b, c, d); - /* 0xe6: - * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15 - * and ZMM16-ZMM31 state are enabled by OS) - * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS) - */ - if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) { - xbzrle_encode_buffer_func = xbzrle_encode_buffer_avx512; - } - } - } -} -#endif - XBZRLECacheStats xbzrle_counters; /* used by the search for pages to send */ @@ -660,9 +632,9 @@ static int save_xbzrle_page(RAMState *rs, PageSearchStatus *pss, memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE); /* XBZRLE encoding (if there is no overflow) */ - encoded_len = xbzrle_encode_buffer_func(prev_cached_page, XBZRLE.current_buf, - TARGET_PAGE_SIZE, XBZRLE.encoded_buf, - TARGET_PAGE_SIZE); + encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf, + TARGET_PAGE_SIZE, XBZRLE.encoded_buf, + TARGET_PAGE_SIZE); /* * Update the cache contents, so that it corresponds to the data diff --git a/migration/xbzrle.c b/migration/xbzrle.c index 258e4959c9..3eddcf249b 100644 --- a/migration/xbzrle.c +++ b/migration/xbzrle.c @@ -15,6 +15,152 @@ #include "qemu/host-utils.h" #include "xbzrle.h" +#if defined(CONFIG_AVX512BW_OPT) +#include <immintrin.h> +#include "host/cpuinfo.h" + +static int __attribute__((target("avx512bw"))) +xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, + uint8_t *dst, int dlen) +{ + uint32_t zrun_len = 0, nzrun_len = 0; + int d = 0, i = 0, num = 0; + uint8_t *nzrun_start = NULL; + /* add 1 to include residual part in main loop */ + uint32_t count512s = (slen >> 6) + 1; + /* countResidual is tail of data, i.e., countResidual = slen % 64 */ + uint32_t count_residual = slen & 0b111111; + bool never_same = true; + uint64_t mask_residual = 1; + mask_residual <<= count_residual; + mask_residual -= 1; + __m512i r = _mm512_set1_epi32(0); + + while (count512s) { + int bytes_to_check = 64; + uint64_t mask = 0xffffffffffffffff; + if (count512s == 1) { + bytes_to_check = count_residual; + mask = mask_residual; + } + __m512i old_data = _mm512_mask_loadu_epi8(r, + mask, old_buf + i); + __m512i new_data = _mm512_mask_loadu_epi8(r, + mask, new_buf + i); + uint64_t comp = _mm512_cmpeq_epi8_mask(old_data, new_data); + count512s--; + + bool is_same = (comp & 0x1); + while (bytes_to_check) { + if (d + 2 > dlen) { + return -1; + } + if (is_same) { + if (nzrun_len) { + d += uleb128_encode_small(dst + d, nzrun_len); + if (d + nzrun_len > dlen) { + return -1; + } + nzrun_start = new_buf + i - nzrun_len; + memcpy(dst + d, nzrun_start, nzrun_len); + d += nzrun_len; + nzrun_len = 0; + } + /* 64 data at a time for speed */ + if (count512s && (comp == 0xffffffffffffffff)) { + i += 64; + zrun_len += 64; + break; + } + never_same = false; + num = ctz64(~comp); + num = (num < bytes_to_check) ? num : bytes_to_check; + zrun_len += num; + bytes_to_check -= num; + comp >>= num; + i += num; + if (bytes_to_check) { + /* still has different data after same data */ + d += uleb128_encode_small(dst + d, zrun_len); + zrun_len = 0; + } else { + break; + } + } + if (never_same || zrun_len) { + /* + * never_same only acts if + * data begins with diff in first count512s + */ + d += uleb128_encode_small(dst + d, zrun_len); + zrun_len = 0; + never_same = false; + } + /* has diff, 64 data at a time for speed */ + if ((bytes_to_check == 64) && (comp == 0x0)) { + i += 64; + nzrun_len += 64; + break; + } + num = ctz64(comp); + num = (num < bytes_to_check) ? num : bytes_to_check; + nzrun_len += num; + bytes_to_check -= num; + comp >>= num; + i += num; + if (bytes_to_check) { + /* mask like 111000 */ + d += uleb128_encode_small(dst + d, nzrun_len); + /* overflow */ + if (d + nzrun_len > dlen) { + return -1; + } + nzrun_start = new_buf + i - nzrun_len; + memcpy(dst + d, nzrun_start, nzrun_len); + d += nzrun_len; + nzrun_len = 0; + is_same = true; + } + } + } + + if (nzrun_len != 0) { + d += uleb128_encode_small(dst + d, nzrun_len); + /* overflow */ + if (d + nzrun_len > dlen) { + return -1; + } + nzrun_start = new_buf + i - nzrun_len; + memcpy(dst + d, nzrun_start, nzrun_len); + d += nzrun_len; + } + return d; +} + +static int xbzrle_encode_buffer_int(uint8_t *old_buf, uint8_t *new_buf, + int slen, uint8_t *dst, int dlen); + +static int (*accel_func)(uint8_t *, uint8_t *, int, uint8_t *, int); + +static void __attribute__((constructor)) init_accel(void) +{ + unsigned info = cpuinfo_init(); + if (info & CPUINFO_AVX512BW) { + accel_func = xbzrle_encode_buffer_avx512; + } else { + accel_func = xbzrle_encode_buffer_int; + } +} + +int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen, + uint8_t *dst, int dlen) +{ + return accel_func(old_buf, new_buf, slen, dst, dlen); +} + +#define xbzrle_encode_buffer xbzrle_encode_buffer_int +#endif + /* page = zrun nzrun | zrun nzrun page @@ -175,125 +321,3 @@ int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen) return d; } - -#if defined(CONFIG_AVX512BW_OPT) -#include <immintrin.h> - -int __attribute__((target("avx512bw"))) -xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, - uint8_t *dst, int dlen) -{ - uint32_t zrun_len = 0, nzrun_len = 0; - int d = 0, i = 0, num = 0; - uint8_t *nzrun_start = NULL; - /* add 1 to include residual part in main loop */ - uint32_t count512s = (slen >> 6) + 1; - /* countResidual is tail of data, i.e., countResidual = slen % 64 */ - uint32_t count_residual = slen & 0b111111; - bool never_same = true; - uint64_t mask_residual = 1; - mask_residual <<= count_residual; - mask_residual -= 1; - __m512i r = _mm512_set1_epi32(0); - - while (count512s) { - int bytes_to_check = 64; - uint64_t mask = 0xffffffffffffffff; - if (count512s == 1) { - bytes_to_check = count_residual; - mask = mask_residual; - } - __m512i old_data = _mm512_mask_loadu_epi8(r, - mask, old_buf + i); - __m512i new_data = _mm512_mask_loadu_epi8(r, - mask, new_buf + i); - uint64_t comp = _mm512_cmpeq_epi8_mask(old_data, new_data); - count512s--; - - bool is_same = (comp & 0x1); - while (bytes_to_check) { - if (d + 2 > dlen) { - return -1; - } - if (is_same) { - if (nzrun_len) { - d += uleb128_encode_small(dst + d, nzrun_len); - if (d + nzrun_len > dlen) { - return -1; - } - nzrun_start = new_buf + i - nzrun_len; - memcpy(dst + d, nzrun_start, nzrun_len); - d += nzrun_len; - nzrun_len = 0; - } - /* 64 data at a time for speed */ - if (count512s && (comp == 0xffffffffffffffff)) { - i += 64; - zrun_len += 64; - break; - } - never_same = false; - num = ctz64(~comp); - num = (num < bytes_to_check) ? num : bytes_to_check; - zrun_len += num; - bytes_to_check -= num; - comp >>= num; - i += num; - if (bytes_to_check) { - /* still has different data after same data */ - d += uleb128_encode_small(dst + d, zrun_len); - zrun_len = 0; - } else { - break; - } - } - if (never_same || zrun_len) { - /* - * never_same only acts if - * data begins with diff in first count512s - */ - d += uleb128_encode_small(dst + d, zrun_len); - zrun_len = 0; - never_same = false; - } - /* has diff, 64 data at a time for speed */ - if ((bytes_to_check == 64) && (comp == 0x0)) { - i += 64; - nzrun_len += 64; - break; - } - num = ctz64(comp); - num = (num < bytes_to_check) ? num : bytes_to_check; - nzrun_len += num; - bytes_to_check -= num; - comp >>= num; - i += num; - if (bytes_to_check) { - /* mask like 111000 */ - d += uleb128_encode_small(dst + d, nzrun_len); - /* overflow */ - if (d + nzrun_len > dlen) { - return -1; - } - nzrun_start = new_buf + i - nzrun_len; - memcpy(dst + d, nzrun_start, nzrun_len); - d += nzrun_len; - nzrun_len = 0; - is_same = true; - } - } - } - - if (nzrun_len != 0) { - d += uleb128_encode_small(dst + d, nzrun_len); - /* overflow */ - if (d + nzrun_len > dlen) { - return -1; - } - nzrun_start = new_buf + i - nzrun_len; - memcpy(dst + d, nzrun_start, nzrun_len); - d += nzrun_len; - } - return d; -} -#endif diff --git a/migration/xbzrle.h b/migration/xbzrle.h index 6feb49160a..39e651b9ec 100644 --- a/migration/xbzrle.h +++ b/migration/xbzrle.h @@ -18,8 +18,5 @@ int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen, uint8_t *dst, int dlen); int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen); -#if defined(CONFIG_AVX512BW_OPT) -int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, - uint8_t *dst, int dlen); -#endif + #endif diff --git a/net/eth.c b/net/eth.c index 70bcd8e355..649e66bb1f 100644 --- a/net/eth.c +++ b/net/eth.c @@ -21,26 +21,16 @@ #include "net/checksum.h" #include "net/tap.h" -void eth_setup_vlan_headers_ex(struct eth_header *ehdr, uint16_t vlan_tag, - uint16_t vlan_ethtype, bool *is_new) +void eth_setup_vlan_headers(struct eth_header *ehdr, size_t *ehdr_size, + uint16_t vlan_tag, uint16_t vlan_ethtype) { struct vlan_header *vhdr = PKT_GET_VLAN_HDR(ehdr); - switch (be16_to_cpu(ehdr->h_proto)) { - case ETH_P_VLAN: - case ETH_P_DVLAN: - /* vlan hdr exists */ - *is_new = false; - break; - - default: - /* No VLAN header, put a new one */ - vhdr->h_proto = ehdr->h_proto; - ehdr->h_proto = cpu_to_be16(vlan_ethtype); - *is_new = true; - break; - } + memmove(vhdr + 1, vhdr, *ehdr_size - ETH_HLEN); vhdr->h_tci = cpu_to_be16(vlan_tag); + vhdr->h_proto = ehdr->h_proto; + ehdr->h_proto = cpu_to_be16(vlan_ethtype); + *ehdr_size += sizeof(*vhdr); } uint8_t @@ -136,7 +126,7 @@ _eth_tcp_has_data(bool is_ip4, return l4len > TCP_HEADER_DATA_OFFSET(tcp); } -void eth_get_protocols(const struct iovec *iov, int iovcnt, +void eth_get_protocols(const struct iovec *iov, size_t iovcnt, size_t iovoff, bool *hasip4, bool *hasip6, size_t *l3hdr_off, size_t *l4hdr_off, @@ -147,26 +137,24 @@ void eth_get_protocols(const struct iovec *iov, int iovcnt, { int proto; bool fragment = false; - size_t l2hdr_len = eth_get_l2_hdr_length_iov(iov, iovcnt); size_t input_size = iov_size(iov, iovcnt); size_t copied; uint8_t ip_p; *hasip4 = *hasip6 = false; + *l3hdr_off = iovoff + eth_get_l2_hdr_length_iov(iov, iovcnt, iovoff); l4hdr_info->proto = ETH_L4_HDR_PROTO_INVALID; - proto = eth_get_l3_proto(iov, iovcnt, l2hdr_len); - - *l3hdr_off = l2hdr_len; + proto = eth_get_l3_proto(iov, iovcnt, *l3hdr_off); if (proto == ETH_P_IP) { struct ip_header *iphdr = &ip4hdr_info->ip4_hdr; - if (input_size < l2hdr_len) { + if (input_size < *l3hdr_off) { return; } - copied = iov_to_buf(iov, iovcnt, l2hdr_len, iphdr, sizeof(*iphdr)); + copied = iov_to_buf(iov, iovcnt, *l3hdr_off, iphdr, sizeof(*iphdr)); if (copied < sizeof(*iphdr) || IP_HEADER_VERSION(iphdr) != IP_HEADER_VERSION_4) { return; @@ -175,17 +163,17 @@ void eth_get_protocols(const struct iovec *iov, int iovcnt, *hasip4 = true; ip_p = iphdr->ip_p; ip4hdr_info->fragment = IP4_IS_FRAGMENT(iphdr); - *l4hdr_off = l2hdr_len + IP_HDR_GET_LEN(iphdr); + *l4hdr_off = *l3hdr_off + IP_HDR_GET_LEN(iphdr); fragment = ip4hdr_info->fragment; } else if (proto == ETH_P_IPV6) { - if (!eth_parse_ipv6_hdr(iov, iovcnt, l2hdr_len, ip6hdr_info)) { + if (!eth_parse_ipv6_hdr(iov, iovcnt, *l3hdr_off, ip6hdr_info)) { return; } *hasip6 = true; ip_p = ip6hdr_info->l4proto; - *l4hdr_off = l2hdr_len + ip6hdr_info->full_hdr_len; + *l4hdr_off = *l3hdr_off + ip6hdr_info->full_hdr_len; fragment = ip6hdr_info->fragment; } else { return; @@ -223,16 +211,20 @@ void eth_get_protocols(const struct iovec *iov, int iovcnt, *l5hdr_off = *l4hdr_off + sizeof(l4hdr_info->hdr.udp); } break; + + case IP_PROTO_SCTP: + l4hdr_info->proto = ETH_L4_HDR_PROTO_SCTP; + break; } } size_t eth_strip_vlan(const struct iovec *iov, int iovcnt, size_t iovoff, - uint8_t *new_ehdr_buf, + void *new_ehdr_buf, uint16_t *payload_offset, uint16_t *tci) { struct vlan_header vlan_hdr; - struct eth_header *new_ehdr = (struct eth_header *) new_ehdr_buf; + struct eth_header *new_ehdr = new_ehdr_buf; size_t copied = iov_to_buf(iov, iovcnt, iovoff, new_ehdr, sizeof(*new_ehdr)); @@ -277,36 +269,50 @@ eth_strip_vlan(const struct iovec *iov, int iovcnt, size_t iovoff, } size_t -eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff, - uint16_t vet, uint8_t *new_ehdr_buf, +eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff, int index, + uint16_t vet, uint16_t vet_ext, void *new_ehdr_buf, uint16_t *payload_offset, uint16_t *tci) { struct vlan_header vlan_hdr; - struct eth_header *new_ehdr = (struct eth_header *) new_ehdr_buf; - - size_t copied = iov_to_buf(iov, iovcnt, iovoff, - new_ehdr, sizeof(*new_ehdr)); - - if (copied < sizeof(*new_ehdr)) { - return 0; - } + uint16_t *new_ehdr_proto; + size_t new_ehdr_size; + size_t copied; - if (be16_to_cpu(new_ehdr->h_proto) == vet) { - copied = iov_to_buf(iov, iovcnt, iovoff + sizeof(*new_ehdr), - &vlan_hdr, sizeof(vlan_hdr)); + switch (index) { + case 0: + new_ehdr_proto = &PKT_GET_ETH_HDR(new_ehdr_buf)->h_proto; + new_ehdr_size = sizeof(struct eth_header); + copied = iov_to_buf(iov, iovcnt, iovoff, new_ehdr_buf, new_ehdr_size); + break; - if (copied < sizeof(vlan_hdr)) { + case 1: + new_ehdr_proto = &PKT_GET_VLAN_HDR(new_ehdr_buf)->h_proto; + new_ehdr_size = sizeof(struct eth_header) + sizeof(struct vlan_header); + copied = iov_to_buf(iov, iovcnt, iovoff, new_ehdr_buf, new_ehdr_size); + if (be16_to_cpu(PKT_GET_ETH_HDR(new_ehdr_buf)->h_proto) != vet_ext) { return 0; } + break; - new_ehdr->h_proto = vlan_hdr.h_proto; + default: + return 0; + } - *tci = be16_to_cpu(vlan_hdr.h_tci); - *payload_offset = iovoff + sizeof(*new_ehdr) + sizeof(vlan_hdr); - return sizeof(struct eth_header); + if (copied < new_ehdr_size || be16_to_cpu(*new_ehdr_proto) != vet) { + return 0; } - return 0; + copied = iov_to_buf(iov, iovcnt, iovoff + new_ehdr_size, + &vlan_hdr, sizeof(vlan_hdr)); + if (copied < sizeof(vlan_hdr)) { + return 0; + } + + *new_ehdr_proto = vlan_hdr.h_proto; + *payload_offset = iovoff + new_ehdr_size + sizeof(vlan_hdr); + *tci = be16_to_cpu(vlan_hdr.h_tci); + + return new_ehdr_size; } void diff --git a/qapi/qom.json b/qapi/qom.json index f61badaba4..7f92ea43e8 100644 --- a/qapi/qom.json +++ b/qapi/qom.json @@ -647,6 +647,10 @@ # selects a default alignment (currently the page size). # (default: 0) # +# @offset: the offset into the target file that the region starts at. You +# can use this option to back multiple regions with a single file. Must +# be a multiple of the page size. (default: 0) (since 8.1) +# # @discard-data: if true, the file contents can be destroyed when QEMU # exits, to avoid unnecessarily flushing data to the backing file. # Note that @discard-data is only an optimization, and QEMU might @@ -668,6 +672,7 @@ { 'struct': 'MemoryBackendFileProperties', 'base': 'MemoryBackendProperties', 'data': { '*align': 'size', + '*offset': 'size', '*discard-data': 'bool', 'mem-path': 'str', '*pmem': { 'type': 'bool', 'if': 'CONFIG_LIBPMEM' }, diff --git a/qemu-options.hx b/qemu-options.hx index e4566149ee..b37eb9662b 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -4949,7 +4949,7 @@ SRST they are specified. Note that the 'id' property must be set. These objects are placed in the '/objects' path. - ``-object memory-backend-file,id=id,size=size,mem-path=dir,share=on|off,discard-data=on|off,merge=on|off,dump=on|off,prealloc=on|off,host-nodes=host-nodes,policy=default|preferred|bind|interleave,align=align,readonly=on|off`` + ``-object memory-backend-file,id=id,size=size,mem-path=dir,share=on|off,discard-data=on|off,merge=on|off,dump=on|off,prealloc=on|off,host-nodes=host-nodes,policy=default|preferred|bind|interleave,align=align,offset=offset,readonly=on|off`` Creates a memory file backend object, which can be used to back the guest RAM with huge pages. @@ -5019,6 +5019,10 @@ SRST such cases, users can specify the required alignment via this option. + The ``offset`` option specifies the offset into the target file + that the region starts at. You can use this parameter to back + multiple regions with a single file. + The ``pmem`` option specifies whether the backing file specified by ``mem-path`` is in host persistent memory that can be accessed using the SNIA NVM programming model (e.g. Intel diff --git a/scripts/ci/gitlab-kubernetes-runners/values.yaml b/scripts/ci/gitlab-kubernetes-runners/values.yaml new file mode 100644 index 0000000000..204a96a842 --- /dev/null +++ b/scripts/ci/gitlab-kubernetes-runners/values.yaml @@ -0,0 +1,30 @@ +gitlabUrl: "https://gitlab.com/" +runnerRegistrationToken: "" +rbac: + create: true +concurrent: 200 +runners: + privileged: true + config: | + [[runners]] + limit = 100 + environment = [ + "DOCKER_HOST=tcp://docker:2376", + "DOCKER_TLS_CERTDIR=/certs", + "DOCKER_TLS_VERIFY=1", + "DOCKER_CERT_PATH=/certs/client" + ] + [runners.kubernetes] + poll_timeout = 1200 + image = "ubuntu:20.04" + cpu_request = "0.5" + service_cpu_request = "0.5" + helper_cpu_request = "0.25" + cpu_request_overwrite_max_allowed = "7" + memory_request_overwrite_max_allowed = "30Gi" + [[runners.kubernetes.volumes.empty_dir]] + name = "docker-certs" + mount_path = "/certs/client" + medium = "Memory" + [runners.kubernetes.node_selector] + agentpool = "jobs" diff --git a/scripts/ci/org.centos/stream/8/x86_64/test-avocado b/scripts/ci/org.centos/stream/8/x86_64/test-avocado index 7bb5b317b6..73e7a1a312 100755 --- a/scripts/ci/org.centos/stream/8/x86_64/test-avocado +++ b/scripts/ci/org.centos/stream/8/x86_64/test-avocado @@ -30,7 +30,8 @@ make get-vm-images tests/avocado/cpu_queries.py:QueryCPUModelExpansion.test \ tests/avocado/empty_cpu_model.py:EmptyCPUModel.test \ tests/avocado/hotplug_cpu.py:HotPlugCPU.test \ - tests/avocado/igb.py:IGB.test \ + tests/avocado/netdev-ethtool.py:NetDevEthtool.test_igb \ + tests/avocado/netdev-ethtool.py:NetDevEthtool.test_igb_nomsi \ tests/avocado/info_usernet.py:InfoUsernet.test_hostfwd \ tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu \ tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu_pt \ diff --git a/softmmu/memory.c b/softmmu/memory.c index 9ee4131d75..7d9494ce70 100644 --- a/softmmu/memory.c +++ b/softmmu/memory.c @@ -1619,6 +1619,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, uint64_t align, uint32_t ram_flags, const char *path, + ram_addr_t offset, bool readonly, Error **errp) { @@ -1630,7 +1631,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, mr->destructor = memory_region_destructor_ram; mr->align = align; mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path, - readonly, &err); + offset, readonly, &err); if (err) { mr->size = int128_zero(); object_unparent(OBJECT(mr)); diff --git a/softmmu/physmem.c b/softmmu/physmem.c index efaed36773..9d7e172260 100644 --- a/softmmu/physmem.c +++ b/softmmu/physmem.c @@ -1369,6 +1369,11 @@ static void *file_ram_alloc(RAMBlock *block, error_setg(errp, "alignment 0x%" PRIx64 " must be a power of two", block->mr->align); return NULL; + } else if (offset % block->page_size) { + error_setg(errp, "offset 0x%" PRIx64 + " must be multiples of page size 0x%zx", + offset, block->page_size); + return NULL; } block->mr->align = MAX(block->page_size, block->mr->align); #if defined(__s390x__) @@ -1400,7 +1405,7 @@ static void *file_ram_alloc(RAMBlock *block, * those labels. Therefore, extending the non-empty backend file * is disabled as well. */ - if (truncate && ftruncate(fd, memory)) { + if (truncate && ftruncate(fd, offset + memory)) { perror("ftruncate"); } @@ -1416,6 +1421,7 @@ static void *file_ram_alloc(RAMBlock *block, } block->fd = fd; + block->fd_offset = offset; return area; } #endif @@ -1889,7 +1895,7 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, size = HOST_PAGE_ALIGN(size); file_size = get_file_size(fd); - if (file_size > 0 && file_size < size) { + if (file_size > offset && file_size < (offset + size)) { error_setg(errp, "backing store size 0x%" PRIx64 " does not match 'size' option 0x" RAM_ADDR_FMT, file_size, size); @@ -1929,7 +1935,7 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, const char *mem_path, - bool readonly, Error **errp) + off_t offset, bool readonly, Error **errp) { int fd; bool created; @@ -1941,7 +1947,8 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, return NULL; } - block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, 0, readonly, errp); + block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, readonly, + errp); if (!block) { if (created) { unlink(mem_path); @@ -2075,7 +2082,7 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length) flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0; if (block->fd >= 0) { area = mmap(vaddr, length, PROT_READ | PROT_WRITE, - flags, block->fd, offset); + flags, block->fd, offset + block->fd_offset); } else { flags |= MAP_ANONYMOUS; area = mmap(vaddr, length, PROT_READ | PROT_WRITE, diff --git a/target/arm/tcg/m_helper.c b/target/arm/tcg/m_helper.c index 9758f225d6..9cef70e5c9 100644 --- a/target/arm/tcg/m_helper.c +++ b/target/arm/tcg/m_helper.c @@ -1937,8 +1937,8 @@ static bool do_v7m_function_return(ARMCPU *cpu) */ mmu_idx = arm_v7m_mmu_idx_for_secstate(env, true); oi = make_memop_idx(MO_LEUL, arm_to_core_mmu_idx(mmu_idx)); - newpc = cpu_ldl_le_mmu(env, frameptr, oi, 0); - newpsr = cpu_ldl_le_mmu(env, frameptr + 4, oi, 0); + newpc = cpu_ldl_mmu(env, frameptr, oi, 0); + newpsr = cpu_ldl_mmu(env, frameptr + 4, oi, 0); /* Consistency checks on new IPSR */ newpsr_exc = newpsr & XPSR_EXCP; diff --git a/target/ppc/cpu.h b/target/ppc/cpu.h index 1c02596d9f..0f9f2e1a0c 100644 --- a/target/ppc/cpu.h +++ b/target/ppc/cpu.h @@ -1124,7 +1124,6 @@ struct CPUArchState { /* used to speed-up TLB assist handlers */ target_ulong nip; /* next instruction pointer */ - uint64_t retxh; /* high part of 128-bit helper return */ /* when a memory exception occurs, the access type is stored here */ int access_type; diff --git a/target/ppc/helper.h b/target/ppc/helper.h index 0beaca5c7a..38efbc351c 100644 --- a/target/ppc/helper.h +++ b/target/ppc/helper.h @@ -810,12 +810,3 @@ DEF_HELPER_4(DSCLIQ, void, env, fprp, fprp, i32) DEF_HELPER_1(tbegin, void, env) DEF_HELPER_FLAGS_1(fixup_thrm, TCG_CALL_NO_RWG, void, env) - -#ifdef TARGET_PPC64 -DEF_HELPER_FLAGS_3(lq_le_parallel, TCG_CALL_NO_WG, i64, env, tl, i32) -DEF_HELPER_FLAGS_3(lq_be_parallel, TCG_CALL_NO_WG, i64, env, tl, i32) -DEF_HELPER_FLAGS_5(stq_le_parallel, TCG_CALL_NO_WG, - void, env, tl, i64, i64, i32) -DEF_HELPER_FLAGS_5(stq_be_parallel, TCG_CALL_NO_WG, - void, env, tl, i64, i64, i32) -#endif diff --git a/target/ppc/mem_helper.c b/target/ppc/mem_helper.c index 1578887a8f..46eae65819 100644 --- a/target/ppc/mem_helper.c +++ b/target/ppc/mem_helper.c @@ -367,54 +367,6 @@ target_ulong helper_lscbx(CPUPPCState *env, target_ulong addr, uint32_t reg, return i; } -#ifdef TARGET_PPC64 -uint64_t helper_lq_le_parallel(CPUPPCState *env, target_ulong addr, - uint32_t opidx) -{ - Int128 ret; - - /* We will have raised EXCP_ATOMIC from the translator. */ - assert(HAVE_ATOMIC128); - ret = cpu_atomic_ldo_le_mmu(env, addr, opidx, GETPC()); - env->retxh = int128_gethi(ret); - return int128_getlo(ret); -} - -uint64_t helper_lq_be_parallel(CPUPPCState *env, target_ulong addr, - uint32_t opidx) -{ - Int128 ret; - - /* We will have raised EXCP_ATOMIC from the translator. */ - assert(HAVE_ATOMIC128); - ret = cpu_atomic_ldo_be_mmu(env, addr, opidx, GETPC()); - env->retxh = int128_gethi(ret); - return int128_getlo(ret); -} - -void helper_stq_le_parallel(CPUPPCState *env, target_ulong addr, - uint64_t lo, uint64_t hi, uint32_t opidx) -{ - Int128 val; - - /* We will have raised EXCP_ATOMIC from the translator. */ - assert(HAVE_ATOMIC128); - val = int128_make128(lo, hi); - cpu_atomic_sto_le_mmu(env, addr, val, opidx, GETPC()); -} - -void helper_stq_be_parallel(CPUPPCState *env, target_ulong addr, - uint64_t lo, uint64_t hi, uint32_t opidx) -{ - Int128 val; - - /* We will have raised EXCP_ATOMIC from the translator. */ - assert(HAVE_ATOMIC128); - val = int128_make128(lo, hi); - cpu_atomic_sto_be_mmu(env, addr, val, opidx, GETPC()); -} -#endif - /*****************************************************************************/ /* Altivec extension helpers */ #if HOST_BIG_ENDIAN diff --git a/target/ppc/translate.c b/target/ppc/translate.c index f603f1a939..1720570b9b 100644 --- a/target/ppc/translate.c +++ b/target/ppc/translate.c @@ -3757,6 +3757,7 @@ static void gen_lqarx(DisasContext *ctx) { int rd = rD(ctx->opcode); TCGv EA, hi, lo; + TCGv_i128 t16; if (unlikely((rd & 1) || (rd == rA(ctx->opcode)) || (rd == rB(ctx->opcode)))) { @@ -3772,36 +3773,9 @@ static void gen_lqarx(DisasContext *ctx) lo = cpu_gpr[rd + 1]; hi = cpu_gpr[rd]; - if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { - if (HAVE_ATOMIC128) { - TCGv_i32 oi = tcg_temp_new_i32(); - if (ctx->le_mode) { - tcg_gen_movi_i32(oi, make_memop_idx(MO_LE | MO_128 | MO_ALIGN, - ctx->mem_idx)); - gen_helper_lq_le_parallel(lo, cpu_env, EA, oi); - } else { - tcg_gen_movi_i32(oi, make_memop_idx(MO_BE | MO_128 | MO_ALIGN, - ctx->mem_idx)); - gen_helper_lq_be_parallel(lo, cpu_env, EA, oi); - } - tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUPPCState, retxh)); - } else { - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; - return; - } - } else if (ctx->le_mode) { - tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_LEUQ | MO_ALIGN_16); - tcg_gen_mov_tl(cpu_reserve, EA); - gen_addr_add(ctx, EA, EA, 8); - tcg_gen_qemu_ld_i64(hi, EA, ctx->mem_idx, MO_LEUQ); - } else { - tcg_gen_qemu_ld_i64(hi, EA, ctx->mem_idx, MO_BEUQ | MO_ALIGN_16); - tcg_gen_mov_tl(cpu_reserve, EA); - gen_addr_add(ctx, EA, EA, 8); - tcg_gen_qemu_ld_i64(lo, EA, ctx->mem_idx, MO_BEUQ); - } + t16 = tcg_temp_new_i128(); + tcg_gen_qemu_ld_i128(t16, EA, ctx->mem_idx, DEF_MEMOP(MO_128 | MO_ALIGN)); + tcg_gen_extr_i128_i64(lo, hi, t16); tcg_gen_st_tl(hi, cpu_env, offsetof(CPUPPCState, reserve_val)); tcg_gen_st_tl(lo, cpu_env, offsetof(CPUPPCState, reserve_val2)); diff --git a/target/ppc/translate/fixedpoint-impl.c.inc b/target/ppc/translate/fixedpoint-impl.c.inc index 02d86b77a8..f47f1a50e8 100644 --- a/target/ppc/translate/fixedpoint-impl.c.inc +++ b/target/ppc/translate/fixedpoint-impl.c.inc @@ -72,7 +72,7 @@ static bool do_ldst_quad(DisasContext *ctx, arg_D *a, bool store, bool prefixed) #if defined(TARGET_PPC64) TCGv ea; TCGv_i64 low_addr_gpr, high_addr_gpr; - MemOp mop; + TCGv_i128 t16; REQUIRE_INSNS_FLAGS(ctx, 64BX); @@ -101,51 +101,14 @@ static bool do_ldst_quad(DisasContext *ctx, arg_D *a, bool store, bool prefixed) low_addr_gpr = cpu_gpr[a->rt + 1]; high_addr_gpr = cpu_gpr[a->rt]; } + t16 = tcg_temp_new_i128(); - if (tb_cflags(ctx->base.tb) & CF_PARALLEL) { - if (HAVE_ATOMIC128) { - mop = DEF_MEMOP(MO_128); - TCGv_i32 oi = tcg_constant_i32(make_memop_idx(mop, ctx->mem_idx)); - if (store) { - if (ctx->le_mode) { - gen_helper_stq_le_parallel(cpu_env, ea, low_addr_gpr, - high_addr_gpr, oi); - } else { - gen_helper_stq_be_parallel(cpu_env, ea, high_addr_gpr, - low_addr_gpr, oi); - - } - } else { - if (ctx->le_mode) { - gen_helper_lq_le_parallel(low_addr_gpr, cpu_env, ea, oi); - tcg_gen_ld_i64(high_addr_gpr, cpu_env, - offsetof(CPUPPCState, retxh)); - } else { - gen_helper_lq_be_parallel(high_addr_gpr, cpu_env, ea, oi); - tcg_gen_ld_i64(low_addr_gpr, cpu_env, - offsetof(CPUPPCState, retxh)); - } - } - } else { - /* Restart with exclusive lock. */ - gen_helper_exit_atomic(cpu_env); - ctx->base.is_jmp = DISAS_NORETURN; - } + if (store) { + tcg_gen_concat_i64_i128(t16, low_addr_gpr, high_addr_gpr); + tcg_gen_qemu_st_i128(t16, ea, ctx->mem_idx, DEF_MEMOP(MO_128)); } else { - mop = DEF_MEMOP(MO_UQ); - if (store) { - tcg_gen_qemu_st_i64(low_addr_gpr, ea, ctx->mem_idx, mop); - } else { - tcg_gen_qemu_ld_i64(low_addr_gpr, ea, ctx->mem_idx, mop); - } - - gen_addr_add(ctx, ea, ea, 8); - - if (store) { - tcg_gen_qemu_st_i64(high_addr_gpr, ea, ctx->mem_idx, mop); - } else { - tcg_gen_qemu_ld_i64(high_addr_gpr, ea, ctx->mem_idx, mop); - } + tcg_gen_qemu_ld_i128(t16, ea, ctx->mem_idx, DEF_MEMOP(MO_128)); + tcg_gen_extr_i128_i64(low_addr_gpr, high_addr_gpr, t16); } #else qemu_build_not_reached(); diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h index c47e7adcb1..f130c29f83 100644 --- a/target/s390x/cpu.h +++ b/target/s390x/cpu.h @@ -76,9 +76,6 @@ struct CPUArchState { float_status fpu_status; /* passed to softfloat lib */ - /* The low part of a 128-bit return, or remainder of a divide. */ - uint64_t retxl; - PSW psw; S390CrashReason crash_reason; diff --git a/target/s390x/helper.h b/target/s390x/helper.h index 341bc51ec2..7529e725f2 100644 --- a/target/s390x/helper.h +++ b/target/s390x/helper.h @@ -108,10 +108,6 @@ DEF_HELPER_FLAGS_2(sfas, TCG_CALL_NO_WG, void, env, i64) DEF_HELPER_FLAGS_2(srnm, TCG_CALL_NO_WG, void, env, i64) DEF_HELPER_FLAGS_1(popcnt, TCG_CALL_NO_RWG_SE, i64, i64) DEF_HELPER_2(stfle, i32, env, i64) -DEF_HELPER_FLAGS_2(lpq, TCG_CALL_NO_WG, i64, env, i64) -DEF_HELPER_FLAGS_2(lpq_parallel, TCG_CALL_NO_WG, i64, env, i64) -DEF_HELPER_FLAGS_4(stpq, TCG_CALL_NO_WG, void, env, i64, i64, i64) -DEF_HELPER_FLAGS_4(stpq_parallel, TCG_CALL_NO_WG, void, env, i64, i64, i64) DEF_HELPER_4(mvcos, i32, env, i64, i64, i64) DEF_HELPER_4(cu12, i32, env, i32, i32, i32) DEF_HELPER_4(cu14, i32, env, i32, i32, i32) diff --git a/target/s390x/tcg/insn-data.h.inc b/target/s390x/tcg/insn-data.h.inc index 1f1ac742a9..bcc70d99ba 100644 --- a/target/s390x/tcg/insn-data.h.inc +++ b/target/s390x/tcg/insn-data.h.inc @@ -570,7 +570,7 @@ D(0xc804, LPD, SSF, ILA, 0, 0, new_P, r3_P32, lpd, 0, MO_TEUL) D(0xc805, LPDG, SSF, ILA, 0, 0, new_P, r3_P64, lpd, 0, MO_TEUQ) /* LOAD PAIR FROM QUADWORD */ - C(0xe38f, LPQ, RXY_a, Z, 0, a2, r1_P, 0, lpq, 0) + C(0xe38f, LPQ, RXY_a, Z, 0, a2, 0, r1_D64, lpq, 0) /* LOAD POSITIVE */ C(0x1000, LPR, RR_a, Z, 0, r2_32s, new, r1_32, abs, abs32) C(0xb900, LPGR, RRE, Z, 0, r2, r1, 0, abs, abs64) diff --git a/target/s390x/tcg/mem_helper.c b/target/s390x/tcg/mem_helper.c index 8b58b8d88d..d02ec861d8 100644 --- a/target/s390x/tcg/mem_helper.c +++ b/target/s390x/tcg/mem_helper.c @@ -1737,6 +1737,11 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2, bool parallel) { uint32_t mem_idx = cpu_mmu_index(env, false); + MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, mem_idx); + MemOpIdx oi8 = make_memop_idx(MO_TE | MO_64, mem_idx); + MemOpIdx oi4 = make_memop_idx(MO_TE | MO_32, mem_idx); + MemOpIdx oi2 = make_memop_idx(MO_TE | MO_16, mem_idx); + MemOpIdx oi1 = make_memop_idx(MO_8, mem_idx); uintptr_t ra = GETPC(); uint32_t fc = extract32(env->regs[0], 0, 8); uint32_t sc = extract32(env->regs[0], 8, 8); @@ -1775,34 +1780,30 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, max = 3; #endif if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) || - (HAVE_ATOMIC128 ? 0 : sc > max)) { + (HAVE_ATOMIC128_RW ? 0 : sc > max)) { cpu_loop_exit_atomic(env_cpu(env), ra); } } - /* All loads happen before all stores. For simplicity, load the entire - store value area from the parameter list. */ - svh = cpu_ldq_data_ra(env, pl + 16, ra); - svl = cpu_ldq_data_ra(env, pl + 24, ra); + /* + * All loads happen before all stores. For simplicity, load the entire + * store value area from the parameter list. + */ + svh = cpu_ldq_mmu(env, pl + 16, oi8, ra); + svl = cpu_ldq_mmu(env, pl + 24, oi8, ra); switch (fc) { case 0: { - uint32_t nv = cpu_ldl_data_ra(env, pl, ra); + uint32_t nv = cpu_ldl_mmu(env, pl, oi4, ra); uint32_t cv = env->regs[r3]; uint32_t ov; if (parallel) { -#ifdef CONFIG_USER_ONLY - uint32_t *haddr = g2h(env_cpu(env), a1); - ov = qatomic_cmpxchg__nocheck(haddr, cv, nv); -#else - MemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx); - ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi, ra); -#endif + ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi4, ra); } else { - ov = cpu_ldl_data_ra(env, a1, ra); - cpu_stl_data_ra(env, a1, (ov == cv ? nv : ov), ra); + ov = cpu_ldl_mmu(env, a1, oi4, ra); + cpu_stl_mmu(env, a1, (ov == cv ? nv : ov), oi4, ra); } cc = (ov != cv); env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov); @@ -1811,21 +1812,20 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, case 1: { - uint64_t nv = cpu_ldq_data_ra(env, pl, ra); + uint64_t nv = cpu_ldq_mmu(env, pl, oi8, ra); uint64_t cv = env->regs[r3]; uint64_t ov; if (parallel) { #ifdef CONFIG_ATOMIC64 - MemOpIdx oi = make_memop_idx(MO_TEUQ | MO_ALIGN, mem_idx); - ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi, ra); + ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi8, ra); #else /* Note that we asserted !parallel above. */ g_assert_not_reached(); #endif } else { - ov = cpu_ldq_data_ra(env, a1, ra); - cpu_stq_data_ra(env, a1, (ov == cv ? nv : ov), ra); + ov = cpu_ldq_mmu(env, a1, oi8, ra); + cpu_stq_mmu(env, a1, (ov == cv ? nv : ov), oi8, ra); } cc = (ov != cv); env->regs[r3] = ov; @@ -1834,27 +1834,19 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, case 2: { - uint64_t nvh = cpu_ldq_data_ra(env, pl, ra); - uint64_t nvl = cpu_ldq_data_ra(env, pl + 8, ra); - Int128 nv = int128_make128(nvl, nvh); + Int128 nv = cpu_ld16_mmu(env, pl, oi16, ra); Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]); Int128 ov; if (!parallel) { - uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra); - uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra); - - ov = int128_make128(ol, oh); + ov = cpu_ld16_mmu(env, a1, oi16, ra); cc = !int128_eq(ov, cv); if (cc) { nv = ov; } - - cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra); - cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra); + cpu_st16_mmu(env, a1, nv, oi16, ra); } else if (HAVE_CMPXCHG128) { - MemOpIdx oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx); - ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra); + ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi16, ra); cc = !int128_eq(ov, cv); } else { /* Note that we asserted !parallel above. */ @@ -1876,29 +1868,19 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, if (cc == 0) { switch (sc) { case 0: - cpu_stb_data_ra(env, a2, svh >> 56, ra); + cpu_stb_mmu(env, a2, svh >> 56, oi1, ra); break; case 1: - cpu_stw_data_ra(env, a2, svh >> 48, ra); + cpu_stw_mmu(env, a2, svh >> 48, oi2, ra); break; case 2: - cpu_stl_data_ra(env, a2, svh >> 32, ra); + cpu_stl_mmu(env, a2, svh >> 32, oi4, ra); break; case 3: - cpu_stq_data_ra(env, a2, svh, ra); + cpu_stq_mmu(env, a2, svh, oi8, ra); break; case 4: - if (!parallel) { - cpu_stq_data_ra(env, a2 + 0, svh, ra); - cpu_stq_data_ra(env, a2 + 8, svl, ra); - } else if (HAVE_ATOMIC128) { - MemOpIdx oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx); - Int128 sv = int128_make128(svl, svh); - cpu_atomic_sto_be_mmu(env, a2, sv, oi, ra); - } else { - /* Note that we asserted !parallel above. */ - g_assert_not_reached(); - } + cpu_st16_mmu(env, a2, int128_make128(svl, svh), oi16, ra); break; default: g_assert_not_reached(); @@ -2398,67 +2380,6 @@ uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr) } #endif -/* load pair from quadword */ -uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr) -{ - uintptr_t ra = GETPC(); - uint64_t hi, lo; - - check_alignment(env, addr, 16, ra); - hi = cpu_ldq_data_ra(env, addr + 0, ra); - lo = cpu_ldq_data_ra(env, addr + 8, ra); - - env->retxl = lo; - return hi; -} - -uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) -{ - uintptr_t ra = GETPC(); - uint64_t hi, lo; - int mem_idx; - MemOpIdx oi; - Int128 v; - - assert(HAVE_ATOMIC128); - - mem_idx = cpu_mmu_index(env, false); - oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx); - v = cpu_atomic_ldo_be_mmu(env, addr, oi, ra); - hi = int128_gethi(v); - lo = int128_getlo(v); - - env->retxl = lo; - return hi; -} - -/* store pair to quadword */ -void HELPER(stpq)(CPUS390XState *env, uint64_t addr, - uint64_t low, uint64_t high) -{ - uintptr_t ra = GETPC(); - - check_alignment(env, addr, 16, ra); - cpu_stq_data_ra(env, addr + 0, high, ra); - cpu_stq_data_ra(env, addr + 8, low, ra); -} - -void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, - uint64_t low, uint64_t high) -{ - uintptr_t ra = GETPC(); - int mem_idx; - MemOpIdx oi; - Int128 v; - - assert(HAVE_ATOMIC128); - - mem_idx = cpu_mmu_index(env, false); - oi = make_memop_idx(MO_TEUQ | MO_ALIGN_16, mem_idx); - v = int128_make128(low, high); - cpu_atomic_sto_be_mmu(env, addr, v, oi, ra); -} - /* Execute instruction. This instruction executes an insn modified with the contents of r1. It does not change the executed instruction in memory; it does not change the program counter. diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index d6670e6a87..3eb3708d55 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -335,11 +335,6 @@ static void store_freg32_i64(int reg, TCGv_i64 v) tcg_gen_st32_i64(v, cpu_env, freg32_offset(reg)); } -static void return_low128(TCGv_i64 dest) -{ - tcg_gen_ld_i64(dest, cpu_env, offsetof(CPUS390XState, retxl)); -} - static void update_psw_addr(DisasContext *s) { /* psw.addr */ @@ -3130,15 +3125,9 @@ static DisasJumpType op_lpd(DisasContext *s, DisasOps *o) static DisasJumpType op_lpq(DisasContext *s, DisasOps *o) { - if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { - gen_helper_lpq(o->out, cpu_env, o->in2); - } else if (HAVE_ATOMIC128) { - gen_helper_lpq_parallel(o->out, cpu_env, o->in2); - } else { - gen_helper_exit_atomic(cpu_env); - return DISAS_NORETURN; - } - return_low128(o->out2); + o->out_128 = tcg_temp_new_i128(); + tcg_gen_qemu_ld_i128(o->out_128, o->in2, get_mem_index(s), + MO_TE | MO_128 | MO_ALIGN); return DISAS_NEXT; } @@ -4533,14 +4522,11 @@ static DisasJumpType op_stmh(DisasContext *s, DisasOps *o) static DisasJumpType op_stpq(DisasContext *s, DisasOps *o) { - if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { - gen_helper_stpq(cpu_env, o->in2, o->out2, o->out); - } else if (HAVE_ATOMIC128) { - gen_helper_stpq_parallel(cpu_env, o->in2, o->out2, o->out); - } else { - gen_helper_exit_atomic(cpu_env); - return DISAS_NORETURN; - } + TCGv_i128 t16 = tcg_temp_new_i128(); + + tcg_gen_concat_i64_i128(t16, o->out2, o->out); + tcg_gen_qemu_st_i128(t16, o->in2, get_mem_index(s), + MO_TE | MO_128 | MO_ALIGN); return DISAS_NEXT; } diff --git a/target/sh4/translate.c b/target/sh4/translate.c index 0dedbb8210..d9accfa1e7 100644 --- a/target/sh4/translate.c +++ b/target/sh4/translate.c @@ -17,8 +17,6 @@ * License along with this library; if not, see <http://www.gnu.org/licenses/>. */ -#define DEBUG_DISAS - #include "qemu/osdep.h" #include "cpu.h" #include "disas/disas.h" diff --git a/target/sparc/ldst_helper.c b/target/sparc/ldst_helper.c index 7972d56a72..981a47d8bb 100644 --- a/target/sparc/ldst_helper.c +++ b/target/sparc/ldst_helper.c @@ -1334,25 +1334,13 @@ uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr, ret = cpu_ldb_mmu(env, addr, oi, GETPC()); break; case 2: - if (asi & 8) { - ret = cpu_ldw_le_mmu(env, addr, oi, GETPC()); - } else { - ret = cpu_ldw_be_mmu(env, addr, oi, GETPC()); - } + ret = cpu_ldw_mmu(env, addr, oi, GETPC()); break; case 4: - if (asi & 8) { - ret = cpu_ldl_le_mmu(env, addr, oi, GETPC()); - } else { - ret = cpu_ldl_be_mmu(env, addr, oi, GETPC()); - } + ret = cpu_ldl_mmu(env, addr, oi, GETPC()); break; case 8: - if (asi & 8) { - ret = cpu_ldq_le_mmu(env, addr, oi, GETPC()); - } else { - ret = cpu_ldq_be_mmu(env, addr, oi, GETPC()); - } + ret = cpu_ldq_mmu(env, addr, oi, GETPC()); break; default: g_assert_not_reached(); diff --git a/target/sparc/translate.c b/target/sparc/translate.c index 414e014b11..9377798490 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -34,8 +34,6 @@ #include "asi.h" -#define DEBUG_DISAS - #define DYNAMIC_PC 1 /* dynamic pc value */ #define JUMP_PC 2 /* dynamic pc value which takes only two values according to jump_pc[T2] */ diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index bc6b99a1bd..84283665e7 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -13,12 +13,6 @@ #include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" #include "qemu/bitops.h" -#ifdef __linux__ -#include <asm/hwcap.h> -#endif -#ifdef CONFIG_DARWIN -#include <sys/sysctl.h> -#endif /* We're going to re-use TCGType in setting of the SF bit, which controls the size of the operation performed. If we know the values match, it @@ -77,9 +71,6 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) return TCG_REG_X0 + slot; } -bool have_lse; -bool have_lse2; - #define TCG_REG_TMP TCG_REG_X30 #define TCG_VEC_TMP TCG_REG_V31 @@ -2878,39 +2869,8 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) } } -#ifdef CONFIG_DARWIN -static bool sysctl_for_bool(const char *name) -{ - int val = 0; - size_t len = sizeof(val); - - if (sysctlbyname(name, &val, &len, NULL, 0) == 0) { - return val != 0; - } - - /* - * We might in the future ask for properties not present in older kernels, - * but we're only asking about static properties, all of which should be - * 'int'. So we shouln't see ENOMEM (val too small), or any of the other - * more exotic errors. - */ - assert(errno == ENOENT); - return false; -} -#endif - static void tcg_target_init(TCGContext *s) { -#ifdef __linux__ - unsigned long hwcap = qemu_getauxval(AT_HWCAP); - have_lse = hwcap & HWCAP_ATOMICS; - have_lse2 = hwcap & HWCAP_USCAT; -#endif -#ifdef CONFIG_DARWIN - have_lse = sysctl_for_bool("hw.optional.arm.FEAT_LSE"); - have_lse2 = sysctl_for_bool("hw.optional.arm.FEAT_LSE2"); -#endif - tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffffu; tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffffu; tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull; diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index 74ee2ed255..d5f7614880 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -13,6 +13,8 @@ #ifndef AARCH64_TCG_TARGET_H #define AARCH64_TCG_TARGET_H +#include "host/cpuinfo.h" + #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 24 #define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) @@ -57,8 +59,8 @@ typedef enum { #define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_EVEN #define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL -extern bool have_lse; -extern bool have_lse2; +#define have_lse (cpuinfo & CPUINFO_LSE) +#define have_lse2 (cpuinfo & CPUINFO_LSE2) /* optional instructions */ #define TCG_TARGET_HAS_div_i32 1 diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index 8b9a5f00e5..bfe9d98b7e 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -158,42 +158,14 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) # define SOFTMMU_RESERVE_REGS 0 #endif -/* The host compiler should supply <cpuid.h> to enable runtime features - detection, as we're not going to go so far as our own inline assembly. - If not available, default values will be assumed. */ -#if defined(CONFIG_CPUID_H) -#include "qemu/cpuid.h" -#endif - /* For 64-bit, we always know that CMOV is available. */ #if TCG_TARGET_REG_BITS == 64 -# define have_cmov 1 -#elif defined(CONFIG_CPUID_H) -static bool have_cmov; -#else -# define have_cmov 0 -#endif - -/* We need these symbols in tcg-target.h, and we can't properly conditionalize - it there. Therefore we always define the variable. */ -bool have_bmi1; -bool have_popcnt; -bool have_avx1; -bool have_avx2; -bool have_avx512bw; -bool have_avx512dq; -bool have_avx512vbmi2; -bool have_avx512vl; -bool have_movbe; -bool have_atomic16; - -#ifdef CONFIG_CPUID_H -static bool have_bmi2; -static bool have_lzcnt; +# define have_cmov true #else -# define have_bmi2 0 -# define have_lzcnt 0 +# define have_cmov (cpuinfo & CPUINFO_CMOV) #endif +#define have_bmi2 (cpuinfo & CPUINFO_BMI2) +#define have_lzcnt (cpuinfo & CPUINFO_LZCNT) static const tcg_insn_unit *tb_ret_addr; @@ -3961,93 +3933,6 @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count) static void tcg_target_init(TCGContext *s) { -#ifdef CONFIG_CPUID_H - unsigned a, b, c, d, b7 = 0, c7 = 0; - unsigned max = __get_cpuid_max(0, 0); - - if (max >= 7) { - /* BMI1 is available on AMD Piledriver and Intel Haswell CPUs. */ - __cpuid_count(7, 0, a, b7, c7, d); - have_bmi1 = (b7 & bit_BMI) != 0; - have_bmi2 = (b7 & bit_BMI2) != 0; - } - - if (max >= 1) { - __cpuid(1, a, b, c, d); -#ifndef have_cmov - /* For 32-bit, 99% certainty that we're running on hardware that - supports cmov, but we still need to check. In case cmov is not - available, we'll use a small forward branch. */ - have_cmov = (d & bit_CMOV) != 0; -#endif - - /* MOVBE is only available on Intel Atom and Haswell CPUs, so we - need to probe for it. */ - have_movbe = (c & bit_MOVBE) != 0; - have_popcnt = (c & bit_POPCNT) != 0; - - /* There are a number of things we must check before we can be - sure of not hitting invalid opcode. */ - if (c & bit_OSXSAVE) { - unsigned bv = xgetbv_low(0); - - if ((bv & 6) == 6) { - have_avx1 = (c & bit_AVX) != 0; - have_avx2 = (b7 & bit_AVX2) != 0; - - /* - * There are interesting instructions in AVX512, so long - * as we have AVX512VL, which indicates support for EVEX - * on sizes smaller than 512 bits. We are required to - * check that OPMASK and all extended ZMM state are enabled - * even if we're not using them -- the insns will fault. - */ - if ((bv & 0xe0) == 0xe0 - && (b7 & bit_AVX512F) - && (b7 & bit_AVX512VL)) { - have_avx512vl = true; - have_avx512bw = (b7 & bit_AVX512BW) != 0; - have_avx512dq = (b7 & bit_AVX512DQ) != 0; - have_avx512vbmi2 = (c7 & bit_AVX512VBMI2) != 0; - } - - /* - * The Intel SDM has added: - * Processors that enumerate support for Intel® AVX - * (by setting the feature flag CPUID.01H:ECX.AVX[bit 28]) - * guarantee that the 16-byte memory operations performed - * by the following instructions will always be carried - * out atomically: - * - MOVAPD, MOVAPS, and MOVDQA. - * - VMOVAPD, VMOVAPS, and VMOVDQA when encoded with VEX.128. - * - VMOVAPD, VMOVAPS, VMOVDQA32, and VMOVDQA64 when encoded - * with EVEX.128 and k0 (masking disabled). - * Note that these instructions require the linear addresses - * of their memory operands to be 16-byte aligned. - * - * AMD has provided an even stronger guarantee that processors - * with AVX provide 16-byte atomicity for all cachable, - * naturally aligned single loads and stores, e.g. MOVDQU. - * - * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 - */ - if (have_avx1) { - __cpuid(0, a, b, c, d); - have_atomic16 = (c == signature_INTEL_ecx || - c == signature_AMD_ecx); - } - } - } - } - - max = __get_cpuid_max(0x8000000, 0); - if (max >= 1) { - __cpuid(0x80000001, a, b, c, d); - /* LZCNT was introduced with AMD Barcelona and Intel Haswell CPUs. */ - have_lzcnt = (c & bit_LZCNT) != 0; - } -#endif /* CONFIG_CPUID_H */ - tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; if (TCG_TARGET_REG_BITS == 64) { tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS; diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index 0b5a2c68c5..0106946996 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -25,6 +25,8 @@ #ifndef I386_TCG_TARGET_H #define I386_TCG_TARGET_H +#include "host/cpuinfo.h" + #define TCG_TARGET_INSN_UNIT_SIZE 1 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 31 @@ -111,16 +113,22 @@ typedef enum { # define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_BY_REF #endif -extern bool have_bmi1; -extern bool have_popcnt; -extern bool have_avx1; -extern bool have_avx2; -extern bool have_avx512bw; -extern bool have_avx512dq; -extern bool have_avx512vbmi2; -extern bool have_avx512vl; -extern bool have_movbe; -extern bool have_atomic16; +#define have_bmi1 (cpuinfo & CPUINFO_BMI1) +#define have_popcnt (cpuinfo & CPUINFO_POPCNT) +#define have_avx1 (cpuinfo & CPUINFO_AVX1) +#define have_avx2 (cpuinfo & CPUINFO_AVX2) +#define have_movbe (cpuinfo & CPUINFO_MOVBE) +#define have_atomic16 (cpuinfo & CPUINFO_ATOMIC_VMOVDQA) + +/* + * There are interesting instructions in AVX512, so long as we have AVX512VL, + * which indicates support for EVEX on sizes smaller than 512 bits. + */ +#define have_avx512vl ((cpuinfo & CPUINFO_AVX512VL) && \ + (cpuinfo & CPUINFO_AVX512F)) +#define have_avx512bw ((cpuinfo & CPUINFO_AVX512BW) && have_avx512vl) +#define have_avx512dq ((cpuinfo & CPUINFO_AVX512DQ) && have_avx512vl) +#define have_avx512vbmi2 ((cpuinfo & CPUINFO_AVX512VBMI2) && have_avx512vl) /* optional instructions */ #define TCG_TARGET_HAS_div2_i32 1 diff --git a/tcg/tcg-op-ldst.c b/tcg/tcg-op-ldst.c index f4e508cb68..3d27c347d7 100644 --- a/tcg/tcg-op-ldst.c +++ b/tcg/tcg-op-ldst.c @@ -975,13 +975,11 @@ static void tcg_gen_nonatomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr, { if (TCG_TARGET_REG_BITS == 32) { /* Inline expansion below is simply too large for 32-bit hosts. */ - gen_atomic_cx_i128 gen = ((memop & MO_BSWAP) == MO_LE - ? gen_helper_nonatomic_cmpxchgo_le - : gen_helper_nonatomic_cmpxchgo_be); MemOpIdx oi = make_memop_idx(memop, idx); TCGv_i64 a64 = maybe_extend_addr64(addr); - gen(retv, cpu_env, a64, cmpv, newv, tcg_constant_i32(oi)); + gen_helper_nonatomic_cmpxchgo(retv, cpu_env, a64, cmpv, newv, + tcg_constant_i32(oi)); maybe_free_addr64(a64); } else { TCGv_i128 oldv = tcg_temp_ebb_new_i128(); diff --git a/tcg/tcg.c b/tcg/tcg.c index 0b0fe9c7ad..ac30d484f5 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -22,9 +22,6 @@ * THE SOFTWARE. */ -/* define it to use liveness analysis (better code) */ -#define USE_TCG_OPTIMIZATIONS - #include "qemu/osdep.h" /* Define to jump the ELF file used to communicate with GDB. */ @@ -1451,7 +1448,6 @@ void tcg_prologue_init(TCGContext *s) (uintptr_t)s->code_buf, prologue_size); #endif -#ifdef DEBUG_DISAS if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { FILE *logfile = qemu_log_trylock(); if (logfile) { @@ -1483,7 +1479,6 @@ void tcg_prologue_init(TCGContext *s) qemu_log_unlock(logfile); } } -#endif #ifndef CONFIG_TCG_INTERPRETER /* @@ -5998,7 +5993,6 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) } #endif -#ifdef DEBUG_DISAS if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) && qemu_log_in_addr_range(pc_start))) { FILE *logfile = qemu_log_trylock(); @@ -6009,7 +6003,6 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) qemu_log_unlock(logfile); } } -#endif #ifdef CONFIG_DEBUG_TCG /* Ensure all labels referenced have been emitted. */ @@ -6032,9 +6025,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); #endif -#ifdef USE_TCG_OPTIMIZATIONS tcg_optimize(s); -#endif #ifdef CONFIG_PROFILER qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); @@ -6046,7 +6037,6 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) liveness_pass_1(s); if (s->nb_indirects > 0) { -#ifdef DEBUG_DISAS if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) && qemu_log_in_addr_range(pc_start))) { FILE *logfile = qemu_log_trylock(); @@ -6057,7 +6047,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) qemu_log_unlock(logfile); } } -#endif + /* Replace indirect temps with direct temps. */ if (liveness_pass_2(s)) { /* If changes were made, re-run liveness. */ @@ -6069,7 +6059,6 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); #endif -#ifdef DEBUG_DISAS if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) && qemu_log_in_addr_range(pc_start))) { FILE *logfile = qemu_log_trylock(); @@ -6080,7 +6069,6 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) qemu_log_unlock(logfile); } } -#endif /* Initialize goto_tb jump offsets. */ tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; diff --git a/tests/avocado/netdev-ethtool.py b/tests/avocado/netdev-ethtool.py index f7e9464184..5f33288f81 100644 --- a/tests/avocado/netdev-ethtool.py +++ b/tests/avocado/netdev-ethtool.py @@ -7,7 +7,6 @@ from avocado import skip from avocado_qemu import QemuSystemTest -from avocado_qemu import exec_command, exec_command_and_wait_for_pattern from avocado_qemu import wait_for_console_pattern class NetDevEthtool(QemuSystemTest): @@ -30,7 +29,7 @@ class NetDevEthtool(QemuSystemTest): # URL into a unique one return self.fetch_asset(name=name, locations=(url), asset_hash=sha1) - def common_test_code(self, netdev, extra_args=None, kvm=False): + def common_test_code(self, netdev, extra_args=None): # This custom kernel has drivers for all the supported network # devices we can emulate in QEMU @@ -58,9 +57,6 @@ class NetDevEthtool(QemuSystemTest): '-drive', drive, '-device', netdev) - if kvm: - self.vm.add_args('-accel', 'kvm') - self.vm.set_console(console_index=0) self.vm.launch() @@ -71,10 +67,6 @@ class NetDevEthtool(QemuSystemTest): # no need to gracefully shutdown, just finish self.vm.kill() - # Skip testing for MSI for now. Allegedly it was fixed by: - # 28e96556ba (igb: Allocate MSI-X vector when testing) - # but I'm seeing oops in the kernel - @skip("Kernel bug with MSI enabled") def test_igb(self): """ :avocado: tags=device:igb @@ -87,13 +79,6 @@ class NetDevEthtool(QemuSystemTest): """ self.common_test_code("igb", "pci=nomsi") - def test_igb_nomsi_kvm(self): - """ - :avocado: tags=device:igb - """ - self.require_accelerator('kvm') - self.common_test_code("igb", "pci=nomsi", True) - # It seems the other popular cards we model in QEMU currently fail # the pattern test with: # diff --git a/tests/bench/meson.build b/tests/bench/meson.build index 4e6b469066..3c799dbd98 100644 --- a/tests/bench/meson.build +++ b/tests/bench/meson.build @@ -3,12 +3,6 @@ qht_bench = executable('qht-bench', sources: 'qht-bench.c', dependencies: [qemuutil]) -if have_system -xbzrle_bench = executable('xbzrle-bench', - sources: 'xbzrle-bench.c', - dependencies: [qemuutil,migration]) -endif - qtree_bench = executable('qtree-bench', sources: 'qtree-bench.c', dependencies: [qemuutil]) diff --git a/tests/bench/xbzrle-bench.c b/tests/bench/xbzrle-bench.c deleted file mode 100644 index 8848a3a32d..0000000000 --- a/tests/bench/xbzrle-bench.c +++ /dev/null @@ -1,469 +0,0 @@ -/* - * Xor Based Zero Run Length Encoding unit tests. - * - * Copyright 2013 Red Hat, Inc. and/or its affiliates - * - * Authors: - * Orit Wasserman <owasserm@redhat.com> - * - * This work is licensed under the terms of the GNU GPL, version 2 or later. - * See the COPYING file in the top-level directory. - * - */ -#include "qemu/osdep.h" -#include "qemu/cutils.h" -#include "../migration/xbzrle.h" - -#if defined(CONFIG_AVX512BW_OPT) -#define XBZRLE_PAGE_SIZE 4096 -static bool is_cpu_support_avx512bw; -#include "qemu/cpuid.h" -static void __attribute__((constructor)) init_cpu_flag(void) -{ - unsigned max = __get_cpuid_max(0, NULL); - int a, b, c, d; - is_cpu_support_avx512bw = false; - if (max >= 1) { - __cpuid(1, a, b, c, d); - /* We must check that AVX is not just available, but usable. */ - if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) { - int bv; - __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0)); - __cpuid_count(7, 0, a, b, c, d); - /* 0xe6: - * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15 - * and ZMM16-ZMM31 state are enabled by OS) - * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS) - */ - if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) { - is_cpu_support_avx512bw = true; - } - } - } - return ; -} - -struct ResTime { - float t_raw; - float t_512; -}; - - -/* Function prototypes -int xbzrle_encode_buffer_avx512(uint8_t *old_buf, uint8_t *new_buf, int slen, - uint8_t *dst, int dlen); -*/ -static void encode_decode_zero(struct ResTime *res) -{ - uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed512 = g_malloc0(XBZRLE_PAGE_SIZE); - int i = 0; - int dlen = 0, dlen512 = 0; - int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1006); - - for (i = diff_len; i > 0; i--) { - buffer[1000 + i] = i; - buffer512[1000 + i] = i; - } - - buffer[1000 + diff_len + 3] = 103; - buffer[1000 + diff_len + 5] = 105; - - buffer512[1000 + diff_len + 3] = 103; - buffer512[1000 + diff_len + 5] = 105; - - /* encode zero page */ - time_t t_start, t_end, t_start512, t_end512; - t_start = clock(); - dlen = xbzrle_encode_buffer(buffer, buffer, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); - t_end = clock(); - float time_val = difftime(t_end, t_start); - g_assert(dlen == 0); - - t_start512 = clock(); - dlen512 = xbzrle_encode_buffer_avx512(buffer512, buffer512, XBZRLE_PAGE_SIZE, - compressed512, XBZRLE_PAGE_SIZE); - t_end512 = clock(); - float time_val512 = difftime(t_end512, t_start512); - g_assert(dlen512 == 0); - - res->t_raw = time_val; - res->t_512 = time_val512; - - g_free(buffer); - g_free(compressed); - g_free(buffer512); - g_free(compressed512); - -} - -static void test_encode_decode_zero_avx512(void) -{ - int i; - float time_raw = 0.0, time_512 = 0.0; - struct ResTime res; - for (i = 0; i < 10000; i++) { - encode_decode_zero(&res); - time_raw += res.t_raw; - time_512 += res.t_512; - } - printf("Zero test:\n"); - printf("Raw xbzrle_encode time is %f ms\n", time_raw); - printf("512 xbzrle_encode time is %f ms\n", time_512); -} - -static void encode_decode_unchanged(struct ResTime *res) -{ - uint8_t *compressed = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed512 = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); - int i = 0; - int dlen = 0, dlen512 = 0; - int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1006); - - for (i = diff_len; i > 0; i--) { - test[1000 + i] = i + 4; - test512[1000 + i] = i + 4; - } - - test[1000 + diff_len + 3] = 107; - test[1000 + diff_len + 5] = 109; - - test512[1000 + diff_len + 3] = 107; - test512[1000 + diff_len + 5] = 109; - - /* test unchanged buffer */ - time_t t_start, t_end, t_start512, t_end512; - t_start = clock(); - dlen = xbzrle_encode_buffer(test, test, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); - t_end = clock(); - float time_val = difftime(t_end, t_start); - g_assert(dlen == 0); - - t_start512 = clock(); - dlen512 = xbzrle_encode_buffer_avx512(test512, test512, XBZRLE_PAGE_SIZE, - compressed512, XBZRLE_PAGE_SIZE); - t_end512 = clock(); - float time_val512 = difftime(t_end512, t_start512); - g_assert(dlen512 == 0); - - res->t_raw = time_val; - res->t_512 = time_val512; - - g_free(test); - g_free(compressed); - g_free(test512); - g_free(compressed512); - -} - -static void test_encode_decode_unchanged_avx512(void) -{ - int i; - float time_raw = 0.0, time_512 = 0.0; - struct ResTime res; - for (i = 0; i < 10000; i++) { - encode_decode_unchanged(&res); - time_raw += res.t_raw; - time_512 += res.t_512; - } - printf("Unchanged test:\n"); - printf("Raw xbzrle_encode time is %f ms\n", time_raw); - printf("512 xbzrle_encode time is %f ms\n", time_512); -} - -static void encode_decode_1_byte(struct ResTime *res) -{ - uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed = g_malloc(XBZRLE_PAGE_SIZE); - uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed512 = g_malloc(XBZRLE_PAGE_SIZE); - int dlen = 0, rc = 0, dlen512 = 0, rc512 = 0; - uint8_t buf[2]; - uint8_t buf512[2]; - - test[XBZRLE_PAGE_SIZE - 1] = 1; - test512[XBZRLE_PAGE_SIZE - 1] = 1; - - time_t t_start, t_end, t_start512, t_end512; - t_start = clock(); - dlen = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); - t_end = clock(); - float time_val = difftime(t_end, t_start); - g_assert(dlen == (uleb128_encode_small(&buf[0], 4095) + 2)); - - rc = xbzrle_decode_buffer(compressed, dlen, buffer, XBZRLE_PAGE_SIZE); - g_assert(rc == XBZRLE_PAGE_SIZE); - g_assert(memcmp(test, buffer, XBZRLE_PAGE_SIZE) == 0); - - t_start512 = clock(); - dlen512 = xbzrle_encode_buffer_avx512(buffer512, test512, XBZRLE_PAGE_SIZE, - compressed512, XBZRLE_PAGE_SIZE); - t_end512 = clock(); - float time_val512 = difftime(t_end512, t_start512); - g_assert(dlen512 == (uleb128_encode_small(&buf512[0], 4095) + 2)); - - rc512 = xbzrle_decode_buffer(compressed512, dlen512, buffer512, - XBZRLE_PAGE_SIZE); - g_assert(rc512 == XBZRLE_PAGE_SIZE); - g_assert(memcmp(test512, buffer512, XBZRLE_PAGE_SIZE) == 0); - - res->t_raw = time_val; - res->t_512 = time_val512; - - g_free(buffer); - g_free(compressed); - g_free(test); - g_free(buffer512); - g_free(compressed512); - g_free(test512); - -} - -static void test_encode_decode_1_byte_avx512(void) -{ - int i; - float time_raw = 0.0, time_512 = 0.0; - struct ResTime res; - for (i = 0; i < 10000; i++) { - encode_decode_1_byte(&res); - time_raw += res.t_raw; - time_512 += res.t_512; - } - printf("1 byte test:\n"); - printf("Raw xbzrle_encode time is %f ms\n", time_raw); - printf("512 xbzrle_encode time is %f ms\n", time_512); -} - -static void encode_decode_overflow(struct ResTime *res) -{ - uint8_t *compressed = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed512 = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); - int i = 0, rc = 0, rc512 = 0; - - for (i = 0; i < XBZRLE_PAGE_SIZE / 2 - 1; i++) { - test[i * 2] = 1; - test512[i * 2] = 1; - } - - /* encode overflow */ - time_t t_start, t_end, t_start512, t_end512; - t_start = clock(); - rc = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); - t_end = clock(); - float time_val = difftime(t_end, t_start); - g_assert(rc == -1); - - t_start512 = clock(); - rc512 = xbzrle_encode_buffer_avx512(buffer512, test512, XBZRLE_PAGE_SIZE, - compressed512, XBZRLE_PAGE_SIZE); - t_end512 = clock(); - float time_val512 = difftime(t_end512, t_start512); - g_assert(rc512 == -1); - - res->t_raw = time_val; - res->t_512 = time_val512; - - g_free(buffer); - g_free(compressed); - g_free(test); - g_free(buffer512); - g_free(compressed512); - g_free(test512); - -} - -static void test_encode_decode_overflow_avx512(void) -{ - int i; - float time_raw = 0.0, time_512 = 0.0; - struct ResTime res; - for (i = 0; i < 10000; i++) { - encode_decode_overflow(&res); - time_raw += res.t_raw; - time_512 += res.t_512; - } - printf("Overflow test:\n"); - printf("Raw xbzrle_encode time is %f ms\n", time_raw); - printf("512 xbzrle_encode time is %f ms\n", time_512); -} - -static void encode_decode_range_avx512(struct ResTime *res) -{ - uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed = g_malloc(XBZRLE_PAGE_SIZE); - uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed512 = g_malloc(XBZRLE_PAGE_SIZE); - uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); - int i = 0, rc = 0, rc512 = 0; - int dlen = 0, dlen512 = 0; - - int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1006); - - for (i = diff_len; i > 0; i--) { - buffer[1000 + i] = i; - test[1000 + i] = i + 4; - buffer512[1000 + i] = i; - test512[1000 + i] = i + 4; - } - - buffer[1000 + diff_len + 3] = 103; - test[1000 + diff_len + 3] = 107; - - buffer[1000 + diff_len + 5] = 105; - test[1000 + diff_len + 5] = 109; - - buffer512[1000 + diff_len + 3] = 103; - test512[1000 + diff_len + 3] = 107; - - buffer512[1000 + diff_len + 5] = 105; - test512[1000 + diff_len + 5] = 109; - - /* test encode/decode */ - time_t t_start, t_end, t_start512, t_end512; - t_start = clock(); - dlen = xbzrle_encode_buffer(test, buffer, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); - t_end = clock(); - float time_val = difftime(t_end, t_start); - rc = xbzrle_decode_buffer(compressed, dlen, test, XBZRLE_PAGE_SIZE); - g_assert(rc < XBZRLE_PAGE_SIZE); - g_assert(memcmp(test, buffer, XBZRLE_PAGE_SIZE) == 0); - - t_start512 = clock(); - dlen512 = xbzrle_encode_buffer_avx512(test512, buffer512, XBZRLE_PAGE_SIZE, - compressed512, XBZRLE_PAGE_SIZE); - t_end512 = clock(); - float time_val512 = difftime(t_end512, t_start512); - rc512 = xbzrle_decode_buffer(compressed512, dlen512, test512, XBZRLE_PAGE_SIZE); - g_assert(rc512 < XBZRLE_PAGE_SIZE); - g_assert(memcmp(test512, buffer512, XBZRLE_PAGE_SIZE) == 0); - - res->t_raw = time_val; - res->t_512 = time_val512; - - g_free(buffer); - g_free(compressed); - g_free(test); - g_free(buffer512); - g_free(compressed512); - g_free(test512); - -} - -static void test_encode_decode_avx512(void) -{ - int i; - float time_raw = 0.0, time_512 = 0.0; - struct ResTime res; - for (i = 0; i < 10000; i++) { - encode_decode_range_avx512(&res); - time_raw += res.t_raw; - time_512 += res.t_512; - } - printf("Encode decode test:\n"); - printf("Raw xbzrle_encode time is %f ms\n", time_raw); - printf("512 xbzrle_encode time is %f ms\n", time_512); -} - -static void encode_decode_random(struct ResTime *res) -{ - uint8_t *buffer = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed = g_malloc(XBZRLE_PAGE_SIZE); - uint8_t *test = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *buffer512 = g_malloc0(XBZRLE_PAGE_SIZE); - uint8_t *compressed512 = g_malloc(XBZRLE_PAGE_SIZE); - uint8_t *test512 = g_malloc0(XBZRLE_PAGE_SIZE); - int i = 0, rc = 0, rc512 = 0; - int dlen = 0, dlen512 = 0; - - int diff_len = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1); - /* store the index of diff */ - int dirty_index[diff_len]; - for (int j = 0; j < diff_len; j++) { - dirty_index[j] = g_test_rand_int_range(0, XBZRLE_PAGE_SIZE - 1); - } - for (i = diff_len - 1; i >= 0; i--) { - buffer[dirty_index[i]] = i; - test[dirty_index[i]] = i + 4; - buffer512[dirty_index[i]] = i; - test512[dirty_index[i]] = i + 4; - } - - time_t t_start, t_end, t_start512, t_end512; - t_start = clock(); - dlen = xbzrle_encode_buffer(test, buffer, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); - t_end = clock(); - float time_val = difftime(t_end, t_start); - rc = xbzrle_decode_buffer(compressed, dlen, test, XBZRLE_PAGE_SIZE); - g_assert(rc < XBZRLE_PAGE_SIZE); - - t_start512 = clock(); - dlen512 = xbzrle_encode_buffer_avx512(test512, buffer512, XBZRLE_PAGE_SIZE, - compressed512, XBZRLE_PAGE_SIZE); - t_end512 = clock(); - float time_val512 = difftime(t_end512, t_start512); - rc512 = xbzrle_decode_buffer(compressed512, dlen512, test512, XBZRLE_PAGE_SIZE); - g_assert(rc512 < XBZRLE_PAGE_SIZE); - - res->t_raw = time_val; - res->t_512 = time_val512; - - g_free(buffer); - g_free(compressed); - g_free(test); - g_free(buffer512); - g_free(compressed512); - g_free(test512); - -} - -static void test_encode_decode_random_avx512(void) -{ - int i; - float time_raw = 0.0, time_512 = 0.0; - struct ResTime res; - for (i = 0; i < 10000; i++) { - encode_decode_random(&res); - time_raw += res.t_raw; - time_512 += res.t_512; - } - printf("Random test:\n"); - printf("Raw xbzrle_encode time is %f ms\n", time_raw); - printf("512 xbzrle_encode time is %f ms\n", time_512); -} -#endif - -int main(int argc, char **argv) -{ - g_test_init(&argc, &argv, NULL); - g_test_rand_int(); - #if defined(CONFIG_AVX512BW_OPT) - if (likely(is_cpu_support_avx512bw)) { - g_test_add_func("/xbzrle/encode_decode_zero", test_encode_decode_zero_avx512); - g_test_add_func("/xbzrle/encode_decode_unchanged", - test_encode_decode_unchanged_avx512); - g_test_add_func("/xbzrle/encode_decode_1_byte", test_encode_decode_1_byte_avx512); - g_test_add_func("/xbzrle/encode_decode_overflow", - test_encode_decode_overflow_avx512); - g_test_add_func("/xbzrle/encode_decode", test_encode_decode_avx512); - g_test_add_func("/xbzrle/encode_decode_random", test_encode_decode_random_avx512); - } - #endif - return g_test_run(); -} diff --git a/tests/qtest/libqos/igb.c b/tests/qtest/libqos/igb.c index 12fb531bf0..a603468beb 100644 --- a/tests/qtest/libqos/igb.c +++ b/tests/qtest/libqos/igb.c @@ -114,6 +114,7 @@ static void igb_pci_start_hw(QOSGraphObject *obj) e1000e_macreg_write(&d->e1000e, E1000_RCTL, E1000_RCTL_EN); /* Enable all interrupts */ + e1000e_macreg_write(&d->e1000e, E1000_GPIE, E1000_GPIE_MSIX_MODE); e1000e_macreg_write(&d->e1000e, E1000_IMS, 0xFFFFFFFF); e1000e_macreg_write(&d->e1000e, E1000_EIMS, 0xFFFFFFFF); diff --git a/tests/unit/test-xbzrle.c b/tests/unit/test-xbzrle.c index 547046d093..b6996de69a 100644 --- a/tests/unit/test-xbzrle.c +++ b/tests/unit/test-xbzrle.c @@ -16,35 +16,6 @@ #define XBZRLE_PAGE_SIZE 4096 -int (*xbzrle_encode_buffer_func)(uint8_t *, uint8_t *, int, - uint8_t *, int) = xbzrle_encode_buffer; -#if defined(CONFIG_AVX512BW_OPT) -#include "qemu/cpuid.h" -static void __attribute__((constructor)) init_cpu_flag(void) -{ - unsigned max = __get_cpuid_max(0, NULL); - int a, b, c, d; - if (max >= 1) { - __cpuid(1, a, b, c, d); - /* We must check that AVX is not just available, but usable. */ - if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) { - int bv; - __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0)); - __cpuid_count(7, 0, a, b, c, d); - /* 0xe6: - * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15 - * and ZMM16-ZMM31 state are enabled by OS) - * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS) - */ - if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) { - xbzrle_encode_buffer_func = xbzrle_encode_buffer_avx512; - } - } - } - return ; -} -#endif - static void test_uleb(void) { uint32_t i, val; @@ -83,8 +54,8 @@ static void test_encode_decode_zero(void) buffer[1000 + diff_len + 5] = 105; /* encode zero page */ - dlen = xbzrle_encode_buffer_func(buffer, buffer, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); + dlen = xbzrle_encode_buffer(buffer, buffer, XBZRLE_PAGE_SIZE, + compressed, XBZRLE_PAGE_SIZE); g_assert(dlen == 0); g_free(buffer); @@ -107,8 +78,8 @@ static void test_encode_decode_unchanged(void) test[1000 + diff_len + 5] = 109; /* test unchanged buffer */ - dlen = xbzrle_encode_buffer_func(test, test, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); + dlen = xbzrle_encode_buffer(test, test, XBZRLE_PAGE_SIZE, + compressed, XBZRLE_PAGE_SIZE); g_assert(dlen == 0); g_free(test); @@ -125,8 +96,8 @@ static void test_encode_decode_1_byte(void) test[XBZRLE_PAGE_SIZE - 1] = 1; - dlen = xbzrle_encode_buffer_func(buffer, test, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); + dlen = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, + compressed, XBZRLE_PAGE_SIZE); g_assert(dlen == (uleb128_encode_small(&buf[0], 4095) + 2)); rc = xbzrle_decode_buffer(compressed, dlen, buffer, XBZRLE_PAGE_SIZE); @@ -150,8 +121,8 @@ static void test_encode_decode_overflow(void) } /* encode overflow */ - rc = xbzrle_encode_buffer_func(buffer, test, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); + rc = xbzrle_encode_buffer(buffer, test, XBZRLE_PAGE_SIZE, + compressed, XBZRLE_PAGE_SIZE); g_assert(rc == -1); g_free(buffer); @@ -181,8 +152,8 @@ static void encode_decode_range(void) test[1000 + diff_len + 5] = 109; /* test encode/decode */ - dlen = xbzrle_encode_buffer_func(test, buffer, XBZRLE_PAGE_SIZE, compressed, - XBZRLE_PAGE_SIZE); + dlen = xbzrle_encode_buffer(test, buffer, XBZRLE_PAGE_SIZE, + compressed, XBZRLE_PAGE_SIZE); rc = xbzrle_decode_buffer(compressed, dlen, test, XBZRLE_PAGE_SIZE); g_assert(rc < XBZRLE_PAGE_SIZE); diff --git a/util/bufferiszero.c b/util/bufferiszero.c index 1886bc5ba4..3e6a5dfd63 100644 --- a/util/bufferiszero.c +++ b/util/bufferiszero.c @@ -24,6 +24,7 @@ #include "qemu/osdep.h" #include "qemu/cutils.h" #include "qemu/bswap.h" +#include "host/cpuinfo.h" static bool buffer_zero_int(const void *buf, size_t len) @@ -184,111 +185,75 @@ buffer_zero_avx512(const void *buf, size_t len) } #endif /* CONFIG_AVX512F_OPT */ - -/* Note that for test_buffer_is_zero_next_accel, the most preferred - * ISA must have the least significant bit. - */ -#define CACHE_AVX512F 1 -#define CACHE_AVX2 2 -#define CACHE_SSE4 4 -#define CACHE_SSE2 8 - -/* Make sure that these variables are appropriately initialized when +/* + * Make sure that these variables are appropriately initialized when * SSE2 is enabled on the compiler command-line, but the compiler is * too old to support CONFIG_AVX2_OPT. */ #if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT) -# define INIT_CACHE 0 -# define INIT_ACCEL buffer_zero_int +# define INIT_USED 0 +# define INIT_LENGTH 0 +# define INIT_ACCEL buffer_zero_int #else # ifndef __SSE2__ # error "ISA selection confusion" # endif -# define INIT_CACHE CACHE_SSE2 -# define INIT_ACCEL buffer_zero_sse2 +# define INIT_USED CPUINFO_SSE2 +# define INIT_LENGTH 64 +# define INIT_ACCEL buffer_zero_sse2 #endif -static unsigned cpuid_cache = INIT_CACHE; +static unsigned used_accel = INIT_USED; +static unsigned length_to_accel = INIT_LENGTH; static bool (*buffer_accel)(const void *, size_t) = INIT_ACCEL; -static int length_to_accel = 64; -static void init_accel(unsigned cache) +static unsigned __attribute__((noinline)) +select_accel_cpuinfo(unsigned info) { - bool (*fn)(const void *, size_t) = buffer_zero_int; - if (cache & CACHE_SSE2) { - fn = buffer_zero_sse2; - length_to_accel = 64; - } + /* Array is sorted in order of algorithm preference. */ + static const struct { + unsigned bit; + unsigned len; + bool (*fn)(const void *, size_t); + } all[] = { +#ifdef CONFIG_AVX512F_OPT + { CPUINFO_AVX512F, 256, buffer_zero_avx512 }, +#endif #ifdef CONFIG_AVX2_OPT - if (cache & CACHE_SSE4) { - fn = buffer_zero_sse4; - length_to_accel = 64; - } - if (cache & CACHE_AVX2) { - fn = buffer_zero_avx2; - length_to_accel = 128; - } + { CPUINFO_AVX2, 128, buffer_zero_avx2 }, + { CPUINFO_SSE4, 64, buffer_zero_sse4 }, #endif -#ifdef CONFIG_AVX512F_OPT - if (cache & CACHE_AVX512F) { - fn = buffer_zero_avx512; - length_to_accel = 256; + { CPUINFO_SSE2, 64, buffer_zero_sse2 }, + { CPUINFO_ALWAYS, 0, buffer_zero_int }, + }; + + for (unsigned i = 0; i < ARRAY_SIZE(all); ++i) { + if (info & all[i].bit) { + length_to_accel = all[i].len; + buffer_accel = all[i].fn; + return all[i].bit; + } } -#endif - buffer_accel = fn; + return 0; } #if defined(CONFIG_AVX512F_OPT) || defined(CONFIG_AVX2_OPT) -#include "qemu/cpuid.h" - -static void __attribute__((constructor)) init_cpuid_cache(void) +static void __attribute__((constructor)) init_accel(void) { - unsigned max = __get_cpuid_max(0, NULL); - int a, b, c, d; - unsigned cache = 0; - - if (max >= 1) { - __cpuid(1, a, b, c, d); - if (d & bit_SSE2) { - cache |= CACHE_SSE2; - } - if (c & bit_SSE4_1) { - cache |= CACHE_SSE4; - } - - /* We must check that AVX is not just available, but usable. */ - if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) { - unsigned bv = xgetbv_low(0); - __cpuid_count(7, 0, a, b, c, d); - if ((bv & 0x6) == 0x6 && (b & bit_AVX2)) { - cache |= CACHE_AVX2; - } - /* 0xe6: - * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15 - * and ZMM16-ZMM31 state are enabled by OS) - * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS) - */ - if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512F)) { - cache |= CACHE_AVX512F; - } - } - } - cpuid_cache = cache; - init_accel(cache); + used_accel = select_accel_cpuinfo(cpuinfo_init()); } #endif /* CONFIG_AVX2_OPT */ bool test_buffer_is_zero_next_accel(void) { - /* If no bits set, we just tested buffer_zero_int, and there - are no more acceleration options to test. */ - if (cpuid_cache == 0) { - return false; - } - /* Disable the accelerator we used before and select a new one. */ - cpuid_cache &= cpuid_cache - 1; - init_accel(cpuid_cache); - return true; + /* + * Accumulate the accelerators that we've already tested, and + * remove them from the set to test this round. We'll get back + * a zero from select_accel_cpuinfo when there are no more. + */ + unsigned used = select_accel_cpuinfo(cpuinfo & ~used_accel); + used_accel |= used; + return used; } static bool select_accel_fn(const void *buf, size_t len) diff --git a/util/cpuinfo-aarch64.c b/util/cpuinfo-aarch64.c new file mode 100644 index 0000000000..f99acb7884 --- /dev/null +++ b/util/cpuinfo-aarch64.c @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * Host specific cpu indentification for AArch64. + */ + +#include "qemu/osdep.h" +#include "host/cpuinfo.h" + +#ifdef CONFIG_LINUX +# ifdef CONFIG_GETAUXVAL +# include <sys/auxv.h> +# else +# include <asm/hwcap.h> +# include "elf.h" +# endif +#endif +#ifdef CONFIG_DARWIN +# include <sys/sysctl.h> +#endif + +unsigned cpuinfo; + +#ifdef CONFIG_DARWIN +static bool sysctl_for_bool(const char *name) +{ + int val = 0; + size_t len = sizeof(val); + + if (sysctlbyname(name, &val, &len, NULL, 0) == 0) { + return val != 0; + } + + /* + * We might in the future ask for properties not present in older kernels, + * but we're only asking about static properties, all of which should be + * 'int'. So we shouln't see ENOMEM (val too small), or any of the other + * more exotic errors. + */ + assert(errno == ENOENT); + return false; +} +#endif + +/* Called both as constructor and (possibly) via other constructors. */ +unsigned __attribute__((constructor)) cpuinfo_init(void) +{ + unsigned info = cpuinfo; + + if (info) { + return info; + } + + info = CPUINFO_ALWAYS; + +#ifdef CONFIG_LINUX + unsigned long hwcap = qemu_getauxval(AT_HWCAP); + info |= (hwcap & HWCAP_ATOMICS ? CPUINFO_LSE : 0); + info |= (hwcap & HWCAP_USCAT ? CPUINFO_LSE2 : 0); +#endif +#ifdef CONFIG_DARWIN + info |= sysctl_for_bool("hw.optional.arm.FEAT_LSE") * CPUINFO_LSE; + info |= sysctl_for_bool("hw.optional.arm.FEAT_LSE2") * CPUINFO_LSE2; +#endif + + cpuinfo = info; + return info; +} diff --git a/util/cpuinfo-i386.c b/util/cpuinfo-i386.c new file mode 100644 index 0000000000..ab6143d9e7 --- /dev/null +++ b/util/cpuinfo-i386.c @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + * Host specific cpu indentification for x86. + */ + +#include "qemu/osdep.h" +#include "host/cpuinfo.h" +#ifdef CONFIG_CPUID_H +# include "qemu/cpuid.h" +#endif + +unsigned cpuinfo; + +/* Called both as constructor and (possibly) via other constructors. */ +unsigned __attribute__((constructor)) cpuinfo_init(void) +{ + unsigned info = cpuinfo; + + if (info) { + return info; + } + +#ifdef CONFIG_CPUID_H + unsigned max, a, b, c, d, b7 = 0, c7 = 0; + + max = __get_cpuid_max(0, 0); + + if (max >= 7) { + __cpuid_count(7, 0, a, b7, c7, d); + info |= (b7 & bit_BMI ? CPUINFO_BMI1 : 0); + info |= (b7 & bit_BMI2 ? CPUINFO_BMI2 : 0); + } + + if (max >= 1) { + __cpuid(1, a, b, c, d); + + info |= (d & bit_CMOV ? CPUINFO_CMOV : 0); + info |= (d & bit_SSE2 ? CPUINFO_SSE2 : 0); + info |= (c & bit_SSE4_1 ? CPUINFO_SSE4 : 0); + info |= (c & bit_MOVBE ? CPUINFO_MOVBE : 0); + info |= (c & bit_POPCNT ? CPUINFO_POPCNT : 0); + + /* For AVX features, we must check available and usable. */ + if ((c & bit_AVX) && (c & bit_OSXSAVE)) { + unsigned bv = xgetbv_low(0); + + if ((bv & 6) == 6) { + info |= CPUINFO_AVX1; + info |= (b7 & bit_AVX2 ? CPUINFO_AVX2 : 0); + + if ((bv & 0xe0) == 0xe0) { + info |= (b7 & bit_AVX512F ? CPUINFO_AVX512F : 0); + info |= (b7 & bit_AVX512VL ? CPUINFO_AVX512VL : 0); + info |= (b7 & bit_AVX512BW ? CPUINFO_AVX512BW : 0); + info |= (b7 & bit_AVX512DQ ? CPUINFO_AVX512DQ : 0); + info |= (c7 & bit_AVX512VBMI2 ? CPUINFO_AVX512VBMI2 : 0); + } + + /* + * The Intel SDM has added: + * Processors that enumerate support for Intel® AVX + * (by setting the feature flag CPUID.01H:ECX.AVX[bit 28]) + * guarantee that the 16-byte memory operations performed + * by the following instructions will always be carried + * out atomically: + * - MOVAPD, MOVAPS, and MOVDQA. + * - VMOVAPD, VMOVAPS, and VMOVDQA when encoded with VEX.128. + * - VMOVAPD, VMOVAPS, VMOVDQA32, and VMOVDQA64 when encoded + * with EVEX.128 and k0 (masking disabled). + * Note that these instructions require the linear addresses + * of their memory operands to be 16-byte aligned. + * + * AMD has provided an even stronger guarantee that processors + * with AVX provide 16-byte atomicity for all cachable, + * naturally aligned single loads and stores, e.g. MOVDQU. + * + * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 + */ + __cpuid(0, a, b, c, d); + if (c == signature_INTEL_ecx) { + info |= CPUINFO_ATOMIC_VMOVDQA; + } else if (c == signature_AMD_ecx) { + info |= CPUINFO_ATOMIC_VMOVDQA | CPUINFO_ATOMIC_VMOVDQU; + } + } + } + } + + max = __get_cpuid_max(0x8000000, 0); + if (max >= 1) { + __cpuid(0x80000001, a, b, c, d); + info |= (c & bit_LZCNT ? CPUINFO_LZCNT : 0); + } +#endif + + info |= CPUINFO_ALWAYS; + cpuinfo = info; + return info; +} diff --git a/util/crc32c.c b/util/crc32c.c index 762657d853..ea7f345de8 100644 --- a/util/crc32c.c +++ b/util/crc32c.c @@ -113,3 +113,11 @@ uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length) return crc^0xffffffff; } +uint32_t iov_crc32c(uint32_t crc, const struct iovec *iov, size_t iov_cnt) +{ + while (iov_cnt--) { + crc = crc32c(crc, iov->iov_base, iov->iov_len) ^ 0xffffffff; + iov++; + } + return crc ^ 0xffffffff; +} diff --git a/util/meson.build b/util/meson.build index e1f1c39e10..3a93071d27 100644 --- a/util/meson.build +++ b/util/meson.build @@ -108,3 +108,9 @@ if have_block endif util_ss.add(when: 'CONFIG_LINUX', if_true: files('vfio-helpers.c')) endif + +if cpu == 'aarch64' + util_ss.add(files('cpuinfo-aarch64.c')) +elif cpu in ['x86', 'x86_64'] + util_ss.add(files('cpuinfo-i386.c')) +endif diff --git a/util/vfio-helpers.c b/util/vfio-helpers.c index 2d8af38f88..f8bab46c68 100644 --- a/util/vfio-helpers.c +++ b/util/vfio-helpers.c @@ -106,15 +106,17 @@ struct QEMUVFIOState { */ static char *sysfs_find_group_file(const char *device, Error **errp) { + g_autoptr(GError) gerr = NULL; char *sysfs_link; char *sysfs_group; char *p; char *path = NULL; sysfs_link = g_strdup_printf("/sys/bus/pci/devices/%s/iommu_group", device); - sysfs_group = g_malloc0(PATH_MAX); - if (readlink(sysfs_link, sysfs_group, PATH_MAX - 1) == -1) { - error_setg_errno(errp, errno, "Failed to find iommu group sysfs path"); + sysfs_group = g_file_read_link(sysfs_link, &gerr); + if (gerr) { + error_setg(errp, "Failed to find iommu group sysfs path: %s", + gerr->message); goto out; } p = strrchr(sysfs_group, '/'); |