diff options
Diffstat (limited to 'accel')
| -rw-r--r-- | accel/stubs/tcg-stub.c | 5 | ||||
| -rw-r--r-- | accel/tcg/cputlb.c | 46 | ||||
| -rw-r--r-- | accel/tcg/plugin-gen.c | 4 | ||||
| -rw-r--r-- | accel/tcg/tb-maint.c | 59 | ||||
| -rw-r--r-- | accel/tcg/tcg-all.c | 21 | ||||
| -rw-r--r-- | accel/tcg/translate-all.c | 6 |
6 files changed, 87 insertions, 54 deletions
diff --git a/accel/stubs/tcg-stub.c b/accel/stubs/tcg-stub.c index 3b76b8b17c..77055e3964 100644 --- a/accel/stubs/tcg-stub.c +++ b/accel/stubs/tcg-stub.c @@ -17,8 +17,3 @@ G_NORETURN void cpu_loop_exit(CPUState *cpu) { g_assert_not_reached(); } - -G_NORETURN void cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc) -{ - g_assert_not_reached(); -} diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 87e14bde4f..2a6aa01c57 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -129,7 +129,7 @@ static inline uint64_t tlb_addr_write(const CPUTLBEntry *entry) static inline uintptr_t tlb_index(CPUState *cpu, uintptr_t mmu_idx, vaddr addr) { - uintptr_t size_mask = cpu->neg.tlb.f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS; + uintptr_t size_mask = cpu_tlb_fast(cpu, mmu_idx)->mask >> CPU_TLB_ENTRY_BITS; return (addr >> TARGET_PAGE_BITS) & size_mask; } @@ -138,7 +138,7 @@ static inline uintptr_t tlb_index(CPUState *cpu, uintptr_t mmu_idx, static inline CPUTLBEntry *tlb_entry(CPUState *cpu, uintptr_t mmu_idx, vaddr addr) { - return &cpu->neg.tlb.f[mmu_idx].table[tlb_index(cpu, mmu_idx, addr)]; + return &cpu_tlb_fast(cpu, mmu_idx)->table[tlb_index(cpu, mmu_idx, addr)]; } static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns, @@ -292,7 +292,7 @@ static void tlb_flush_one_mmuidx_locked(CPUState *cpu, int mmu_idx, int64_t now) { CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx]; - CPUTLBDescFast *fast = &cpu->neg.tlb.f[mmu_idx]; + CPUTLBDescFast *fast = cpu_tlb_fast(cpu, mmu_idx); tlb_mmu_resize_locked(desc, fast, now); tlb_mmu_flush_locked(desc, fast); @@ -331,7 +331,7 @@ void tlb_init(CPUState *cpu) cpu->neg.tlb.c.dirty = 0; for (i = 0; i < NB_MMU_MODES; i++) { - tlb_mmu_init(&cpu->neg.tlb.d[i], &cpu->neg.tlb.f[i], now); + tlb_mmu_init(&cpu->neg.tlb.d[i], cpu_tlb_fast(cpu, i), now); } } @@ -342,7 +342,7 @@ void tlb_destroy(CPUState *cpu) qemu_spin_destroy(&cpu->neg.tlb.c.lock); for (i = 0; i < NB_MMU_MODES; i++) { CPUTLBDesc *desc = &cpu->neg.tlb.d[i]; - CPUTLBDescFast *fast = &cpu->neg.tlb.f[i]; + CPUTLBDescFast *fast = cpu_tlb_fast(cpu, i); g_free(fast->table); g_free(desc->fulltlb); @@ -370,8 +370,8 @@ static void flush_all_helper(CPUState *src, run_on_cpu_func fn, static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) { - uint16_t asked = data.host_int; - uint16_t all_dirty, work, to_clean; + MMUIdxMap asked = data.host_int; + MMUIdxMap all_dirty, work, to_clean; int64_t now = get_clock_realtime(); assert_cpu_is_self(cpu); @@ -408,7 +408,7 @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) } } -void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap) +void tlb_flush_by_mmuidx(CPUState *cpu, MMUIdxMap idxmap) { tlb_debug("mmu_idx: 0x%" PRIx16 "\n", idxmap); @@ -422,7 +422,7 @@ void tlb_flush(CPUState *cpu) tlb_flush_by_mmuidx(cpu, ALL_MMUIDX_BITS); } -void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu, uint16_t idxmap) +void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu, MMUIdxMap idxmap) { const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work; @@ -531,7 +531,7 @@ static void tlb_flush_page_locked(CPUState *cpu, int midx, vaddr page) */ static void tlb_flush_page_by_mmuidx_async_0(CPUState *cpu, vaddr addr, - uint16_t idxmap) + MMUIdxMap idxmap) { int mmu_idx; @@ -570,14 +570,14 @@ static void tlb_flush_page_by_mmuidx_async_1(CPUState *cpu, { vaddr addr_and_idxmap = data.target_ptr; vaddr addr = addr_and_idxmap & TARGET_PAGE_MASK; - uint16_t idxmap = addr_and_idxmap & ~TARGET_PAGE_MASK; + MMUIdxMap idxmap = addr_and_idxmap & ~TARGET_PAGE_MASK; tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap); } typedef struct { vaddr addr; - uint16_t idxmap; + MMUIdxMap idxmap; } TLBFlushPageByMMUIdxData; /** @@ -599,7 +599,7 @@ static void tlb_flush_page_by_mmuidx_async_2(CPUState *cpu, g_free(d); } -void tlb_flush_page_by_mmuidx(CPUState *cpu, vaddr addr, uint16_t idxmap) +void tlb_flush_page_by_mmuidx(CPUState *cpu, vaddr addr, MMUIdxMap idxmap) { tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%" PRIx16 "\n", addr, idxmap); @@ -618,7 +618,7 @@ void tlb_flush_page(CPUState *cpu, vaddr addr) void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu, vaddr addr, - uint16_t idxmap) + MMUIdxMap idxmap) { tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%"PRIx16"\n", addr, idxmap); @@ -667,7 +667,7 @@ static void tlb_flush_range_locked(CPUState *cpu, int midx, unsigned bits) { CPUTLBDesc *d = &cpu->neg.tlb.d[midx]; - CPUTLBDescFast *f = &cpu->neg.tlb.f[midx]; + CPUTLBDescFast *f = cpu_tlb_fast(cpu, midx); vaddr mask = MAKE_64BIT_MASK(0, bits); /* @@ -715,8 +715,8 @@ static void tlb_flush_range_locked(CPUState *cpu, int midx, typedef struct { vaddr addr; vaddr len; - uint16_t idxmap; - uint16_t bits; + MMUIdxMap idxmap; + unsigned bits; } TLBFlushRangeData; static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu, @@ -766,7 +766,7 @@ static void tlb_flush_range_by_mmuidx_async_1(CPUState *cpu, } void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr, - vaddr len, uint16_t idxmap, + vaddr len, MMUIdxMap idxmap, unsigned bits) { TLBFlushRangeData d; @@ -797,7 +797,7 @@ void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr, } void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, vaddr addr, - uint16_t idxmap, unsigned bits) + MMUIdxMap idxmap, unsigned bits) { tlb_flush_range_by_mmuidx(cpu, addr, TARGET_PAGE_SIZE, idxmap, bits); } @@ -805,7 +805,7 @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, vaddr addr, void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu, vaddr addr, vaddr len, - uint16_t idxmap, + MMUIdxMap idxmap, unsigned bits) { TLBFlushRangeData d, *p; @@ -847,7 +847,7 @@ void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu, void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, vaddr addr, - uint16_t idxmap, + MMUIdxMap idxmap, unsigned bits) { tlb_flush_range_by_mmuidx_all_cpus_synced(src_cpu, addr, TARGET_PAGE_SIZE, @@ -923,7 +923,7 @@ void tlb_reset_dirty(CPUState *cpu, uintptr_t start, uintptr_t length) qemu_spin_lock(&cpu->neg.tlb.c.lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx]; - CPUTLBDescFast *fast = &cpu->neg.tlb.f[mmu_idx]; + CPUTLBDescFast *fast = cpu_tlb_fast(cpu, mmu_idx); unsigned int n = tlb_n_entries(fast); unsigned int i; @@ -1316,7 +1316,7 @@ static bool victim_tlb_hit(CPUState *cpu, size_t mmu_idx, size_t index, if (cmp == page) { /* Found entry in victim tlb, swap tlb and iotlb. */ - CPUTLBEntry tmptlb, *tlb = &cpu->neg.tlb.f[mmu_idx].table[index]; + CPUTLBEntry tmptlb, *tlb = &cpu_tlb_fast(cpu, mmu_idx)->table[index]; qemu_spin_lock(&cpu->neg.tlb.c.lock); copy_tlb_helper_locked(&tmptlb, tlb); diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index 9920381a84..1ffcb4b2d2 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -102,8 +102,8 @@ static TCGv_i32 gen_cpu_index(void) /* * Optimize when we run with a single vcpu. All values using cpu_index, * including scoreboard index, will be optimized out. - * User-mode calls tb_flush when setting this flag. In system-mode, all - * vcpus are created before generating code. + * User-mode flushes all TBs when setting this flag. + * In system-mode, all vcpus are created before generating code. */ if (!tcg_cflags_has(current_cpu, CF_PARALLEL)) { return tcg_constant_i32(current_cpu->cpu_index); diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c index 0048316f99..5a8d0784e7 100644 --- a/accel/tcg/tb-maint.c +++ b/accel/tcg/tb-maint.c @@ -36,6 +36,9 @@ #include "internal-common.h" #ifdef CONFIG_USER_ONLY #include "user/page-protection.h" +#define runstate_is_running() true +#else +#include "system/runstate.h" #endif @@ -88,7 +91,10 @@ static IntervalTreeRoot tb_root; static void tb_remove_all(void) { - assert_memory_lock(); + /* + * Only called from tb_flush__exclusive_or_serial, where we have already + * asserted that we're in an exclusive state. + */ memset(&tb_root, 0, sizeof(tb_root)); } @@ -756,17 +762,19 @@ static void tb_remove(TranslationBlock *tb) } #endif /* CONFIG_USER_ONLY */ -/* flush all the translation blocks */ -static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) +/* + * Flush all the translation blocks. + * Must be called from a context in which no cpus are running, + * e.g. start_exclusive() or vm_stop(). + */ +void tb_flush__exclusive_or_serial(void) { - bool did_flush = false; + CPUState *cpu; - mmap_lock(); - /* If it is already been done on request of another CPU, just retry. */ - if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { - goto done; - } - did_flush = true; + assert(tcg_enabled()); + /* Note that cpu_in_serial_context checks cpu_in_exclusive_context. */ + assert(!runstate_is_running() || + (current_cpu && cpu_in_serial_context(current_cpu))); CPU_FOREACH(cpu) { tcg_flush_jmp_cache(cpu); @@ -778,25 +786,23 @@ static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) tcg_region_reset_all(); /* XXX: flush processor icache at this point if cache flush is expensive */ qatomic_inc(&tb_ctx.tb_flush_count); + qemu_plugin_flush_cb(); +} -done: - mmap_unlock(); - if (did_flush) { - qemu_plugin_flush_cb(); +static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) +{ + /* If it is already been done on request of another CPU, just retry. */ + if (tb_ctx.tb_flush_count == tb_flush_count.host_int) { + tb_flush__exclusive_or_serial(); } } -void tb_flush(CPUState *cpu) +void queue_tb_flush(CPUState *cs) { if (tcg_enabled()) { unsigned tb_flush_count = qatomic_read(&tb_ctx.tb_flush_count); - - if (cpu_in_serial_context(cpu)) { - do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count)); - } else { - async_safe_run_on_cpu(cpu, do_tb_flush, - RUN_ON_CPU_HOST_INT(tb_flush_count)); - } + async_safe_run_on_cpu(cs, do_tb_flush, + RUN_ON_CPU_HOST_INT(tb_flush_count)); } } @@ -836,6 +842,14 @@ static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig) * We first acquired the lock, and since the destination pointer matches, * we know for sure that @orig is in the jmp list. */ + if (dest == orig) { + /* + * In the case of a TB that links to itself, removing the entry + * from the list means that it won't be present later during + * tb_jmp_unlink -- unlink now. + */ + tb_reset_jump(orig, n_orig); + } pprev = &dest->jmp_list_head; TB_FOR_EACH_JMP(dest, tb, n) { if (tb == orig && n == n_orig) { @@ -1154,7 +1168,6 @@ tb_invalidate_phys_page_range__locked(CPUState *cpu, page_collection_unlock(pages); /* Force execution of one insn next time. */ cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu); - mmap_unlock(); cpu_loop_exit_noexc(cpu); } } diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c index 5125e1a4e2..18ea0c58b0 100644 --- a/accel/tcg/tcg-all.c +++ b/accel/tcg/tcg-all.c @@ -38,6 +38,8 @@ #include "qemu/target-info.h" #ifndef CONFIG_USER_ONLY #include "hw/boards.h" +#include "exec/tb-flush.h" +#include "system/runstate.h" #endif #include "accel/accel-ops.h" #include "accel/accel-cpu-ops.h" @@ -82,6 +84,23 @@ static void tcg_accel_instance_init(Object *obj) bool one_insn_per_tb; +#ifndef CONFIG_USER_ONLY +static void tcg_vm_change_state(void *opaque, bool running, RunState state) +{ + if (state == RUN_STATE_RESTORE_VM) { + /* + * loadvm will update the content of RAM, bypassing the usual + * mechanisms that ensure we flush TBs for writes to memory + * we've translated code from, so we must flush all TBs. + * + * vm_stop() has just stopped all cpus, so we are exclusive. + */ + assert(!running); + tb_flush__exclusive_or_serial(); + } +} +#endif + static int tcg_init_machine(AccelState *as, MachineState *ms) { TCGState *s = TCG_STATE(as); @@ -124,6 +143,8 @@ static int tcg_init_machine(AccelState *as, MachineState *ms) default: g_assert_not_reached(); } + + qemu_add_vm_change_state_handler(tcg_vm_change_state, NULL); #endif tcg_allowed = true; diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index d468667b0d..da9d7f1675 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -289,7 +289,11 @@ TranslationBlock *tb_gen_code(CPUState *cpu, TCGTBCPUState s) tb = tcg_tb_alloc(tcg_ctx); if (unlikely(!tb)) { /* flush must be done */ - tb_flush(cpu); + if (cpu_in_serial_context(cpu)) { + tb_flush__exclusive_or_serial(); + goto buffer_overflow; + } + queue_tb_flush(cpu); mmap_unlock(); /* Make the execution loop process the flush as soon as possible. */ cpu->exception_index = EXCP_INTERRUPT; |