diff options
Diffstat (limited to 'accel')
| -rw-r--r-- | accel/tcg/atomic_template.h | 32 | ||||
| -rw-r--r-- | accel/tcg/cpu-exec.c | 95 | ||||
| -rw-r--r-- | accel/tcg/cputlb.c | 1 | ||||
| -rw-r--r-- | accel/tcg/translate-all.c | 52 | ||||
| -rw-r--r-- | accel/tcg/user-exec.c | 58 |
5 files changed, 156 insertions, 82 deletions
diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index b400b2a3d3..1c7c17526c 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -62,7 +62,9 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) { DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; - return atomic_cmpxchg__nocheck(haddr, cmpv, newv); + DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); + ATOMIC_MMU_CLEANUP; + return ret; } #if DATA_SIZE >= 16 @@ -70,6 +72,7 @@ ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) { DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; __atomic_load(haddr, &val, __ATOMIC_RELAXED); + ATOMIC_MMU_CLEANUP; return val; } @@ -78,13 +81,16 @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, { DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; __atomic_store(haddr, &val, __ATOMIC_RELAXED); + ATOMIC_MMU_CLEANUP; } #else ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) { DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; - return atomic_xchg__nocheck(haddr, val); + DATA_TYPE ret = atomic_xchg__nocheck(haddr, val); + ATOMIC_MMU_CLEANUP; + return ret; } #define GEN_ATOMIC_HELPER(X) \ @@ -92,8 +98,10 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE val EXTRA_ARGS) \ { \ DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ - return atomic_##X(haddr, val); \ -} \ + DATA_TYPE ret = atomic_##X(haddr, val); \ + ATOMIC_MMU_CLEANUP; \ + return ret; \ +} GEN_ATOMIC_HELPER(fetch_add) GEN_ATOMIC_HELPER(fetch_and) @@ -123,7 +131,9 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) { DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; - return BSWAP(atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv))); + DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); + ATOMIC_MMU_CLEANUP; + return BSWAP(ret); } #if DATA_SIZE >= 16 @@ -131,6 +141,7 @@ ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) { DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; __atomic_load(haddr, &val, __ATOMIC_RELAXED); + ATOMIC_MMU_CLEANUP; return BSWAP(val); } @@ -140,13 +151,16 @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; val = BSWAP(val); __atomic_store(haddr, &val, __ATOMIC_RELAXED); + ATOMIC_MMU_CLEANUP; } #else ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val EXTRA_ARGS) { DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; - return BSWAP(atomic_xchg__nocheck(haddr, BSWAP(val))); + ABI_TYPE ret = atomic_xchg__nocheck(haddr, BSWAP(val)); + ATOMIC_MMU_CLEANUP; + return BSWAP(ret); } #define GEN_ATOMIC_HELPER(X) \ @@ -154,7 +168,9 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ABI_TYPE val EXTRA_ARGS) \ { \ DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ - return BSWAP(atomic_##X(haddr, BSWAP(val))); \ + DATA_TYPE ret = atomic_##X(haddr, BSWAP(val)); \ + ATOMIC_MMU_CLEANUP; \ + return BSWAP(ret); \ } GEN_ATOMIC_HELPER(fetch_and) @@ -180,6 +196,7 @@ ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr, sto = BSWAP(ret + val); ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto); if (ldn == ldo) { + ATOMIC_MMU_CLEANUP; return ret; } ldo = ldn; @@ -198,6 +215,7 @@ ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr, sto = BSWAP(ret); ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto); if (ldn == ldo) { + ATOMIC_MMU_CLEANUP; return ret; } ldo = ldn; diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index 61297f8f4a..9b544d88c8 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -470,48 +470,51 @@ static inline void cpu_handle_debug_exception(CPUState *cpu) static inline bool cpu_handle_exception(CPUState *cpu, int *ret) { - if (cpu->exception_index >= 0) { - if (cpu->exception_index >= EXCP_INTERRUPT) { - /* exit request from the cpu execution loop */ - *ret = cpu->exception_index; - if (*ret == EXCP_DEBUG) { - cpu_handle_debug_exception(cpu); - } - cpu->exception_index = -1; - return true; - } else { + if (cpu->exception_index < 0) { +#ifndef CONFIG_USER_ONLY + if (replay_has_exception() + && cpu->icount_decr.u16.low + cpu->icount_extra == 0) { + /* try to cause an exception pending in the log */ + cpu_exec_nocache(cpu, 1, tb_find(cpu, NULL, 0, curr_cflags()), true); + } +#endif + if (cpu->exception_index < 0) { + return false; + } + } + + if (cpu->exception_index >= EXCP_INTERRUPT) { + /* exit request from the cpu execution loop */ + *ret = cpu->exception_index; + if (*ret == EXCP_DEBUG) { + cpu_handle_debug_exception(cpu); + } + cpu->exception_index = -1; + return true; + } else { #if defined(CONFIG_USER_ONLY) - /* if user mode only, we simulate a fake exception - which will be handled outside the cpu execution - loop */ + /* if user mode only, we simulate a fake exception + which will be handled outside the cpu execution + loop */ #if defined(TARGET_I386) + CPUClass *cc = CPU_GET_CLASS(cpu); + cc->do_interrupt(cpu); +#endif + *ret = cpu->exception_index; + cpu->exception_index = -1; + return true; +#else + if (replay_exception()) { CPUClass *cc = CPU_GET_CLASS(cpu); + qemu_mutex_lock_iothread(); cc->do_interrupt(cpu); -#endif - *ret = cpu->exception_index; + qemu_mutex_unlock_iothread(); cpu->exception_index = -1; + } else if (!replay_has_interrupt()) { + /* give a chance to iothread in replay mode */ + *ret = EXCP_INTERRUPT; return true; -#else - if (replay_exception()) { - CPUClass *cc = CPU_GET_CLASS(cpu); - qemu_mutex_lock_iothread(); - cc->do_interrupt(cpu); - qemu_mutex_unlock_iothread(); - cpu->exception_index = -1; - } else if (!replay_has_interrupt()) { - /* give a chance to iothread in replay mode */ - *ret = EXCP_INTERRUPT; - return true; - } -#endif } -#ifndef CONFIG_USER_ONLY - } else if (replay_has_exception() - && cpu->icount_decr.u16.low + cpu->icount_extra == 0) { - /* try to cause an exception pending in the log */ - cpu_exec_nocache(cpu, 1, tb_find(cpu, NULL, 0, curr_cflags()), true); - *ret = -1; - return true; #endif } @@ -522,6 +525,19 @@ static inline bool cpu_handle_interrupt(CPUState *cpu, TranslationBlock **last_tb) { CPUClass *cc = CPU_GET_CLASS(cpu); + int32_t insns_left; + + /* Clear the interrupt flag now since we're processing + * cpu->interrupt_request and cpu->exit_request. + */ + insns_left = atomic_read(&cpu->icount_decr.u32); + atomic_set(&cpu->icount_decr.u16.high, 0); + if (unlikely(insns_left < 0)) { + /* Ensure the zeroing of icount_decr comes before the next read + * of cpu->exit_request or cpu->interrupt_request. + */ + smp_mb(); + } if (unlikely(atomic_read(&cpu->interrupt_request))) { int interrupt_request; @@ -618,17 +634,14 @@ static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb, *last_tb = NULL; insns_left = atomic_read(&cpu->icount_decr.u32); - atomic_set(&cpu->icount_decr.u16.high, 0); if (insns_left < 0) { /* Something asked us to stop executing chained TBs; just * continue round the main loop. Whatever requested the exit * will also have set something else (eg exit_request or - * interrupt_request) which we will handle next time around - * the loop. But we need to ensure the zeroing of icount_decr - * comes before the next read of cpu->exit_request - * or cpu->interrupt_request. + * interrupt_request) which will be handled by + * cpu_handle_interrupt. cpu_handle_interrupt will also + * clear cpu->icount_decr.u16.high. */ - smp_mb(); return; } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index a23919c3a8..d071ca4d14 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -1041,6 +1041,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, #define ATOMIC_NAME(X) \ HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu)) #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr) +#define ATOMIC_MMU_CLEANUP do { } while (0) #define DATA_SIZE 1 #include "atomic_template.h" diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 34c5e28d07..e7f0329a52 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -352,36 +352,42 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, return 0; } -bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr) +bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc) { TranslationBlock *tb; bool r = false; + uintptr_t check_offset; - /* A retaddr of zero is invalid so we really shouldn't have ended - * up here. The target code has likely forgotten to check retaddr - * != 0 before attempting to restore state. We return early to - * avoid blowing up on a recursive tb_lock(). The target must have - * previously survived a failed cpu_restore_state because - * tb_find_pc(0) would have failed anyway. It still should be - * fixed though. + /* The host_pc has to be in the region of current code buffer. If + * it is not we will not be able to resolve it here. The two cases + * where host_pc will not be correct are: + * + * - fault during translation (instruction fetch) + * - fault from helper (not using GETPC() macro) + * + * Either way we need return early to avoid blowing up on a + * recursive tb_lock() as we can't resolve it here. + * + * We are using unsigned arithmetic so if host_pc < + * tcg_init_ctx.code_gen_buffer check_offset will wrap to way + * above the code_gen_buffer_size */ - - if (!retaddr) { - return r; - } - - tb_lock(); - tb = tb_find_pc(retaddr); - if (tb) { - cpu_restore_state_from_tb(cpu, tb, retaddr); - if (tb->cflags & CF_NOCACHE) { - /* one-shot translation, invalidate it immediately */ - tb_phys_invalidate(tb, -1); - tb_remove(tb); + check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer; + + if (check_offset < tcg_init_ctx.code_gen_buffer_size) { + tb_lock(); + tb = tb_find_pc(host_pc); + if (tb) { + cpu_restore_state_from_tb(cpu, tb, host_pc); + if (tb->cflags & CF_NOCACHE) { + /* one-shot translation, invalidate it immediately */ + tb_phys_invalidate(tb, -1); + tb_remove(tb); + } + r = true; } - r = true; + tb_unlock(); } - tb_unlock(); return r; } diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 492ea0826c..0324ba8ad1 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -39,6 +39,8 @@ #include <sys/ucontext.h> #endif +__thread uintptr_t helper_retaddr; + //#define DEBUG_SIGNAL /* exit the current TB from a signal handler. The host registers are @@ -62,6 +64,27 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address, CPUClass *cc; int ret; + /* We must handle PC addresses from two different sources: + * a call return address and a signal frame address. + * + * Within cpu_restore_state_from_tb we assume the former and adjust + * the address by -GETPC_ADJ so that the address is within the call + * insn so that addr does not accidentally match the beginning of the + * next guest insn. + * + * However, when the PC comes from the signal frame, it points to + * the actual faulting host insn and not a call insn. Subtracting + * GETPC_ADJ in that case may accidentally match the previous guest insn. + * + * So for the later case, adjust forward to compensate for what + * will be done later by cpu_restore_state_from_tb. + */ + if (helper_retaddr) { + pc = helper_retaddr; + } else { + pc += GETPC_ADJ; + } + /* For synchronous signals we expect to be coming from the vCPU * thread (so current_cpu should be valid) and either from running * code or during translation which can fault as we cross pages. @@ -84,21 +107,24 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address, switch (page_unprotect(h2g(address), pc)) { case 0: /* Fault not caused by a page marked unwritable to protect - * cached translations, must be the guest binary's problem + * cached translations, must be the guest binary's problem. */ break; case 1: /* Fault caused by protection of cached translation; TBs - * invalidated, so resume execution + * invalidated, so resume execution. Retain helper_retaddr + * for a possible second fault. */ return 1; case 2: /* Fault caused by protection of cached translation, and the * currently executing TB was modified and must be exited - * immediately. + * immediately. Clear helper_retaddr for next execution. */ + helper_retaddr = 0; cpu_exit_tb_from_sighandler(cpu, old_set); - g_assert_not_reached(); + /* NORETURN */ + default: g_assert_not_reached(); } @@ -112,17 +138,25 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address, /* see if it is an MMU fault */ g_assert(cc->handle_mmu_fault); ret = cc->handle_mmu_fault(cpu, address, is_write, MMU_USER_IDX); + + if (ret == 0) { + /* The MMU fault was handled without causing real CPU fault. + * Retain helper_retaddr for a possible second fault. + */ + return 1; + } + + /* All other paths lead to cpu_exit; clear helper_retaddr + * for next execution. + */ + helper_retaddr = 0; + if (ret < 0) { return 0; /* not an MMU fault */ } - if (ret == 0) { - return 1; /* the MMU fault was handled without causing real CPU fault */ - } - /* Now we have a real cpu fault. Since this is the exact location of - * the exception, we must undo the adjustment done by cpu_restore_state - * for handling call return addresses. */ - cpu_restore_state(cpu, pc + GETPC_ADJ); + /* Now we have a real cpu fault. */ + cpu_restore_state(cpu, pc); sigprocmask(SIG_SETMASK, old_set, NULL); cpu_loop_exit(cpu); @@ -585,11 +619,13 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, if (unlikely(addr & (size - 1))) { cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr); } + helper_retaddr = retaddr; return g2h(addr); } /* Macro to call the above, with local variables from the use context. */ #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC()) +#define ATOMIC_MMU_CLEANUP do { helper_retaddr = 0; } while (0) #define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END)) #define EXTRA_ARGS |