diff options
Diffstat (limited to 'include/exec')
| -rw-r--r-- | include/exec/cpu-defs.h | 35 | ||||
| -rw-r--r-- | include/exec/cpu_ldst.h | 104 | ||||
| -rw-r--r-- | include/exec/cputlb.h | 3 | ||||
| -rw-r--r-- | include/exec/exec-all.h | 6 | ||||
| -rw-r--r-- | include/exec/memattrs.h | 4 | ||||
| -rw-r--r-- | include/exec/memory.h | 25 | ||||
| -rw-r--r-- | include/exec/ram_addr.h | 138 |
7 files changed, 247 insertions, 68 deletions
diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index 3f56546066..d5aecaf49e 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -27,6 +27,7 @@ #include <inttypes.h> #include "qemu/osdep.h" #include "qemu/queue.h" +#include "tcg-target.h" #ifndef CONFIG_USER_ONLY #include "exec/hwaddr.h" #endif @@ -70,8 +71,6 @@ typedef uint64_t target_ulong; #define TB_JMP_PAGE_MASK (TB_JMP_CACHE_SIZE - TB_JMP_PAGE_SIZE) #if !defined(CONFIG_USER_ONLY) -#define CPU_TLB_BITS 8 -#define CPU_TLB_SIZE (1 << CPU_TLB_BITS) /* use a fully associative victim tlb of 8 entries */ #define CPU_VTLB_SIZE 8 @@ -81,6 +80,38 @@ typedef uint64_t target_ulong; #define CPU_TLB_ENTRY_BITS 5 #endif +/* TCG_TARGET_TLB_DISPLACEMENT_BITS is used in CPU_TLB_BITS to ensure that + * the TLB is not unnecessarily small, but still small enough for the + * TLB lookup instruction sequence used by the TCG target. + * + * TCG will have to generate an operand as large as the distance between + * env and the tlb_table[NB_MMU_MODES - 1][0].addend. For simplicity, + * the TCG targets just round everything up to the next power of two, and + * count bits. This works because: 1) the size of each TLB is a largish + * power of two, 2) and because the limit of the displacement is really close + * to a power of two, 3) the offset of tlb_table[0][0] inside env is smaller + * than the size of a TLB. + * + * For example, the maximum displacement 0xFFF0 on PPC and MIPS, but TCG + * just says "the displacement is 16 bits". TCG_TARGET_TLB_DISPLACEMENT_BITS + * then ensures that tlb_table at least 0x8000 bytes large ("not unnecessarily + * small": 2^15). The operand then will come up smaller than 0xFFF0 without + * any particular care, because the TLB for a single MMU mode is larger than + * 0x10000-0xFFF0=16 bytes. In the end, the maximum value of the operand + * could be something like 0xC000 (the offset of the last TLB table) plus + * 0x18 (the offset of the addend field in each TLB entry) plus the offset + * of tlb_table inside env (which is non-trivial but not huge). + */ +#define CPU_TLB_BITS \ + MIN(8, \ + TCG_TARGET_TLB_DISPLACEMENT_BITS - CPU_TLB_ENTRY_BITS - \ + (NB_MMU_MODES <= 1 ? 0 : \ + NB_MMU_MODES <= 2 ? 1 : \ + NB_MMU_MODES <= 4 ? 2 : \ + NB_MMU_MODES <= 8 ? 3 : 4)) + +#define CPU_TLB_SIZE (1 << CPU_TLB_BITS) + typedef struct CPUTLBEntry { /* bit TARGET_LONG_BITS to TARGET_PAGE_BITS : virtual address bit TARGET_PAGE_BITS-1..4 : Nonzero for accesses that should not diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index 1673287189..0ec398c0f8 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -263,12 +263,104 @@ uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx); #undef MEMSUFFIX #endif /* (NB_MMU_MODES >= 7) */ -#if (NB_MMU_MODES > 7) -/* Note that supporting NB_MMU_MODES == 9 would require - * changes to at least the ARM TCG backend. - */ -#error "NB_MMU_MODES > 7 is not supported for now" -#endif /* (NB_MMU_MODES > 7) */ +#if (NB_MMU_MODES >= 8) && defined(MMU_MODE7_SUFFIX) + +#define CPU_MMU_INDEX 7 +#define MEMSUFFIX MMU_MODE7_SUFFIX +#define DATA_SIZE 1 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 2 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 4 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 8 +#include "exec/cpu_ldst_template.h" +#undef CPU_MMU_INDEX +#undef MEMSUFFIX +#endif /* (NB_MMU_MODES >= 8) */ + +#if (NB_MMU_MODES >= 9) && defined(MMU_MODE8_SUFFIX) + +#define CPU_MMU_INDEX 8 +#define MEMSUFFIX MMU_MODE8_SUFFIX +#define DATA_SIZE 1 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 2 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 4 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 8 +#include "exec/cpu_ldst_template.h" +#undef CPU_MMU_INDEX +#undef MEMSUFFIX +#endif /* (NB_MMU_MODES >= 9) */ + +#if (NB_MMU_MODES >= 10) && defined(MMU_MODE9_SUFFIX) + +#define CPU_MMU_INDEX 9 +#define MEMSUFFIX MMU_MODE9_SUFFIX +#define DATA_SIZE 1 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 2 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 4 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 8 +#include "exec/cpu_ldst_template.h" +#undef CPU_MMU_INDEX +#undef MEMSUFFIX +#endif /* (NB_MMU_MODES >= 10) */ + +#if (NB_MMU_MODES >= 11) && defined(MMU_MODE10_SUFFIX) + +#define CPU_MMU_INDEX 10 +#define MEMSUFFIX MMU_MODE10_SUFFIX +#define DATA_SIZE 1 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 2 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 4 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 8 +#include "exec/cpu_ldst_template.h" +#undef CPU_MMU_INDEX +#undef MEMSUFFIX +#endif /* (NB_MMU_MODES >= 11) */ + +#if (NB_MMU_MODES >= 12) && defined(MMU_MODE11_SUFFIX) + +#define CPU_MMU_INDEX 11 +#define MEMSUFFIX MMU_MODE11_SUFFIX +#define DATA_SIZE 1 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 2 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 4 +#include "exec/cpu_ldst_template.h" + +#define DATA_SIZE 8 +#include "exec/cpu_ldst_template.h" +#undef CPU_MMU_INDEX +#undef MEMSUFFIX +#endif /* (NB_MMU_MODES >= 12) */ + +#if (NB_MMU_MODES > 12) +#error "NB_MMU_MODES > 12 is not supported for now" +#endif /* (NB_MMU_MODES > 12) */ /* these access are slower, they must be as rare as possible */ #define CPU_MMU_INDEX (cpu_mmu_index(env)) diff --git a/include/exec/cputlb.h b/include/exec/cputlb.h index e0da9d7ad3..360815e1b4 100644 --- a/include/exec/cputlb.h +++ b/include/exec/cputlb.h @@ -22,8 +22,7 @@ #if !defined(CONFIG_USER_ONLY) /* cputlb.c */ void tlb_protect_code(ram_addr_t ram_addr); -void tlb_unprotect_code_phys(CPUState *cpu, ram_addr_t ram_addr, - target_ulong vaddr); +void tlb_unprotect_code(ram_addr_t ram_addr); void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start, uintptr_t length); void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length); diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index b58cd47ced..2f7a4f1700 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -90,11 +90,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, int cflags); void cpu_exec_init(CPUArchState *env); void QEMU_NORETURN cpu_loop_exit(CPUState *cpu); -int page_unprotect(target_ulong address, uintptr_t pc, void *puc); -void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, - int is_cpu_write_access); -void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end, - int is_cpu_write_access); + #if !defined(CONFIG_USER_ONLY) bool qemu_in_vcpu_thread(void); void cpu_reload_memory_map(CPUState *cpu); diff --git a/include/exec/memattrs.h b/include/exec/memattrs.h index 96dc440423..f8537a8d91 100644 --- a/include/exec/memattrs.h +++ b/include/exec/memattrs.h @@ -29,7 +29,9 @@ typedef struct MemTxAttrs { * "didn't specify" if necessary. */ unsigned int unspecified:1; - /* ARM/AMBA TrustZone Secure access */ + /* ARM/AMBA: TrustZone Secure access + * x86: System Management Mode access + */ unsigned int secure:1; /* Memory access is usermode (unprivileged) */ unsigned int user:1; diff --git a/include/exec/memory.h b/include/exec/memory.h index b61c84f62a..8ae004eb06 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -206,8 +206,10 @@ struct MemoryListener { void (*region_add)(MemoryListener *listener, MemoryRegionSection *section); void (*region_del)(MemoryListener *listener, MemoryRegionSection *section); void (*region_nop)(MemoryListener *listener, MemoryRegionSection *section); - void (*log_start)(MemoryListener *listener, MemoryRegionSection *section); - void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section); + void (*log_start)(MemoryListener *listener, MemoryRegionSection *section, + int old, int new); + void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section, + int old, int new); void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section); void (*log_global_start)(MemoryListener *listener); void (*log_global_stop)(MemoryListener *listener); @@ -591,11 +593,23 @@ const char *memory_region_name(const MemoryRegion *mr); /** * memory_region_is_logging: return whether a memory region is logging writes * - * Returns %true if the memory region is logging writes + * Returns %true if the memory region is logging writes for the given client * * @mr: the memory region being queried + * @client: the client being queried */ -bool memory_region_is_logging(MemoryRegion *mr); +bool memory_region_is_logging(MemoryRegion *mr, uint8_t client); + +/** + * memory_region_get_dirty_log_mask: return the clients for which a + * memory region is logging writes. + * + * Returns a bitmap of clients, in which the DIRTY_MEMORY_* constants + * are the bit indices. + * + * @mr: the memory region being queried + */ +uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr); /** * memory_region_is_rom: check whether a memory region is ROM @@ -647,8 +661,7 @@ void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, * * @mr: the memory region being updated. * @log: whether dirty logging is to be enabled or disabled. - * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or - * %DIRTY_MEMORY_VGA. + * @client: the user of the logging information; %DIRTY_MEMORY_VGA only. */ void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client); diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index ff558a4734..c113f21140 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -41,6 +41,9 @@ void qemu_ram_free_from_ptr(ram_addr_t addr); int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp); +#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) +#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) + static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, ram_addr_t length, unsigned client) @@ -56,7 +59,7 @@ static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, return next < end; } -static inline bool cpu_physical_memory_get_clean(ram_addr_t start, +static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, ram_addr_t length, unsigned client) { @@ -68,7 +71,7 @@ static inline bool cpu_physical_memory_get_clean(ram_addr_t start, page = start >> TARGET_PAGE_BITS; next = find_next_zero_bit(ram_list.dirty_memory[client], end, page); - return next < end; + return next >= end; } static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, @@ -86,44 +89,52 @@ static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) return !(vga && code && migration); } -static inline bool cpu_physical_memory_range_includes_clean(ram_addr_t start, - ram_addr_t length) +static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, + ram_addr_t length, + uint8_t mask) { - bool vga = cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_VGA); - bool code = cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_CODE); - bool migration = - cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_MIGRATION); - return vga || code || migration; + uint8_t ret = 0; + + if (mask & (1 << DIRTY_MEMORY_VGA) && + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { + ret |= (1 << DIRTY_MEMORY_VGA); + } + if (mask & (1 << DIRTY_MEMORY_CODE) && + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { + ret |= (1 << DIRTY_MEMORY_CODE); + } + if (mask & (1 << DIRTY_MEMORY_MIGRATION) && + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { + ret |= (1 << DIRTY_MEMORY_MIGRATION); + } + return ret; } static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) { assert(client < DIRTY_MEMORY_NUM); - set_bit(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]); -} - -static inline void cpu_physical_memory_set_dirty_range_nocode(ram_addr_t start, - ram_addr_t length) -{ - unsigned long end, page; - - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); + set_bit_atomic(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]); } static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, - ram_addr_t length) + ram_addr_t length, + uint8_t mask) { unsigned long end, page; + unsigned long **d = ram_list.dirty_memory; end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; page = start >> TARGET_PAGE_BITS; - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_CODE], page, end - page); + if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { + bitmap_set_atomic(d[DIRTY_MEMORY_MIGRATION], page, end - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { + bitmap_set_atomic(d[DIRTY_MEMORY_VGA], page, end - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { + bitmap_set_atomic(d[DIRTY_MEMORY_CODE], page, end - page); + } xen_modified_memory(start, length); } @@ -149,14 +160,18 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, for (k = 0; k < nr; k++) { if (bitmap[k]) { unsigned long temp = leul_to_cpu(bitmap[k]); + unsigned long **d = ram_list.dirty_memory; - ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION][page + k] |= temp; - ram_list.dirty_memory[DIRTY_MEMORY_VGA][page + k] |= temp; - ram_list.dirty_memory[DIRTY_MEMORY_CODE][page + k] |= temp; + atomic_or(&d[DIRTY_MEMORY_MIGRATION][page + k], temp); + atomic_or(&d[DIRTY_MEMORY_VGA][page + k], temp); + if (tcg_enabled()) { + atomic_or(&d[DIRTY_MEMORY_CODE][page + k], temp); + } } } - xen_modified_memory(start, pages); + xen_modified_memory(start, pages << TARGET_PAGE_BITS); } else { + uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; /* * bitmap-traveling is faster than memory-traveling (for addr...) * especially when most of the memory is not dirty. @@ -171,7 +186,7 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, addr = page_number * TARGET_PAGE_SIZE; ram_addr = start + addr; cpu_physical_memory_set_dirty_range(ram_addr, - TARGET_PAGE_SIZE * hpratio); + TARGET_PAGE_SIZE * hpratio, clients); } while (c != 0); } } @@ -179,29 +194,60 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, } #endif /* not _WIN32 */ -static inline void cpu_physical_memory_clear_dirty_range_type(ram_addr_t start, - ram_addr_t length, - unsigned client) -{ - unsigned long end, page; - - assert(client < DIRTY_MEMORY_NUM); - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - bitmap_clear(ram_list.dirty_memory[client], page, end - page); -} +bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, + ram_addr_t length, + unsigned client); static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, ram_addr_t length) { - cpu_physical_memory_clear_dirty_range_type(start, length, DIRTY_MEMORY_MIGRATION); - cpu_physical_memory_clear_dirty_range_type(start, length, DIRTY_MEMORY_VGA); - cpu_physical_memory_clear_dirty_range_type(start, length, DIRTY_MEMORY_CODE); + cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); + cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); + cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); } -void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length, - unsigned client); +static inline +uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, + ram_addr_t start, + ram_addr_t length) +{ + ram_addr_t addr; + unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); + uint64_t num_dirty = 0; + + /* start address is aligned at the start of a word? */ + if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) { + int k; + int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); + unsigned long *src = ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]; + + for (k = page; k < page + nr; k++) { + if (src[k]) { + unsigned long bits = atomic_xchg(&src[k], 0); + unsigned long new_dirty; + new_dirty = ~dest[k]; + dest[k] |= bits; + new_dirty &= bits; + num_dirty += ctpopl(new_dirty); + } + } + } else { + for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { + if (cpu_physical_memory_test_and_clear_dirty( + start + addr, + TARGET_PAGE_SIZE, + DIRTY_MEMORY_MIGRATION)) { + long k = (start + addr) >> TARGET_PAGE_BITS; + if (!test_and_set_bit(k, dest)) { + num_dirty++; + } + } + } + } + + return num_dirty; +} #endif #endif |