diff options
Diffstat (limited to 'include')
39 files changed, 718 insertions, 576 deletions
diff --git a/include/hw/core/accel-cpu.h b/include/accel/accel-cpu-target.h index 24dad45ab9..37dde7fae3 100644 --- a/include/hw/core/accel-cpu.h +++ b/include/accel/accel-cpu-target.h @@ -8,8 +8,8 @@ * See the COPYING file in the top-level directory. */ -#ifndef ACCEL_CPU_H -#define ACCEL_CPU_H +#ifndef ACCEL_CPU_TARGET_H +#define ACCEL_CPU_TARGET_H /* * This header is used to define new accelerator-specific target-specific @@ -20,6 +20,9 @@ * subclasses in target/, or the accel implementation itself in accel/ */ +#include "qom/object.h" +#include "cpu.h" + #define TYPE_ACCEL_CPU "accel-" CPU_RESOLVING_TYPE #define ACCEL_CPU_NAME(name) (name "-" TYPE_ACCEL_CPU) typedef struct AccelCPUClass AccelCPUClass; diff --git a/include/hw/core/tcg-cpu-ops.h b/include/accel/tcg/cpu-ops.h index 2e3f1690f1..2e3f1690f1 100644 --- a/include/hw/core/tcg-cpu-ops.h +++ b/include/accel/tcg/cpu-ops.h diff --git a/include/accel/tcg/getpc.h b/include/accel/tcg/getpc.h new file mode 100644 index 0000000000..8a97ce34e7 --- /dev/null +++ b/include/accel/tcg/getpc.h @@ -0,0 +1,24 @@ +/* + * Get host pc for helper unwinding. + * + * Copyright (c) 2003 Fabrice Bellard + * SPDX-License-Identifier: LGPL-2.1-or-later + */ + +#ifndef ACCEL_TCG_GETPC_H +#define ACCEL_TCG_GETPC_H + +#ifndef CONFIG_TCG +#error Can only include this header with TCG +#endif + +/* GETPC is the true target of the return instruction that we'll execute. */ +#ifdef CONFIG_TCG_INTERPRETER +extern __thread uintptr_t tci_tb_ptr; +# define GETPC() tci_tb_ptr +#else +# define GETPC() \ + ((uintptr_t)__builtin_extract_return_addr(__builtin_return_address(0))) +#endif + +#endif /* ACCEL_TCG_GETPC_H */ diff --git a/include/block/aio.h b/include/block/aio.h index 43883a8a33..b2ab3514de 100644 --- a/include/block/aio.h +++ b/include/block/aio.h @@ -54,7 +54,7 @@ typedef void QEMUBHFunc(void *opaque); typedef bool AioPollFn(void *opaque); typedef void IOHandler(void *opaque); -struct ThreadPool; +struct ThreadPoolAio; struct LinuxAioState; typedef struct LuringState LuringState; @@ -207,7 +207,7 @@ struct AioContext { /* Thread pool for performing work and receiving completion callbacks. * Has its own locking. */ - struct ThreadPool *thread_pool; + struct ThreadPoolAio *thread_pool; #ifdef CONFIG_LINUX_AIO struct LinuxAioState *linux_aio; @@ -500,8 +500,8 @@ void aio_set_event_notifier_poll(AioContext *ctx, */ GSource *aio_get_g_source(AioContext *ctx); -/* Return the ThreadPool bound to this AioContext */ -struct ThreadPool *aio_get_thread_pool(AioContext *ctx); +/* Return the ThreadPoolAio bound to this AioContext */ +struct ThreadPoolAio *aio_get_thread_pool(AioContext *ctx); /* Setup the LinuxAioState bound to this AioContext */ struct LinuxAioState *aio_setup_linux_aio(AioContext *ctx, Error **errp); diff --git a/include/block/block_int-common.h b/include/block/block_int-common.h index bb91a0f62f..ebb4e56a50 100644 --- a/include/block/block_int-common.h +++ b/include/block/block_int-common.h @@ -28,7 +28,6 @@ #include "block/block-common.h" #include "block/block-global-state.h" #include "block/snapshot.h" -#include "qemu/clang-tsa.h" #include "qemu/iov.h" #include "qemu/rcu.h" #include "qemu/stats64.h" diff --git a/include/block/graph-lock.h b/include/block/graph-lock.h index dc8d949184..2c26c72108 100644 --- a/include/block/graph-lock.h +++ b/include/block/graph-lock.h @@ -20,8 +20,6 @@ #ifndef GRAPH_LOCK_H #define GRAPH_LOCK_H -#include "qemu/clang-tsa.h" - /** * Graph Lock API * This API provides a rwlock used to protect block layer diff --git a/include/block/thread-pool.h b/include/block/thread-pool.h index 948ff5f30c..dd48cf07e8 100644 --- a/include/block/thread-pool.h +++ b/include/block/thread-pool.h @@ -24,20 +24,70 @@ typedef int ThreadPoolFunc(void *opaque); -typedef struct ThreadPool ThreadPool; +typedef struct ThreadPoolAio ThreadPoolAio; -ThreadPool *thread_pool_new(struct AioContext *ctx); -void thread_pool_free(ThreadPool *pool); +ThreadPoolAio *thread_pool_new_aio(struct AioContext *ctx); +void thread_pool_free_aio(ThreadPoolAio *pool); /* - * thread_pool_submit* API: submit I/O requests in the thread's + * thread_pool_submit_{aio,co} API: submit I/O requests in the thread's * current AioContext. */ BlockAIOCB *thread_pool_submit_aio(ThreadPoolFunc *func, void *arg, BlockCompletionFunc *cb, void *opaque); int coroutine_fn thread_pool_submit_co(ThreadPoolFunc *func, void *arg); -void thread_pool_submit(ThreadPoolFunc *func, void *arg); +void thread_pool_update_params(ThreadPoolAio *pool, struct AioContext *ctx); + +/* ------------------------------------------- */ +/* Generic thread pool types and methods below */ +typedef struct ThreadPool ThreadPool; + +/* Create a new thread pool. Never returns NULL. */ +ThreadPool *thread_pool_new(void); + +/* + * Free the thread pool. + * Waits for all the previously submitted work to complete before performing + * the actual freeing operation. + */ +void thread_pool_free(ThreadPool *pool); + +/* + * Submit a new work (task) for the pool. + * + * @opaque_destroy is an optional GDestroyNotify for the @opaque argument + * to the work function at @func. + */ +void thread_pool_submit(ThreadPool *pool, ThreadPoolFunc *func, + void *opaque, GDestroyNotify opaque_destroy); + +/* + * Submit a new work (task) for the pool, making sure it starts getting + * processed immediately, launching a new thread for it if necessary. + * + * @opaque_destroy is an optional GDestroyNotify for the @opaque argument + * to the work function at @func. + */ +void thread_pool_submit_immediate(ThreadPool *pool, ThreadPoolFunc *func, + void *opaque, GDestroyNotify opaque_destroy); + +/* + * Wait for all previously submitted work to complete before returning. + * + * Can be used as a barrier between two sets of tasks executed on a thread + * pool without destroying it or in a performance sensitive path where the + * caller just wants to wait for all tasks to complete while deferring the + * pool free operation for later, less performance sensitive time. + */ +void thread_pool_wait(ThreadPool *pool); -void thread_pool_update_params(ThreadPool *pool, struct AioContext *ctx); +/* Set the maximum number of threads in the pool. */ +bool thread_pool_set_max_threads(ThreadPool *pool, int max_threads); + +/* + * Adjust the maximum number of threads in the pool to give each task its + * own thread (exactly one thread per task). + */ +bool thread_pool_adjust_max_threads_to_work(ThreadPool *pool); #endif diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index 09f537d06f..8cd6c00cf8 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -21,6 +21,7 @@ #include "exec/page-protection.h" #include "exec/cpu-common.h" +#include "exec/cpu-interrupt.h" #include "exec/memory.h" #include "exec/tswap.h" #include "hw/core/cpu.h" @@ -105,81 +106,10 @@ static inline void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val /* page related stuff */ #include "exec/cpu-defs.h" -#ifdef TARGET_PAGE_BITS_VARY -# include "exec/page-vary.h" -extern const TargetPageBits target_page; -# ifdef CONFIG_DEBUG_TCG -# define TARGET_PAGE_BITS ({ assert(target_page.decided); \ - target_page.bits; }) -# define TARGET_PAGE_MASK ({ assert(target_page.decided); \ - (target_long)target_page.mask; }) -# else -# define TARGET_PAGE_BITS target_page.bits -# define TARGET_PAGE_MASK ((target_long)target_page.mask) -# endif -# define TARGET_PAGE_SIZE (-(int)TARGET_PAGE_MASK) -#else -# define TARGET_PAGE_BITS_MIN TARGET_PAGE_BITS -# define TARGET_PAGE_SIZE (1 << TARGET_PAGE_BITS) -# define TARGET_PAGE_MASK ((target_long)-1 << TARGET_PAGE_BITS) -#endif - -#define TARGET_PAGE_ALIGN(addr) ROUND_UP((addr), TARGET_PAGE_SIZE) +#include "exec/target_page.h" CPUArchState *cpu_copy(CPUArchState *env); -/* Flags for use in ENV->INTERRUPT_PENDING. - - The numbers assigned here are non-sequential in order to preserve - binary compatibility with the vmstate dump. Bit 0 (0x0001) was - previously used for CPU_INTERRUPT_EXIT, and is cleared when loading - the vmstate dump. */ - -/* External hardware interrupt pending. This is typically used for - interrupts from devices. */ -#define CPU_INTERRUPT_HARD 0x0002 - -/* Exit the current TB. This is typically used when some system-level device - makes some change to the memory mapping. E.g. the a20 line change. */ -#define CPU_INTERRUPT_EXITTB 0x0004 - -/* Halt the CPU. */ -#define CPU_INTERRUPT_HALT 0x0020 - -/* Debug event pending. */ -#define CPU_INTERRUPT_DEBUG 0x0080 - -/* Reset signal. */ -#define CPU_INTERRUPT_RESET 0x0400 - -/* Several target-specific external hardware interrupts. Each target/cpu.h - should define proper names based on these defines. */ -#define CPU_INTERRUPT_TGT_EXT_0 0x0008 -#define CPU_INTERRUPT_TGT_EXT_1 0x0010 -#define CPU_INTERRUPT_TGT_EXT_2 0x0040 -#define CPU_INTERRUPT_TGT_EXT_3 0x0200 -#define CPU_INTERRUPT_TGT_EXT_4 0x1000 - -/* Several target-specific internal interrupts. These differ from the - preceding target-specific interrupts in that they are intended to - originate from within the cpu itself, typically in response to some - instruction being executed. These, therefore, are not masked while - single-stepping within the debugger. */ -#define CPU_INTERRUPT_TGT_INT_0 0x0100 -#define CPU_INTERRUPT_TGT_INT_1 0x0800 -#define CPU_INTERRUPT_TGT_INT_2 0x2000 - -/* First unused bit: 0x4000. */ - -/* The set of all bits that should be masked when single-stepping. */ -#define CPU_INTERRUPT_SSTEP_MASK \ - (CPU_INTERRUPT_HARD \ - | CPU_INTERRUPT_TGT_EXT_0 \ - | CPU_INTERRUPT_TGT_EXT_1 \ - | CPU_INTERRUPT_TGT_EXT_2 \ - | CPU_INTERRUPT_TGT_EXT_3 \ - | CPU_INTERRUPT_TGT_EXT_4) - #include "cpu.h" #ifdef CONFIG_USER_ONLY @@ -249,29 +179,6 @@ static inline int cpu_mmu_index(CPUState *cs, bool ifetch) /* The two sets of flags must not overlap. */ QEMU_BUILD_BUG_ON(TLB_FLAGS_MASK & TLB_SLOW_FLAGS_MASK); -/** - * tlb_hit_page: return true if page aligned @addr is a hit against the - * TLB entry @tlb_addr - * - * @addr: virtual address to test (must be page aligned) - * @tlb_addr: TLB entry address (a CPUTLBEntry addr_read/write/code value) - */ -static inline bool tlb_hit_page(uint64_t tlb_addr, vaddr addr) -{ - return addr == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK)); -} - -/** - * tlb_hit: return true if @addr is a hit against the TLB entry @tlb_addr - * - * @addr: virtual address to test (need not be page aligned) - * @tlb_addr: TLB entry address (a CPUTLBEntry addr_read/write/code value) - */ -static inline bool tlb_hit(uint64_t tlb_addr, vaddr addr) -{ - return tlb_hit_page(tlb_addr, addr & TARGET_PAGE_MASK); -} - #endif /* !CONFIG_USER_ONLY */ /* Validate correct placement of CPUArchState. */ diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index ae18398fa9..9f955f53fd 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -46,30 +46,4 @@ #include "exec/target_long.h" -#if defined(CONFIG_SOFTMMU) && defined(CONFIG_TCG) -#define CPU_TLB_DYN_MIN_BITS 6 -#define CPU_TLB_DYN_DEFAULT_BITS 8 - -# if HOST_LONG_BITS == 32 -/* Make sure we do not require a double-word shift for the TLB load */ -# define CPU_TLB_DYN_MAX_BITS (32 - TARGET_PAGE_BITS) -# else /* HOST_LONG_BITS == 64 */ -/* - * Assuming TARGET_PAGE_BITS==12, with 2**22 entries we can cover 2**(22+12) == - * 2**34 == 16G of address space. This is roughly what one would expect a - * TLB to cover in a modern (as of 2018) x86_64 CPU. For instance, Intel - * Skylake's Level-2 STLB has 16 1G entries. - * Also, make sure we do not size the TLB past the guest's address space. - */ -# ifdef TARGET_PAGE_BITS_VARY -# define CPU_TLB_DYN_MAX_BITS \ - MIN(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS) -# else -# define CPU_TLB_DYN_MAX_BITS \ - MIN_CONST(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS) -# endif -# endif - -#endif /* CONFIG_SOFTMMU && CONFIG_TCG */ - #endif diff --git a/include/exec/cpu-interrupt.h b/include/exec/cpu-interrupt.h new file mode 100644 index 0000000000..40715193ca --- /dev/null +++ b/include/exec/cpu-interrupt.h @@ -0,0 +1,70 @@ +/* + * Flags for use with cpu_interrupt() + * + * Copyright (c) 2003 Fabrice Bellard + * SPDX-License-Identifier: LGPL-2.1-or-later + */ + +#ifndef CPU_INTERRUPT_H +#define CPU_INTERRUPT_H + +/* + * The numbers assigned here are non-sequential in order to preserve binary + * compatibility with the vmstate dump. Bit 0 (0x0001) was previously used + * for CPU_INTERRUPT_EXIT, and is cleared when loading the vmstate dump. + */ + +/* + * External hardware interrupt pending. + * This is typically used for interrupts from devices. + */ +#define CPU_INTERRUPT_HARD 0x0002 + +/* + * Exit the current TB. This is typically used when some system-level device + * makes some change to the memory mapping. E.g. the a20 line change. + */ +#define CPU_INTERRUPT_EXITTB 0x0004 + +/* Halt the CPU. */ +#define CPU_INTERRUPT_HALT 0x0020 + +/* Debug event pending. */ +#define CPU_INTERRUPT_DEBUG 0x0080 + +/* Reset signal. */ +#define CPU_INTERRUPT_RESET 0x0400 + +/* + * Several target-specific external hardware interrupts. Each target/cpu.h + * should define proper names based on these defines. + */ +#define CPU_INTERRUPT_TGT_EXT_0 0x0008 +#define CPU_INTERRUPT_TGT_EXT_1 0x0010 +#define CPU_INTERRUPT_TGT_EXT_2 0x0040 +#define CPU_INTERRUPT_TGT_EXT_3 0x0200 +#define CPU_INTERRUPT_TGT_EXT_4 0x1000 + +/* + * Several target-specific internal interrupts. These differ from the + * preceding target-specific interrupts in that they are intended to + * originate from within the cpu itself, typically in response to some + * instruction being executed. These, therefore, are not masked while + * single-stepping within the debugger. + */ +#define CPU_INTERRUPT_TGT_INT_0 0x0100 +#define CPU_INTERRUPT_TGT_INT_1 0x0800 +#define CPU_INTERRUPT_TGT_INT_2 0x2000 + +/* First unused bit: 0x4000. */ + +/* The set of all bits that should be masked when single-stepping. */ +#define CPU_INTERRUPT_SSTEP_MASK \ + (CPU_INTERRUPT_HARD \ + | CPU_INTERRUPT_TGT_EXT_0 \ + | CPU_INTERRUPT_TGT_EXT_1 \ + | CPU_INTERRUPT_TGT_EXT_2 \ + | CPU_INTERRUPT_TGT_EXT_3 \ + | CPU_INTERRUPT_TGT_EXT_4) + +#endif /* CPU_INTERRUPT_H */ diff --git a/include/exec/cputlb.h b/include/exec/cputlb.h index ef18642a32..8125f6809c 100644 --- a/include/exec/cputlb.h +++ b/include/exec/cputlb.h @@ -21,15 +21,266 @@ #define CPUTLB_H #include "exec/cpu-common.h" +#include "exec/hwaddr.h" +#include "exec/memattrs.h" +#include "exec/vaddr.h" -#ifdef CONFIG_TCG - -#if !defined(CONFIG_USER_ONLY) -/* cputlb.c */ +#if defined(CONFIG_TCG) && !defined(CONFIG_USER_ONLY) void tlb_protect_code(ram_addr_t ram_addr); void tlb_unprotect_code(ram_addr_t ram_addr); #endif -#endif /* CONFIG_TCG */ - +#ifndef CONFIG_USER_ONLY +void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length); +void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length); #endif + +/** + * tlb_set_page_full: + * @cpu: CPU context + * @mmu_idx: mmu index of the tlb to modify + * @addr: virtual address of the entry to add + * @full: the details of the tlb entry + * + * Add an entry to @cpu tlb index @mmu_idx. All of the fields of + * @full must be filled, except for xlat_section, and constitute + * the complete description of the translated page. + * + * This is generally called by the target tlb_fill function after + * having performed a successful page table walk to find the physical + * address and attributes for the translation. + * + * At most one entry for a given virtual address is permitted. Only a + * single TARGET_PAGE_SIZE region is mapped; @full->lg_page_size is only + * used by tlb_flush_page. + */ +void tlb_set_page_full(CPUState *cpu, int mmu_idx, vaddr addr, + CPUTLBEntryFull *full); + +/** + * tlb_set_page_with_attrs: + * @cpu: CPU to add this TLB entry for + * @addr: virtual address of page to add entry for + * @paddr: physical address of the page + * @attrs: memory transaction attributes + * @prot: access permissions (PAGE_READ/PAGE_WRITE/PAGE_EXEC bits) + * @mmu_idx: MMU index to insert TLB entry for + * @size: size of the page in bytes + * + * Add an entry to this CPU's TLB (a mapping from virtual address + * @addr to physical address @paddr) with the specified memory + * transaction attributes. This is generally called by the target CPU + * specific code after it has been called through the tlb_fill() + * entry point and performed a successful page table walk to find + * the physical address and attributes for the virtual address + * which provoked the TLB miss. + * + * At most one entry for a given virtual address is permitted. Only a + * single TARGET_PAGE_SIZE region is mapped; the supplied @size is only + * used by tlb_flush_page. + */ +void tlb_set_page_with_attrs(CPUState *cpu, vaddr addr, + hwaddr paddr, MemTxAttrs attrs, + int prot, int mmu_idx, vaddr size); + +/** + * tlb_set_page: + * + * This function is equivalent to calling tlb_set_page_with_attrs() + * with an @attrs argument of MEMTXATTRS_UNSPECIFIED. It's provided + * as a convenience for CPUs which don't use memory transaction attributes. + */ +void tlb_set_page(CPUState *cpu, vaddr addr, + hwaddr paddr, int prot, + int mmu_idx, vaddr size); + +#if defined(CONFIG_TCG) && !defined(CONFIG_USER_ONLY) +/** + * tlb_flush_page: + * @cpu: CPU whose TLB should be flushed + * @addr: virtual address of page to be flushed + * + * Flush one page from the TLB of the specified CPU, for all + * MMU indexes. + */ +void tlb_flush_page(CPUState *cpu, vaddr addr); + +/** + * tlb_flush_page_all_cpus_synced: + * @cpu: src CPU of the flush + * @addr: virtual address of page to be flushed + * + * Flush one page from the TLB of all CPUs, for all + * MMU indexes. + * + * When this function returns, no CPUs will subsequently perform + * translations using the flushed TLBs. + */ +void tlb_flush_page_all_cpus_synced(CPUState *src, vaddr addr); + +/** + * tlb_flush: + * @cpu: CPU whose TLB should be flushed + * + * Flush the entire TLB for the specified CPU. Most CPU architectures + * allow the implementation to drop entries from the TLB at any time + * so this is generally safe. If more selective flushing is required + * use one of the other functions for efficiency. + */ +void tlb_flush(CPUState *cpu); + +/** + * tlb_flush_all_cpus_synced: + * @cpu: src CPU of the flush + * + * Flush the entire TLB for all CPUs, for all MMU indexes. + * + * When this function returns, no CPUs will subsequently perform + * translations using the flushed TLBs. + */ +void tlb_flush_all_cpus_synced(CPUState *src_cpu); + +/** + * tlb_flush_page_by_mmuidx: + * @cpu: CPU whose TLB should be flushed + * @addr: virtual address of page to be flushed + * @idxmap: bitmap of MMU indexes to flush + * + * Flush one page from the TLB of the specified CPU, for the specified + * MMU indexes. + */ +void tlb_flush_page_by_mmuidx(CPUState *cpu, vaddr addr, + uint16_t idxmap); + +/** + * tlb_flush_page_by_mmuidx_all_cpus_synced: + * @cpu: Originating CPU of the flush + * @addr: virtual address of page to be flushed + * @idxmap: bitmap of MMU indexes to flush + * + * Flush one page from the TLB of all CPUs, for the specified + * MMU indexes. + * + * When this function returns, no CPUs will subsequently perform + * translations using the flushed TLBs. + */ +void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu, vaddr addr, + uint16_t idxmap); + +/** + * tlb_flush_by_mmuidx: + * @cpu: CPU whose TLB should be flushed + * @wait: If true ensure synchronisation by exiting the cpu_loop + * @idxmap: bitmap of MMU indexes to flush + * + * Flush all entries from the TLB of the specified CPU, for the specified + * MMU indexes. + */ +void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap); + +/** + * tlb_flush_by_mmuidx_all_cpus_synced: + * @cpu: Originating CPU of the flush + * @idxmap: bitmap of MMU indexes to flush + * + * Flush all entries from the TLB of all CPUs, for the specified + * MMU indexes. + * + * When this function returns, no CPUs will subsequently perform + * translations using the flushed TLBs. + */ +void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, uint16_t idxmap); + +/** + * tlb_flush_page_bits_by_mmuidx + * @cpu: CPU whose TLB should be flushed + * @addr: virtual address of page to be flushed + * @idxmap: bitmap of mmu indexes to flush + * @bits: number of significant bits in address + * + * Similar to tlb_flush_page_mask, but with a bitmap of indexes. + */ +void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, vaddr addr, + uint16_t idxmap, unsigned bits); + +/* Similarly, with broadcast and syncing. */ +void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *cpu, vaddr addr, + uint16_t idxmap, + unsigned bits); + +/** + * tlb_flush_range_by_mmuidx + * @cpu: CPU whose TLB should be flushed + * @addr: virtual address of the start of the range to be flushed + * @len: length of range to be flushed + * @idxmap: bitmap of mmu indexes to flush + * @bits: number of significant bits in address + * + * For each mmuidx in @idxmap, flush all pages within [@addr,@addr+@len), + * comparing only the low @bits worth of each virtual page. + */ +void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr, + vaddr len, uint16_t idxmap, + unsigned bits); + +/* Similarly, with broadcast and syncing. */ +void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *cpu, + vaddr addr, + vaddr len, + uint16_t idxmap, + unsigned bits); +#else +static inline void tlb_flush_page(CPUState *cpu, vaddr addr) +{ +} +static inline void tlb_flush_page_all_cpus_synced(CPUState *src, vaddr addr) +{ +} +static inline void tlb_flush(CPUState *cpu) +{ +} +static inline void tlb_flush_all_cpus_synced(CPUState *src_cpu) +{ +} +static inline void tlb_flush_page_by_mmuidx(CPUState *cpu, + vaddr addr, uint16_t idxmap) +{ +} + +static inline void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap) +{ +} +static inline void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu, + vaddr addr, + uint16_t idxmap) +{ +} +static inline void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, + uint16_t idxmap) +{ +} +static inline void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, + vaddr addr, + uint16_t idxmap, + unsigned bits) +{ +} +static inline void +tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *cpu, vaddr addr, + uint16_t idxmap, unsigned bits) +{ +} +static inline void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr, + vaddr len, uint16_t idxmap, + unsigned bits) +{ +} +static inline void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *cpu, + vaddr addr, + vaddr len, + uint16_t idxmap, + unsigned bits) +{ +} +#endif /* CONFIG_TCG && !CONFIG_USER_ONLY */ +#endif /* CPUTLB_H */ diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index d9045c9ac4..dd5c40f223 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -27,263 +27,8 @@ #include "exec/mmu-access-type.h" #include "exec/translation-block.h" -#if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG) -/* cputlb.c */ -/** - * tlb_init - initialize a CPU's TLB - * @cpu: CPU whose TLB should be initialized - */ -void tlb_init(CPUState *cpu); -/** - * tlb_destroy - destroy a CPU's TLB - * @cpu: CPU whose TLB should be destroyed - */ -void tlb_destroy(CPUState *cpu); -/** - * tlb_flush_page: - * @cpu: CPU whose TLB should be flushed - * @addr: virtual address of page to be flushed - * - * Flush one page from the TLB of the specified CPU, for all - * MMU indexes. - */ -void tlb_flush_page(CPUState *cpu, vaddr addr); -/** - * tlb_flush_page_all_cpus_synced: - * @cpu: src CPU of the flush - * @addr: virtual address of page to be flushed - * - * Flush one page from the TLB of all CPUs, for all - * MMU indexes. - * - * When this function returns, no CPUs will subsequently perform - * translations using the flushed TLBs. - */ -void tlb_flush_page_all_cpus_synced(CPUState *src, vaddr addr); -/** - * tlb_flush: - * @cpu: CPU whose TLB should be flushed - * - * Flush the entire TLB for the specified CPU. Most CPU architectures - * allow the implementation to drop entries from the TLB at any time - * so this is generally safe. If more selective flushing is required - * use one of the other functions for efficiency. - */ -void tlb_flush(CPUState *cpu); -/** - * tlb_flush_all_cpus_synced: - * @cpu: src CPU of the flush - * - * Flush the entire TLB for all CPUs, for all MMU indexes. - * - * When this function returns, no CPUs will subsequently perform - * translations using the flushed TLBs. - */ -void tlb_flush_all_cpus_synced(CPUState *src_cpu); -/** - * tlb_flush_page_by_mmuidx: - * @cpu: CPU whose TLB should be flushed - * @addr: virtual address of page to be flushed - * @idxmap: bitmap of MMU indexes to flush - * - * Flush one page from the TLB of the specified CPU, for the specified - * MMU indexes. - */ -void tlb_flush_page_by_mmuidx(CPUState *cpu, vaddr addr, - uint16_t idxmap); -/** - * tlb_flush_page_by_mmuidx_all_cpus_synced: - * @cpu: Originating CPU of the flush - * @addr: virtual address of page to be flushed - * @idxmap: bitmap of MMU indexes to flush - * - * Flush one page from the TLB of all CPUs, for the specified - * MMU indexes. - * - * When this function returns, no CPUs will subsequently perform - * translations using the flushed TLBs. - */ -void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu, vaddr addr, - uint16_t idxmap); -/** - * tlb_flush_by_mmuidx: - * @cpu: CPU whose TLB should be flushed - * @wait: If true ensure synchronisation by exiting the cpu_loop - * @idxmap: bitmap of MMU indexes to flush - * - * Flush all entries from the TLB of the specified CPU, for the specified - * MMU indexes. - */ -void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap); -/** - * tlb_flush_by_mmuidx_all_cpus_synced: - * @cpu: Originating CPU of the flush - * @idxmap: bitmap of MMU indexes to flush - * - * Flush all entries from the TLB of all CPUs, for the specified - * MMU indexes. - * - * When this function returns, no CPUs will subsequently perform - * translations using the flushed TLBs. - */ -void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, uint16_t idxmap); - -/** - * tlb_flush_page_bits_by_mmuidx - * @cpu: CPU whose TLB should be flushed - * @addr: virtual address of page to be flushed - * @idxmap: bitmap of mmu indexes to flush - * @bits: number of significant bits in address - * - * Similar to tlb_flush_page_mask, but with a bitmap of indexes. - */ -void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, vaddr addr, - uint16_t idxmap, unsigned bits); - -/* Similarly, with broadcast and syncing. */ -void tlb_flush_page_bits_by_mmuidx_all_cpus_synced - (CPUState *cpu, vaddr addr, uint16_t idxmap, unsigned bits); - -/** - * tlb_flush_range_by_mmuidx - * @cpu: CPU whose TLB should be flushed - * @addr: virtual address of the start of the range to be flushed - * @len: length of range to be flushed - * @idxmap: bitmap of mmu indexes to flush - * @bits: number of significant bits in address - * - * For each mmuidx in @idxmap, flush all pages within [@addr,@addr+@len), - * comparing only the low @bits worth of each virtual page. - */ -void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr, - vaddr len, uint16_t idxmap, - unsigned bits); - -/* Similarly, with broadcast and syncing. */ -void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *cpu, - vaddr addr, - vaddr len, - uint16_t idxmap, - unsigned bits); - -/** - * tlb_set_page_full: - * @cpu: CPU context - * @mmu_idx: mmu index of the tlb to modify - * @addr: virtual address of the entry to add - * @full: the details of the tlb entry - * - * Add an entry to @cpu tlb index @mmu_idx. All of the fields of - * @full must be filled, except for xlat_section, and constitute - * the complete description of the translated page. - * - * This is generally called by the target tlb_fill function after - * having performed a successful page table walk to find the physical - * address and attributes for the translation. - * - * At most one entry for a given virtual address is permitted. Only a - * single TARGET_PAGE_SIZE region is mapped; @full->lg_page_size is only - * used by tlb_flush_page. - */ -void tlb_set_page_full(CPUState *cpu, int mmu_idx, vaddr addr, - CPUTLBEntryFull *full); - -/** - * tlb_set_page_with_attrs: - * @cpu: CPU to add this TLB entry for - * @addr: virtual address of page to add entry for - * @paddr: physical address of the page - * @attrs: memory transaction attributes - * @prot: access permissions (PAGE_READ/PAGE_WRITE/PAGE_EXEC bits) - * @mmu_idx: MMU index to insert TLB entry for - * @size: size of the page in bytes - * - * Add an entry to this CPU's TLB (a mapping from virtual address - * @addr to physical address @paddr) with the specified memory - * transaction attributes. This is generally called by the target CPU - * specific code after it has been called through the tlb_fill() - * entry point and performed a successful page table walk to find - * the physical address and attributes for the virtual address - * which provoked the TLB miss. - * - * At most one entry for a given virtual address is permitted. Only a - * single TARGET_PAGE_SIZE region is mapped; the supplied @size is only - * used by tlb_flush_page. - */ -void tlb_set_page_with_attrs(CPUState *cpu, vaddr addr, - hwaddr paddr, MemTxAttrs attrs, - int prot, int mmu_idx, vaddr size); -/* tlb_set_page: - * - * This function is equivalent to calling tlb_set_page_with_attrs() - * with an @attrs argument of MEMTXATTRS_UNSPECIFIED. It's provided - * as a convenience for CPUs which don't use memory transaction attributes. - */ -void tlb_set_page(CPUState *cpu, vaddr addr, - hwaddr paddr, int prot, - int mmu_idx, vaddr size); -#else -static inline void tlb_init(CPUState *cpu) -{ -} -static inline void tlb_destroy(CPUState *cpu) -{ -} -static inline void tlb_flush_page(CPUState *cpu, vaddr addr) -{ -} -static inline void tlb_flush_page_all_cpus_synced(CPUState *src, vaddr addr) -{ -} -static inline void tlb_flush(CPUState *cpu) -{ -} -static inline void tlb_flush_all_cpus_synced(CPUState *src_cpu) -{ -} -static inline void tlb_flush_page_by_mmuidx(CPUState *cpu, - vaddr addr, uint16_t idxmap) -{ -} - -static inline void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap) -{ -} -static inline void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu, - vaddr addr, - uint16_t idxmap) -{ -} -static inline void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, - uint16_t idxmap) -{ -} -static inline void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, - vaddr addr, - uint16_t idxmap, - unsigned bits) -{ -} -static inline void -tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *cpu, vaddr addr, - uint16_t idxmap, unsigned bits) -{ -} -static inline void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr, - vaddr len, uint16_t idxmap, - unsigned bits) -{ -} -static inline void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *cpu, - vaddr addr, - vaddr len, - uint16_t idxmap, - unsigned bits) -{ -} -#endif - #if defined(CONFIG_TCG) +#include "accel/tcg/getpc.h" /** * probe_access: @@ -433,24 +178,6 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr); void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last); void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr); -/* GETPC is the true target of the return instruction that we'll execute. */ -#if defined(CONFIG_TCG_INTERPRETER) -extern __thread uintptr_t tci_tb_ptr; -# define GETPC() tci_tb_ptr -#else -# define GETPC() \ - ((uintptr_t)__builtin_extract_return_addr(__builtin_return_address(0))) -#endif - -/* The true return address will often point to a host insn that is part of - the next translated guest insn. Adjust the address backward to point to - the middle of the call insn. Subtracting one would do the job except for - several compressed mode architectures (arm, mips) which set the low bit - to indicate the compressed mode; subtracting two works around that. It - is also the case that there are no host isas that contain a call insn - smaller than 4 bytes, so we don't worry about special-casing this. */ -#define GETPC_ADJ 2 - #if !defined(CONFIG_USER_ONLY) /** @@ -502,9 +229,6 @@ static inline tb_page_addr_t get_page_addr_code(CPUArchState *env, #if !defined(CONFIG_USER_ONLY) -void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length); -void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length); - MemoryRegionSection * address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr, hwaddr *xlat, hwaddr *plen, diff --git a/include/exec/memop.h b/include/exec/memop.h index acdb40a9b3..407a47d82c 100644 --- a/include/exec/memop.h +++ b/include/exec/memop.h @@ -91,8 +91,12 @@ typedef enum MemOp { * Depending on alignment, one or both will be single-copy atomic. * This is the atomicity e.g. of Arm FEAT_LSE2 LDP. * MO_ATOM_SUBALIGN: the operation is single-copy atomic by parts - * by the alignment. E.g. if the address is 0 mod 4, then each - * 4-byte subobject is single-copy atomic. + * by the alignment. E.g. if an 8-byte value is accessed at an + * address which is 0 mod 8, then the whole 8-byte access is + * single-copy atomic; otherwise, if it is accessed at 0 mod 4 + * then each 4-byte subobject is single-copy atomic; otherwise + * if it is accessed at 0 mod 2 then the four 2-byte subobjects + * are single-copy atomic. * This is the atomicity e.g. of IBM Power. * MO_ATOM_NONE: the operation has no atomicity requirements. * diff --git a/include/exec/page-protection.h b/include/exec/page-protection.h index bae3355f62..3e0a8a0333 100644 --- a/include/exec/page-protection.h +++ b/include/exec/page-protection.h @@ -40,8 +40,6 @@ #ifdef CONFIG_USER_ONLY -#include "qemu/clang-tsa.h" - void TSA_NO_TSA mmap_lock(void); void TSA_NO_TSA mmap_unlock(void); bool have_mmap_lock(void); diff --git a/include/exec/poison.h b/include/exec/poison.h index f4283f693a..8ed04b3108 100644 --- a/include/exec/poison.h +++ b/include/exec/poison.h @@ -44,23 +44,7 @@ #pragma GCC poison TARGET_FMT_ld #pragma GCC poison TARGET_FMT_lu -#pragma GCC poison TARGET_PAGE_SIZE -#pragma GCC poison TARGET_PAGE_MASK -#pragma GCC poison TARGET_PAGE_BITS -#pragma GCC poison TARGET_PAGE_ALIGN - -#pragma GCC poison CPU_INTERRUPT_HARD -#pragma GCC poison CPU_INTERRUPT_EXITTB -#pragma GCC poison CPU_INTERRUPT_HALT -#pragma GCC poison CPU_INTERRUPT_DEBUG -#pragma GCC poison CPU_INTERRUPT_TGT_EXT_0 -#pragma GCC poison CPU_INTERRUPT_TGT_EXT_1 -#pragma GCC poison CPU_INTERRUPT_TGT_EXT_2 -#pragma GCC poison CPU_INTERRUPT_TGT_EXT_3 -#pragma GCC poison CPU_INTERRUPT_TGT_EXT_4 -#pragma GCC poison CPU_INTERRUPT_TGT_INT_0 -#pragma GCC poison CPU_INTERRUPT_TGT_INT_1 -#pragma GCC poison CPU_INTERRUPT_TGT_INT_2 +#pragma GCC poison TARGET_PHYS_ADDR_SPACE_BITS #pragma GCC poison CONFIG_ALPHA_DIS #pragma GCC poison CONFIG_HPPA_DIS diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 94bb3ccbe4..3d8df4edf1 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -23,6 +23,7 @@ #include "cpu.h" #include "system/xen.h" #include "system/tcg.h" +#include "exec/cputlb.h" #include "exec/ramlist.h" #include "exec/ramblock.h" #include "exec/exec-all.h" diff --git a/include/exec/target_page.h b/include/exec/target_page.h index 98ffbb5c23..8e89e5cbe6 100644 --- a/include/exec/target_page.h +++ b/include/exec/target_page.h @@ -14,10 +14,56 @@ #ifndef EXEC_TARGET_PAGE_H #define EXEC_TARGET_PAGE_H -size_t qemu_target_page_size(void); -int qemu_target_page_mask(void); -int qemu_target_page_bits(void); -int qemu_target_page_bits_min(void); +/* + * If compiling per-target, get the real values. + * For generic code, reuse the mechanism for variable page size. + */ +#ifdef COMPILING_PER_TARGET +#include "cpu-param.h" +#include "exec/target_long.h" +#define TARGET_PAGE_TYPE target_long +#else +#define TARGET_PAGE_BITS_VARY +#define TARGET_PAGE_TYPE int +#endif + +#ifdef TARGET_PAGE_BITS_VARY +# include "exec/page-vary.h" +extern const TargetPageBits target_page; +# ifdef CONFIG_DEBUG_TCG +# define TARGET_PAGE_BITS ({ assert(target_page.decided); \ + target_page.bits; }) +# define TARGET_PAGE_MASK ({ assert(target_page.decided); \ + (TARGET_PAGE_TYPE)target_page.mask; }) +# else +# define TARGET_PAGE_BITS target_page.bits +# define TARGET_PAGE_MASK ((TARGET_PAGE_TYPE)target_page.mask) +# endif +# define TARGET_PAGE_SIZE (-(int)TARGET_PAGE_MASK) +#else +# define TARGET_PAGE_BITS_MIN TARGET_PAGE_BITS +# define TARGET_PAGE_SIZE (1 << TARGET_PAGE_BITS) +# define TARGET_PAGE_MASK ((TARGET_PAGE_TYPE)-1 << TARGET_PAGE_BITS) +#endif +#define TARGET_PAGE_ALIGN(addr) ROUND_UP((addr), TARGET_PAGE_SIZE) + +static inline size_t qemu_target_page_size(void) +{ + return TARGET_PAGE_SIZE; +} + +static inline int qemu_target_page_mask(void) +{ + return TARGET_PAGE_MASK; +} + +static inline int qemu_target_page_bits(void) +{ + return TARGET_PAGE_BITS; +} + +int qemu_target_page_bits_min(void); size_t qemu_target_pages_to_MiB(size_t pages); + #endif diff --git a/include/hw/arm/bsa.h b/include/hw/arm/bsa.h index 8eaab603c0..13ed2d2ac1 100644 --- a/include/hw/arm/bsa.h +++ b/include/hw/arm/bsa.h @@ -22,6 +22,8 @@ #define QEMU_ARM_BSA_H /* These are architectural INTID values */ +#define ARCH_TIMER_S_EL2_VIRT_IRQ 19 +#define ARCH_TIMER_S_EL2_IRQ 20 #define VIRTUAL_PMU_IRQ 23 #define ARCH_GIC_MAINT_IRQ 25 #define ARCH_TIMER_NS_EL2_IRQ 26 diff --git a/include/hw/arm/smmu-common.h b/include/hw/arm/smmu-common.h index d1a4a64551..e5e2d09294 100644 --- a/include/hw/arm/smmu-common.h +++ b/include/hw/arm/smmu-common.h @@ -110,7 +110,6 @@ typedef struct SMMUTransCfg { /* Used by stage-1 only. */ bool aa64; /* arch64 or aarch32 translation table */ bool record_faults; /* record fault events */ - uint64_t ttb; /* TT base address */ uint8_t oas; /* output address width */ uint8_t tbi; /* Top Byte Ignore */ int asid; @@ -143,6 +142,11 @@ typedef struct SMMUIOTLBKey { uint8_t level; } SMMUIOTLBKey; +typedef struct SMMUSIDRange { + uint32_t start; + uint32_t end; +} SMMUSIDRange; + struct SMMUState { /* <private> */ SysBusDevice dev; @@ -220,6 +224,7 @@ void smmu_iotlb_inv_iova(SMMUState *s, int asid, int vmid, dma_addr_t iova, uint8_t tg, uint64_t num_pages, uint8_t ttl); void smmu_iotlb_inv_ipa(SMMUState *s, int vmid, dma_addr_t ipa, uint8_t tg, uint64_t num_pages, uint8_t ttl); +void smmu_configs_inv_sid_range(SMMUState *s, SMMUSIDRange sid_range); /* Unmap the range of all the notifiers registered to any IOMMU mr */ void smmu_inv_notifiers_all(SMMUState *s); diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index fb397cdfc5..9dd6ac7c76 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -124,7 +124,9 @@ struct SysemuCPUOps; * @get_pc: Callback for getting the Program Counter register. * As above, with the semantics of the target architecture. * @gdb_read_register: Callback for letting GDB read a register. + * No more than @gdb_num_core_regs registers can be read. * @gdb_write_register: Callback for letting GDB write a register. + * No more than @gdb_num_core_regs registers can be written. * @gdb_adjust_breakpoint: Callback for adjusting the address of a * breakpoint. Used by AVR to handle a gdb mis-feature with * its Harvard architecture split code and data. @@ -614,6 +616,8 @@ extern bool mttcg_enabled; */ bool cpu_paging_enabled(const CPUState *cpu); +#if !defined(CONFIG_USER_ONLY) + /** * cpu_get_memory_mapping: * @cpu: The CPU whose memory mappings are to be obtained. @@ -625,8 +629,6 @@ bool cpu_paging_enabled(const CPUState *cpu); bool cpu_get_memory_mapping(CPUState *cpu, MemoryMappingList *list, Error **errp); -#if !defined(CONFIG_USER_ONLY) - /** * cpu_write_elf64_note: * @f: pointer to a function that writes memory to a file diff --git a/include/hw/pci/pci.h b/include/hw/pci/pci.h index 4002bbeebd..c220cc8449 100644 --- a/include/hw/pci/pci.h +++ b/include/hw/pci/pci.h @@ -216,6 +216,8 @@ enum { QEMU_PCIE_ARI_NEXTFN_1 = (1 << QEMU_PCIE_ARI_NEXTFN_1_BITNR), #define QEMU_PCIE_EXT_TAG_BITNR 13 QEMU_PCIE_EXT_TAG = (1 << QEMU_PCIE_EXT_TAG_BITNR), +#define QEMU_PCI_CAP_PM_BITNR 14 + QEMU_PCI_CAP_PM = (1 << QEMU_PCI_CAP_PM_BITNR), }; typedef struct PCIINTxRoute { @@ -676,5 +678,6 @@ static inline void pci_irq_deassert(PCIDevice *pci_dev) MSIMessage pci_get_msi_message(PCIDevice *dev, int vector); void pci_set_enabled(PCIDevice *pci_dev, bool state); void pci_set_power(PCIDevice *pci_dev, bool state); +int pci_pm_init(PCIDevice *pci_dev, uint8_t offset, Error **errp); #endif diff --git a/include/hw/pci/pci_device.h b/include/hw/pci/pci_device.h index add208edfa..345b12eaac 100644 --- a/include/hw/pci/pci_device.h +++ b/include/hw/pci/pci_device.h @@ -105,6 +105,9 @@ struct PCIDevice { /* Capability bits */ uint32_t cap_present; + /* Offset of PM capability in config space */ + uint8_t pm_cap; + /* Offset of MSI-X capability in config space */ uint8_t msix_cap; diff --git a/include/hw/pci/pcie.h b/include/hw/pci/pcie.h index b8d59732bc..70a5de09de 100644 --- a/include/hw/pci/pcie.h +++ b/include/hw/pci/pcie.h @@ -58,8 +58,6 @@ typedef enum { struct PCIExpressDevice { /* Offset of express capability in config space */ uint8_t exp_cap; - /* Offset of Power Management capability in config space */ - uint8_t pm_cap; /* SLOT */ bool hpev_notified; /* Logical AND of conditions for hot plug event. diff --git a/include/hw/qdev-properties.h b/include/hw/qdev-properties.h index bf27375a3c..15fcec5260 100644 --- a/include/hw/qdev-properties.h +++ b/include/hw/qdev-properties.h @@ -30,7 +30,7 @@ struct Property { }; struct PropertyInfo { - const char *name; + const char *type; const char *description; const QEnumLookup *enum_table; bool realized_set_allowed; /* allow setting property on realized device */ @@ -49,7 +49,6 @@ struct PropertyInfo { extern const PropertyInfo qdev_prop_bit; extern const PropertyInfo qdev_prop_bit64; extern const PropertyInfo qdev_prop_bool; -extern const PropertyInfo qdev_prop_enum; extern const PropertyInfo qdev_prop_uint8; extern const PropertyInfo qdev_prop_uint16; extern const PropertyInfo qdev_prop_uint32; diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h index 2c43ea123f..04944d4fed 100644 --- a/include/hw/s390x/s390-pci-bus.h +++ b/include/hw/s390x/s390-pci-bus.h @@ -277,6 +277,7 @@ struct S390PCIIOMMU { AddressSpace as; MemoryRegion mr; IOMMUMemoryRegion iommu_mr; + MemoryRegion *dm_mr; bool enabled; uint64_t g_iota; uint64_t pba; @@ -362,6 +363,7 @@ struct S390PCIBusDevice { bool interp; bool forwarding_assist; bool aif; + bool rtr_avail; QTAILQ_ENTRY(S390PCIBusDevice) link; }; @@ -389,6 +391,7 @@ int pci_chsc_sei_nt2_have_event(void); void s390_pci_sclp_configure(SCCB *sccb); void s390_pci_sclp_deconfigure(SCCB *sccb); void s390_pci_iommu_enable(S390PCIIOMMU *iommu); +void s390_pci_iommu_direct_map_enable(S390PCIIOMMU *iommu); void s390_pci_iommu_disable(S390PCIIOMMU *iommu); void s390_pci_generate_error_event(uint16_t pec, uint32_t fh, uint32_t fid, uint64_t faddr, uint32_t e); diff --git a/include/hw/s390x/s390-pci-clp.h b/include/hw/s390x/s390-pci-clp.h index 03b7f9ba5f..6a635d693b 100644 --- a/include/hw/s390x/s390-pci-clp.h +++ b/include/hw/s390x/s390-pci-clp.h @@ -158,6 +158,7 @@ typedef struct ClpRspQueryPciGrp { #define CLP_RSP_QPCIG_MASK_NOI 0xfff uint16_t i; uint8_t version; +#define CLP_RSP_QPCIG_MASK_RTR 0x20 #define CLP_RSP_QPCIG_MASK_FRAME 0x2 #define CLP_RSP_QPCIG_MASK_REFRESH 0x1 uint8_t fr; diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index ac35136a11..04b123a6c9 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -36,6 +36,23 @@ #define VFIO_MSG_PREFIX "vfio %s: " +/* + * Flags to be used as unique delimiters for VFIO devices in the migration + * stream. These flags are composed as: + * 0xffffffff => MSB 32-bit all 1s + * 0xef10 => Magic ID, represents emulated (virtual) function IO + * 0x0000 => 16-bits reserved for flags + * + * The beginning of state information is marked by _DEV_CONFIG_STATE, + * _DEV_SETUP_STATE, or _DEV_DATA_STATE, respectively. The end of a + * certain state information is marked by _END_OF_STATE. + */ +#define VFIO_MIG_FLAG_END_OF_STATE (0xffffffffef100001ULL) +#define VFIO_MIG_FLAG_DEV_CONFIG_STATE (0xffffffffef100002ULL) +#define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL) +#define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL) +#define VFIO_MIG_FLAG_DEV_INIT_DATA_SENT (0xffffffffef100005ULL) + enum { VFIO_DEVICE_TYPE_PCI = 0, VFIO_DEVICE_TYPE_PLATFORM = 1, @@ -61,6 +78,8 @@ typedef struct VFIORegion { uint8_t nr; /* cache the region number for debug */ } VFIORegion; +typedef struct VFIOMultifd VFIOMultifd; + typedef struct VFIOMigration { struct VFIODevice *vbasedev; VMChangeStateEntry *vm_state; @@ -72,6 +91,8 @@ typedef struct VFIOMigration { uint64_t mig_flags; uint64_t precopy_init_size; uint64_t precopy_dirty_size; + bool multifd_transfer; + VFIOMultifd *multifd; bool initial_data_sent; bool event_save_iterate_started; @@ -133,6 +154,7 @@ typedef struct VFIODevice { bool no_mmap; bool ram_block_discard_allowed; OnOffAuto enable_migration; + OnOffAuto migration_multifd_transfer; bool migration_events; VFIODeviceOps *ops; unsigned int num_irqs; @@ -274,9 +296,13 @@ void vfio_unblock_multiple_devices_migration(void); bool vfio_viommu_preset(VFIODevice *vbasedev); int64_t vfio_mig_bytes_transferred(void); void vfio_reset_bytes_transferred(void); +void vfio_mig_add_bytes_transferred(unsigned long val); bool vfio_device_state_is_running(VFIODevice *vbasedev); bool vfio_device_state_is_precopy(VFIODevice *vbasedev); +int vfio_save_device_config_state(QEMUFile *f, void *opaque, Error **errp); +int vfio_load_device_config_state(QEMUFile *f, void *opaque); + #ifdef CONFIG_LINUX int vfio_get_region_info(VFIODevice *vbasedev, int index, struct vfio_region_info **info); @@ -291,6 +317,11 @@ struct vfio_info_cap_header * vfio_get_device_info_cap(struct vfio_device_info *info, uint16_t id); struct vfio_info_cap_header * vfio_get_cap(void *ptr, uint32_t cap_offset, uint16_t id); + +int vfio_migration_set_state(VFIODevice *vbasedev, + enum vfio_device_mig_state new_state, + enum vfio_device_mig_state recover_state, + Error **errp); #endif bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp); diff --git a/include/migration/client-options.h b/include/migration/client-options.h index 59f4b55cf4..289c9d7762 100644 --- a/include/migration/client-options.h +++ b/include/migration/client-options.h @@ -10,6 +10,10 @@ #ifndef QEMU_MIGRATION_CLIENT_OPTIONS_H #define QEMU_MIGRATION_CLIENT_OPTIONS_H + +/* properties */ +bool migrate_send_switchover_start(void); + /* capabilities */ bool migrate_background_snapshot(void); diff --git a/include/migration/misc.h b/include/migration/misc.h index c660be8095..8fd36eba1d 100644 --- a/include/migration/misc.h +++ b/include/migration/misc.h @@ -45,9 +45,12 @@ bool migrate_ram_is_ignored(RAMBlock *block); /* migration/block.c */ AnnounceParameters *migrate_announce_params(void); + /* migration/savevm.c */ void dump_vmstate_json_to_file(FILE *out_fp); +void qemu_loadvm_start_load_thread(MigrationLoadThread function, + void *opaque); /* migration/migration.c */ void migration_object_init(void); @@ -115,4 +118,26 @@ bool migrate_is_uri(const char *uri); bool migrate_uri_parse(const char *uri, MigrationChannel **channel, Error **errp); +/* migration/multifd-device-state.c */ +typedef struct SaveLiveCompletePrecopyThreadData { + SaveLiveCompletePrecopyThreadHandler hdlr; + char *idstr; + uint32_t instance_id; + void *handler_opaque; +} SaveLiveCompletePrecopyThreadData; + +bool multifd_queue_device_state(char *idstr, uint32_t instance_id, + char *data, size_t len); +bool multifd_device_state_supported(void); + +void +multifd_spawn_device_state_save_thread(SaveLiveCompletePrecopyThreadHandler hdlr, + char *idstr, uint32_t instance_id, + void *opaque); + +bool multifd_device_state_save_thread_should_exit(void); + +void multifd_abort_device_state_save_threads(void); +bool multifd_join_device_state_save_threads(void); + #endif diff --git a/include/migration/register.h b/include/migration/register.h index f60e797894..c041ce32f2 100644 --- a/include/migration/register.h +++ b/include/migration/register.h @@ -69,7 +69,9 @@ typedef struct SaveVMHandlers { /** * @save_cleanup * - * Uninitializes the data structures on the source + * Uninitializes the data structures on the source. + * Note that this handler can be called even if save_setup + * wasn't called earlier. * * @opaque: data pointer passed to register_savevm_live() */ @@ -103,6 +105,25 @@ typedef struct SaveVMHandlers { */ int (*save_live_complete_precopy)(QEMUFile *f, void *opaque); + /** + * @save_live_complete_precopy_thread (invoked in a separate thread) + * + * Called at the end of a precopy phase from a separate worker thread + * in configurations where multifd device state transfer is supported + * in order to perform asynchronous transmission of the remaining data in + * parallel with @save_live_complete_precopy handlers. + * When postcopy is enabled, devices that support postcopy will skip this + * step. + * + * @d: a #SaveLiveCompletePrecopyThreadData containing parameters that the + * handler may need, including this device section idstr and instance_id, + * and opaque data pointer passed to register_savevm_live(). + * @errp: pointer to Error*, to store an error if it happens. + * + * Returns true to indicate success and false for errors. + */ + SaveLiveCompletePrecopyThreadHandler save_live_complete_precopy_thread; + /* This runs both outside and inside the BQL. */ /** @@ -228,6 +249,21 @@ typedef struct SaveVMHandlers { int (*load_state)(QEMUFile *f, void *opaque, int version_id); /** + * @load_state_buffer (invoked outside the BQL) + * + * Load device state buffer provided to qemu_loadvm_load_state_buffer(). + * + * @opaque: data pointer passed to register_savevm_live() + * @buf: the data buffer to load + * @len: the data length in buffer + * @errp: pointer to Error*, to store an error if it happens. + * + * Returns true to indicate success and false for errors. + */ + bool (*load_state_buffer)(void *opaque, char *buf, size_t len, + Error **errp); + + /** * @load_setup * * Initializes the data structures on the destination. @@ -244,6 +280,8 @@ typedef struct SaveVMHandlers { * @load_cleanup * * Uninitializes the data structures on the destination. + * Note that this handler can be called even if load_setup + * wasn't called earlier. * * @opaque: data pointer passed to register_savevm_live() * @@ -275,6 +313,18 @@ typedef struct SaveVMHandlers { * otherwise */ bool (*switchover_ack_needed)(void *opaque); + + /** + * @switchover_start + * + * Notifies that the switchover has started. Called only on + * the destination. + * + * @opaque: data pointer passed to register_savevm_live() + * + * Returns zero to indicate success and negative for error + */ + int (*switchover_start)(void *opaque); } SaveVMHandlers; /** diff --git a/include/qapi/error.h b/include/qapi/error.h index f5fe216262..41e3816380 100644 --- a/include/qapi/error.h +++ b/include/qapi/error.h @@ -437,6 +437,8 @@ Error *error_copy(const Error *err); */ void error_free(Error *err); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(Error, error_free) + /* * Convenience function to assert that *@errp is set, then silently free it. */ diff --git a/include/qemu/atomic128.h b/include/qemu/atomic128.h index 88af6d4ea3..31e5c48d8f 100644 --- a/include/qemu/atomic128.h +++ b/include/qemu/atomic128.h @@ -13,6 +13,7 @@ #ifndef QEMU_ATOMIC128_H #define QEMU_ATOMIC128_H +#include "qemu/atomic.h" #include "qemu/int128.h" /* @@ -58,7 +59,7 @@ * Therefore, special case each platform. */ -#include "host/atomic128-cas.h" -#include "host/atomic128-ldst.h" +#include "host/atomic128-cas.h.inc" +#include "host/atomic128-ldst.h.inc" #endif /* QEMU_ATOMIC128_H */ diff --git a/include/qemu/clang-tsa.h b/include/qemu/clang-tsa.h deleted file mode 100644 index ba06fb8c92..0000000000 --- a/include/qemu/clang-tsa.h +++ /dev/null @@ -1,114 +0,0 @@ -#ifndef CLANG_TSA_H -#define CLANG_TSA_H - -/* - * Copyright 2018 Jarkko Hietaniemi <jhi@iki.fi> - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without - * limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -/* http://clang.llvm.org/docs/ThreadSafetyAnalysis.html - * - * TSA is available since clang 3.6-ish. - */ -#ifdef __clang__ -# define TSA(x) __attribute__((x)) -#else -# define TSA(x) /* No TSA, make TSA attributes no-ops. */ -#endif - -/* TSA_CAPABILITY() is used to annotate typedefs: - * - * typedef pthread_mutex_t TSA_CAPABILITY("mutex") tsa_mutex; - */ -#define TSA_CAPABILITY(x) TSA(capability(x)) - -/* TSA_GUARDED_BY() is used to annotate global variables, - * the data is guarded: - * - * Foo foo TSA_GUARDED_BY(mutex); - */ -#define TSA_GUARDED_BY(x) TSA(guarded_by(x)) - -/* TSA_PT_GUARDED_BY() is used to annotate global pointers, the data - * behind the pointer is guarded. - * - * Foo* ptr TSA_PT_GUARDED_BY(mutex); - */ -#define TSA_PT_GUARDED_BY(x) TSA(pt_guarded_by(x)) - -/* The TSA_REQUIRES() is used to annotate functions: the caller of the - * function MUST hold the resource, the function will NOT release it. - * - * More than one mutex may be specified, comma-separated. - * - * void Foo(void) TSA_REQUIRES(mutex); - */ -#define TSA_REQUIRES(...) TSA(requires_capability(__VA_ARGS__)) -#define TSA_REQUIRES_SHARED(...) TSA(requires_shared_capability(__VA_ARGS__)) - -/* TSA_EXCLUDES() is used to annotate functions: the caller of the - * function MUST NOT hold resource, the function first acquires the - * resource, and then releases it. - * - * More than one mutex may be specified, comma-separated. - * - * void Foo(void) TSA_EXCLUDES(mutex); - */ -#define TSA_EXCLUDES(...) TSA(locks_excluded(__VA_ARGS__)) - -/* TSA_ACQUIRE() is used to annotate functions: the caller of the - * function MUST NOT hold the resource, the function will acquire the - * resource, but NOT release it. - * - * More than one mutex may be specified, comma-separated. - * - * void Foo(void) TSA_ACQUIRE(mutex); - */ -#define TSA_ACQUIRE(...) TSA(acquire_capability(__VA_ARGS__)) -#define TSA_ACQUIRE_SHARED(...) TSA(acquire_shared_capability(__VA_ARGS__)) - -/* TSA_RELEASE() is used to annotate functions: the caller of the - * function MUST hold the resource, but the function will then release it. - * - * More than one mutex may be specified, comma-separated. - * - * void Foo(void) TSA_RELEASE(mutex); - */ -#define TSA_RELEASE(...) TSA(release_capability(__VA_ARGS__)) -#define TSA_RELEASE_SHARED(...) TSA(release_shared_capability(__VA_ARGS__)) - -/* TSA_NO_TSA is used to annotate functions. Use only when you need to. - * - * void Foo(void) TSA_NO_TSA; - */ -#define TSA_NO_TSA TSA(no_thread_safety_analysis) - -/* - * TSA_ASSERT() is used to annotate functions: This function will assert that - * the lock is held. When it returns, the caller of the function is assumed to - * already hold the resource. - * - * More than one mutex may be specified, comma-separated. - */ -#define TSA_ASSERT(...) TSA(assert_capability(__VA_ARGS__)) -#define TSA_ASSERT_SHARED(...) TSA(assert_shared_capability(__VA_ARGS__)) - -#endif /* #ifndef CLANG_TSA_H */ diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h index d904408e5e..496dac5ac1 100644 --- a/include/qemu/compiler.h +++ b/include/qemu/compiler.h @@ -208,6 +208,102 @@ #endif /* + * http://clang.llvm.org/docs/ThreadSafetyAnalysis.html + * + * TSA is available since clang 3.6-ish. + */ +#ifdef __clang__ +# define TSA(x) __attribute__((x)) +#else +# define TSA(x) /* No TSA, make TSA attributes no-ops. */ +#endif + +/* + * TSA_CAPABILITY() is used to annotate typedefs: + * + * typedef pthread_mutex_t TSA_CAPABILITY("mutex") tsa_mutex; + */ +#define TSA_CAPABILITY(x) TSA(capability(x)) + +/* + * TSA_GUARDED_BY() is used to annotate global variables, + * the data is guarded: + * + * Foo foo TSA_GUARDED_BY(mutex); + */ +#define TSA_GUARDED_BY(x) TSA(guarded_by(x)) + +/* + * TSA_PT_GUARDED_BY() is used to annotate global pointers, the data + * behind the pointer is guarded. + * + * Foo* ptr TSA_PT_GUARDED_BY(mutex); + */ +#define TSA_PT_GUARDED_BY(x) TSA(pt_guarded_by(x)) + +/* + * The TSA_REQUIRES() is used to annotate functions: the caller of the + * function MUST hold the resource, the function will NOT release it. + * + * More than one mutex may be specified, comma-separated. + * + * void Foo(void) TSA_REQUIRES(mutex); + */ +#define TSA_REQUIRES(...) TSA(requires_capability(__VA_ARGS__)) +#define TSA_REQUIRES_SHARED(...) TSA(requires_shared_capability(__VA_ARGS__)) + +/* + * TSA_EXCLUDES() is used to annotate functions: the caller of the + * function MUST NOT hold resource, the function first acquires the + * resource, and then releases it. + * + * More than one mutex may be specified, comma-separated. + * + * void Foo(void) TSA_EXCLUDES(mutex); + */ +#define TSA_EXCLUDES(...) TSA(locks_excluded(__VA_ARGS__)) + +/* + * TSA_ACQUIRE() is used to annotate functions: the caller of the + * function MUST NOT hold the resource, the function will acquire the + * resource, but NOT release it. + * + * More than one mutex may be specified, comma-separated. + * + * void Foo(void) TSA_ACQUIRE(mutex); + */ +#define TSA_ACQUIRE(...) TSA(acquire_capability(__VA_ARGS__)) +#define TSA_ACQUIRE_SHARED(...) TSA(acquire_shared_capability(__VA_ARGS__)) + +/* + * TSA_RELEASE() is used to annotate functions: the caller of the + * function MUST hold the resource, but the function will then release it. + * + * More than one mutex may be specified, comma-separated. + * + * void Foo(void) TSA_RELEASE(mutex); + */ +#define TSA_RELEASE(...) TSA(release_capability(__VA_ARGS__)) +#define TSA_RELEASE_SHARED(...) TSA(release_shared_capability(__VA_ARGS__)) + +/* + * TSA_NO_TSA is used to annotate functions. Use only when you need to. + * + * void Foo(void) TSA_NO_TSA; + */ +#define TSA_NO_TSA TSA(no_thread_safety_analysis) + +/* + * TSA_ASSERT() is used to annotate functions: This function will assert that + * the lock is held. When it returns, the caller of the function is assumed to + * already hold the resource. + * + * More than one mutex may be specified, comma-separated. + */ +#define TSA_ASSERT(...) TSA(assert_capability(__VA_ARGS__)) +#define TSA_ASSERT_SHARED(...) TSA(assert_shared_capability(__VA_ARGS__)) + +/* * Ugly CPP trick that is like "defined FOO", but also works in C * code. Useful to replace #ifdef with "if" statements; assumes * the symbol was defined with Meson's "config.set()", so it is empty diff --git a/include/qemu/thread.h b/include/qemu/thread.h index 7eba27a704..6f800aad31 100644 --- a/include/qemu/thread.h +++ b/include/qemu/thread.h @@ -3,7 +3,6 @@ #include "qemu/processor.h" #include "qemu/atomic.h" -#include "qemu/clang-tsa.h" typedef struct QemuCond QemuCond; typedef struct QemuSemaphore QemuSemaphore; diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h index 3d84efcac4..507f0814d5 100644 --- a/include/qemu/typedefs.h +++ b/include/qemu/typedefs.h @@ -22,6 +22,7 @@ * Please keep this list in case-insensitive alphabetical order. */ typedef struct AccelCPUState AccelCPUState; +typedef struct AccelOpsClass AccelOpsClass; typedef struct AccelState AccelState; typedef struct AddressSpace AddressSpace; typedef struct AioContext AioContext; @@ -108,6 +109,7 @@ typedef struct QString QString; typedef struct RAMBlock RAMBlock; typedef struct Range Range; typedef struct ReservedRegion ReservedRegion; +typedef struct SaveLiveCompletePrecopyThreadData SaveLiveCompletePrecopyThreadData; typedef struct SHPCDevice SHPCDevice; typedef struct SSIBus SSIBus; typedef struct TCGCPUOps TCGCPUOps; @@ -131,5 +133,9 @@ typedef struct IRQState *qemu_irq; * Function types */ typedef void (*qemu_irq_handler)(void *opaque, int n, int level); +typedef bool (*MigrationLoadThread)(void *opaque, bool *should_quit, + Error **errp); +typedef bool (*SaveLiveCompletePrecopyThreadHandler)(SaveLiveCompletePrecopyThreadData *d, + Error **errp); #endif /* QEMU_TYPEDEFS_H */ diff --git a/include/system/accel-ops.h b/include/system/accel-ops.h index 137fb96d44..4c99d25aef 100644 --- a/include/system/accel-ops.h +++ b/include/system/accel-ops.h @@ -17,7 +17,6 @@ #define TYPE_ACCEL_OPS "accel" ACCEL_OPS_SUFFIX #define ACCEL_OPS_NAME(name) (name "-" TYPE_ACCEL_OPS) -typedef struct AccelOpsClass AccelOpsClass; DECLARE_CLASS_CHECKERS(AccelOpsClass, ACCEL_OPS, TYPE_ACCEL_OPS) /** diff --git a/include/system/arch_init.h b/include/system/arch_init.h index 5b1c1026f3..d8b7744048 100644 --- a/include/system/arch_init.h +++ b/include/system/arch_init.h @@ -27,6 +27,4 @@ enum { extern const uint32_t arch_type; -void qemu_init_arch_modules(void); - #endif diff --git a/include/system/cpus.h b/include/system/cpus.h index 3d8fd368f3..3226c765d0 100644 --- a/include/system/cpus.h +++ b/include/system/cpus.h @@ -1,8 +1,6 @@ #ifndef QEMU_CPUS_H #define QEMU_CPUS_H -#include "system/accel-ops.h" - /* register accel-specific operations */ void cpus_register_accel(const AccelOpsClass *i); @@ -38,8 +36,6 @@ void resume_all_vcpus(void); void pause_all_vcpus(void); void cpu_stop_current(void); -extern int icount_align_option; - /* Unblock cpu */ void qemu_cpu_kick_self(void); |