diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/exec/cpu-common.h | 21 | ||||
| -rw-r--r-- | include/system/memory.h | 11 | ||||
| -rw-r--r-- | include/system/physmem.h | 54 | ||||
| -rw-r--r-- | include/system/ram_addr.h | 426 | ||||
| -rw-r--r-- | include/system/ramblock.h | 27 |
5 files changed, 89 insertions, 450 deletions
diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index b96ac49844..67e15c8e50 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -131,26 +131,14 @@ void cpu_address_space_init(CPUState *cpu, int asidx, */ void cpu_destroy_address_spaces(CPUState *cpu); -void cpu_physical_memory_rw(hwaddr addr, void *buf, - hwaddr len, bool is_write); -static inline void cpu_physical_memory_read(hwaddr addr, - void *buf, hwaddr len) -{ - cpu_physical_memory_rw(addr, buf, len, false); -} -static inline void cpu_physical_memory_write(hwaddr addr, - const void *buf, hwaddr len) -{ - cpu_physical_memory_rw(addr, (void *)buf, len, true); -} +void cpu_physical_memory_read(hwaddr addr, void *buf, hwaddr len); +void cpu_physical_memory_write(hwaddr addr, const void *buf, hwaddr len); void *cpu_physical_memory_map(hwaddr addr, hwaddr *plen, bool is_write); void cpu_physical_memory_unmap(void *buffer, hwaddr len, bool is_write, hwaddr access_len); -bool cpu_physical_memory_is_io(hwaddr phys_addr); - /* Coalesced MMIO regions are areas where write operations can be reordered. * This usually implies that write operations are side-effect free. This allows * batching which can make a major impact on performance when using @@ -158,14 +146,9 @@ bool cpu_physical_memory_is_io(hwaddr phys_addr); */ void qemu_flush_coalesced_mmio_buffer(void); -void cpu_flush_icache_range(hwaddr start, hwaddr len); - typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque); int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque); -int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length); -int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start, - size_t length); /* Returns: 0 on success, -1 on error */ int cpu_memory_rw_debug(CPUState *cpu, vaddr addr, diff --git a/include/system/memory.h b/include/system/memory.h index 08daf0fc59..3bd5ffa5e0 100644 --- a/include/system/memory.h +++ b/include/system/memory.h @@ -2995,6 +2995,8 @@ void address_space_cache_invalidate(MemoryRegionCache *cache, */ void address_space_cache_destroy(MemoryRegionCache *cache); +void address_space_flush_icache_range(AddressSpace *as, hwaddr addr, hwaddr len); + /* address_space_get_iotlb_entry: translate an address into an IOTLB * entry. Should be called from an RCU critical section. */ @@ -3047,6 +3049,15 @@ static inline MemoryRegion *address_space_translate(AddressSpace *as, bool address_space_access_valid(AddressSpace *as, hwaddr addr, hwaddr len, bool is_write, MemTxAttrs attrs); +/** + * address_space_is_io: check whether an guest physical addresses + * whithin an address space is I/O memory. + * + * @as: #AddressSpace to be accessed + * @addr: address within that address space + */ +bool address_space_is_io(AddressSpace *as, hwaddr addr); + /* address_space_map: map a physical memory region into a host virtual address * * May map a subset of the requested range, given by and returned in @plen. diff --git a/include/system/physmem.h b/include/system/physmem.h new file mode 100644 index 0000000000..879f6eae38 --- /dev/null +++ b/include/system/physmem.h @@ -0,0 +1,54 @@ +/* + * QEMU physical memory interfaces (target independent). + * + * Copyright (c) 2003 Fabrice Bellard + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ +#ifndef QEMU_SYSTEM_PHYSMEM_H +#define QEMU_SYSTEM_PHYSMEM_H + +#include "exec/hwaddr.h" +#include "exec/ramlist.h" + +#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) +#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) + +bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); + +bool physical_memory_is_clean(ram_addr_t addr); + +uint8_t physical_memory_range_includes_clean(ram_addr_t start, + ram_addr_t length, + uint8_t mask); + +void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); + +void physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, + uint8_t mask); + +/* + * Contrary to physical_memory_sync_dirty_bitmap() this function returns + * the number of dirty pages in @bitmap passed as argument. On the other hand, + * physical_memory_sync_dirty_bitmap() returns newly dirtied pages that + * weren't set in the global migration bitmap. + */ +uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap, + ram_addr_t start, + ram_addr_t pages); + +void physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length); + +bool physical_memory_test_and_clear_dirty(ram_addr_t start, + ram_addr_t length, + unsigned client); + +DirtyBitmapSnapshot * +physical_memory_snapshot_and_clear_dirty(MemoryRegion *mr, hwaddr offset, + hwaddr length, unsigned client); + +bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, + ram_addr_t start, + ram_addr_t length); + +#endif diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 15a1b1a4fa..683485980c 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -19,17 +19,9 @@ #ifndef SYSTEM_RAM_ADDR_H #define SYSTEM_RAM_ADDR_H -#include "system/xen.h" -#include "system/tcg.h" -#include "exec/cputlb.h" -#include "exec/ramlist.h" #include "system/ramblock.h" -#include "system/memory.h" #include "exec/target_page.h" -#include "qemu/rcu.h" - #include "exec/hwaddr.h" -#include "exec/cpu-common.h" extern uint64_t total_dirty_pages; @@ -80,17 +72,6 @@ static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page) return bitmap_test_and_clear(rb->clear_bmap, page >> shift, 1); } -static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) -{ - return (b && b->host && offset < b->used_length) ? true : false; -} - -static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) -{ - assert(offset_in_ramblock(block, offset)); - return (char *)block->host + offset; -} - static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr, RAMBlock *rb) { @@ -99,8 +80,6 @@ static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr, return host_addr_offset >> TARGET_PAGE_BITS; } -bool ramblock_is_pmem(RAMBlock *rb); - /** * qemu_ram_alloc_from_file, * qemu_ram_alloc_from_fd: Allocate a ram block from the specified backing @@ -153,409 +132,4 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) qemu_ram_msync(block, 0, block->used_length); } -#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) -#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) - -static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, - ram_addr_t length, - unsigned client) -{ - DirtyMemoryBlocks *blocks; - unsigned long end, page; - unsigned long idx, offset, base; - bool dirty = false; - - assert(client < DIRTY_MEMORY_NUM); - - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - - WITH_RCU_READ_LOCK_GUARD() { - blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - unsigned long num = next - base; - unsigned long found = find_next_bit(blocks->blocks[idx], - num, offset); - if (found < num) { - dirty = true; - break; - } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; - } - } - - return dirty; -} - -static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, - ram_addr_t length, - unsigned client) -{ - DirtyMemoryBlocks *blocks; - unsigned long end, page; - unsigned long idx, offset, base; - bool dirty = true; - - assert(client < DIRTY_MEMORY_NUM); - - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - - RCU_READ_LOCK_GUARD(); - - blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - unsigned long num = next - base; - unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset); - if (found < num) { - dirty = false; - break; - } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; - } - - return dirty; -} - -static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, - unsigned client) -{ - return cpu_physical_memory_get_dirty(addr, 1, client); -} - -static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) -{ - bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); - bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); - bool migration = - cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); - return !(vga && code && migration); -} - -static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, - ram_addr_t length, - uint8_t mask) -{ - uint8_t ret = 0; - - if (mask & (1 << DIRTY_MEMORY_VGA) && - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { - ret |= (1 << DIRTY_MEMORY_VGA); - } - if (mask & (1 << DIRTY_MEMORY_CODE) && - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { - ret |= (1 << DIRTY_MEMORY_CODE); - } - if (mask & (1 << DIRTY_MEMORY_MIGRATION) && - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { - ret |= (1 << DIRTY_MEMORY_MIGRATION); - } - return ret; -} - -static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, - unsigned client) -{ - unsigned long page, idx, offset; - DirtyMemoryBlocks *blocks; - - assert(client < DIRTY_MEMORY_NUM); - - page = addr >> TARGET_PAGE_BITS; - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - - RCU_READ_LOCK_GUARD(); - - blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - - set_bit_atomic(offset, blocks->blocks[idx]); -} - -static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, - ram_addr_t length, - uint8_t mask) -{ - DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; - unsigned long end, page; - unsigned long idx, offset, base; - int i; - - if (!mask && !xen_enabled()) { - return; - } - - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - - WITH_RCU_READ_LOCK_GUARD() { - for (i = 0; i < DIRTY_MEMORY_NUM; i++) { - blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]); - } - - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - - if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], - offset, next - page); - } - if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], - offset, next - page); - } - if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], - offset, next - page); - } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; - } - } - - if (xen_enabled()) { - xen_hvm_modified_memory(start, length); - } -} - -#if !defined(_WIN32) - -/* - * Contrary to cpu_physical_memory_sync_dirty_bitmap() this function returns - * the number of dirty pages in @bitmap passed as argument. On the other hand, - * cpu_physical_memory_sync_dirty_bitmap() returns newly dirtied pages that - * weren't set in the global migration bitmap. - */ -static inline -uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, - ram_addr_t start, - ram_addr_t pages) -{ - unsigned long i, j; - unsigned long page_number, c, nbits; - hwaddr addr; - ram_addr_t ram_addr; - uint64_t num_dirty = 0; - unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; - unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE; - unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); - - /* start address is aligned at the start of a word? */ - if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && - (hpratio == 1)) { - unsigned long **blocks[DIRTY_MEMORY_NUM]; - unsigned long idx; - unsigned long offset; - long k; - long nr = BITS_TO_LONGS(pages); - - idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; - offset = BIT_WORD((start >> TARGET_PAGE_BITS) % - DIRTY_MEMORY_BLOCK_SIZE); - - WITH_RCU_READ_LOCK_GUARD() { - for (i = 0; i < DIRTY_MEMORY_NUM; i++) { - blocks[i] = - qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks; - } - - for (k = 0; k < nr; k++) { - if (bitmap[k]) { - unsigned long temp = leul_to_cpu(bitmap[k]); - - nbits = ctpopl(temp); - qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); - - if (global_dirty_tracking) { - qatomic_or( - &blocks[DIRTY_MEMORY_MIGRATION][idx][offset], - temp); - if (unlikely( - global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { - total_dirty_pages += nbits; - } - } - - num_dirty += nbits; - - if (tcg_enabled()) { - qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], - temp); - } - } - - if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { - offset = 0; - idx++; - } - } - } - - if (xen_enabled()) { - xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); - } - } else { - uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; - - if (!global_dirty_tracking) { - clients &= ~(1 << DIRTY_MEMORY_MIGRATION); - } - - /* - * bitmap-traveling is faster than memory-traveling (for addr...) - * especially when most of the memory is not dirty. - */ - for (i = 0; i < len; i++) { - if (bitmap[i] != 0) { - c = leul_to_cpu(bitmap[i]); - nbits = ctpopl(c); - if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { - total_dirty_pages += nbits; - } - num_dirty += nbits; - do { - j = ctzl(c); - c &= ~(1ul << j); - page_number = (i * HOST_LONG_BITS + j) * hpratio; - addr = page_number * TARGET_PAGE_SIZE; - ram_addr = start + addr; - cpu_physical_memory_set_dirty_range(ram_addr, - TARGET_PAGE_SIZE * hpratio, clients); - } while (c != 0); - } - } - } - - return num_dirty; -} -#endif /* not _WIN32 */ - -static inline void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, - ram_addr_t length) -{ - if (tcg_enabled()) { - tlb_reset_dirty_range_all(start, length); - } - -} -bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, - ram_addr_t length, - unsigned client); - -DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty - (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client); - -bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, - ram_addr_t start, - ram_addr_t length); - -static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, - ram_addr_t length) -{ - cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); - cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); - cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); -} - - -/* Called with RCU critical section */ -static inline -uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, - ram_addr_t start, - ram_addr_t length) -{ - ram_addr_t addr; - unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS); - uint64_t num_dirty = 0; - unsigned long *dest = rb->bmap; - - /* start address and length is aligned at the start of a word? */ - if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) == - (start + rb->offset) && - !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) { - int k; - int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); - unsigned long * const *src; - unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; - unsigned long offset = BIT_WORD((word * BITS_PER_LONG) % - DIRTY_MEMORY_BLOCK_SIZE); - unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); - - src = qatomic_rcu_read( - &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; - - for (k = page; k < page + nr; k++) { - if (src[idx][offset]) { - unsigned long bits = qatomic_xchg(&src[idx][offset], 0); - unsigned long new_dirty; - new_dirty = ~dest[k]; - dest[k] |= bits; - new_dirty &= bits; - num_dirty += ctpopl(new_dirty); - } - - if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { - offset = 0; - idx++; - } - } - if (num_dirty) { - cpu_physical_memory_dirty_bits_cleared(start, length); - } - - if (rb->clear_bmap) { - /* - * Postpone the dirty bitmap clear to the point before we - * really send the pages, also we will split the clear - * dirty procedure into smaller chunks. - */ - clear_bmap_set(rb, start >> TARGET_PAGE_BITS, - length >> TARGET_PAGE_BITS); - } else { - /* Slow path - still do that in a huge chunk */ - memory_region_clear_dirty_bitmap(rb->mr, start, length); - } - } else { - ram_addr_t offset = rb->offset; - - for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { - if (cpu_physical_memory_test_and_clear_dirty( - start + addr + offset, - TARGET_PAGE_SIZE, - DIRTY_MEMORY_MIGRATION)) { - long k = (start + addr) >> TARGET_PAGE_BITS; - if (!test_and_set_bit(k, dest)) { - num_dirty++; - } - } - } - } - - return num_dirty; -} - #endif diff --git a/include/system/ramblock.h b/include/system/ramblock.h index 87e847e184..76694fe1b5 100644 --- a/include/system/ramblock.h +++ b/include/system/ramblock.h @@ -11,11 +11,6 @@ * */ -/* - * This header is for use by exec.c and memory.c ONLY. Do not include it. - * The functions declared here will be removed soon. - */ - #ifndef SYSTEM_RAMBLOCK_H #define SYSTEM_RAMBLOCK_H @@ -108,9 +103,31 @@ struct RamBlockAttributes { QLIST_HEAD(, RamDiscardListener) rdl_list; }; +/* @offset: the offset within the RAMBlock */ +int ram_block_discard_range(RAMBlock *rb, uint64_t offset, size_t length); +/* @offset: the offset within the RAMBlock */ +int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t offset, + size_t length); + RamBlockAttributes *ram_block_attributes_create(RAMBlock *ram_block); void ram_block_attributes_destroy(RamBlockAttributes *attr); int ram_block_attributes_state_change(RamBlockAttributes *attr, uint64_t offset, uint64_t size, bool to_discard); +/** + * ram_block_is_pmem: Whether the RAM block is of persistent memory + */ +bool ram_block_is_pmem(RAMBlock *rb); + +static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) +{ + return b && b->host && (offset < b->used_length); +} + +static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) +{ + assert(offset_in_ramblock(block, offset)); + return (char *)block->host + offset; +} + #endif |