From 05e015f73c3b5c50c237d3d8e555e25cfa543a5c Mon Sep 17 00:00:00 2001 From: KONRAD Frederic Date: Thu, 21 Sep 2017 12:04:20 +0200 Subject: memory: avoid a name clash with access macro This avoids a name clash with the access macro on windows 64: make CHK version_gen.h CC aarch64-softmmu/memory.o /home/konrad/qemu/memory.c: In function 'access_with_adjusted_size': /home/konrad/qemu/memory.c:591:73: error: macro "access" passed 7 arguments, \ but takes just 2 (size - access_size - i) * 8, access_mask, attrs); ^ Signed-off-by: KONRAD Frederic Message-Id: <1505988260-8483-1-git-send-email-frederic.konrad@adacore.com> Signed-off-by: Paolo Bonzini --- memory.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index b9920a6540..2b90117c60 100644 --- a/memory.c +++ b/memory.c @@ -560,13 +560,14 @@ static MemTxResult access_with_adjusted_size(hwaddr addr, unsigned size, unsigned access_size_min, unsigned access_size_max, - MemTxResult (*access)(MemoryRegion *mr, - hwaddr addr, - uint64_t *value, - unsigned size, - unsigned shift, - uint64_t mask, - MemTxAttrs attrs), + MemTxResult (*access_fn) + (MemoryRegion *mr, + hwaddr addr, + uint64_t *value, + unsigned size, + unsigned shift, + uint64_t mask, + MemTxAttrs attrs), MemoryRegion *mr, MemTxAttrs attrs) { @@ -587,12 +588,12 @@ static MemTxResult access_with_adjusted_size(hwaddr addr, access_mask = -1ULL >> (64 - access_size * 8); if (memory_region_big_endian(mr)) { for (i = 0; i < size; i += access_size) { - r |= access(mr, addr + i, value, access_size, + r |= access_fn(mr, addr + i, value, access_size, (size - access_size - i) * 8, access_mask, attrs); } } else { for (i = 0; i < size; i += access_size) { - r |= access(mr, addr + i, value, access_size, i * 8, + r |= access_fn(mr, addr + i, value, access_size, i * 8, access_mask, attrs); } } -- cgit 1.4.1 From 447b0d0b9ee8a0ac216c3186e0f3c427a1001f0c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 21 Sep 2017 14:32:47 +0200 Subject: memory: avoid "resurrection" of dead FlatViews It's possible for address_space_get_flatview() as it currently stands to cause a use-after-free for the returned FlatView, if the reference count is incremented after the FlatView has been replaced by a writer: thread 1 thread 2 RCU thread ------------------------------------------------------------- rcu_read_lock read as->current_map set as->current_map flatview_unref '--> call_rcu flatview_ref [ref=1] rcu_read_unlock flatview_destroy Since FlatViews are not updated very often, we can just detect the situation using a new atomic op atomic_fetch_inc_nonzero, similar to Linux's atomic_inc_not_zero, which performs the refcount increment only if it hasn't already hit zero. This is similar to Linux commit de09a9771a53 ("CRED: Fix get_task_cred() and task_state() to not resurrect dead credentials", 2010-07-29). Signed-off-by: Paolo Bonzini --- docs/devel/atomics.txt | 1 + include/qemu/atomic.h | 8 ++++++++ memory.c | 12 ++++++++---- 3 files changed, 17 insertions(+), 4 deletions(-) (limited to 'memory.c') diff --git a/docs/devel/atomics.txt b/docs/devel/atomics.txt index 048e5f23cb..10c5fa37e8 100644 --- a/docs/devel/atomics.txt +++ b/docs/devel/atomics.txt @@ -64,6 +64,7 @@ operations: typeof(*ptr) atomic_fetch_and(ptr, val) typeof(*ptr) atomic_fetch_or(ptr, val) typeof(*ptr) atomic_fetch_xor(ptr, val) + typeof(*ptr) atomic_fetch_inc_nonzero(ptr) typeof(*ptr) atomic_xchg(ptr, val) typeof(*ptr) atomic_cmpxchg(ptr, old, new) diff --git a/include/qemu/atomic.h b/include/qemu/atomic.h index b6b62fb771..d73c9e14d7 100644 --- a/include/qemu/atomic.h +++ b/include/qemu/atomic.h @@ -442,4 +442,12 @@ } while(0) #endif +#define atomic_fetch_inc_nonzero(ptr) ({ \ + typeof_strip_qual(*ptr) _oldn = atomic_read(ptr); \ + while (_oldn && atomic_cmpxchg(ptr, _oldn, _oldn + 1) != _oldn) { \ + _oldn = atomic_read(ptr); \ + } \ + _oldn; \ +}) + #endif /* QEMU_ATOMIC_H */ diff --git a/memory.c b/memory.c index 2b90117c60..51f54ab430 100644 --- a/memory.c +++ b/memory.c @@ -294,9 +294,9 @@ static void flatview_destroy(FlatView *view) g_free(view); } -static void flatview_ref(FlatView *view) +static bool flatview_ref(FlatView *view) { - atomic_inc(&view->ref); + return atomic_fetch_inc_nonzero(&view->ref) > 0; } static void flatview_unref(FlatView *view) @@ -773,8 +773,12 @@ static FlatView *address_space_get_flatview(AddressSpace *as) FlatView *view; rcu_read_lock(); - view = atomic_rcu_read(&as->current_map); - flatview_ref(view); + do { + view = atomic_rcu_read(&as->current_map); + /* If somebody has replaced as->current_map concurrently, + * flatview_ref returns false. + */ + } while (!flatview_ref(view)); rcu_read_unlock(); return view; } -- cgit 1.4.1 From 9a62e24f45bc97f8eaf198caf58906b47c50a8d5 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:50:54 +1000 Subject: memory: Open code FlatView rendering We are going to share FlatView's between AddressSpace's and per-AS memory listeners won't suit the purpose anymore so open code the dispatch tree rendering. Since there is a good chance that dispatch_listener was the only listener, this avoids address_space_update_topology_pass() if there is no registered listeners; this should improve starting time. This should cause no behavioural change. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-3-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- exec.c | 27 +++------------------------ include/exec/memory-internal.h | 6 ++++-- include/exec/memory.h | 1 - memory.c | 19 ++++++++++++++----- 4 files changed, 21 insertions(+), 32 deletions(-) (limited to 'memory.c') diff --git a/exec.c b/exec.c index fd8994b25d..1626d254bb 100644 --- a/exec.c +++ b/exec.c @@ -1347,9 +1347,8 @@ static void register_multipage(AddressSpaceDispatch *d, phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index); } -static void mem_add(MemoryListener *listener, MemoryRegionSection *section) +void mem_add(AddressSpace *as, MemoryRegionSection *section) { - AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener); AddressSpaceDispatch *d = as->next_dispatch; MemoryRegionSection now = *section, remain = *section; Int128 page_size = int128_make64(TARGET_PAGE_SIZE); @@ -2673,9 +2672,8 @@ static void io_mem_init(void) NULL, UINT64_MAX); } -static void mem_begin(MemoryListener *listener) +void mem_begin(AddressSpace *as) { - AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener); AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1); uint16_t n; @@ -2699,9 +2697,8 @@ static void address_space_dispatch_free(AddressSpaceDispatch *d) g_free(d); } -static void mem_commit(MemoryListener *listener) +void mem_commit(AddressSpace *as) { - AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener); AddressSpaceDispatch *cur = as->dispatch; AddressSpaceDispatch *next = as->next_dispatch; @@ -2731,24 +2728,6 @@ static void tcg_commit(MemoryListener *listener) tlb_flush(cpuas->cpu); } -void address_space_init_dispatch(AddressSpace *as) -{ - as->dispatch = NULL; - as->dispatch_listener = (MemoryListener) { - .begin = mem_begin, - .commit = mem_commit, - .region_add = mem_add, - .region_nop = mem_add, - .priority = 0, - }; - memory_listener_register(&as->dispatch_listener, as); -} - -void address_space_unregister(AddressSpace *as) -{ - memory_listener_unregister(&as->dispatch_listener); -} - void address_space_destroy_dispatch(AddressSpace *as) { AddressSpaceDispatch *d = as->dispatch; diff --git a/include/exec/memory-internal.h b/include/exec/memory-internal.h index fb467acdba..9abde2f11c 100644 --- a/include/exec/memory-internal.h +++ b/include/exec/memory-internal.h @@ -22,8 +22,6 @@ #ifndef CONFIG_USER_ONLY typedef struct AddressSpaceDispatch AddressSpaceDispatch; -void address_space_init_dispatch(AddressSpace *as); -void address_space_unregister(AddressSpace *as); void address_space_destroy_dispatch(AddressSpace *as); extern const MemoryRegionOps unassigned_mem_ops; @@ -31,5 +29,9 @@ extern const MemoryRegionOps unassigned_mem_ops; bool memory_region_access_valid(MemoryRegion *mr, hwaddr addr, unsigned size, bool is_write); +void mem_add(AddressSpace *as, MemoryRegionSection *section); +void mem_begin(AddressSpace *as); +void mem_commit(AddressSpace *as); + #endif #endif diff --git a/include/exec/memory.h b/include/exec/memory.h index 1dcd3122d7..9581f7a7db 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -318,7 +318,6 @@ struct AddressSpace { struct MemoryRegionIoeventfd *ioeventfds; struct AddressSpaceDispatch *dispatch; struct AddressSpaceDispatch *next_dispatch; - MemoryListener dispatch_listener; QTAILQ_HEAD(memory_listeners_as, MemoryListener) listeners; QTAILQ_ENTRY(AddressSpace) address_spaces_link; }; diff --git a/memory.c b/memory.c index 51f54ab430..3241e449e7 100644 --- a/memory.c +++ b/memory.c @@ -884,14 +884,24 @@ static void address_space_update_topology_pass(AddressSpace *as, } } - static void address_space_update_topology(AddressSpace *as) { FlatView *old_view = address_space_get_flatview(as); FlatView *new_view = generate_memory_topology(as->root); + int i; - address_space_update_topology_pass(as, old_view, new_view, false); - address_space_update_topology_pass(as, old_view, new_view, true); + mem_begin(as); + for (i = 0; i < new_view->nr; i++) { + MemoryRegionSection mrs = + section_from_flat_range(&new_view->ranges[i], as); + mem_add(as, &mrs); + } + mem_commit(as); + + if (!QTAILQ_EMPTY(&as->listeners)) { + address_space_update_topology_pass(as, old_view, new_view, false); + address_space_update_topology_pass(as, old_view, new_view, true); + } /* Writes are protected by the BQL. */ atomic_rcu_set(&as->current_map, new_view); @@ -2626,7 +2636,7 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) QTAILQ_INIT(&as->listeners); QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); as->name = g_strdup(name ? name : "anonymous"); - address_space_init_dispatch(as); + as->dispatch = NULL; memory_region_update_pending |= root->enabled; memory_region_transaction_commit(); } @@ -2677,7 +2687,6 @@ void address_space_destroy(AddressSpace *as) as->root = NULL; memory_region_transaction_commit(); QTAILQ_REMOVE(&address_spaces, as, address_spaces_link); - address_space_unregister(as); /* At this point, as->dispatch and as->current_map are dummy * entries that the guest should never use. Wait for the old -- cgit 1.4.1 From cc94cd6d36602d976a5e7bc29134d1eaefb4102e Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:50:55 +1000 Subject: memory: Move FlatView allocation to a helper This moves a FlatView allocation and initialization to a helper. While we are nere, replace g_new with g_new0 to not to bother if we add new fields in the future. This should cause no behavioural change. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-4-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- memory.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index 3241e449e7..eec668eec7 100644 --- a/memory.c +++ b/memory.c @@ -258,12 +258,14 @@ static bool flatrange_equal(FlatRange *a, FlatRange *b) && a->readonly == b->readonly; } -static void flatview_init(FlatView *view) +static FlatView *flatview_new(void) { + FlatView *view; + + view = g_new0(FlatView, 1); view->ref = 1; - view->ranges = NULL; - view->nr = 0; - view->nr_allocated = 0; + + return view; } /* Insert a range into a given position. Caller is responsible for maintaining @@ -707,8 +709,7 @@ static FlatView *generate_memory_topology(MemoryRegion *mr) { FlatView *view; - view = g_new(FlatView, 1); - flatview_init(view); + view = flatview_new(); if (mr) { render_memory_region(view, mr, int128_zero(), @@ -2629,8 +2630,7 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) as->ref_count = 1; as->root = root; as->malloced = false; - as->current_map = g_new(FlatView, 1); - flatview_init(as->current_map); + as->current_map = flatview_new(); as->ioeventfd_nb = 0; as->ioeventfds = NULL; QTAILQ_INIT(&as->listeners); -- cgit 1.4.1 From 66a6df1dc6d5b28cc3e65db0d71683fbdddc6b62 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:50:56 +1000 Subject: memory: Move AddressSpaceDispatch from AddressSpace to FlatView As we are going to share FlatView's between AddressSpace's, and AddressSpaceDispatch is a structure to perform quick lookup in FlatView, this moves ASD to FlatView. After previosly open coded ASD rendering, we can also remove as->next_dispatch as the new FlatView pointer is stored on a stack and set to an AS atomically. flatview_destroy() is executed under RCU instead of address_space_dispatch_free() now. This makes mem_begin/mem_commit to work with ASD and mem_add with FV as later on mem_add will be taking FV as an argument anyway. This should cause no behavioural change. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-5-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- exec.c | 41 +++++++++++------------------------------ include/exec/memory-internal.h | 12 +++++++----- include/exec/memory.h | 2 -- memory.c | 31 ++++++++++++++++++++++++------- 4 files changed, 42 insertions(+), 44 deletions(-) (limited to 'memory.c') diff --git a/exec.c b/exec.c index 1626d254bb..afd64127e6 100644 --- a/exec.c +++ b/exec.c @@ -187,8 +187,6 @@ typedef struct PhysPageMap { } PhysPageMap; struct AddressSpaceDispatch { - struct rcu_head rcu; - MemoryRegionSection *mru_section; /* This is a multi-level map on the physical address space. * The bottom level has pointers to MemoryRegionSections. @@ -485,7 +483,7 @@ static MemoryRegionSection address_space_do_translate(AddressSpace *as, IOMMUMemoryRegionClass *imrc; for (;;) { - AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch); + AddressSpaceDispatch *d = address_space_to_dispatch(as); section = address_space_translate_internal(d, addr, &addr, plen, is_mmio); iommu_mr = memory_region_get_iommu(section->mr); @@ -1222,7 +1220,7 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu, } else { AddressSpaceDispatch *d; - d = atomic_rcu_read(§ion->address_space->dispatch); + d = address_space_to_dispatch(section->address_space); iotlb = section - d->map.sections; iotlb += xlat; } @@ -1347,9 +1345,9 @@ static void register_multipage(AddressSpaceDispatch *d, phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index); } -void mem_add(AddressSpace *as, MemoryRegionSection *section) +void mem_add(AddressSpace *as, FlatView *fv, MemoryRegionSection *section) { - AddressSpaceDispatch *d = as->next_dispatch; + AddressSpaceDispatch *d = flatview_to_dispatch(fv); MemoryRegionSection now = *section, remain = *section; Int128 page_size = int128_make64(TARGET_PAGE_SIZE); @@ -2672,7 +2670,7 @@ static void io_mem_init(void) NULL, UINT64_MAX); } -void mem_begin(AddressSpace *as) +AddressSpaceDispatch *mem_begin(AddressSpace *as) { AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1); uint16_t n; @@ -2688,26 +2686,19 @@ void mem_begin(AddressSpace *as) d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 }; d->as = as; - as->next_dispatch = d; + + return d; } -static void address_space_dispatch_free(AddressSpaceDispatch *d) +void address_space_dispatch_free(AddressSpaceDispatch *d) { phys_sections_free(&d->map); g_free(d); } -void mem_commit(AddressSpace *as) +void mem_commit(AddressSpaceDispatch *d) { - AddressSpaceDispatch *cur = as->dispatch; - AddressSpaceDispatch *next = as->next_dispatch; - - phys_page_compact_all(next, next->map.nodes_nb); - - atomic_rcu_set(&as->dispatch, next); - if (cur) { - call_rcu(cur, address_space_dispatch_free, rcu); - } + phys_page_compact_all(d, d->map.nodes_nb); } static void tcg_commit(MemoryListener *listener) @@ -2723,21 +2714,11 @@ static void tcg_commit(MemoryListener *listener) * We reload the dispatch pointer now because cpu_reloading_memory_map() * may have split the RCU critical section. */ - d = atomic_rcu_read(&cpuas->as->dispatch); + d = address_space_to_dispatch(cpuas->as); atomic_rcu_set(&cpuas->memory_dispatch, d); tlb_flush(cpuas->cpu); } -void address_space_destroy_dispatch(AddressSpace *as) -{ - AddressSpaceDispatch *d = as->dispatch; - - atomic_rcu_set(&as->dispatch, NULL); - if (d) { - call_rcu(d, address_space_dispatch_free, rcu); - } -} - static void memory_map_init(void) { system_memory = g_malloc(sizeof(*system_memory)); diff --git a/include/exec/memory-internal.h b/include/exec/memory-internal.h index 9abde2f11c..6e08eda256 100644 --- a/include/exec/memory-internal.h +++ b/include/exec/memory-internal.h @@ -22,16 +22,18 @@ #ifndef CONFIG_USER_ONLY typedef struct AddressSpaceDispatch AddressSpaceDispatch; -void address_space_destroy_dispatch(AddressSpace *as); - extern const MemoryRegionOps unassigned_mem_ops; bool memory_region_access_valid(MemoryRegion *mr, hwaddr addr, unsigned size, bool is_write); -void mem_add(AddressSpace *as, MemoryRegionSection *section); -void mem_begin(AddressSpace *as); -void mem_commit(AddressSpace *as); +void mem_add(AddressSpace *as, FlatView *fv, MemoryRegionSection *section); +AddressSpaceDispatch *mem_begin(AddressSpace *as); +void mem_commit(AddressSpaceDispatch *d); + +AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as); +AddressSpaceDispatch *flatview_to_dispatch(FlatView *fv); +void address_space_dispatch_free(AddressSpaceDispatch *d); #endif #endif diff --git a/include/exec/memory.h b/include/exec/memory.h index 9581f7a7db..2346f8b863 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -316,8 +316,6 @@ struct AddressSpace { int ioeventfd_nb; struct MemoryRegionIoeventfd *ioeventfds; - struct AddressSpaceDispatch *dispatch; - struct AddressSpaceDispatch *next_dispatch; QTAILQ_HEAD(memory_listeners_as, MemoryListener) listeners; QTAILQ_ENTRY(AddressSpace) address_spaces_link; }; diff --git a/memory.c b/memory.c index eec668eec7..962e9b961f 100644 --- a/memory.c +++ b/memory.c @@ -229,6 +229,7 @@ struct FlatView { FlatRange *ranges; unsigned nr; unsigned nr_allocated; + struct AddressSpaceDispatch *dispatch; }; typedef struct AddressSpaceOps AddressSpaceOps; @@ -289,6 +290,9 @@ static void flatview_destroy(FlatView *view) { int i; + if (view->dispatch) { + address_space_dispatch_free(view->dispatch); + } for (i = 0; i < view->nr; i++) { memory_region_unref(view->ranges[i].mr); } @@ -304,10 +308,25 @@ static bool flatview_ref(FlatView *view) static void flatview_unref(FlatView *view) { if (atomic_fetch_dec(&view->ref) == 1) { - flatview_destroy(view); + call_rcu(view, flatview_destroy, rcu); } } +static FlatView *address_space_to_flatview(AddressSpace *as) +{ + return atomic_rcu_read(&as->current_map); +} + +AddressSpaceDispatch *flatview_to_dispatch(FlatView *fv) +{ + return fv->dispatch; +} + +AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as) +{ + return flatview_to_dispatch(address_space_to_flatview(as)); +} + static bool can_merge(FlatRange *r1, FlatRange *r2) { return int128_eq(addrrange_end(r1->addr), r2->addr.start) @@ -891,13 +910,13 @@ static void address_space_update_topology(AddressSpace *as) FlatView *new_view = generate_memory_topology(as->root); int i; - mem_begin(as); + new_view->dispatch = mem_begin(as); for (i = 0; i < new_view->nr; i++) { MemoryRegionSection mrs = section_from_flat_range(&new_view->ranges[i], as); - mem_add(as, &mrs); + mem_add(as, new_view, &mrs); } - mem_commit(as); + mem_commit(new_view->dispatch); if (!QTAILQ_EMPTY(&as->listeners)) { address_space_update_topology_pass(as, old_view, new_view, false); @@ -906,7 +925,7 @@ static void address_space_update_topology(AddressSpace *as) /* Writes are protected by the BQL. */ atomic_rcu_set(&as->current_map, new_view); - call_rcu(old_view, flatview_unref, rcu); + flatview_unref(old_view); /* Note that all the old MemoryRegions are still alive up to this * point. This relieves most MemoryListeners from the need to @@ -2636,7 +2655,6 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) QTAILQ_INIT(&as->listeners); QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); as->name = g_strdup(name ? name : "anonymous"); - as->dispatch = NULL; memory_region_update_pending |= root->enabled; memory_region_transaction_commit(); } @@ -2645,7 +2663,6 @@ static void do_address_space_destroy(AddressSpace *as) { bool do_free = as->malloced; - address_space_destroy_dispatch(as); assert(QTAILQ_EMPTY(&as->listeners)); flatview_unref(as->current_map); -- cgit 1.4.1 From 166206845f7fd75e720e6feea0bb01957c8da07f Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:50:58 +1000 Subject: memory: Switch memory from using AddressSpace to FlatView FlatView's will be shared between AddressSpace's and subpage_t and MemoryRegionSection cannot store AS anymore, hence this change. In particular, for: typedef struct subpage_t { MemoryRegion iomem; - AddressSpace *as; + FlatView *fv; hwaddr base; uint16_t sub_section[]; } subpage_t; struct MemoryRegionSection { MemoryRegion *mr; - AddressSpace *address_space; + FlatView *fv; hwaddr offset_within_region; Int128 size; hwaddr offset_within_address_space; bool readonly; }; This should cause no behavioural change. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-7-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- exec.c | 180 ++++++++++++++++++++++++----------------- hw/intc/openpic_kvm.c | 2 +- include/exec/memory-internal.h | 2 +- include/exec/memory.h | 51 ++++++++---- memory.c | 33 ++++---- 5 files changed, 159 insertions(+), 109 deletions(-) (limited to 'memory.c') diff --git a/exec.c b/exec.c index a54dde7835..d2b9f60494 100644 --- a/exec.c +++ b/exec.c @@ -198,7 +198,7 @@ struct AddressSpaceDispatch { #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK) typedef struct subpage_t { MemoryRegion iomem; - AddressSpace *as; + FlatView *fv; hwaddr base; uint16_t sub_section[]; } subpage_t; @@ -468,13 +468,13 @@ address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *x } /* Called from RCU critical section */ -static MemoryRegionSection address_space_do_translate(AddressSpace *as, - hwaddr addr, - hwaddr *xlat, - hwaddr *plen, - bool is_write, - bool is_mmio, - AddressSpace **target_as) +static MemoryRegionSection flatview_do_translate(FlatView *fv, + hwaddr addr, + hwaddr *xlat, + hwaddr *plen, + bool is_write, + bool is_mmio, + AddressSpace **target_as) { IOMMUTLBEntry iotlb; MemoryRegionSection *section; @@ -482,8 +482,9 @@ static MemoryRegionSection address_space_do_translate(AddressSpace *as, IOMMUMemoryRegionClass *imrc; for (;;) { - AddressSpaceDispatch *d = address_space_to_dispatch(as); - section = address_space_translate_internal(d, addr, &addr, plen, is_mmio); + section = address_space_translate_internal( + flatview_to_dispatch(fv), addr, &addr, + plen, is_mmio); iommu_mr = memory_region_get_iommu(section->mr); if (!iommu_mr) { @@ -500,7 +501,7 @@ static MemoryRegionSection address_space_do_translate(AddressSpace *as, goto translate_fail; } - as = iotlb.target_as; + fv = address_space_to_flatview(iotlb.target_as); *target_as = iotlb.target_as; } @@ -523,8 +524,8 @@ IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr, plen = (hwaddr)-1; /* This can never be MMIO. */ - section = address_space_do_translate(as, addr, &xlat, &plen, - is_write, false, &as); + section = flatview_do_translate(address_space_to_flatview(as), addr, + &xlat, &plen, is_write, false, &as); /* Illegal translation */ if (section.mr == &io_mem_unassigned) { @@ -560,16 +561,15 @@ iotlb_fail: } /* Called from RCU critical section */ -MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr, - hwaddr *xlat, hwaddr *plen, - bool is_write) +MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat, + hwaddr *plen, bool is_write) { MemoryRegion *mr; MemoryRegionSection section; + AddressSpace *as = NULL; /* This can be MMIO, so setup MMIO bit. */ - section = address_space_do_translate(as, addr, xlat, plen, is_write, true, - &as); + section = flatview_do_translate(fv, addr, xlat, plen, is_write, true, &as); mr = section.mr; if (xen_enabled() && memory_access_is_direct(mr, is_write)) { @@ -1219,7 +1219,7 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu, } else { AddressSpaceDispatch *d; - d = address_space_to_dispatch(section->address_space); + d = flatview_to_dispatch(section->fv); iotlb = section - d->map.sections; iotlb += xlat; } @@ -1245,7 +1245,7 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu, static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, uint16_t section); -static subpage_t *subpage_init(AddressSpace *as, hwaddr base); +static subpage_t *subpage_init(FlatView *fv, hwaddr base); static void *(*phys_mem_alloc)(size_t size, uint64_t *align) = qemu_anon_ram_alloc; @@ -1302,7 +1302,7 @@ static void phys_sections_free(PhysPageMap *map) g_free(map->nodes); } -static void register_subpage(AddressSpace *as, AddressSpaceDispatch *d, +static void register_subpage(FlatView *fv, AddressSpaceDispatch *d, MemoryRegionSection *section) { subpage_t *subpage; @@ -1318,8 +1318,8 @@ static void register_subpage(AddressSpace *as, AddressSpaceDispatch *d, assert(existing->mr->subpage || existing->mr == &io_mem_unassigned); if (!(existing->mr->subpage)) { - subpage = subpage_init(as, base); - subsection.address_space = as; + subpage = subpage_init(fv, base); + subsection.fv = fv; subsection.mr = &subpage->iomem; phys_page_set(d, base >> TARGET_PAGE_BITS, 1, phys_section_add(&d->map, &subsection)); @@ -1345,7 +1345,7 @@ static void register_multipage(AddressSpaceDispatch *d, phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index); } -void mem_add(AddressSpace *as, FlatView *fv, MemoryRegionSection *section) +void mem_add(FlatView *fv, MemoryRegionSection *section) { AddressSpaceDispatch *d = flatview_to_dispatch(fv); MemoryRegionSection now = *section, remain = *section; @@ -1356,7 +1356,7 @@ void mem_add(AddressSpace *as, FlatView *fv, MemoryRegionSection *section) - now.offset_within_address_space; now.size = int128_min(int128_make64(left), now.size); - register_subpage(as, d, &now); + register_subpage(fv, d, &now); } else { now.size = int128_zero(); } @@ -1366,10 +1366,10 @@ void mem_add(AddressSpace *as, FlatView *fv, MemoryRegionSection *section) remain.offset_within_region += int128_get64(now.size); now = remain; if (int128_lt(remain.size, page_size)) { - register_subpage(as, d, &now); + register_subpage(fv, d, &now); } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) { now.size = page_size; - register_subpage(as, d, &now); + register_subpage(fv, d, &now); } else { now.size = int128_and(now.size, int128_neg(page_size)); register_multipage(d, &now); @@ -2500,6 +2500,11 @@ static const MemoryRegionOps watch_mem_ops = { .endianness = DEVICE_NATIVE_ENDIAN, }; +static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, + const uint8_t *buf, int len); +static bool flatview_access_valid(FlatView *fv, hwaddr addr, int len, + bool is_write); + static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data, unsigned len, MemTxAttrs attrs) { @@ -2511,8 +2516,7 @@ static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data, printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__, subpage, len, addr); #endif - res = address_space_read(subpage->as, addr + subpage->base, - attrs, buf, len); + res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len); if (res) { return res; } @@ -2561,8 +2565,7 @@ static MemTxResult subpage_write(void *opaque, hwaddr addr, default: abort(); } - return address_space_write(subpage->as, addr + subpage->base, - attrs, buf, len); + return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len); } static bool subpage_accepts(void *opaque, hwaddr addr, @@ -2574,8 +2577,8 @@ static bool subpage_accepts(void *opaque, hwaddr addr, __func__, subpage, is_write ? 'w' : 'r', len, addr); #endif - return address_space_access_valid(subpage->as, addr + subpage->base, - len, is_write); + return flatview_access_valid(subpage->fv, addr + subpage->base, + len, is_write); } static const MemoryRegionOps subpage_ops = { @@ -2609,12 +2612,12 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, return 0; } -static subpage_t *subpage_init(AddressSpace *as, hwaddr base) +static subpage_t *subpage_init(FlatView *fv, hwaddr base) { subpage_t *mmio; mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t)); - mmio->as = as; + mmio->fv = fv; mmio->base = base; memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio, NULL, TARGET_PAGE_SIZE); @@ -2628,12 +2631,11 @@ static subpage_t *subpage_init(AddressSpace *as, hwaddr base) return mmio; } -static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as, - MemoryRegion *mr) +static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr) { - assert(as); + assert(fv); MemoryRegionSection section = { - .address_space = as, + .fv = fv, .mr = mr, .offset_within_address_space = 0, .offset_within_region = 0, @@ -2672,16 +2674,17 @@ static void io_mem_init(void) AddressSpaceDispatch *mem_begin(AddressSpace *as) { + FlatView *fv = address_space_to_flatview(as); AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1); uint16_t n; - n = dummy_section(&d->map, as, &io_mem_unassigned); + n = dummy_section(&d->map, fv, &io_mem_unassigned); assert(n == PHYS_SECTION_UNASSIGNED); - n = dummy_section(&d->map, as, &io_mem_notdirty); + n = dummy_section(&d->map, fv, &io_mem_notdirty); assert(n == PHYS_SECTION_NOTDIRTY); - n = dummy_section(&d->map, as, &io_mem_rom); + n = dummy_section(&d->map, fv, &io_mem_rom); assert(n == PHYS_SECTION_ROM); - n = dummy_section(&d->map, as, &io_mem_watch); + n = dummy_section(&d->map, fv, &io_mem_watch); assert(n == PHYS_SECTION_WATCH); d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 }; @@ -2861,11 +2864,11 @@ static bool prepare_mmio_access(MemoryRegion *mr) } /* Called within RCU critical section. */ -static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, - const uint8_t *buf, - int len, hwaddr addr1, - hwaddr l, MemoryRegion *mr) +static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr, + MemTxAttrs attrs, + const uint8_t *buf, + int len, hwaddr addr1, + hwaddr l, MemoryRegion *mr) { uint8_t *ptr; uint64_t val; @@ -2927,14 +2930,14 @@ static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr, } l = len; - mr = address_space_translate(as, addr, &addr1, &l, true); + mr = flatview_translate(fv, addr, &addr1, &l, true); } return result; } -MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, - const uint8_t *buf, int len) +static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, + const uint8_t *buf, int len) { hwaddr l; hwaddr addr1; @@ -2944,20 +2947,27 @@ MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, if (len > 0) { rcu_read_lock(); l = len; - mr = address_space_translate(as, addr, &addr1, &l, true); - result = address_space_write_continue(as, addr, attrs, buf, len, - addr1, l, mr); + mr = flatview_translate(fv, addr, &addr1, &l, true); + result = flatview_write_continue(fv, addr, attrs, buf, len, + addr1, l, mr); rcu_read_unlock(); } return result; } +MemTxResult address_space_write(AddressSpace *as, hwaddr addr, + MemTxAttrs attrs, + const uint8_t *buf, int len) +{ + return flatview_write(address_space_to_flatview(as), addr, attrs, buf, len); +} + /* Called within RCU critical section. */ -MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, uint8_t *buf, - int len, hwaddr addr1, hwaddr l, - MemoryRegion *mr) +MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, + int len, hwaddr addr1, hwaddr l, + MemoryRegion *mr) { uint8_t *ptr; uint64_t val; @@ -3017,14 +3027,14 @@ MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr, } l = len; - mr = address_space_translate(as, addr, &addr1, &l, false); + mr = flatview_translate(fv, addr, &addr1, &l, false); } return result; } -MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, uint8_t *buf, int len) +MemTxResult flatview_read_full(FlatView *fv, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, int len) { hwaddr l; hwaddr addr1; @@ -3034,25 +3044,33 @@ MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, if (len > 0) { rcu_read_lock(); l = len; - mr = address_space_translate(as, addr, &addr1, &l, false); - result = address_space_read_continue(as, addr, attrs, buf, len, - addr1, l, mr); + mr = flatview_translate(fv, addr, &addr1, &l, false); + result = flatview_read_continue(fv, addr, attrs, buf, len, + addr1, l, mr); rcu_read_unlock(); } return result; } -MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, - uint8_t *buf, int len, bool is_write) +static MemTxResult flatview_rw(FlatView *fv, hwaddr addr, MemTxAttrs attrs, + uint8_t *buf, int len, bool is_write) { if (is_write) { - return address_space_write(as, addr, attrs, (uint8_t *)buf, len); + return flatview_write(fv, addr, attrs, (uint8_t *)buf, len); } else { - return address_space_read(as, addr, attrs, (uint8_t *)buf, len); + return flatview_read(fv, addr, attrs, (uint8_t *)buf, len); } } +MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, + int len, bool is_write) +{ + return flatview_rw(address_space_to_flatview(as), + addr, attrs, buf, len, is_write); +} + void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf, int len, int is_write) { @@ -3210,7 +3228,8 @@ static void cpu_notify_map_clients(void) qemu_mutex_unlock(&map_client_list_lock); } -bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write) +static bool flatview_access_valid(FlatView *fv, hwaddr addr, int len, + bool is_write) { MemoryRegion *mr; hwaddr l, xlat; @@ -3218,7 +3237,7 @@ bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_ rcu_read_lock(); while (len > 0) { l = len; - mr = address_space_translate(as, addr, &xlat, &l, is_write); + mr = flatview_translate(fv, addr, &xlat, &l, is_write); if (!memory_access_is_direct(mr, is_write)) { l = memory_access_size(mr, l, addr); if (!memory_region_access_valid(mr, xlat, l, is_write)) { @@ -3234,8 +3253,16 @@ bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_ return true; } +bool address_space_access_valid(AddressSpace *as, hwaddr addr, + int len, bool is_write) +{ + return flatview_access_valid(address_space_to_flatview(as), + addr, len, is_write); +} + static hwaddr -address_space_extend_translation(AddressSpace *as, hwaddr addr, hwaddr target_len, +flatview_extend_translation(FlatView *fv, hwaddr addr, + hwaddr target_len, MemoryRegion *mr, hwaddr base, hwaddr len, bool is_write) { @@ -3252,7 +3279,8 @@ address_space_extend_translation(AddressSpace *as, hwaddr addr, hwaddr target_le } len = target_len; - this_mr = address_space_translate(as, addr, &xlat, &len, is_write); + this_mr = flatview_translate(fv, addr, &xlat, + &len, is_write); if (this_mr != mr || xlat != base + done) { return done; } @@ -3275,6 +3303,7 @@ void *address_space_map(AddressSpace *as, hwaddr l, xlat; MemoryRegion *mr; void *ptr; + FlatView *fv = address_space_to_flatview(as); if (len == 0) { return NULL; @@ -3282,7 +3311,7 @@ void *address_space_map(AddressSpace *as, l = len; rcu_read_lock(); - mr = address_space_translate(as, addr, &xlat, &l, is_write); + mr = flatview_translate(fv, addr, &xlat, &l, is_write); if (!memory_access_is_direct(mr, is_write)) { if (atomic_xchg(&bounce.in_use, true)) { @@ -3298,7 +3327,7 @@ void *address_space_map(AddressSpace *as, memory_region_ref(mr); bounce.mr = mr; if (!is_write) { - address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED, + flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED, bounce.buffer, l); } @@ -3309,7 +3338,8 @@ void *address_space_map(AddressSpace *as, memory_region_ref(mr); - *plen = address_space_extend_translation(as, addr, len, mr, xlat, l, is_write); + *plen = flatview_extend_translation(fv, addr, len, mr, xlat, + l, is_write); ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen, true); rcu_read_unlock(); diff --git a/hw/intc/openpic_kvm.c b/hw/intc/openpic_kvm.c index 0518e017c4..fa83420254 100644 --- a/hw/intc/openpic_kvm.c +++ b/hw/intc/openpic_kvm.c @@ -124,7 +124,7 @@ static void kvm_openpic_region_add(MemoryListener *listener, uint64_t reg_base; int ret; - if (section->address_space != &address_space_memory) { + if (section->fv != address_space_to_flatview(&address_space_memory)) { abort(); } diff --git a/include/exec/memory-internal.h b/include/exec/memory-internal.h index 6e08eda256..1cf8ad9869 100644 --- a/include/exec/memory-internal.h +++ b/include/exec/memory-internal.h @@ -27,7 +27,7 @@ extern const MemoryRegionOps unassigned_mem_ops; bool memory_region_access_valid(MemoryRegion *mr, hwaddr addr, unsigned size, bool is_write); -void mem_add(AddressSpace *as, FlatView *fv, MemoryRegionSection *section); +void mem_add(FlatView *fv, MemoryRegionSection *section); AddressSpaceDispatch *mem_begin(AddressSpace *as); void mem_commit(AddressSpaceDispatch *d); diff --git a/include/exec/memory.h b/include/exec/memory.h index 2346f8b863..7816e5d655 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -48,6 +48,7 @@ typedef struct MemoryRegionOps MemoryRegionOps; typedef struct MemoryRegionMmio MemoryRegionMmio; +typedef struct FlatView FlatView; struct MemoryRegionMmio { CPUReadMemoryFunc *read[3]; @@ -320,6 +321,8 @@ struct AddressSpace { QTAILQ_ENTRY(AddressSpace) address_spaces_link; }; +FlatView *address_space_to_flatview(AddressSpace *as); + /** * MemoryRegionSection: describes a fragment of a #MemoryRegion * @@ -333,7 +336,7 @@ struct AddressSpace { */ struct MemoryRegionSection { MemoryRegion *mr; - AddressSpace *address_space; + FlatView *fv; hwaddr offset_within_region; Int128 size; hwaddr offset_within_address_space; @@ -1842,9 +1845,17 @@ IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr, * @len: pointer to length * @is_write: indicates the transfer direction */ -MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr, - hwaddr *xlat, hwaddr *len, - bool is_write); +MemoryRegion *flatview_translate(FlatView *fv, + hwaddr addr, hwaddr *xlat, + hwaddr *len, bool is_write); + +static inline MemoryRegion *address_space_translate(AddressSpace *as, + hwaddr addr, hwaddr *xlat, + hwaddr *len, bool is_write) +{ + return flatview_translate(address_space_to_flatview(as), + addr, xlat, len, is_write); +} /* address_space_access_valid: check for validity of accessing an address * space range @@ -1895,12 +1906,13 @@ void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len, /* Internal functions, part of the implementation of address_space_read. */ -MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, uint8_t *buf, - int len, hwaddr addr1, hwaddr l, - MemoryRegion *mr); -MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, uint8_t *buf, int len); +MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, + int len, hwaddr addr1, hwaddr l, + MemoryRegion *mr); + +MemTxResult flatview_read_full(FlatView *fv, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, int len); void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr); static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write) @@ -1927,8 +1939,8 @@ static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write) * @buf: buffer with the data transferred */ static inline __attribute__((__always_inline__)) -MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, - uint8_t *buf, int len) +MemTxResult flatview_read(FlatView *fv, hwaddr addr, MemTxAttrs attrs, + uint8_t *buf, int len) { MemTxResult result = MEMTX_OK; hwaddr l, addr1; @@ -1939,22 +1951,29 @@ MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, if (len) { rcu_read_lock(); l = len; - mr = address_space_translate(as, addr, &addr1, &l, false); + mr = flatview_translate(fv, addr, &addr1, &l, false); if (len == l && memory_access_is_direct(mr, false)) { ptr = qemu_map_ram_ptr(mr->ram_block, addr1); memcpy(buf, ptr, len); } else { - result = address_space_read_continue(as, addr, attrs, buf, len, - addr1, l, mr); + result = flatview_read_continue(fv, addr, attrs, buf, len, + addr1, l, mr); } rcu_read_unlock(); } } else { - result = address_space_read_full(as, addr, attrs, buf, len); + result = flatview_read_full(fv, addr, attrs, buf, len); } return result; } +static inline MemTxResult address_space_read(AddressSpace *as, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, + int len) +{ + return flatview_read(address_space_to_flatview(as), addr, attrs, buf, len); +} + /** * address_space_read_cached: read from a cached RAM region * diff --git a/memory.c b/memory.c index 962e9b961f..bf3728ae2f 100644 --- a/memory.c +++ b/memory.c @@ -154,7 +154,8 @@ enum ListenerDirection { Forward, Reverse }; /* No need to ref/unref .mr, the FlatRange keeps it alive. */ #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...) \ do { \ - MemoryRegionSection mrs = section_from_flat_range(fr, as); \ + MemoryRegionSection mrs = section_from_flat_range(fr, \ + address_space_to_flatview(as)); \ MEMORY_LISTENER_CALL(as, callback, dir, &mrs, ##_args); \ } while(0) @@ -208,7 +209,6 @@ static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd a, } typedef struct FlatRange FlatRange; -typedef struct FlatView FlatView; /* Range of memory in the global map. Addresses are absolute. */ struct FlatRange { @@ -238,11 +238,11 @@ typedef struct AddressSpaceOps AddressSpaceOps; for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) static inline MemoryRegionSection -section_from_flat_range(FlatRange *fr, AddressSpace *as) +section_from_flat_range(FlatRange *fr, FlatView *fv) { return (MemoryRegionSection) { .mr = fr->mr, - .address_space = as, + .fv = fv, .offset_within_region = fr->offset_in_region, .size = fr->addr.size, .offset_within_address_space = int128_get64(fr->addr.start), @@ -312,7 +312,7 @@ static void flatview_unref(FlatView *view) } } -static FlatView *address_space_to_flatview(AddressSpace *as) +FlatView *address_space_to_flatview(AddressSpace *as) { return atomic_rcu_read(&as->current_map); } @@ -761,7 +761,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, fds_new[inew]))) { fd = &fds_old[iold]; section = (MemoryRegionSection) { - .address_space = as, + .fv = address_space_to_flatview(as), .offset_within_address_space = int128_get64(fd->addr.start), .size = fd->addr.size, }; @@ -774,7 +774,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, fds_old[iold]))) { fd = &fds_new[inew]; section = (MemoryRegionSection) { - .address_space = as, + .fv = address_space_to_flatview(as), .offset_within_address_space = int128_get64(fd->addr.start), .size = fd->addr.size, }; @@ -794,7 +794,7 @@ static FlatView *address_space_get_flatview(AddressSpace *as) rcu_read_lock(); do { - view = atomic_rcu_read(&as->current_map); + view = address_space_to_flatview(as); /* If somebody has replaced as->current_map concurrently, * flatview_ref returns false. */ @@ -913,8 +913,8 @@ static void address_space_update_topology(AddressSpace *as) new_view->dispatch = mem_begin(as); for (i = 0; i < new_view->nr; i++) { MemoryRegionSection mrs = - section_from_flat_range(&new_view->ranges[i], as); - mem_add(as, new_view, &mrs); + section_from_flat_range(&new_view->ranges[i], new_view); + mem_add(new_view, &mrs); } mem_commit(new_view->dispatch); @@ -1870,7 +1870,7 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) view = address_space_get_flatview(as); FOR_EACH_FLAT_RANGE(fr, view) { if (fr->mr == mr) { - MemoryRegionSection mrs = section_from_flat_range(fr, as); + MemoryRegionSection mrs = section_from_flat_range(fr, view); listener->log_sync(listener, &mrs); } } @@ -1973,7 +1973,7 @@ static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpa FOR_EACH_FLAT_RANGE(fr, view) { if (fr->mr == mr) { section = (MemoryRegionSection) { - .address_space = as, + .fv = view, .offset_within_address_space = int128_get64(fr->addr.start), .size = fr->addr.size, }; @@ -2324,7 +2324,7 @@ static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr, } range = addrrange_make(int128_make64(addr), int128_make64(size)); - view = atomic_rcu_read(&as->current_map); + view = address_space_to_flatview(as); fr = flatview_lookup(view, range); if (!fr) { return ret; @@ -2335,7 +2335,7 @@ static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr, } ret.mr = fr->mr; - ret.address_space = as; + ret.fv = view; range = addrrange_intersection(range, fr->addr); ret.offset_within_region = fr->offset_in_region; ret.offset_within_region += int128_get64(int128_sub(range.start, @@ -2384,7 +2384,8 @@ void memory_global_dirty_log_sync(void) view = address_space_get_flatview(as); FOR_EACH_FLAT_RANGE(fr, view) { if (fr->dirty_log_mask) { - MemoryRegionSection mrs = section_from_flat_range(fr, as); + MemoryRegionSection mrs = section_from_flat_range(fr, view); + listener->log_sync(listener, &mrs); } } @@ -2469,7 +2470,7 @@ static void listener_add_address_space(MemoryListener *listener, FOR_EACH_FLAT_RANGE(fr, view) { MemoryRegionSection section = { .mr = fr->mr, - .address_space = as, + .fv = view, .offset_within_region = fr->offset_in_region, .size = fr->addr.size, .offset_within_address_space = int128_get64(fr->addr.start), -- cgit 1.4.1 From 8629d3fcb77e9775e44d9051bad0fb5187925eae Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:00 +1000 Subject: memory: Rename mem_begin/mem_commit/mem_add helpers This renames some helpers to reflect better what they do. This should cause no behavioural change. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-9-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- exec.c | 12 +++--------- include/exec/memory-internal.h | 6 +++--- memory.c | 6 +++--- 3 files changed, 9 insertions(+), 15 deletions(-) (limited to 'memory.c') diff --git a/exec.c b/exec.c index 548ec71b4c..b085f82503 100644 --- a/exec.c +++ b/exec.c @@ -358,7 +358,7 @@ static void phys_page_compact(PhysPageEntry *lp, Node *nodes) } } -static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb) +void address_space_dispatch_compact(AddressSpaceDispatch *d) { if (d->phys_map.skip) { phys_page_compact(&d->phys_map, d->map.nodes); @@ -1346,7 +1346,7 @@ static void register_multipage(FlatView *fv, phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index); } -void mem_add(FlatView *fv, MemoryRegionSection *section) +void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section) { MemoryRegionSection now = *section, remain = *section; Int128 page_size = int128_make64(TARGET_PAGE_SIZE); @@ -2672,9 +2672,8 @@ static void io_mem_init(void) NULL, UINT64_MAX); } -AddressSpaceDispatch *mem_begin(AddressSpace *as) +AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv) { - FlatView *fv = address_space_to_flatview(as); AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1); uint16_t n; @@ -2698,11 +2697,6 @@ void address_space_dispatch_free(AddressSpaceDispatch *d) g_free(d); } -void mem_commit(AddressSpaceDispatch *d) -{ - phys_page_compact_all(d, d->map.nodes_nb); -} - static void tcg_commit(MemoryListener *listener) { CPUAddressSpace *cpuas; diff --git a/include/exec/memory-internal.h b/include/exec/memory-internal.h index 1cf8ad9869..d4a35c6e96 100644 --- a/include/exec/memory-internal.h +++ b/include/exec/memory-internal.h @@ -27,9 +27,9 @@ extern const MemoryRegionOps unassigned_mem_ops; bool memory_region_access_valid(MemoryRegion *mr, hwaddr addr, unsigned size, bool is_write); -void mem_add(FlatView *fv, MemoryRegionSection *section); -AddressSpaceDispatch *mem_begin(AddressSpace *as); -void mem_commit(AddressSpaceDispatch *d); +void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section); +AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv); +void address_space_dispatch_compact(AddressSpaceDispatch *d); AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as); AddressSpaceDispatch *flatview_to_dispatch(FlatView *fv); diff --git a/memory.c b/memory.c index bf3728ae2f..c374317ba2 100644 --- a/memory.c +++ b/memory.c @@ -910,13 +910,13 @@ static void address_space_update_topology(AddressSpace *as) FlatView *new_view = generate_memory_topology(as->root); int i; - new_view->dispatch = mem_begin(as); + new_view->dispatch = address_space_dispatch_new(new_view); for (i = 0; i < new_view->nr; i++) { MemoryRegionSection mrs = section_from_flat_range(&new_view->ranges[i], new_view); - mem_add(new_view, &mrs); + flatview_add_to_dispatch(new_view, &mrs); } - mem_commit(new_view->dispatch); + address_space_dispatch_compact(new_view->dispatch); if (!QTAILQ_EMPTY(&as->listeners)) { address_space_update_topology_pass(as, old_view, new_view, false); -- cgit 1.4.1 From 89c177bbdd6cf8e50b3fd4831697d50e195d6432 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:01 +1000 Subject: memory: Store physical root MR in FlatView Address spaces get to keep a root MR (alias or not) but FlatView stores the actual MR as this is going to be used later on to decide whether to share a particular FlatView or not. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-10-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- memory.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index c374317ba2..15724db7d6 100644 --- a/memory.c +++ b/memory.c @@ -230,6 +230,7 @@ struct FlatView { unsigned nr; unsigned nr_allocated; struct AddressSpaceDispatch *dispatch; + MemoryRegion *root; }; typedef struct AddressSpaceOps AddressSpaceOps; @@ -259,12 +260,14 @@ static bool flatrange_equal(FlatRange *a, FlatRange *b) && a->readonly == b->readonly; } -static FlatView *flatview_new(void) +static FlatView *flatview_new(MemoryRegion *mr_root) { FlatView *view; view = g_new0(FlatView, 1); view->ref = 1; + view->root = mr_root; + memory_region_ref(mr_root); return view; } @@ -297,6 +300,7 @@ static void flatview_destroy(FlatView *view) memory_region_unref(view->ranges[i].mr); } g_free(view->ranges); + memory_region_unref(view->root); g_free(view); } @@ -723,12 +727,25 @@ static void render_memory_region(FlatView *view, } } +static MemoryRegion *memory_region_get_flatview_root(MemoryRegion *mr) +{ + while (mr->alias && !mr->alias_offset && + int128_ge(mr->size, mr->alias->size)) { + /* The alias is included in its entirety. Use it as + * the "real" root, so that we can share more FlatViews. + */ + mr = mr->alias; + } + + return mr; +} + /* Render a memory topology into a list of disjoint absolute ranges. */ static FlatView *generate_memory_topology(MemoryRegion *mr) { FlatView *view; - view = flatview_new(); + view = flatview_new(mr); if (mr) { render_memory_region(view, mr, int128_zero(), @@ -907,7 +924,8 @@ static void address_space_update_topology_pass(AddressSpace *as, static void address_space_update_topology(AddressSpace *as) { FlatView *old_view = address_space_get_flatview(as); - FlatView *new_view = generate_memory_topology(as->root); + MemoryRegion *physmr = memory_region_get_flatview_root(old_view->root); + FlatView *new_view = generate_memory_topology(physmr); int i; new_view->dispatch = address_space_dispatch_new(new_view); @@ -2650,7 +2668,7 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) as->ref_count = 1; as->root = root; as->malloced = false; - as->current_map = flatview_new(); + as->current_map = flatview_new(root); as->ioeventfd_nb = 0; as->ioeventfds = NULL; QTAILQ_INIT(&as->listeners); -- cgit 1.4.1 From 9bf561e36cf8fed9565011a19ba9ea0100e1811e Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:02 +1000 Subject: memory: Alloc dispatch tree where topology is generared This is to make next patches simpler. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-11-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- memory.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index 15724db7d6..6f6c2332ca 100644 --- a/memory.c +++ b/memory.c @@ -743,6 +743,7 @@ static MemoryRegion *memory_region_get_flatview_root(MemoryRegion *mr) /* Render a memory topology into a list of disjoint absolute ranges. */ static FlatView *generate_memory_topology(MemoryRegion *mr) { + int i; FlatView *view; view = flatview_new(mr); @@ -753,6 +754,14 @@ static FlatView *generate_memory_topology(MemoryRegion *mr) } flatview_simplify(view); + view->dispatch = address_space_dispatch_new(view); + for (i = 0; i < view->nr; i++) { + MemoryRegionSection mrs = + section_from_flat_range(&view->ranges[i], view); + flatview_add_to_dispatch(view, &mrs); + } + address_space_dispatch_compact(view->dispatch); + return view; } @@ -926,15 +935,6 @@ static void address_space_update_topology(AddressSpace *as) FlatView *old_view = address_space_get_flatview(as); MemoryRegion *physmr = memory_region_get_flatview_root(old_view->root); FlatView *new_view = generate_memory_topology(physmr); - int i; - - new_view->dispatch = address_space_dispatch_new(new_view); - for (i = 0; i < new_view->nr; i++) { - MemoryRegionSection mrs = - section_from_flat_range(&new_view->ranges[i], new_view); - flatview_add_to_dispatch(new_view, &mrs); - } - address_space_dispatch_compact(new_view->dispatch); if (!QTAILQ_EMPTY(&as->listeners)) { address_space_update_topology_pass(as, old_view, new_view, false); -- cgit 1.4.1 From 02218487649558ed66c3689d4cc55250a42601d8 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:03 +1000 Subject: memory: Move address_space_update_ioeventfds So it is called (twice) from the same function. This is to make the next patches a bit simpler. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-12-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- memory.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index 6f6c2332ca..bb582d052a 100644 --- a/memory.c +++ b/memory.c @@ -952,8 +952,6 @@ static void address_space_update_topology(AddressSpace *as) * counting is necessary. */ flatview_unref(old_view); - - address_space_update_ioeventfds(as); } void memory_region_transaction_begin(void) @@ -976,6 +974,7 @@ void memory_region_transaction_commit(void) QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { address_space_update_topology(as); + address_space_update_ioeventfds(as); } memory_region_update_pending = false; MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); -- cgit 1.4.1 From 967dc9b1194a9281124b2e1ce67b6c3359a2138f Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:04 +1000 Subject: memory: Share FlatView's and dispatch trees between address spaces This allows sharing flat views between address spaces (AS) when the same root memory region is used when creating a new address space. This is done by walking through all ASes and caching one FlatView per a physical root MR (i.e. not aliased). This removes search for duplicates from address_space_init_shareable() as FlatViews are shared elsewhere and keeping as::ref_count correct seems an unnecessary and useless complication. This should cause no change and memory use or boot time yet. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-13-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- memory.c | 56 +++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 11 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index bb582d052a..8ee3c81862 100644 --- a/memory.c +++ b/memory.c @@ -47,6 +47,8 @@ static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners static QTAILQ_HEAD(, AddressSpace) address_spaces = QTAILQ_HEAD_INITIALIZER(address_spaces); +static GHashTable *flat_views; + typedef struct AddrRange AddrRange; /* @@ -761,6 +763,7 @@ static FlatView *generate_memory_topology(MemoryRegion *mr) flatview_add_to_dispatch(view, &mrs); } address_space_dispatch_compact(view->dispatch); + g_hash_table_replace(flat_views, mr, view); return view; } @@ -930,11 +933,47 @@ static void address_space_update_topology_pass(AddressSpace *as, } } -static void address_space_update_topology(AddressSpace *as) +static void flatviews_init(void) +{ + if (flat_views) { + return; + } + + flat_views = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, + (GDestroyNotify) flatview_unref); +} + +static void flatviews_reset(void) +{ + AddressSpace *as; + + if (flat_views) { + g_hash_table_unref(flat_views); + flat_views = NULL; + } + flatviews_init(); + + /* Render unique FVs */ + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + MemoryRegion *physmr = memory_region_get_flatview_root(as->root); + + if (g_hash_table_lookup(flat_views, physmr)) { + continue; + } + + generate_memory_topology(physmr); + } +} + +static void address_space_set_flatview(AddressSpace *as) { FlatView *old_view = address_space_get_flatview(as); - MemoryRegion *physmr = memory_region_get_flatview_root(old_view->root); - FlatView *new_view = generate_memory_topology(physmr); + MemoryRegion *physmr = memory_region_get_flatview_root(as->root); + FlatView *new_view = g_hash_table_lookup(flat_views, physmr); + + assert(new_view); + + flatview_ref(new_view); if (!QTAILQ_EMPTY(&as->listeners)) { address_space_update_topology_pass(as, old_view, new_view, false); @@ -970,10 +1009,12 @@ void memory_region_transaction_commit(void) --memory_region_transaction_depth; if (!memory_region_transaction_depth) { if (memory_region_update_pending) { + flatviews_reset(); + MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { - address_space_update_topology(as); + address_space_set_flatview(as); address_space_update_ioeventfds(as); } memory_region_update_pending = false; @@ -2696,13 +2737,6 @@ AddressSpace *address_space_init_shareable(MemoryRegion *root, const char *name) { AddressSpace *as; - QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { - if (root == as->root && as->malloced) { - as->ref_count++; - return as; - } - } - as = g_malloc0(sizeof *as); address_space_init(as, root, name); as->malloced = true; -- cgit 1.4.1 From 67ace39b253ed5ae465275bc870f7e495547658b Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:05 +1000 Subject: memory: Do not allocate FlatView in address_space_init This creates a new AS object without any FlatView as memory_region_transaction_commit() may want to reuse the empty FV. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-14-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- memory.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index 8ee3c81862..3f6dd40303 100644 --- a/memory.c +++ b/memory.c @@ -967,22 +967,37 @@ static void flatviews_reset(void) static void address_space_set_flatview(AddressSpace *as) { - FlatView *old_view = address_space_get_flatview(as); + FlatView *old_view = address_space_to_flatview(as); MemoryRegion *physmr = memory_region_get_flatview_root(as->root); FlatView *new_view = g_hash_table_lookup(flat_views, physmr); assert(new_view); + if (old_view == new_view) { + return; + } + + if (old_view) { + flatview_ref(old_view); + } + flatview_ref(new_view); if (!QTAILQ_EMPTY(&as->listeners)) { - address_space_update_topology_pass(as, old_view, new_view, false); - address_space_update_topology_pass(as, old_view, new_view, true); + FlatView tmpview = { .nr = 0 }, *old_view2 = old_view; + + if (!old_view2) { + old_view2 = &tmpview; + } + address_space_update_topology_pass(as, old_view2, new_view, false); + address_space_update_topology_pass(as, old_view2, new_view, true); } /* Writes are protected by the BQL. */ atomic_rcu_set(&as->current_map, new_view); - flatview_unref(old_view); + if (old_view) { + flatview_unref(old_view); + } /* Note that all the old MemoryRegions are still alive up to this * point. This relieves most MemoryListeners from the need to @@ -990,7 +1005,9 @@ static void address_space_set_flatview(AddressSpace *as) * outside the iothread mutex, in which case precise reference * counting is necessary. */ - flatview_unref(old_view); + if (old_view) { + flatview_unref(old_view); + } } void memory_region_transaction_begin(void) @@ -2708,7 +2725,7 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) as->ref_count = 1; as->root = root; as->malloced = false; - as->current_map = flatview_new(root); + as->current_map = NULL; as->ioeventfd_nb = 0; as->ioeventfds = NULL; QTAILQ_INIT(&as->listeners); -- cgit 1.4.1 From 5e8fd947e2670c3c18f139de6a83fafcb56abbcc Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:06 +1000 Subject: memory: Rework "info mtree" to print flat views and dispatch trees This adds a new "-d" switch to "info mtree" to print dispatch tree internals. This changes the way "-f" is handled - it prints now flat views and associated address spaces. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-15-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- exec.c | 84 +++++++++++++++++++++++++++++++++++++++ hmp-commands-info.hx | 7 ++-- include/exec/memory-internal.h | 4 ++ include/exec/memory.h | 3 +- memory.c | 90 +++++++++++++++++++++++++++++++++++++----- monitor.c | 3 +- 6 files changed, 176 insertions(+), 15 deletions(-) (limited to 'memory.c') diff --git a/exec.c b/exec.c index b085f82503..7a80460725 100644 --- a/exec.c +++ b/exec.c @@ -3616,3 +3616,87 @@ void page_size_init(void) } qemu_host_page_mask = -(intptr_t)qemu_host_page_size; } + +#if !defined(CONFIG_USER_ONLY) + +static void mtree_print_phys_entries(fprintf_function mon, void *f, + int start, int end, int skip, int ptr) +{ + if (start == end - 1) { + mon(f, "\t%3d ", start); + } else { + mon(f, "\t%3d..%-3d ", start, end - 1); + } + mon(f, " skip=%d ", skip); + if (ptr == PHYS_MAP_NODE_NIL) { + mon(f, " ptr=NIL"); + } else if (!skip) { + mon(f, " ptr=#%d", ptr); + } else { + mon(f, " ptr=[%d]", ptr); + } + mon(f, "\n"); +} + +#define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \ + int128_sub((size), int128_one())) : 0) + +void mtree_print_dispatch(fprintf_function mon, void *f, + AddressSpaceDispatch *d, MemoryRegion *root) +{ + int i; + + mon(f, " Dispatch\n"); + mon(f, " Physical sections\n"); + + for (i = 0; i < d->map.sections_nb; ++i) { + MemoryRegionSection *s = d->map.sections + i; + const char *names[] = { " [unassigned]", " [not dirty]", + " [ROM]", " [watch]" }; + + mon(f, " #%d @" TARGET_FMT_plx ".." TARGET_FMT_plx " %s%s%s%s%s", + i, + s->offset_within_address_space, + s->offset_within_address_space + MR_SIZE(s->mr->size), + s->mr->name ? s->mr->name : "(noname)", + i < ARRAY_SIZE(names) ? names[i] : "", + s->mr == root ? " [ROOT]" : "", + s == d->mru_section ? " [MRU]" : "", + s->mr->is_iommu ? " [iommu]" : ""); + + if (s->mr->alias) { + mon(f, " alias=%s", s->mr->alias->name ? + s->mr->alias->name : "noname"); + } + mon(f, "\n"); + } + + mon(f, " Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n", + P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip); + for (i = 0; i < d->map.nodes_nb; ++i) { + int j, jprev; + PhysPageEntry prev; + Node *n = d->map.nodes + i; + + mon(f, " [%d]\n", i); + + for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) { + PhysPageEntry *pe = *n + j; + + if (pe->ptr == prev.ptr && pe->skip == prev.skip) { + continue; + } + + mtree_print_phys_entries(mon, f, jprev, j, prev.skip, prev.ptr); + + jprev = j; + prev = *pe; + } + + if (jprev != ARRAY_SIZE(*n)) { + mtree_print_phys_entries(mon, f, jprev, j, prev.skip, prev.ptr); + } + } +} + +#endif diff --git a/hmp-commands-info.hx b/hmp-commands-info.hx index 1c6772597d..4f1ece93e5 100644 --- a/hmp-commands-info.hx +++ b/hmp-commands-info.hx @@ -250,9 +250,10 @@ ETEXI { .name = "mtree", - .args_type = "flatview:-f", - .params = "[-f]", - .help = "show memory tree (-f: dump flat view for address spaces)", + .args_type = "flatview:-f,dispatch_tree:-d", + .params = "[-f][-d]", + .help = "show memory tree (-f: dump flat view for address spaces;" + "-d: dump dispatch tree, valid with -f only)", .cmd = hmp_info_mtree, }, diff --git a/include/exec/memory-internal.h b/include/exec/memory-internal.h index d4a35c6e96..647e9bd5c4 100644 --- a/include/exec/memory-internal.h +++ b/include/exec/memory-internal.h @@ -35,5 +35,9 @@ AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as); AddressSpaceDispatch *flatview_to_dispatch(FlatView *fv); void address_space_dispatch_free(AddressSpaceDispatch *d); +void mtree_print_dispatch(fprintf_function mon, void *f, + struct AddressSpaceDispatch *d, + MemoryRegion *root); + #endif #endif diff --git a/include/exec/memory.h b/include/exec/memory.h index 7816e5d655..2f4f56cf40 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -1515,7 +1515,8 @@ void memory_global_dirty_log_start(void); */ void memory_global_dirty_log_stop(void); -void mtree_info(fprintf_function mon_printf, void *f, bool flatview); +void mtree_info(fprintf_function mon_printf, void *f, bool flatview, + bool dispatch_tree); /** * memory_region_request_mmio_ptr: request a pointer to an mmio diff --git a/memory.c b/memory.c index 3f6dd40303..6729fb3ab3 100644 --- a/memory.c +++ b/memory.c @@ -2907,18 +2907,44 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, } } -static void mtree_print_flatview(fprintf_function p, void *f, - AddressSpace *as) +struct FlatViewInfo { + fprintf_function mon_printf; + void *f; + int counter; + bool dispatch_tree; +}; + +static void mtree_print_flatview(gpointer key, gpointer value, + gpointer user_data) { - FlatView *view = address_space_get_flatview(as); + FlatView *view = key; + GArray *fv_address_spaces = value; + struct FlatViewInfo *fvi = user_data; + fprintf_function p = fvi->mon_printf; + void *f = fvi->f; FlatRange *range = &view->ranges[0]; MemoryRegion *mr; int n = view->nr; + int i; + AddressSpace *as; + + p(f, "FlatView #%d\n", fvi->counter); + ++fvi->counter; + + for (i = 0; i < fv_address_spaces->len; ++i) { + as = g_array_index(fv_address_spaces, AddressSpace*, i); + p(f, " AS \"%s\", root: %s", as->name, memory_region_name(as->root)); + if (as->root->alias) { + p(f, ", alias %s", memory_region_name(as->root->alias)); + } + p(f, "\n"); + } + + p(f, " Root memory region: %s\n", + view->root ? memory_region_name(view->root) : "(none)"); if (n <= 0) { - p(f, MTREE_INDENT "No rendered FlatView for " - "address space '%s'\n", as->name); - flatview_unref(view); + p(f, MTREE_INDENT "No rendered FlatView\n\n"); return; } @@ -2945,21 +2971,65 @@ static void mtree_print_flatview(fprintf_function p, void *f, range++; } +#if !defined(CONFIG_USER_ONLY) + if (fvi->dispatch_tree && view->root) { + mtree_print_dispatch(p, f, view->dispatch, view->root); + } +#endif + + p(f, "\n"); +} + +static gboolean mtree_info_flatview_free(gpointer key, gpointer value, + gpointer user_data) +{ + FlatView *view = key; + GArray *fv_address_spaces = value; + + g_array_unref(fv_address_spaces); flatview_unref(view); + + return true; } -void mtree_info(fprintf_function mon_printf, void *f, bool flatview) +void mtree_info(fprintf_function mon_printf, void *f, bool flatview, + bool dispatch_tree) { MemoryRegionListHead ml_head; MemoryRegionList *ml, *ml2; AddressSpace *as; if (flatview) { + FlatView *view; + struct FlatViewInfo fvi = { + .mon_printf = mon_printf, + .f = f, + .counter = 0, + .dispatch_tree = dispatch_tree + }; + GArray *fv_address_spaces; + GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal); + + /* Gather all FVs in one table */ QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { - mon_printf(f, "address-space (flat view): %s\n", as->name); - mtree_print_flatview(mon_printf, f, as); - mon_printf(f, "\n"); + view = address_space_get_flatview(as); + + fv_address_spaces = g_hash_table_lookup(views, view); + if (!fv_address_spaces) { + fv_address_spaces = g_array_new(false, false, sizeof(as)); + g_hash_table_insert(views, view, fv_address_spaces); + } + + g_array_append_val(fv_address_spaces, as); } + + /* Print */ + g_hash_table_foreach(views, mtree_print_flatview, &fvi); + + /* Free */ + g_hash_table_foreach_remove(views, mtree_info_flatview_free, 0); + g_hash_table_unref(views); + return; } diff --git a/monitor.c b/monitor.c index 058045b3cb..f4856b9268 100644 --- a/monitor.c +++ b/monitor.c @@ -1703,8 +1703,9 @@ static void hmp_boot_set(Monitor *mon, const QDict *qdict) static void hmp_info_mtree(Monitor *mon, const QDict *qdict) { bool flatview = qdict_get_try_bool(qdict, "flatview", false); + bool dispatch_tree = qdict_get_try_bool(qdict, "dispatch_tree", false); - mtree_info((fprintf_function)monitor_printf, mon, flatview); + mtree_info((fprintf_function)monitor_printf, mon, flatview, dispatch_tree); } static void hmp_info_numa(Monitor *mon, const QDict *qdict) -- cgit 1.4.1 From b516572f31c0ea0937cd9d11d9bd72dd83809886 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:08 +1000 Subject: memory: Get rid of address_space_init_shareable Since FlatViews are shared now and ASes not, this gets rid of address_space_init_shareable(). This should cause no behavioural change. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-17-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- cpus.c | 5 +++-- hw/arm/armv7m.c | 9 ++++----- include/exec/memory.h | 19 ------------------- include/hw/arm/armv7m.h | 2 +- memory.c | 21 --------------------- target/arm/cpu.c | 16 ++++++++-------- target/i386/cpu.c | 5 +++-- 7 files changed, 19 insertions(+), 58 deletions(-) (limited to 'memory.c') diff --git a/cpus.c b/cpus.c index 9bed61eefc..c9a624003a 100644 --- a/cpus.c +++ b/cpus.c @@ -1764,8 +1764,9 @@ void qemu_init_vcpu(CPUState *cpu) /* If the target cpu hasn't set up any address spaces itself, * give it the default one. */ - AddressSpace *as = address_space_init_shareable(cpu->memory, - "cpu-memory"); + AddressSpace *as = g_new0(AddressSpace, 1); + + address_space_init(as, cpu->memory, "cpu-memory"); cpu->num_ases = 1; cpu_address_space_init(cpu, as, 0); } diff --git a/hw/arm/armv7m.c b/hw/arm/armv7m.c index 57a680687a..bb2dfc942b 100644 --- a/hw/arm/armv7m.c +++ b/hw/arm/armv7m.c @@ -41,7 +41,7 @@ static MemTxResult bitband_read(void *opaque, hwaddr offset, /* Find address in underlying memory and round down to multiple of size */ addr = bitband_addr(s, offset) & (-size); - res = address_space_read(s->source_as, addr, attrs, buf, size); + res = address_space_read(&s->source_as, addr, attrs, buf, size); if (res) { return res; } @@ -66,7 +66,7 @@ static MemTxResult bitband_write(void *opaque, hwaddr offset, uint64_t value, /* Find address in underlying memory and round down to multiple of size */ addr = bitband_addr(s, offset) & (-size); - res = address_space_read(s->source_as, addr, attrs, buf, size); + res = address_space_read(&s->source_as, addr, attrs, buf, size); if (res) { return res; } @@ -79,7 +79,7 @@ static MemTxResult bitband_write(void *opaque, hwaddr offset, uint64_t value, } else { buf[bitpos >> 3] &= ~bit; } - return address_space_write(s->source_as, addr, attrs, buf, size); + return address_space_write(&s->source_as, addr, attrs, buf, size); } static const MemoryRegionOps bitband_ops = { @@ -111,8 +111,7 @@ static void bitband_realize(DeviceState *dev, Error **errp) return; } - s->source_as = address_space_init_shareable(s->source_memory, - "bitband-source"); + address_space_init(&s->source_as, s->source_memory, "bitband-source"); } /* Board init. */ diff --git a/include/exec/memory.h b/include/exec/memory.h index 2f4f56cf40..402824c6f2 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -309,8 +309,6 @@ struct AddressSpace { struct rcu_head rcu; char *name; MemoryRegion *root; - int ref_count; - bool malloced; /* Accessed via RCU. */ struct FlatView *current_map; @@ -1585,23 +1583,6 @@ MemTxResult memory_region_dispatch_write(MemoryRegion *mr, */ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name); -/** - * address_space_init_shareable: return an address space for a memory region, - * creating it if it does not already exist - * - * @root: a #MemoryRegion that routes addresses for the address space - * @name: an address space name. The name is only used for debugging - * output. - * - * This function will return a pointer to an existing AddressSpace - * which was initialized with the specified MemoryRegion, or it will - * create and initialize one if it does not already exist. The ASes - * are reference-counted, so the memory will be freed automatically - * when the AddressSpace is destroyed via address_space_destroy. - */ -AddressSpace *address_space_init_shareable(MemoryRegion *root, - const char *name); - /** * address_space_destroy: destroy an address space * diff --git a/include/hw/arm/armv7m.h b/include/hw/arm/armv7m.h index 9ad316c76e..35ab757264 100644 --- a/include/hw/arm/armv7m.h +++ b/include/hw/arm/armv7m.h @@ -21,7 +21,7 @@ typedef struct { SysBusDevice parent_obj; /*< public >*/ - AddressSpace *source_as; + AddressSpace source_as; MemoryRegion iomem; uint32_t base; MemoryRegion *source_memory; diff --git a/memory.c b/memory.c index 6729fb3ab3..aa7355bafb 100644 --- a/memory.c +++ b/memory.c @@ -2722,9 +2722,7 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) { memory_region_ref(root); memory_region_transaction_begin(); - as->ref_count = 1; as->root = root; - as->malloced = false; as->current_map = NULL; as->ioeventfd_nb = 0; as->ioeventfds = NULL; @@ -2737,37 +2735,18 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) static void do_address_space_destroy(AddressSpace *as) { - bool do_free = as->malloced; - assert(QTAILQ_EMPTY(&as->listeners)); flatview_unref(as->current_map); g_free(as->name); g_free(as->ioeventfds); memory_region_unref(as->root); - if (do_free) { - g_free(as); - } -} - -AddressSpace *address_space_init_shareable(MemoryRegion *root, const char *name) -{ - AddressSpace *as; - - as = g_malloc0(sizeof *as); - address_space_init(as, root, name); - as->malloced = true; - return as; } void address_space_destroy(AddressSpace *as) { MemoryRegion *root = as->root; - as->ref_count--; - if (as->ref_count) { - return; - } /* Flush out anything from MemoryListeners listening in on this */ memory_region_transaction_begin(); as->root = NULL; diff --git a/target/arm/cpu.c b/target/arm/cpu.c index 20a3445bda..f61ca660e6 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -684,6 +684,9 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) CPUARMState *env = &cpu->env; int pagebits; Error *local_err = NULL; +#ifndef CONFIG_USER_ONLY + AddressSpace *as; +#endif cpu_exec_realizefn(cs, &local_err); if (local_err != NULL) { @@ -874,24 +877,21 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) #ifndef CONFIG_USER_ONLY if (cpu->has_el3 || arm_feature(env, ARM_FEATURE_M_SECURITY)) { - AddressSpace *as; + as = g_new0(AddressSpace, 1); cs->num_ases = 2; if (!cpu->secure_memory) { cpu->secure_memory = cs->memory; } - as = address_space_init_shareable(cpu->secure_memory, - "cpu-secure-memory"); + address_space_init(as, cpu->secure_memory, "cpu-secure-memory"); cpu_address_space_init(cs, as, ARMASIdx_S); } else { cs->num_ases = 1; } - - cpu_address_space_init(cs, - address_space_init_shareable(cs->memory, - "cpu-memory"), - ARMASIdx_NS); + as = g_new0(AddressSpace, 1); + address_space_init(as, cs->memory, "cpu-memory"); + cpu_address_space_init(cs, as, ARMASIdx_NS); #endif qemu_init_vcpu(cs); diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 0aa28fc775..98732cd65f 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -3738,10 +3738,11 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp) #ifndef CONFIG_USER_ONLY if (tcg_enabled()) { - AddressSpace *as_normal = address_space_init_shareable(cs->memory, - "cpu-memory"); + AddressSpace *as_normal = g_new0(AddressSpace, 1); AddressSpace *as_smm = g_new(AddressSpace, 1); + address_space_init(as_normal, cs->memory, "cpu-memory"); + cpu->cpu_as_mem = g_new(MemoryRegion, 1); cpu->cpu_as_root = g_new(MemoryRegion, 1); -- cgit 1.4.1 From 202fc01b05572ecb258fdf4c5bd56cf6de8140c7 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:09 +1000 Subject: memory: Create FlatView directly This avoids usual memory_region_transaction_commit() which rebuilds all FVs. On POWER8 with 255 CPUs, 255 virtio-net, 40 PCI bridges guest this brings down the boot time from 25s to 20s and reduces the amount of temporary FVs allocated during machine constructon (~800000 -> ~640000) and amount of temporary dispatch trees (~370000 -> ~300000), the total memory footprint goes down (18G -> 17G). Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-18-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- memory.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index aa7355bafb..706c38508f 100644 --- a/memory.c +++ b/memory.c @@ -1010,6 +1010,17 @@ static void address_space_set_flatview(AddressSpace *as) } } +static void address_space_update_topology(AddressSpace *as) +{ + MemoryRegion *physmr = memory_region_get_flatview_root(as->root); + + flatviews_init(); + if (!g_hash_table_lookup(flat_views, physmr)) { + generate_memory_topology(physmr); + } + address_space_set_flatview(as); +} + void memory_region_transaction_begin(void) { qemu_flush_coalesced_mmio_buffer(); @@ -2721,7 +2732,6 @@ void memory_region_invalidate_mmio_ptr(MemoryRegion *mr, hwaddr offset, void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) { memory_region_ref(root); - memory_region_transaction_begin(); as->root = root; as->current_map = NULL; as->ioeventfd_nb = 0; @@ -2729,8 +2739,8 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) QTAILQ_INIT(&as->listeners); QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); as->name = g_strdup(name ? name : "anonymous"); - memory_region_update_pending |= root->enabled; - memory_region_transaction_commit(); + address_space_update_topology(as); + address_space_update_ioeventfds(as); } static void do_address_space_destroy(AddressSpace *as) -- cgit 1.4.1 From 02d9651d6a46479e9d70b72dca34e43605d06cda Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 21 Sep 2017 12:34:00 +0200 Subject: memory: trace FlatView creation and destruction Signed-off-by: Paolo Bonzini --- include/exec/memory.h | 1 - include/qemu/typedefs.h | 1 + memory.c | 3 +++ trace-events | 3 +++ 4 files changed, 7 insertions(+), 1 deletion(-) (limited to 'memory.c') diff --git a/include/exec/memory.h b/include/exec/memory.h index 402824c6f2..5ed4042f87 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -48,7 +48,6 @@ typedef struct MemoryRegionOps MemoryRegionOps; typedef struct MemoryRegionMmio MemoryRegionMmio; -typedef struct FlatView FlatView; struct MemoryRegionMmio { CPUReadMemoryFunc *read[3]; diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h index 163550214c..980d2b330e 100644 --- a/include/qemu/typedefs.h +++ b/include/qemu/typedefs.h @@ -30,6 +30,7 @@ typedef struct DisplaySurface DisplaySurface; typedef struct DriveInfo DriveInfo; typedef struct Error Error; typedef struct EventNotifier EventNotifier; +typedef struct FlatView FlatView; typedef struct FWCfgEntry FWCfgEntry; typedef struct FWCfgIoState FWCfgIoState; typedef struct FWCfgMemState FWCfgMemState; diff --git a/memory.c b/memory.c index 706c38508f..68cdf8bad0 100644 --- a/memory.c +++ b/memory.c @@ -270,6 +270,7 @@ static FlatView *flatview_new(MemoryRegion *mr_root) view->ref = 1; view->root = mr_root; memory_region_ref(mr_root); + trace_flatview_new(view, mr_root); return view; } @@ -295,6 +296,7 @@ static void flatview_destroy(FlatView *view) { int i; + trace_flatview_destroy(view, view->root); if (view->dispatch) { address_space_dispatch_free(view->dispatch); } @@ -314,6 +316,7 @@ static bool flatview_ref(FlatView *view) static void flatview_unref(FlatView *view) { if (atomic_fetch_dec(&view->ref) == 1) { + trace_flatview_destroy_rcu(view, view->root); call_rcu(view, flatview_destroy, rcu); } } diff --git a/trace-events b/trace-events index 1f50f56d9d..1d2eb5d3e4 100644 --- a/trace-events +++ b/trace-events @@ -64,6 +64,9 @@ memory_region_tb_read(int cpu_index, uint64_t addr, uint64_t value, unsigned siz memory_region_tb_write(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr 0x%"PRIx64" value 0x%"PRIx64" size %u" memory_region_ram_device_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u" memory_region_ram_device_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u" +flatview_new(FlatView *view, MemoryRegion *root) "%p (root %p)" +flatview_destroy(FlatView *view, MemoryRegion *root) "%p (root %p)" +flatview_destroy_rcu(FlatView *view, MemoryRegion *root) "%p (root %p)" ### Guest events, keep at bottom -- cgit 1.4.1 From e673ba9af9bf8fd8e0f44025ac738b8285b3ed27 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 21 Sep 2017 12:28:16 +0200 Subject: memory: seek FlatView sharing candidates among children subregions A container can be used instead of an alias to allow switching between multiple subregions. In this case we cannot directly share the subregions (since they only belong to a single parent), but if the subregions are aliases we can in turn walk those. This is not enough to remove all source of quadratic FlatView creation, but it enables sharing of the PCI bus master FlatViews (and their AddressSpaceDispatch structures) across all PCI devices. For 112 virtio-net-pci devices, boot time is reduced from 25 to 10 seconds and memory consumption from 1.4 to 1 G. Signed-off-by: Paolo Bonzini --- memory.c | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index 68cdf8bad0..15b1bd7d89 100644 --- a/memory.c +++ b/memory.c @@ -734,12 +734,40 @@ static void render_memory_region(FlatView *view, static MemoryRegion *memory_region_get_flatview_root(MemoryRegion *mr) { - while (mr->alias && !mr->alias_offset && - int128_ge(mr->size, mr->alias->size)) { - /* The alias is included in its entirety. Use it as - * the "real" root, so that we can share more FlatViews. - */ - mr = mr->alias; + while (mr->enabled) { + if (mr->alias) { + if (!mr->alias_offset && int128_ge(mr->size, mr->alias->size)) { + /* The alias is included in its entirety. Use it as + * the "real" root, so that we can share more FlatViews. + */ + mr = mr->alias; + continue; + } + } else if (!mr->terminates) { + unsigned int found = 0; + MemoryRegion *child, *next = NULL; + QTAILQ_FOREACH(child, &mr->subregions, subregions_link) { + if (child->enabled) { + if (++found > 1) { + next = NULL; + break; + } + if (!child->addr && int128_ge(mr->size, child->size)) { + /* A child is included in its entirety. If it's the only + * enabled one, use it in the hope of finding an alias down the + * way. This will also let us share FlatViews. + */ + next = child; + } + } + } + if (next) { + mr = next; + continue; + } + } + + break; } return mr; -- cgit 1.4.1 From 092aa2fc65b7a35121616aad8f39d47b8f921618 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Thu, 21 Sep 2017 18:51:07 +1000 Subject: memory: Share special empty FlatView This shares an cached empty FlatView among address spaces. The empty FV is used every time when a root MR renders into a FV without memory sections which happens when MR or its children are not enabled or zero-sized. The empty_view is not NULL to keep the rest of memory API intact; it also has a dispatch tree for the same reason. On POWER8 with 255 CPUs, 255 virtio-net, 40 PCI bridges guest this halves the amount of FlatView's in use (557 -> 260) and dispatch tables (~800000 -> ~370000). In an unrelated experiment with 112 non-virtio devices on x86 ("-M pc"), only 4 FlatViews are alive, and about ~2000 are created at startup. Signed-off-by: Alexey Kardashevskiy Message-Id: <20170921085110.25598-16-aik@ozlabs.ru> Signed-off-by: Paolo Bonzini --- memory.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'memory.c') diff --git a/memory.c b/memory.c index 15b1bd7d89..5e6351a6c1 100644 --- a/memory.c +++ b/memory.c @@ -317,6 +317,7 @@ static void flatview_unref(FlatView *view) { if (atomic_fetch_dec(&view->ref) == 1) { trace_flatview_destroy_rcu(view, view->root); + assert(view->root); call_rcu(view, flatview_destroy, rcu); } } @@ -761,16 +762,19 @@ static MemoryRegion *memory_region_get_flatview_root(MemoryRegion *mr) } } } + if (found == 0) { + return NULL; + } if (next) { mr = next; continue; } } - break; + return mr; } - return mr; + return NULL; } /* Render a memory topology into a list of disjoint absolute ranges. */ @@ -966,12 +970,22 @@ static void address_space_update_topology_pass(AddressSpace *as, static void flatviews_init(void) { + static FlatView *empty_view; + if (flat_views) { return; } flat_views = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, (GDestroyNotify) flatview_unref); + if (!empty_view) { + empty_view = generate_memory_topology(NULL); + /* We keep it alive forever in the global variable. */ + flatview_ref(empty_view); + } else { + g_hash_table_replace(flat_views, NULL, empty_view); + flatview_ref(empty_view); + } } static void flatviews_reset(void) -- cgit 1.4.1