diff options
Diffstat (limited to 'migration/ram.c')
| -rw-r--r-- | migration/ram.c | 231 |
1 files changed, 200 insertions, 31 deletions
diff --git a/migration/ram.c b/migration/ram.c index 59191c1ed2..35bd6213e9 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -159,18 +159,44 @@ out: return ret; } +static bool ramblock_is_ignored(RAMBlock *block) +{ + return !qemu_ram_is_migratable(block) || + (migrate_ignore_shared() && qemu_ram_is_shared(block)); +} + /* Should be holding either ram_list.mutex, or the RCU lock. */ +#define RAMBLOCK_FOREACH_NOT_IGNORED(block) \ + INTERNAL_RAMBLOCK_FOREACH(block) \ + if (ramblock_is_ignored(block)) {} else + #define RAMBLOCK_FOREACH_MIGRATABLE(block) \ INTERNAL_RAMBLOCK_FOREACH(block) \ if (!qemu_ram_is_migratable(block)) {} else #undef RAMBLOCK_FOREACH +int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque) +{ + RAMBlock *block; + int ret = 0; + + rcu_read_lock(); + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + ret = func(block, opaque); + if (ret) { + break; + } + } + rcu_read_unlock(); + return ret; +} + static void ramblock_recv_map_init(void) { RAMBlock *rb; - RAMBLOCK_FOREACH_MIGRATABLE(rb) { + RAMBLOCK_FOREACH_NOT_IGNORED(rb) { assert(!rb->receivedmap); rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits()); } @@ -290,6 +316,8 @@ struct RAMState { uint32_t last_version; /* We are in the first round */ bool ram_bulk_stage; + /* The free page optimization is enabled */ + bool fpo_enabled; /* How many times we have dirty too many pages */ int dirty_rate_high_cnt; /* these variables are used for bitmap sync */ @@ -316,7 +344,7 @@ struct RAMState { uint64_t target_page_count; /* number of dirty bits in the bitmap */ uint64_t migration_dirty_pages; - /* protects modification of the bitmap */ + /* Protects modification of the bitmap and migration dirty pages */ QemuMutex bitmap_mutex; /* The RAMBlock used in the last src_page_requests */ RAMBlock *last_req_rb; @@ -328,6 +356,41 @@ typedef struct RAMState RAMState; static RAMState *ram_state; +static NotifierWithReturnList precopy_notifier_list; + +void precopy_infrastructure_init(void) +{ + notifier_with_return_list_init(&precopy_notifier_list); +} + +void precopy_add_notifier(NotifierWithReturn *n) +{ + notifier_with_return_list_add(&precopy_notifier_list, n); +} + +void precopy_remove_notifier(NotifierWithReturn *n) +{ + notifier_with_return_remove(n); +} + +int precopy_notify(PrecopyNotifyReason reason, Error **errp) +{ + PrecopyNotifyData pnd; + pnd.reason = reason; + pnd.errp = errp; + + return notifier_with_return_list_notify(&precopy_notifier_list, &pnd); +} + +void precopy_enable_free_page_optimization(void) +{ + if (!ram_state) { + return; + } + + ram_state->fpo_enabled = true; +} + uint64_t ram_bytes_remaining(void) { return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) : @@ -1545,11 +1608,15 @@ unsigned long migration_bitmap_find_dirty(RAMState *rs, RAMBlock *rb, unsigned long *bitmap = rb->bmap; unsigned long next; - if (!qemu_ram_is_migratable(rb)) { + if (ramblock_is_ignored(rb)) { return size; } - if (rs->ram_bulk_stage && start > 0) { + /* + * When the free page optimization is enabled, we need to check the bitmap + * to send the non-free pages rather than all the pages in the bulk stage. + */ + if (!rs->fpo_enabled && rs->ram_bulk_stage && start > 0) { next = start + 1; } else { next = find_next_bit(bitmap, size, start); @@ -1564,11 +1631,14 @@ static inline bool migration_bitmap_clear_dirty(RAMState *rs, { bool ret; + qemu_mutex_lock(&rs->bitmap_mutex); ret = test_and_clear_bit(page, rb->bmap); if (ret) { rs->migration_dirty_pages--; } + qemu_mutex_unlock(&rs->bitmap_mutex); + return ret; } @@ -1594,7 +1664,7 @@ uint64_t ram_pagesize_summary(void) RAMBlock *block; uint64_t summary = 0; - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { summary |= block->page_size; } @@ -1664,7 +1734,7 @@ static void migration_bitmap_sync(RAMState *rs) qemu_mutex_lock(&rs->bitmap_mutex); rcu_read_lock(); - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { migration_bitmap_sync_range(rs, block, 0, block->used_length); } ram_counters.remaining = ram_bytes_remaining(); @@ -1712,6 +1782,25 @@ static void migration_bitmap_sync(RAMState *rs) } } +static void migration_bitmap_sync_precopy(RAMState *rs) +{ + Error *local_err = NULL; + + /* + * The current notifier usage is just an optimization to migration, so we + * don't stop the normal migration process in the error case. + */ + if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) { + error_report_err(local_err); + } + + migration_bitmap_sync(rs); + + if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) { + error_report_err(local_err); + } +} + /** * save_zero_page_to_file: send the zero page to the file * @@ -2388,7 +2477,7 @@ static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss, size_t pagesize_bits = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS; - if (!qemu_ram_is_migratable(pss->block)) { + if (ramblock_is_ignored(pss->block)) { error_report("block %s should not be migrated !", pss->block->idstr); return 0; } @@ -2486,19 +2575,30 @@ void acct_update_position(QEMUFile *f, size_t size, bool zero) } } -uint64_t ram_bytes_total(void) +static uint64_t ram_bytes_total_common(bool count_ignored) { RAMBlock *block; uint64_t total = 0; rcu_read_lock(); - RAMBLOCK_FOREACH_MIGRATABLE(block) { - total += block->used_length; + if (count_ignored) { + RAMBLOCK_FOREACH_MIGRATABLE(block) { + total += block->used_length; + } + } else { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + total += block->used_length; + } } rcu_read_unlock(); return total; } +uint64_t ram_bytes_total(void) +{ + return ram_bytes_total_common(false); +} + static void xbzrle_load_setup(void) { XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE); @@ -2547,7 +2647,7 @@ static void ram_save_cleanup(void *opaque) */ memory_global_dirty_log_stop(); - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { g_free(block->bmap); block->bmap = NULL; g_free(block->unsentmap); @@ -2566,6 +2666,7 @@ static void ram_state_reset(RAMState *rs) rs->last_page = 0; rs->last_version = ram_list.version; rs->ram_bulk_stage = true; + rs->fpo_enabled = false; } #define MAX_WAIT 50 /* ms, half buffered_file limit */ @@ -2610,7 +2711,7 @@ void ram_postcopy_migrated_memory_release(MigrationState *ms) { struct RAMBlock *block; - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { unsigned long *bitmap = block->bmap; unsigned long range = block->used_length >> TARGET_PAGE_BITS; unsigned long run_start = find_next_zero_bit(bitmap, range, 0); @@ -2688,7 +2789,7 @@ static int postcopy_each_ram_send_discard(MigrationState *ms) struct RAMBlock *block; int ret; - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { PostcopyDiscardState *pds = postcopy_discard_send_init(ms, block->idstr); @@ -2896,7 +2997,7 @@ int ram_postcopy_send_discard_bitmap(MigrationState *ms) rs->last_sent_block = NULL; rs->last_page = 0; - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { unsigned long pages = block->used_length >> TARGET_PAGE_BITS; unsigned long *bitmap = block->bmap; unsigned long *unsentmap = block->unsentmap; @@ -3062,7 +3163,7 @@ static void ram_list_init_bitmaps(void) /* Skip setting bitmap if there is no RAM */ if (ram_bytes_total()) { - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { pages = block->max_length >> TARGET_PAGE_BITS; block->bmap = bitmap_new(pages); bitmap_set(block->bmap, 0, pages); @@ -3083,7 +3184,7 @@ static void ram_init_bitmaps(RAMState *rs) ram_list_init_bitmaps(); memory_global_dirty_log_start(); - migration_bitmap_sync(rs); + migration_bitmap_sync_precopy(rs); rcu_read_unlock(); qemu_mutex_unlock_ramlist(); @@ -3117,7 +3218,7 @@ static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out) * about dirty page logging as well. */ - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { pages += bitmap_count_one(block->bmap, block->used_length >> TARGET_PAGE_BITS); } @@ -3142,6 +3243,53 @@ static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out) } /* + * This function clears bits of the free pages reported by the caller from the + * migration dirty bitmap. @addr is the host address corresponding to the + * start of the continuous guest free pages, and @len is the total bytes of + * those pages. + */ +void qemu_guest_free_page_hint(void *addr, size_t len) +{ + RAMBlock *block; + ram_addr_t offset; + size_t used_len, start, npages; + MigrationState *s = migrate_get_current(); + + /* This function is currently expected to be used during live migration */ + if (!migration_is_setup_or_active(s->state)) { + return; + } + + for (; len > 0; len -= used_len, addr += used_len) { + block = qemu_ram_block_from_host(addr, false, &offset); + if (unlikely(!block || offset >= block->used_length)) { + /* + * The implementation might not support RAMBlock resize during + * live migration, but it could happen in theory with future + * updates. So we add a check here to capture that case. + */ + error_report_once("%s unexpected error", __func__); + return; + } + + if (len <= block->used_length - offset) { + used_len = len; + } else { + used_len = block->used_length - offset; + } + + start = offset >> TARGET_PAGE_BITS; + npages = used_len >> TARGET_PAGE_BITS; + + qemu_mutex_lock(&ram_state->bitmap_mutex); + ram_state->migration_dirty_pages -= + bitmap_count_one_with_offset(block->bmap, start, npages); + bitmap_clear(block->bmap, start, npages); + qemu_mutex_unlock(&ram_state->bitmap_mutex); + } +} + +/* * Each of ram_save_setup, ram_save_iterate and ram_save_complete has * long-running RCU critical section. When rcu-reclaims in the code * start to become numerous it will be necessary to reduce the @@ -3176,7 +3324,7 @@ static int ram_save_setup(QEMUFile *f, void *opaque) rcu_read_lock(); - qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE); + qemu_put_be64(f, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE); RAMBLOCK_FOREACH_MIGRATABLE(block) { qemu_put_byte(f, strlen(block->idstr)); @@ -3185,6 +3333,10 @@ static int ram_save_setup(QEMUFile *f, void *opaque) if (migrate_postcopy_ram() && block->page_size != qemu_host_page_size) { qemu_put_be64(f, block->page_size); } + if (migrate_ignore_shared()) { + qemu_put_be64(f, block->mr->addr); + qemu_put_byte(f, ramblock_is_ignored(block) ? 1 : 0); + } } rcu_read_unlock(); @@ -3312,7 +3464,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque) rcu_read_lock(); if (!migration_in_postcopy()) { - migration_bitmap_sync(rs); + migration_bitmap_sync_precopy(rs); } ram_control_before_iterate(f, RAM_CONTROL_FINISH); @@ -3361,7 +3513,7 @@ static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size, remaining_size < max_size) { qemu_mutex_lock_iothread(); rcu_read_lock(); - migration_bitmap_sync(rs); + migration_bitmap_sync_precopy(rs); rcu_read_unlock(); qemu_mutex_unlock_iothread(); remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE; @@ -3443,7 +3595,7 @@ static inline RAMBlock *ram_block_from_stream(QEMUFile *f, int flags) return NULL; } - if (!qemu_ram_is_migratable(block)) { + if (ramblock_is_ignored(block)) { error_report("block %s should not be migrated !", id); return NULL; } @@ -3698,7 +3850,7 @@ int colo_init_ram_cache(void) RAMBlock *block; rcu_read_lock(); - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { block->colo_cache = qemu_anon_ram_alloc(block->used_length, NULL, false); @@ -3719,7 +3871,7 @@ int colo_init_ram_cache(void) if (ram_bytes_total()) { RAMBlock *block; - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { unsigned long pages = block->max_length >> TARGET_PAGE_BITS; block->bmap = bitmap_new(pages); @@ -3734,7 +3886,7 @@ int colo_init_ram_cache(void) out_locked: - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { if (block->colo_cache) { qemu_anon_ram_free(block->colo_cache, block->used_length); block->colo_cache = NULL; @@ -3751,14 +3903,14 @@ void colo_release_ram_cache(void) RAMBlock *block; memory_global_dirty_log_stop(); - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { g_free(block->bmap); block->bmap = NULL; } rcu_read_lock(); - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { if (block->colo_cache) { qemu_anon_ram_free(block->colo_cache, block->used_length); block->colo_cache = NULL; @@ -3794,7 +3946,7 @@ static int ram_load_cleanup(void *opaque) { RAMBlock *rb; - RAMBLOCK_FOREACH_MIGRATABLE(rb) { + RAMBLOCK_FOREACH_NOT_IGNORED(rb) { if (ramblock_is_pmem(rb)) { pmem_persist(rb->host, rb->used_length); } @@ -3803,7 +3955,7 @@ static int ram_load_cleanup(void *opaque) xbzrle_load_cleanup(); compress_threads_load_cleanup(); - RAMBLOCK_FOREACH_MIGRATABLE(rb) { + RAMBLOCK_FOREACH_NOT_IGNORED(rb) { g_free(rb->receivedmap); rb->receivedmap = NULL; } @@ -4003,7 +4155,7 @@ static void colo_flush_ram_cache(void) memory_global_dirty_log_sync(); rcu_read_lock(); - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { migration_bitmap_sync_range(ram_state, block, 0, block->used_length); } rcu_read_unlock(); @@ -4146,6 +4298,23 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id) ret = -EINVAL; } } + if (migrate_ignore_shared()) { + hwaddr addr = qemu_get_be64(f); + bool ignored = qemu_get_byte(f); + if (ignored != ramblock_is_ignored(block)) { + error_report("RAM block %s should %s be migrated", + id, ignored ? "" : "not"); + ret = -EINVAL; + } + if (ramblock_is_ignored(block) && + block->mr->addr != addr) { + error_report("Mismatched GPAs for block %s " + "%" PRId64 "!= %" PRId64, + id, (uint64_t)addr, + (uint64_t)block->mr->addr); + ret = -EINVAL; + } + } ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG, block->idstr); } else { @@ -4216,7 +4385,7 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id) static bool ram_has_postcopy(void *opaque) { RAMBlock *rb; - RAMBLOCK_FOREACH_MIGRATABLE(rb) { + RAMBLOCK_FOREACH_NOT_IGNORED(rb) { if (ramblock_is_pmem(rb)) { info_report("Block: %s, host: %p is a nvdimm memory, postcopy" "is not supported now!", rb->idstr, rb->host); @@ -4236,7 +4405,7 @@ static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs) trace_ram_dirty_bitmap_sync_start(); - RAMBLOCK_FOREACH_MIGRATABLE(block) { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { qemu_savevm_send_recv_bitmap(file, block->idstr); trace_ram_dirty_bitmap_request(block->idstr); ramblock_count++; |