diff options
39 files changed, 1696 insertions, 126 deletions
diff --git a/hw/arm/virt.c b/hw/arm/virt.c index ecfee362a1..acf9bfbece 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -2546,10 +2546,17 @@ static void machvirt_machine_init(void) } type_init(machvirt_machine_init); +static void virt_machine_5_2_options(MachineClass *mc) +{ +} +DEFINE_VIRT_MACHINE_AS_LATEST(5, 2) + static void virt_machine_5_1_options(MachineClass *mc) { + virt_machine_5_2_options(mc); + compat_props_add(mc->compat_props, hw_compat_5_1, hw_compat_5_1_len); } -DEFINE_VIRT_MACHINE_AS_LATEST(5, 1) +DEFINE_VIRT_MACHINE(5, 1) static void virt_machine_5_0_options(MachineClass *mc) { diff --git a/hw/core/machine-qmp-cmds.c b/hw/core/machine-qmp-cmds.c index 963088b798..21551221ad 100644 --- a/hw/core/machine-qmp-cmds.c +++ b/hw/core/machine-qmp-cmds.c @@ -238,6 +238,10 @@ MachineInfoList *qmp_query_machines(Error **errp) info->default_cpu_type = g_strdup(mc->default_cpu_type); info->has_default_cpu_type = true; } + if (mc->default_ram_id) { + info->default_ram_id = g_strdup(mc->default_ram_id); + info->has_default_ram_id = true; + } entry = g_malloc0(sizeof(*entry)); entry->value = info; diff --git a/hw/core/machine.c b/hw/core/machine.c index 8d1a90c6cf..cf5f2dfaeb 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -28,6 +28,9 @@ #include "hw/mem/nvdimm.h" #include "migration/vmstate.h" +GlobalProperty hw_compat_5_1[] = {}; +const size_t hw_compat_5_1_len = G_N_ELEMENTS(hw_compat_5_1); + GlobalProperty hw_compat_5_0[] = { { "pci-host-bridge", "x-config-reg-migration-enabled", "off" }, { "virtio-balloon-device", "page-poison", "false" }, diff --git a/hw/i386/pc.c b/hw/i386/pc.c index cc616e48f3..5d8d5ef8b3 100644 --- a/hw/i386/pc.c +++ b/hw/i386/pc.c @@ -97,6 +97,9 @@ #include "trace.h" #include CONFIG_DEVICES +GlobalProperty pc_compat_5_1[] = {}; +const size_t pc_compat_5_1_len = G_N_ELEMENTS(pc_compat_5_1); + GlobalProperty pc_compat_5_0[] = { }; const size_t pc_compat_5_0_len = G_N_ELEMENTS(pc_compat_5_0); diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c index 456e336934..32b1453e6a 100644 --- a/hw/i386/pc_piix.c +++ b/hw/i386/pc_piix.c @@ -426,7 +426,7 @@ static void pc_i440fx_machine_options(MachineClass *m) machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE); } -static void pc_i440fx_5_1_machine_options(MachineClass *m) +static void pc_i440fx_5_2_machine_options(MachineClass *m) { PCMachineClass *pcmc = PC_MACHINE_CLASS(m); pc_i440fx_machine_options(m); @@ -435,6 +435,18 @@ static void pc_i440fx_5_1_machine_options(MachineClass *m) pcmc->default_cpu_version = 1; } +DEFINE_I440FX_MACHINE(v5_2, "pc-i440fx-5.2", NULL, + pc_i440fx_5_2_machine_options); + +static void pc_i440fx_5_1_machine_options(MachineClass *m) +{ + pc_i440fx_5_2_machine_options(m); + m->alias = NULL; + m->is_default = false; + compat_props_add(m->compat_props, hw_compat_5_1, hw_compat_5_1_len); + compat_props_add(m->compat_props, pc_compat_5_1, pc_compat_5_1_len); +} + DEFINE_I440FX_MACHINE(v5_1, "pc-i440fx-5.1", NULL, pc_i440fx_5_1_machine_options); diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c index a3e607a544..0cb9c18cd4 100644 --- a/hw/i386/pc_q35.c +++ b/hw/i386/pc_q35.c @@ -353,7 +353,7 @@ static void pc_q35_machine_options(MachineClass *m) m->max_cpus = 288; } -static void pc_q35_5_1_machine_options(MachineClass *m) +static void pc_q35_5_2_machine_options(MachineClass *m) { PCMachineClass *pcmc = PC_MACHINE_CLASS(m); pc_q35_machine_options(m); @@ -361,6 +361,17 @@ static void pc_q35_5_1_machine_options(MachineClass *m) pcmc->default_cpu_version = 1; } +DEFINE_Q35_MACHINE(v5_2, "pc-q35-5.2", NULL, + pc_q35_5_2_machine_options); + +static void pc_q35_5_1_machine_options(MachineClass *m) +{ + pc_q35_5_2_machine_options(m); + m->alias = NULL; + compat_props_add(m->compat_props, hw_compat_5_1, hw_compat_5_1_len); + compat_props_add(m->compat_props, pc_compat_5_1, pc_compat_5_1_len); +} + DEFINE_Q35_MACHINE(v5_1, "pc-q35-5.1", NULL, pc_q35_5_1_machine_options); diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index 0ae293ec94..1c8d0981b3 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -4580,14 +4580,25 @@ static void spapr_machine_latest_class_options(MachineClass *mc) type_init(spapr_machine_register_##suffix) /* + * pseries-5.2 + */ +static void spapr_machine_5_2_class_options(MachineClass *mc) +{ + /* Defaults for the latest behaviour inherited from the base class */ +} + +DEFINE_SPAPR_MACHINE(5_2, "5.2", true); + +/* * pseries-5.1 */ static void spapr_machine_5_1_class_options(MachineClass *mc) { - /* Defaults for the latest behaviour inherited from the base class */ + spapr_machine_5_2_class_options(mc); + compat_props_add(mc->compat_props, hw_compat_5_1, hw_compat_5_1_len); } -DEFINE_SPAPR_MACHINE(5_1, "5.1", true); +DEFINE_SPAPR_MACHINE(5_1, "5.1", false); /* * pseries-5.0 diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index e72c61d2ea..f4ea6a9545 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -801,14 +801,26 @@ bool css_migration_enabled(void) } \ type_init(ccw_machine_register_##suffix) +static void ccw_machine_5_2_instance_options(MachineState *machine) +{ +} + +static void ccw_machine_5_2_class_options(MachineClass *mc) +{ +} +DEFINE_CCW_MACHINE(5_2, "5.2", true); + static void ccw_machine_5_1_instance_options(MachineState *machine) { + ccw_machine_5_2_instance_options(machine); } static void ccw_machine_5_1_class_options(MachineClass *mc) { + ccw_machine_5_2_class_options(mc); + compat_props_add(mc->compat_props, hw_compat_5_1, hw_compat_5_1_len); } -DEFINE_CCW_MACHINE(5_1, "5.1", true); +DEFINE_CCW_MACHINE(5_1, "5.1", false); static void ccw_machine_5_0_instance_options(MachineState *machine) { diff --git a/include/hw/boards.h b/include/hw/boards.h index 426ce5f625..bc5b82ad20 100644 --- a/include/hw/boards.h +++ b/include/hw/boards.h @@ -319,6 +319,9 @@ struct MachineState { } \ type_init(machine_initfn##_register_types) +extern GlobalProperty hw_compat_5_1[]; +extern const size_t hw_compat_5_1_len; + extern GlobalProperty hw_compat_5_0[]; extern const size_t hw_compat_5_0_len; diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h index 3d7ed3a55e..fe52e165b2 100644 --- a/include/hw/i386/pc.h +++ b/include/hw/i386/pc.h @@ -193,6 +193,9 @@ void pc_system_firmware_init(PCMachineState *pcms, MemoryRegion *rom_memory); void pc_madt_cpu_entry(AcpiDeviceIf *adev, int uid, const CPUArchIdList *apic_ids, GArray *entry); +extern GlobalProperty pc_compat_5_1[]; +extern const size_t pc_compat_5_1_len; + extern GlobalProperty pc_compat_5_0[]; extern const size_t pc_compat_5_0_len; diff --git a/include/hw/qdev-properties.h b/include/hw/qdev-properties.h index 8f3a98cba6..528310bb22 100644 --- a/include/hw/qdev-properties.h +++ b/include/hw/qdev-properties.h @@ -238,6 +238,8 @@ extern const PropertyInfo qdev_prop_pcie_link_width; /* * Set properties between creation and realization. + * + * Returns: %true on success, %false on error. */ bool qdev_prop_set_drive_err(DeviceState *dev, const char *name, BlockBackend *value, Error **errp); diff --git a/migration/block-dirty-bitmap.c b/migration/block-dirty-bitmap.c index 784330ebe1..549e14daba 100644 --- a/migration/block-dirty-bitmap.c +++ b/migration/block-dirty-bitmap.c @@ -29,10 +29,10 @@ * * # Header (shared for different chunk types) * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags) - * [ 1 byte: node name size ] \ flags & DEVICE_NAME - * [ n bytes: node name ] / - * [ 1 byte: bitmap name size ] \ flags & BITMAP_NAME - * [ n bytes: bitmap name ] / + * [ 1 byte: node alias size ] \ flags & DEVICE_NAME + * [ n bytes: node alias ] / + * [ 1 byte: bitmap alias size ] \ flags & BITMAP_NAME + * [ n bytes: bitmap alias ] / * * # Start of bitmap migration (flags & START) * header @@ -72,7 +72,9 @@ #include "migration/register.h" #include "qemu/hbitmap.h" #include "qemu/cutils.h" +#include "qemu/id.h" #include "qapi/error.h" +#include "qapi/qapi-commands-migration.h" #include "trace.h" #define CHUNK_SIZE (1 << 10) @@ -104,7 +106,8 @@ typedef struct SaveBitmapState { /* Written during setup phase. */ BlockDriverState *bs; - const char *node_name; + char *node_alias; + char *bitmap_alias; BdrvDirtyBitmap *bitmap; uint64_t total_sectors; uint64_t sectors_per_chunk; @@ -138,8 +141,9 @@ typedef struct LoadBitmapState { /* State of the dirty bitmap migration (DBM) during load process */ typedef struct DBMLoadState { uint32_t flags; - char node_name[256]; - char bitmap_name[256]; + char node_alias[256]; + char bitmap_alias[256]; + char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1]; BlockDriverState *bs; BdrvDirtyBitmap *bitmap; @@ -165,6 +169,188 @@ typedef struct DBMState { static DBMState dbm_state; +/* For hash tables that map node/bitmap names to aliases */ +typedef struct AliasMapInnerNode { + char *string; + GHashTable *subtree; +} AliasMapInnerNode; + +static void free_alias_map_inner_node(void *amin_ptr) +{ + AliasMapInnerNode *amin = amin_ptr; + + g_free(amin->string); + g_hash_table_unref(amin->subtree); + g_free(amin); +} + +/** + * Construct an alias map based on the given QMP structure. + * + * (Note that we cannot store such maps in the MigrationParameters + * object, because that struct is defined by the QAPI schema, which + * makes it basically impossible to have dicts with arbitrary keys. + * Therefore, we instead have to construct these maps when migration + * starts.) + * + * @bbm is the block_bitmap_mapping from the migration parameters. + * + * If @name_to_alias is true, the returned hash table will map node + * and bitmap names to their respective aliases (for outgoing + * migration). + * + * If @name_to_alias is false, the returned hash table will map node + * and bitmap aliases to their respective names (for incoming + * migration). + * + * The hash table maps node names/aliases to AliasMapInnerNode + * objects, whose .string is the respective node alias/name, and whose + * .subtree table maps bitmap names/aliases to the respective bitmap + * alias/name. + */ +static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm, + bool name_to_alias, + Error **errp) +{ + GHashTable *alias_map; + size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1; + + alias_map = g_hash_table_new_full(g_str_hash, g_str_equal, + g_free, free_alias_map_inner_node); + + for (; bbm; bbm = bbm->next) { + const BitmapMigrationNodeAlias *bmna = bbm->value; + const BitmapMigrationBitmapAliasList *bmbal; + AliasMapInnerNode *amin; + GHashTable *bitmaps_map; + const char *node_map_from, *node_map_to; + + if (!id_wellformed(bmna->alias)) { + error_setg(errp, "The node alias '%s' is not well-formed", + bmna->alias); + goto fail; + } + + if (strlen(bmna->alias) > UINT8_MAX) { + error_setg(errp, "The node alias '%s' is longer than %u bytes", + bmna->alias, UINT8_MAX); + goto fail; + } + + if (strlen(bmna->node_name) > max_node_name_len) { + error_setg(errp, "The node name '%s' is longer than %zu bytes", + bmna->node_name, max_node_name_len); + goto fail; + } + + if (name_to_alias) { + if (g_hash_table_contains(alias_map, bmna->node_name)) { + error_setg(errp, "The node name '%s' is mapped twice", + bmna->node_name); + goto fail; + } + + node_map_from = bmna->node_name; + node_map_to = bmna->alias; + } else { + if (g_hash_table_contains(alias_map, bmna->alias)) { + error_setg(errp, "The node alias '%s' is used twice", + bmna->alias); + goto fail; + } + + node_map_from = bmna->alias; + node_map_to = bmna->node_name; + } + + bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, + g_free, g_free); + + amin = g_new(AliasMapInnerNode, 1); + *amin = (AliasMapInnerNode){ + .string = g_strdup(node_map_to), + .subtree = bitmaps_map, + }; + + g_hash_table_insert(alias_map, g_strdup(node_map_from), amin); + + for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) { + const BitmapMigrationBitmapAlias *bmba = bmbal->value; + const char *bmap_map_from, *bmap_map_to; + + if (strlen(bmba->alias) > UINT8_MAX) { + error_setg(errp, + "The bitmap alias '%s' is longer than %u bytes", + bmba->alias, UINT8_MAX); + goto fail; + } + + if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) { + error_setg(errp, "The bitmap name '%s' is longer than %d bytes", + bmba->name, BDRV_BITMAP_MAX_NAME_SIZE); + goto fail; + } + + if (name_to_alias) { + bmap_map_from = bmba->name; + bmap_map_to = bmba->alias; + + if (g_hash_table_contains(bitmaps_map, bmba->name)) { + error_setg(errp, "The bitmap '%s'/'%s' is mapped twice", + bmna->node_name, bmba->name); + goto fail; + } + } else { + bmap_map_from = bmba->alias; + bmap_map_to = bmba->name; + + if (g_hash_table_contains(bitmaps_map, bmba->alias)) { + error_setg(errp, "The bitmap alias '%s'/'%s' is used twice", + bmna->alias, bmba->alias); + goto fail; + } + } + + g_hash_table_insert(bitmaps_map, + g_strdup(bmap_map_from), g_strdup(bmap_map_to)); + } + } + + return alias_map; + +fail: + g_hash_table_destroy(alias_map); + return NULL; +} + +/** + * Run construct_alias_map() in both directions to check whether @bbm + * is valid. + * (This function is to be used by migration/migration.c to validate + * the user-specified block-bitmap-mapping migration parameter.) + * + * Returns true if and only if the mapping is valid. + */ +bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm, + Error **errp) +{ + GHashTable *alias_map; + + alias_map = construct_alias_map(bbm, true, errp); + if (!alias_map) { + return false; + } + g_hash_table_destroy(alias_map); + + alias_map = construct_alias_map(bbm, false, errp); + if (!alias_map) { + return false; + } + g_hash_table_destroy(alias_map); + + return true; +} + static uint32_t qemu_get_bitmap_flags(QEMUFile *f) { uint8_t flags = qemu_get_byte(f); @@ -207,11 +393,11 @@ static void send_bitmap_header(QEMUFile *f, DBMSaveState *s, qemu_put_bitmap_flags(f, flags); if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) { - qemu_put_counted_string(f, dbms->node_name); + qemu_put_counted_string(f, dbms->node_alias); } if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) { - qemu_put_counted_string(f, bdrv_dirty_bitmap_name(bitmap)); + qemu_put_counted_string(f, dbms->bitmap_alias); } } @@ -282,18 +468,25 @@ static void dirty_bitmap_do_save_cleanup(DBMSaveState *s) QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry); bdrv_dirty_bitmap_set_busy(dbms->bitmap, false); bdrv_unref(dbms->bs); + g_free(dbms->node_alias); + g_free(dbms->bitmap_alias); g_free(dbms); } } /* Called with iothread lock taken. */ static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs, - const char *bs_name) + const char *bs_name, GHashTable *alias_map) { BdrvDirtyBitmap *bitmap; SaveBitmapState *dbms; + GHashTable *bitmap_aliases; + const char *node_alias, *bitmap_name, *bitmap_alias; Error *local_err = NULL; + /* When an alias map is given, @bs_name must be @bs's node name */ + assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs))); + FOR_EACH_DIRTY_BITMAP(bs, bitmap) { if (bdrv_dirty_bitmap_name(bitmap)) { break; @@ -303,21 +496,39 @@ static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs, return 0; } + bitmap_name = bdrv_dirty_bitmap_name(bitmap); + if (!bs_name || strcmp(bs_name, "") == 0) { error_report("Bitmap '%s' in unnamed node can't be migrated", - bdrv_dirty_bitmap_name(bitmap)); + bitmap_name); return -1; } - if (bs_name[0] == '#') { + if (alias_map) { + const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name); + + if (!amin) { + /* Skip bitmaps on nodes with no alias */ + return 0; + } + + node_alias = amin->string; + bitmap_aliases = amin->subtree; + } else { + node_alias = bs_name; + bitmap_aliases = NULL; + } + + if (node_alias[0] == '#') { error_report("Bitmap '%s' in a node with auto-generated " "name '%s' can't be migrated", - bdrv_dirty_bitmap_name(bitmap), bs_name); + bitmap_name, node_alias); return -1; } FOR_EACH_DIRTY_BITMAP(bs, bitmap) { - if (!bdrv_dirty_bitmap_name(bitmap)) { + bitmap_name = bdrv_dirty_bitmap_name(bitmap); + if (!bitmap_name) { continue; } @@ -326,12 +537,29 @@ static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs, return -1; } + if (bitmap_aliases) { + bitmap_alias = g_hash_table_lookup(bitmap_aliases, bitmap_name); + if (!bitmap_alias) { + /* Skip bitmaps with no alias */ + continue; + } + } else { + if (strlen(bitmap_name) > UINT8_MAX) { + error_report("Cannot migrate bitmap '%s' on node '%s': " + "Name is longer than %u bytes", + bitmap_name, bs_name, UINT8_MAX); + return -1; + } + bitmap_alias = bitmap_name; + } + bdrv_ref(bs); bdrv_dirty_bitmap_set_busy(bitmap, true); dbms = g_new0(SaveBitmapState, 1); dbms->bs = bs; - dbms->node_name = bs_name; + dbms->node_alias = g_strdup(node_alias); + dbms->bitmap_alias = g_strdup(bitmap_alias); dbms->bitmap = bitmap; dbms->total_sectors = bdrv_nb_sectors(bs); dbms->sectors_per_chunk = CHUNK_SIZE * 8 * @@ -356,43 +584,52 @@ static int init_dirty_bitmap_migration(DBMSaveState *s) SaveBitmapState *dbms; GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL); BlockBackend *blk; + const MigrationParameters *mig_params = &migrate_get_current()->parameters; + GHashTable *alias_map = NULL; + + if (mig_params->has_block_bitmap_mapping) { + alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true, + &error_abort); + } s->bulk_completed = false; s->prev_bs = NULL; s->prev_bitmap = NULL; s->no_bitmaps = false; - /* - * Use blockdevice name for direct (or filtered) children of named block - * backends. - */ - for (blk = blk_next(NULL); blk; blk = blk_next(blk)) { - const char *name = blk_name(blk); - - if (!name || strcmp(name, "") == 0) { - continue; - } + if (!alias_map) { + /* + * Use blockdevice name for direct (or filtered) children of named block + * backends. + */ + for (blk = blk_next(NULL); blk; blk = blk_next(blk)) { + const char *name = blk_name(blk); - bs = blk_bs(blk); + if (!name || strcmp(name, "") == 0) { + continue; + } - /* Skip filters without bitmaps */ - while (bs && bs->drv && bs->drv->is_filter && - !bdrv_has_named_bitmaps(bs)) - { - if (bs->backing) { - bs = bs->backing->bs; - } else if (bs->file) { - bs = bs->file->bs; - } else { - bs = NULL; + bs = blk_bs(blk); + + /* Skip filters without bitmaps */ + while (bs && bs->drv && bs->drv->is_filter && + !bdrv_has_named_bitmaps(bs)) + { + if (bs->backing) { + bs = bs->backing->bs; + } else if (bs->file) { + bs = bs->file->bs; + } else { + bs = NULL; + } } - } - if (bs && bs->drv && !bs->drv->is_filter) { - if (add_bitmaps_to_list(s, bs, name)) { - goto fail; + if (bs && bs->drv && !bs->drv->is_filter) { + if (add_bitmaps_to_list(s, bs, name, NULL)) { + goto fail; + } + g_hash_table_add(handled_by_blk, bs); } - g_hash_table_add(handled_by_blk, bs); } } @@ -401,7 +638,7 @@ static int init_dirty_bitmap_migration(DBMSaveState *s) continue; } - if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs))) { + if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) { goto fail; } } @@ -416,11 +653,17 @@ static int init_dirty_bitmap_migration(DBMSaveState *s) } g_hash_table_destroy(handled_by_blk); + if (alias_map) { + g_hash_table_destroy(alias_map); + } return 0; fail: g_hash_table_destroy(handled_by_blk); + if (alias_map) { + g_hash_table_destroy(alias_map); + } dirty_bitmap_do_save_cleanup(s); return -1; @@ -770,8 +1013,10 @@ static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s) return 0; } -static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s) +static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s, + GHashTable *alias_map) { + GHashTable *bitmap_alias_map = NULL; Error *local_err = NULL; bool nothing; s->flags = qemu_get_bitmap_flags(f); @@ -780,28 +1025,75 @@ static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s) nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS); if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) { - if (!qemu_get_counted_string(f, s->node_name)) { - error_report("Unable to read node name string"); + if (!qemu_get_counted_string(f, s->node_alias)) { + error_report("Unable to read node alias string"); return -EINVAL; } + if (!s->cancelled) { - s->bs = bdrv_lookup_bs(s->node_name, s->node_name, &local_err); + if (alias_map) { + const AliasMapInnerNode *amin; + + amin = g_hash_table_lookup(alias_map, s->node_alias); + if (!amin) { + error_setg(&local_err, "Error: Unknown node alias '%s'", + s->node_alias); + s->bs = NULL; + } else { + bitmap_alias_map = amin->subtree; + s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err); + } + } else { + s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias, + &local_err); + } if (!s->bs) { error_report_err(local_err); cancel_incoming_locked(s); } } - } else if (!s->bs && !nothing && !s->cancelled) { + } else if (s->bs) { + if (alias_map) { + const AliasMapInnerNode *amin; + + /* Must be present in the map, or s->bs would not be set */ + amin = g_hash_table_lookup(alias_map, s->node_alias); + assert(amin != NULL); + + bitmap_alias_map = amin->subtree; + } + } else if (!nothing && !s->cancelled) { error_report("Error: block device name is not set"); cancel_incoming_locked(s); } + assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map); + if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) { - if (!qemu_get_counted_string(f, s->bitmap_name)) { - error_report("Unable to read bitmap name string"); + const char *bitmap_name; + + if (!qemu_get_counted_string(f, s->bitmap_alias)) { + error_report("Unable to read bitmap alias string"); return -EINVAL; } + + if (!s->cancelled) { + if (bitmap_alias_map) { + bitmap_name = g_hash_table_lookup(bitmap_alias_map, + s->bitmap_alias); + if (!bitmap_name) { + error_report("Error: Unknown bitmap alias '%s' on node " + "'%s' (alias '%s')", s->bitmap_alias, + s->bs->node_name, s->node_alias); + cancel_incoming_locked(s); + } + } else { + bitmap_name = s->bitmap_alias; + } + } + if (!s->cancelled) { + g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name)); s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name); /* @@ -811,7 +1103,7 @@ static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s) if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) { error_report("Error: unknown dirty bitmap " "'%s' for block device '%s'", - s->bitmap_name, s->node_name); + s->bitmap_name, s->bs->node_name); cancel_incoming_locked(s); } } @@ -835,6 +1127,8 @@ static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s) */ static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id) { + GHashTable *alias_map = NULL; + const MigrationParameters *mig_params = &migrate_get_current()->parameters; DBMLoadState *s = &((DBMState *)opaque)->load; int ret = 0; @@ -846,13 +1140,18 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id) return -EINVAL; } + if (mig_params->has_block_bitmap_mapping) { + alias_map = construct_alias_map(mig_params->block_bitmap_mapping, + false, &error_abort); + } + do { QEMU_LOCK_GUARD(&s->lock); - ret = dirty_bitmap_load_header(f, s); + ret = dirty_bitmap_load_header(f, s, alias_map); if (ret < 0) { cancel_incoming_locked(s); - return ret; + goto fail; } if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) { @@ -869,12 +1168,17 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id) if (ret) { cancel_incoming_locked(s); - return ret; + goto fail; } } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS)); trace_dirty_bitmap_load_success(); - return 0; + ret = 0; +fail: + if (alias_map) { + g_hash_table_destroy(alias_map); + } + return ret; } static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque) diff --git a/migration/migration.c b/migration/migration.c index 8fe36339db..dbd4afa1e8 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -36,6 +36,7 @@ #include "block/block.h" #include "qapi/error.h" #include "qapi/clone-visitor.h" +#include "qapi/qapi-visit-migration.h" #include "qapi/qapi-visit-sockets.h" #include "qapi/qapi-commands-migration.h" #include "qapi/qapi-events-migration.h" @@ -843,6 +844,13 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) params->has_announce_step = true; params->announce_step = s->parameters.announce_step; + if (s->parameters.has_block_bitmap_mapping) { + params->has_block_bitmap_mapping = true; + params->block_bitmap_mapping = + QAPI_CLONE(BitmapMigrationNodeAliasList, + s->parameters.block_bitmap_mapping); + } + return params; } @@ -1308,6 +1316,13 @@ static bool migrate_params_check(MigrationParameters *params, Error **errp) "is invalid, it must be in the range of 1 to 10000 ms"); return false; } + + if (params->has_block_bitmap_mapping && + !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) { + error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: "); + return false; + } + return true; } @@ -1402,6 +1417,11 @@ static void migrate_params_test_apply(MigrateSetParameters *params, if (params->has_announce_step) { dest->announce_step = params->announce_step; } + + if (params->has_block_bitmap_mapping) { + dest->has_block_bitmap_mapping = true; + dest->block_bitmap_mapping = params->block_bitmap_mapping; + } } static void migrate_params_apply(MigrateSetParameters *params, Error **errp) @@ -1514,6 +1534,16 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) if (params->has_announce_step) { s->parameters.announce_step = params->announce_step; } + + if (params->has_block_bitmap_mapping) { + qapi_free_BitmapMigrationNodeAliasList( + s->parameters.block_bitmap_mapping); + + s->parameters.has_block_bitmap_mapping = true; + s->parameters.block_bitmap_mapping = + QAPI_CLONE(BitmapMigrationNodeAliasList, + params->block_bitmap_mapping); + } } void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp) diff --git a/migration/migration.h b/migration/migration.h index 6c6a931d0d..2ed55c4aef 100644 --- a/migration/migration.h +++ b/migration/migration.h @@ -337,6 +337,9 @@ void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value); void dirty_bitmap_mig_before_vm_start(void); void dirty_bitmap_mig_cancel_outgoing(void); void dirty_bitmap_mig_cancel_incoming(void); +bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm, + Error **errp); + void migrate_add_address(SocketAddress *address); int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque); diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c index ae4b6a4246..7711726fd2 100644 --- a/monitor/hmp-cmds.c +++ b/monitor/hmp-cmds.c @@ -469,6 +469,32 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict) monitor_printf(mon, "%s: '%s'\n", MigrationParameter_str(MIGRATION_PARAMETER_TLS_AUTHZ), params->tls_authz); + + if (params->has_block_bitmap_mapping) { + const BitmapMigrationNodeAliasList *bmnal; + + monitor_printf(mon, "%s:\n", + MigrationParameter_str( + MIGRATION_PARAMETER_BLOCK_BITMAP_MAPPING)); + + for (bmnal = params->block_bitmap_mapping; + bmnal; + bmnal = bmnal->next) + { + const BitmapMigrationNodeAlias *bmna = bmnal->value; + const BitmapMigrationBitmapAliasList *bmbal; + + monitor_printf(mon, " '%s' -> '%s'\n", + bmna->node_name, bmna->alias); + + for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) { + const BitmapMigrationBitmapAlias *bmba = bmbal->value; + + monitor_printf(mon, " '%s' -> '%s'\n", + bmba->name, bmba->alias); + } + } + } } qapi_free_MigrationParameters(params); @@ -1384,6 +1410,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict) p->has_announce_step = true; visit_type_size(v, param, &p->announce_step, &err); break; + case MIGRATION_PARAMETER_BLOCK_BITMAP_MAPPING: + error_setg(&err, "The block-bitmap-mapping parameter can only be set " + "through QMP"); + break; default: assert(0); } diff --git a/pc-bios/bios-256k.bin b/pc-bios/bios-256k.bin index 8fd42903cd..96bdeaa487 100644 --- a/pc-bios/bios-256k.bin +++ b/pc-bios/bios-256k.bin Binary files differdiff --git a/pc-bios/bios.bin b/pc-bios/bios.bin index 528b8ab412..3ecaf4a709 100644 --- a/pc-bios/bios.bin +++ b/pc-bios/bios.bin Binary files differdiff --git a/pc-bios/vgabios-ati.bin b/pc-bios/vgabios-ati.bin index 3dbfe53e5c..118caacde6 100644 --- a/pc-bios/vgabios-ati.bin +++ b/pc-bios/vgabios-ati.bin Binary files differdiff --git a/pc-bios/vgabios-bochs-display.bin b/pc-bios/vgabios-bochs-display.bin index ca4c3db2db..d79124b760 100644 --- a/pc-bios/vgabios-bochs-display.bin +++ b/pc-bios/vgabios-bochs-display.bin Binary files differdiff --git a/pc-bios/vgabios-cirrus.bin b/pc-bios/vgabios-cirrus.bin index b8ee1fe0ff..37bf5e7fe7 100644 --- a/pc-bios/vgabios-cirrus.bin +++ b/pc-bios/vgabios-cirrus.bin Binary files differdiff --git a/pc-bios/vgabios-qxl.bin b/pc-bios/vgabios-qxl.bin index 9829cd1dec..0ff8ff21f0 100644 --- a/pc-bios/vgabios-qxl.bin +++ b/pc-bios/vgabios-qxl.bin Binary files differdiff --git a/pc-bios/vgabios-ramfb.bin b/pc-bios/vgabios-ramfb.bin index 12dcb8e823..df5e9d615a 100644 --- a/pc-bios/vgabios-ramfb.bin +++ b/pc-bios/vgabios-ramfb.bin Binary files differdiff --git a/pc-bios/vgabios-stdvga.bin b/pc-bios/vgabios-stdvga.bin index 68fe2fe820..70f6094153 100644 --- a/pc-bios/vgabios-stdvga.bin +++ b/pc-bios/vgabios-stdvga.bin Binary files differdiff --git a/pc-bios/vgabios-virtio.bin b/pc-bios/vgabios-virtio.bin index 60ce81d37c..65af5be59d 100644 --- a/pc-bios/vgabios-virtio.bin +++ b/pc-bios/vgabios-virtio.bin Binary files differdiff --git a/pc-bios/vgabios-vmware.bin b/pc-bios/vgabios-vmware.bin index b75a3517d1..89d42b32c1 100644 --- a/pc-bios/vgabios-vmware.bin +++ b/pc-bios/vgabios-vmware.bin Binary files differdiff --git a/pc-bios/vgabios.bin b/pc-bios/vgabios.bin index 410436fbdf..320471e18d 100644 --- a/pc-bios/vgabios.bin +++ b/pc-bios/vgabios.bin Binary files differdiff --git a/qapi/machine.json b/qapi/machine.json index 481b1f07ec..abc6fd0477 100644 --- a/qapi/machine.json +++ b/qapi/machine.json @@ -357,13 +357,16 @@ # @default-cpu-type: default CPU model typename if none is requested via # the -cpu argument. (since 4.2) # +# @default-ram-id: the default ID of initial RAM memory backend (since 5.2) +# # Since: 1.2.0 ## { 'struct': 'MachineInfo', 'data': { 'name': 'str', '*alias': 'str', '*is-default': 'bool', 'cpu-max': 'int', 'hotpluggable-cpus': 'bool', 'numa-mem-supported': 'bool', - 'deprecated': 'bool', '*default-cpu-type': 'str' } } + 'deprecated': 'bool', '*default-cpu-type': 'str', + '*default-ram-id': 'str' } } ## # @query-machines: diff --git a/qapi/migration.json b/qapi/migration.json index ea53b23dca..5f6b06172c 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -509,6 +509,44 @@ { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] } ## +# @BitmapMigrationBitmapAlias: +# +# @name: The name of the bitmap. +# +# @alias: An alias name for migration (for example the bitmap name on +# the opposite site). +# +# Since: 5.2 +## +{ 'struct': 'BitmapMigrationBitmapAlias', + 'data': { + 'name': 'str', + 'alias': 'str' + } } + +## +# @BitmapMigrationNodeAlias: +# +# Maps a block node name and the bitmaps it has to aliases for dirty +# bitmap migration. +# +# @node-name: A block node name. +# +# @alias: An alias block node name for migration (for example the +# node name on the opposite site). +# +# @bitmaps: Mappings for the bitmaps on this node. +# +# Since: 5.2 +## +{ 'struct': 'BitmapMigrationNodeAlias', + 'data': { + 'node-name': 'str', + 'alias': 'str', + 'bitmaps': [ 'BitmapMigrationBitmapAlias' ] + } } + +## # @MigrationParameter: # # Migration parameters enumeration @@ -642,6 +680,25 @@ # will consume more CPU. # Defaults to 1. (Since 5.0) # +# @block-bitmap-mapping: Maps block nodes and bitmaps on them to +# aliases for the purpose of dirty bitmap migration. Such +# aliases may for example be the corresponding names on the +# opposite site. +# The mapping must be one-to-one, but not necessarily +# complete: On the source, unmapped bitmaps and all bitmaps +# on unmapped nodes will be ignored. On the destination, +# encountering an unmapped alias in the incoming migration +# stream will result in a report, and all further bitmap +# migration data will then be discarded. +# Note that the destination does not know about bitmaps it +# does not receive, so there is no limitation or requirement +# regarding the number of bitmaps received, or how they are +# named, or on which nodes they are placed. +# By default (when this parameter has never been set), bitmap +# names are mapped to themselves. Nodes are mapped to their +# block device name if there is one, and to their node name +# otherwise. (Since 5.2) +# # Since: 2.4 ## { 'enum': 'MigrationParameter', @@ -656,7 +713,8 @@ 'multifd-channels', 'xbzrle-cache-size', 'max-postcopy-bandwidth', 'max-cpu-throttle', 'multifd-compression', - 'multifd-zlib-level' ,'multifd-zstd-level' ] } + 'multifd-zlib-level' ,'multifd-zstd-level', + 'block-bitmap-mapping' ] } ## # @MigrateSetParameters: @@ -782,6 +840,25 @@ # will consume more CPU. # Defaults to 1. (Since 5.0) # +# @block-bitmap-mapping: Maps block nodes and bitmaps on them to +# aliases for the purpose of dirty bitmap migration. Such +# aliases may for example be the corresponding names on the +# opposite site. +# The mapping must be one-to-one, but not necessarily +# complete: On the source, unmapped bitmaps and all bitmaps +# on unmapped nodes will be ignored. On the destination, +# encountering an unmapped alias in the incoming migration +# stream will result in a report, and all further bitmap +# migration data will then be discarded. +# Note that the destination does not know about bitmaps it +# does not receive, so there is no limitation or requirement +# regarding the number of bitmaps received, or how they are +# named, or on which nodes they are placed. +# By default (when this parameter has never been set), bitmap +# names are mapped to themselves. Nodes are mapped to their +# block device name if there is one, and to their node name +# otherwise. (Since 5.2) +# # Since: 2.4 ## # TODO either fuse back into MigrationParameters, or make @@ -812,7 +889,8 @@ '*max-cpu-throttle': 'int', '*multifd-compression': 'MultiFDCompression', '*multifd-zlib-level': 'int', - '*multifd-zstd-level': 'int' } } + '*multifd-zstd-level': 'int', + '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ] } } ## # @migrate-set-parameters: @@ -958,6 +1036,25 @@ # will consume more CPU. # Defaults to 1. (Since 5.0) # +# @block-bitmap-mapping: Maps block nodes and bitmaps on them to +# aliases for the purpose of dirty bitmap migration. Such +# aliases may for example be the corresponding names on the +# opposite site. +# The mapping must be one-to-one, but not necessarily +# complete: On the source, unmapped bitmaps and all bitmaps +# on unmapped nodes will be ignored. On the destination, +# encountering an unmapped alias in the incoming migration +# stream will result in a report, and all further bitmap +# migration data will then be discarded. +# Note that the destination does not know about bitmaps it +# does not receive, so there is no limitation or requirement +# regarding the number of bitmaps received, or how they are +# named, or on which nodes they are placed. +# By default (when this parameter has never been set), bitmap +# names are mapped to themselves. Nodes are mapped to their +# block device name if there is one, and to their node name +# otherwise. (Since 5.2) +# # Since: 2.4 ## { 'struct': 'MigrationParameters', @@ -986,7 +1083,8 @@ '*max-cpu-throttle': 'uint8', '*multifd-compression': 'MultiFDCompression', '*multifd-zlib-level': 'uint8', - '*multifd-zstd-level': 'uint8' } } + '*multifd-zstd-level': 'uint8', + '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ] } } ## # @query-migrate-parameters: diff --git a/roms/seabios b/roms/seabios -Subproject d9c812dda519a1a73e8370e1b81ddf46eb22ed1 +Subproject 155821a1990b6de78dde5f98fa5ab90e802021e diff --git a/tests/qemu-iotests/300 b/tests/qemu-iotests/300 new file mode 100755 index 0000000000..5b75121b84 --- /dev/null +++ b/tests/qemu-iotests/300 @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 Red Hat, Inc. +# +# Tests for dirty bitmaps migration with node aliases +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import os +import random +import re +from typing import Dict, List, Optional, Union +import iotests +import qemu + +BlockBitmapMapping = List[Dict[str, Union[str, List[Dict[str, str]]]]] + +assert iotests.sock_dir is not None +mig_sock = os.path.join(iotests.sock_dir, 'mig_sock') + + +class TestDirtyBitmapMigration(iotests.QMPTestCase): + src_node_name: str = '' + dst_node_name: str = '' + src_bmap_name: str = '' + dst_bmap_name: str = '' + + def setUp(self) -> None: + self.vm_a = iotests.VM(path_suffix='-a') + self.vm_a.add_blockdev(f'node-name={self.src_node_name},' + 'driver=null-co') + self.vm_a.launch() + + self.vm_b = iotests.VM(path_suffix='-b') + self.vm_b.add_blockdev(f'node-name={self.dst_node_name},' + 'driver=null-co') + self.vm_b.add_incoming(f'unix:{mig_sock}') + self.vm_b.launch() + + result = self.vm_a.qmp('block-dirty-bitmap-add', + node=self.src_node_name, + name=self.src_bmap_name) + self.assert_qmp(result, 'return', {}) + + # Dirty some random megabytes + for _ in range(9): + mb_ofs = random.randrange(1024) + self.vm_a.hmp_qemu_io(self.src_node_name, f'discard {mb_ofs}M 1M') + + result = self.vm_a.qmp('x-debug-block-dirty-bitmap-sha256', + node=self.src_node_name, + name=self.src_bmap_name) + self.bitmap_hash_reference = result['return']['sha256'] + + caps = [{'capability': name, 'state': True} + for name in ('dirty-bitmaps', 'events')] + + for vm in (self.vm_a, self.vm_b): + result = vm.qmp('migrate-set-capabilities', capabilities=caps) + self.assert_qmp(result, 'return', {}) + + def tearDown(self) -> None: + self.vm_a.shutdown() + self.vm_b.shutdown() + try: + os.remove(mig_sock) + except OSError: + pass + + def check_bitmap(self, bitmap_name_valid: bool) -> None: + result = self.vm_b.qmp('x-debug-block-dirty-bitmap-sha256', + node=self.dst_node_name, + name=self.dst_bmap_name) + if bitmap_name_valid: + self.assert_qmp(result, 'return/sha256', + self.bitmap_hash_reference) + else: + self.assert_qmp(result, 'error/desc', + f"Dirty bitmap '{self.dst_bmap_name}' not found") + + def migrate(self, bitmap_name_valid: bool = True, + migration_success: bool = True) -> None: + result = self.vm_a.qmp('migrate', uri=f'unix:{mig_sock}') + self.assert_qmp(result, 'return', {}) + + with iotests.Timeout(5, 'Timeout waiting for migration to complete'): + self.assertEqual(self.vm_a.wait_migration('postmigrate'), + migration_success) + self.assertEqual(self.vm_b.wait_migration('running'), + migration_success) + + if migration_success: + self.check_bitmap(bitmap_name_valid) + + def verify_dest_error(self, msg: Optional[str]) -> None: + """ + Check whether the given error message is present in vm_b's log. + (vm_b is shut down to do so.) + If @msg is None, check that there has not been any error. + """ + self.vm_b.shutdown() + if msg is None: + self.assertNotIn('qemu-system-', self.vm_b.get_log()) + else: + self.assertIn(msg, self.vm_b.get_log()) + + @staticmethod + def mapping(node_name: str, node_alias: str, + bitmap_name: str, bitmap_alias: str) -> BlockBitmapMapping: + return [{ + 'node-name': node_name, + 'alias': node_alias, + 'bitmaps': [{ + 'name': bitmap_name, + 'alias': bitmap_alias + }] + }] + + def set_mapping(self, vm: iotests.VM, mapping: BlockBitmapMapping, + error: Optional[str] = None) -> None: + """ + Invoke migrate-set-parameters on @vm to set the given @mapping. + Check for success if @error is None, or verify the error message + if it is not. + On success, verify that "info migrate_parameters" on HMP returns + our mapping. (Just to check its formatting code.) + """ + result = vm.qmp('migrate-set-parameters', + block_bitmap_mapping=mapping) + + if error is None: + self.assert_qmp(result, 'return', {}) + + result = vm.qmp('human-monitor-command', + command_line='info migrate_parameters') + + m = re.search(r'^block-bitmap-mapping:\r?(\n .*)*\n', + result['return'], flags=re.MULTILINE) + hmp_mapping = m.group(0).replace('\r', '') if m else None + + self.assertEqual(hmp_mapping, self.to_hmp_mapping(mapping)) + else: + self.assert_qmp(result, 'error/desc', error) + + @staticmethod + def to_hmp_mapping(mapping: BlockBitmapMapping) -> str: + result = 'block-bitmap-mapping:\n' + + for node in mapping: + result += f" '{node['node-name']}' -> '{node['alias']}'\n" + + assert isinstance(node['bitmaps'], list) + for bitmap in node['bitmaps']: + result += f" '{bitmap['name']}' -> '{bitmap['alias']}'\n" + + return result + + +class TestAliasMigration(TestDirtyBitmapMigration): + src_node_name = 'node0' + dst_node_name = 'node0' + src_bmap_name = 'bmap0' + dst_bmap_name = 'bmap0' + + def test_migration_without_alias(self) -> None: + self.migrate(self.src_node_name == self.dst_node_name and + self.src_bmap_name == self.dst_bmap_name) + + # Check for error message on the destination + if self.src_node_name != self.dst_node_name: + self.verify_dest_error(f"Cannot find " + f"device={self.src_node_name} nor " + f"node_name={self.src_node_name}") + else: + self.verify_dest_error(None) + + def test_alias_on_src_migration(self) -> None: + mapping = self.mapping(self.src_node_name, self.dst_node_name, + self.src_bmap_name, self.dst_bmap_name) + + self.set_mapping(self.vm_a, mapping) + self.migrate() + self.verify_dest_error(None) + + def test_alias_on_dst_migration(self) -> None: + mapping = self.mapping(self.dst_node_name, self.src_node_name, + self.dst_bmap_name, self.src_bmap_name) + + self.set_mapping(self.vm_b, mapping) + self.migrate() + self.verify_dest_error(None) + + def test_alias_on_both_migration(self) -> None: + src_map = self.mapping(self.src_node_name, 'node-alias', + self.src_bmap_name, 'bmap-alias') + + dst_map = self.mapping(self.dst_node_name, 'node-alias', + self.dst_bmap_name, 'bmap-alias') + + self.set_mapping(self.vm_a, src_map) + self.set_mapping(self.vm_b, dst_map) + self.migrate() + self.verify_dest_error(None) + + +class TestNodeAliasMigration(TestAliasMigration): + src_node_name = 'node-src' + dst_node_name = 'node-dst' + + +class TestBitmapAliasMigration(TestAliasMigration): + src_bmap_name = 'bmap-src' + dst_bmap_name = 'bmap-dst' + + +class TestFullAliasMigration(TestAliasMigration): + src_node_name = 'node-src' + dst_node_name = 'node-dst' + src_bmap_name = 'bmap-src' + dst_bmap_name = 'bmap-dst' + + +class TestLongBitmapNames(TestAliasMigration): + # Giving long bitmap names is OK, as long as there is a short alias for + # migration + src_bmap_name = 'a' * 512 + dst_bmap_name = 'b' * 512 + + # Skip all tests that do not use the intermediate alias + def test_migration_without_alias(self) -> None: + pass + + def test_alias_on_src_migration(self) -> None: + pass + + def test_alias_on_dst_migration(self) -> None: + pass + + +class TestBlockBitmapMappingErrors(TestDirtyBitmapMigration): + src_node_name = 'node0' + dst_node_name = 'node0' + src_bmap_name = 'bmap0' + dst_bmap_name = 'bmap0' + + """ + Note that mapping nodes or bitmaps that do not exist is not an error. + """ + + def test_non_injective_node_mapping(self) -> None: + mapping: BlockBitmapMapping = [ + { + 'node-name': 'node0', + 'alias': 'common-alias', + 'bitmaps': [{ + 'name': 'bmap0', + 'alias': 'bmap-alias0' + }] + }, + { + 'node-name': 'node1', + 'alias': 'common-alias', + 'bitmaps': [{ + 'name': 'bmap1', + 'alias': 'bmap-alias1' + }] + } + ] + + self.set_mapping(self.vm_a, mapping, + "Invalid mapping given for block-bitmap-mapping: " + "The node alias 'common-alias' is used twice") + + def test_non_injective_bitmap_mapping(self) -> None: + mapping: BlockBitmapMapping = [{ + 'node-name': 'node0', + 'alias': 'node-alias0', + 'bitmaps': [ + { + 'name': 'bmap0', + 'alias': 'common-alias' + }, + { + 'name': 'bmap1', + 'alias': 'common-alias' + } + ] + }] + + self.set_mapping(self.vm_a, mapping, + "Invalid mapping given for block-bitmap-mapping: " + "The bitmap alias 'node-alias0'/'common-alias' is " + "used twice") + + def test_ambiguous_node_mapping(self) -> None: + mapping: BlockBitmapMapping = [ + { + 'node-name': 'node0', + 'alias': 'node-alias0', + 'bitmaps': [{ + 'name': 'bmap0', + 'alias': 'bmap-alias0' + }] + }, + { + 'node-name': 'node0', + 'alias': 'node-alias1', + 'bitmaps': [{ + 'name': 'bmap0', + 'alias': 'bmap-alias0' + }] + } + ] + + self.set_mapping(self.vm_a, mapping, + "Invalid mapping given for block-bitmap-mapping: " + "The node name 'node0' is mapped twice") + + def test_ambiguous_bitmap_mapping(self) -> None: + mapping: BlockBitmapMapping = [{ + 'node-name': 'node0', + 'alias': 'node-alias0', + 'bitmaps': [ + { + 'name': 'bmap0', + 'alias': 'bmap-alias0' + }, + { + 'name': 'bmap0', + 'alias': 'bmap-alias1' + } + ] + }] + + self.set_mapping(self.vm_a, mapping, + "Invalid mapping given for block-bitmap-mapping: " + "The bitmap 'node0'/'bmap0' is mapped twice") + + def test_migratee_node_is_not_mapped_on_src(self) -> None: + self.set_mapping(self.vm_a, []) + # Should just ignore all bitmaps on unmapped nodes + self.migrate(False) + self.verify_dest_error(None) + + def test_migratee_node_is_not_mapped_on_dst(self) -> None: + self.set_mapping(self.vm_b, []) + self.migrate(False) + self.verify_dest_error(f"Unknown node alias '{self.src_node_name}'") + + def test_migratee_bitmap_is_not_mapped_on_src(self) -> None: + mapping: BlockBitmapMapping = [{ + 'node-name': self.src_node_name, + 'alias': self.dst_node_name, + 'bitmaps': [] + }] + + self.set_mapping(self.vm_a, mapping) + # Should just ignore all unmapped bitmaps + self.migrate(False) + self.verify_dest_error(None) + + def test_migratee_bitmap_is_not_mapped_on_dst(self) -> None: + mapping: BlockBitmapMapping = [{ + 'node-name': self.dst_node_name, + 'alias': self.src_node_name, + 'bitmaps': [] + }] + + self.set_mapping(self.vm_b, mapping) + self.migrate(False) + self.verify_dest_error(f"Unknown bitmap alias " + f"'{self.src_bmap_name}' " + f"on node '{self.dst_node_name}' " + f"(alias '{self.src_node_name}')") + + def test_unused_mapping_on_dst(self) -> None: + # Let the source not send any bitmaps + self.set_mapping(self.vm_a, []) + + # Establish some mapping on the destination + self.set_mapping(self.vm_b, []) + + # The fact that there is a mapping on B without any bitmaps + # being received should be fine, not fatal + self.migrate(False) + self.verify_dest_error(None) + + def test_non_wellformed_node_alias(self) -> None: + alias = '123-foo' + + mapping: BlockBitmapMapping = [{ + 'node-name': self.src_node_name, + 'alias': alias, + 'bitmaps': [] + }] + + self.set_mapping(self.vm_a, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The node alias '{alias}' is not well-formed") + + def test_node_alias_too_long(self) -> None: + alias = 'a' * 256 + + mapping: BlockBitmapMapping = [{ + 'node-name': self.src_node_name, + 'alias': alias, + 'bitmaps': [] + }] + + self.set_mapping(self.vm_a, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The node alias '{alias}' is longer than 255 bytes") + + def test_bitmap_alias_too_long(self) -> None: + alias = 'a' * 256 + + mapping = self.mapping(self.src_node_name, self.dst_node_name, + self.src_bmap_name, alias) + + self.set_mapping(self.vm_a, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The bitmap alias '{alias}' is longer than 255 " + f"bytes") + + def test_bitmap_name_too_long(self) -> None: + name = 'a' * 256 + + result = self.vm_a.qmp('block-dirty-bitmap-add', + node=self.src_node_name, + name=name) + self.assert_qmp(result, 'return', {}) + + self.migrate(False, False) + + # Check for the error in the source's log + self.vm_a.shutdown() + self.assertIn(f"Cannot migrate bitmap '{name}' on node " + f"'{self.src_node_name}': Name is longer than 255 bytes", + self.vm_a.get_log()) + + # Expect abnormal shutdown of the destination VM because of + # the failed migration + try: + self.vm_b.shutdown() + except qemu.machine.AbnormalShutdown: + pass + + def test_aliased_bitmap_name_too_long(self) -> None: + # Longer than the maximum for bitmap names + self.dst_bmap_name = 'a' * 1024 + + mapping = self.mapping(self.dst_node_name, self.src_node_name, + self.dst_bmap_name, self.src_bmap_name) + + # We would have to create this bitmap during migration, and + # that would fail, because the name is too long. Better to + # catch it early. + self.set_mapping(self.vm_b, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The bitmap name '{self.dst_bmap_name}' is longer " + f"than 1023 bytes") + + def test_node_name_too_long(self) -> None: + # Longer than the maximum for node names + self.dst_node_name = 'a' * 32 + + mapping = self.mapping(self.dst_node_name, self.src_node_name, + self.dst_bmap_name, self.src_bmap_name) + + # During migration, this would appear simply as a node that + # cannot be found. Still better to catch impossible node + # names early (similar to test_non_wellformed_node_alias). + self.set_mapping(self.vm_b, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The node name '{self.dst_node_name}' is longer " + f"than 31 bytes") + + +class TestCrossAliasMigration(TestDirtyBitmapMigration): + """ + Swap aliases, both to see that qemu does not get confused, and + that we can migrate multiple things at once. + + So we migrate this: + node-a.bmap-a -> node-b.bmap-b + node-a.bmap-b -> node-b.bmap-a + node-b.bmap-a -> node-a.bmap-b + node-b.bmap-b -> node-a.bmap-a + """ + + src_node_name = 'node-a' + dst_node_name = 'node-b' + src_bmap_name = 'bmap-a' + dst_bmap_name = 'bmap-b' + + def setUp(self) -> None: + TestDirtyBitmapMigration.setUp(self) + + # Now create another block device and let both have two bitmaps each + result = self.vm_a.qmp('blockdev-add', + node_name='node-b', driver='null-co') + self.assert_qmp(result, 'return', {}) + + result = self.vm_b.qmp('blockdev-add', + node_name='node-a', driver='null-co') + self.assert_qmp(result, 'return', {}) + + bmaps_to_add = (('node-a', 'bmap-b'), + ('node-b', 'bmap-a'), + ('node-b', 'bmap-b')) + + for (node, bmap) in bmaps_to_add: + result = self.vm_a.qmp('block-dirty-bitmap-add', + node=node, name=bmap) + self.assert_qmp(result, 'return', {}) + + @staticmethod + def cross_mapping() -> BlockBitmapMapping: + return [ + { + 'node-name': 'node-a', + 'alias': 'node-b', + 'bitmaps': [ + { + 'name': 'bmap-a', + 'alias': 'bmap-b' + }, + { + 'name': 'bmap-b', + 'alias': 'bmap-a' + } + ] + }, + { + 'node-name': 'node-b', + 'alias': 'node-a', + 'bitmaps': [ + { + 'name': 'bmap-b', + 'alias': 'bmap-a' + }, + { + 'name': 'bmap-a', + 'alias': 'bmap-b' + } + ] + } + ] + + def verify_dest_has_all_bitmaps(self) -> None: + bitmaps = self.vm_b.query_bitmaps() + + # Extract and sort bitmap names + for node in bitmaps: + bitmaps[node] = sorted((bmap['name'] for bmap in bitmaps[node])) + + self.assertEqual(bitmaps, + {'node-a': ['bmap-a', 'bmap-b'], + 'node-b': ['bmap-a', 'bmap-b']}) + + def test_alias_on_src(self) -> None: + self.set_mapping(self.vm_a, self.cross_mapping()) + + # Checks that node-a.bmap-a was migrated to node-b.bmap-b, and + # that is enough + self.migrate() + self.verify_dest_has_all_bitmaps() + self.verify_dest_error(None) + + def test_alias_on_dst(self) -> None: + self.set_mapping(self.vm_b, self.cross_mapping()) + + # Checks that node-a.bmap-a was migrated to node-b.bmap-b, and + # that is enough + self.migrate() + self.verify_dest_has_all_bitmaps() + self.verify_dest_error(None) + + +if __name__ == '__main__': + iotests.main(supported_protocols=['file']) diff --git a/tests/qemu-iotests/300.out b/tests/qemu-iotests/300.out new file mode 100644 index 0000000000..cafb8161f7 --- /dev/null +++ b/tests/qemu-iotests/300.out @@ -0,0 +1,5 @@ +..................................... +---------------------------------------------------------------------- +Ran 37 tests + +OK diff --git a/tests/qemu-iotests/303 b/tests/qemu-iotests/303 new file mode 100755 index 0000000000..6c21774483 --- /dev/null +++ b/tests/qemu-iotests/303 @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Test for dumping of qcow2 image metadata +# +# Copyright (c) 2020 Virtuozzo International GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import iotests +import subprocess +from iotests import qemu_img_create, qemu_io, file_path, log, filter_qemu_io + +iotests.script_initialize(supported_fmts=['qcow2']) + +disk = file_path('disk') +chunk = 1024 * 1024 + + +def create_bitmap(bitmap_number, disabled): + granularity = 1 << (14 + bitmap_number) + bitmap_name = 'bitmap-' + str(bitmap_number) + args = ['bitmap', '--add', '-g', f'{granularity}', '-f', iotests.imgfmt, + disk, bitmap_name] + if disabled: + args.append('--disable') + + iotests.qemu_img_pipe(*args) + + +def write_to_disk(offset, size): + write = f'write {offset} {size}' + log(qemu_io('-c', write, disk), filters=[filter_qemu_io]) + + +def add_bitmap(num, begin, end, disabled): + log(f'Add bitmap {num}') + create_bitmap(num, disabled) + for i in range(begin, end): + write_to_disk((i) * chunk, chunk) + log('') + + +qemu_img_create('-f', iotests.imgfmt, disk, '10M') + +add_bitmap(1, 0, 6, False) +add_bitmap(2, 6, 8, True) +dump = ['qcow2.py', disk, 'dump-header'] +subprocess.run(dump) +# Dump the metadata in JSON format +dump.append('-j') +subprocess.run(dump) diff --git a/tests/qemu-iotests/303.out b/tests/qemu-iotests/303.out new file mode 100644 index 0000000000..7fa1edef0d --- /dev/null +++ b/tests/qemu-iotests/303.out @@ -0,0 +1,158 @@ +Add bitmap 1 +wrote 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 1048576 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 2097152 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 3145728 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 4194304 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 5242880 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + + +Add bitmap 2 +wrote 1048576/1048576 bytes at offset 6291456 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 7340032 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + + +magic 0x514649fb +version 3 +backing_file_offset 0x0 +backing_file_size 0x0 +cluster_bits 16 +size 10485760 +crypt_method 0 +l1_size 1 +l1_table_offset 0x30000 +refcount_table_offset 0x10000 +refcount_table_clusters 1 +nb_snapshots 0 +snapshot_offset 0x0 +incompatible_features [] +compatible_features [] +autoclear_features [0] +refcount_order 4 +header_length 112 + +Header extension: +magic 0x6803f857 (Feature table) +length 336 +data <binary> + +Header extension: +magic 0x23852875 (Bitmaps) +length 24 +nb_bitmaps 2 +reserved32 0 +bitmap_directory_size 0x40 +bitmap_directory_offset 0x9d0000 + +Bitmap name bitmap-1 +bitmap_table_offset 0x9b0000 +bitmap_table_size 1 +flags 0x2 (['auto']) +type 1 +granularity_bits 15 +name_size 8 +extra_data_size 0 +Bitmap table type size offset +0 serialized 65536 10092544 + +Bitmap name bitmap-2 +bitmap_table_offset 0x9c0000 +bitmap_table_size 1 +flags 0x0 ([]) +type 1 +granularity_bits 16 +name_size 8 +extra_data_size 0 +Bitmap table type size offset +0 all-zeroes 0 0 + +{ + "magic": 1363560955, + "version": 3, + "backing_file_offset": 0, + "backing_file_size": 0, + "cluster_bits": 16, + "size": 10485760, + "crypt_method": 0, + "l1_size": 1, + "l1_table_offset": 196608, + "refcount_table_offset": 65536, + "refcount_table_clusters": 1, + "nb_snapshots": 0, + "snapshot_offset": 0, + "incompatible_features": 0, + "compatible_features": 0, + "autoclear_features": 1, + "refcount_order": 4, + "header_length": 112 +} + +[ + { + "name": "Feature table", + "magic": 1745090647, + "length": 336, + "data_str": "<binary>" + }, + { + "name": "Bitmaps", + "magic": 595929205, + "length": 24, + "data": { + "nb_bitmaps": 2, + "reserved32": 0, + "bitmap_directory_size": 64, + "bitmap_directory_offset": 10289152, + "bitmap_directory": [ + { + "name": "bitmap-1", + "bitmap_table_offset": 10158080, + "bitmap_table_size": 1, + "flags": 2, + "type": 1, + "granularity_bits": 15, + "name_size": 8, + "extra_data_size": 0, + "bitmap_table": [ + { + "type": "serialized", + "offset": 10092544, + "reserved": 0 + } + ] + }, + { + "name": "bitmap-2", + "bitmap_table_offset": 10223616, + "bitmap_table_size": 1, + "flags": 0, + "type": 1, + "granularity_bits": 16, + "name_size": 8, + "extra_data_size": 0, + "bitmap_table": [ + { + "type": "all-zeroes", + "offset": 0, + "reserved": 0 + } + ] + } + ] + } + } +] diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index 7f76066640..a53ea7f78b 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -307,6 +307,8 @@ 296 rw 297 meta 299 auto quick +300 migration 301 backing quick 302 quick +303 rw quick 304 rw quick diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 717b5b652c..e197c73ca5 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -729,16 +729,22 @@ class VM(qtest.QEMUQtestMachine): } ])) - def wait_migration(self, expect_runstate): + def wait_migration(self, expect_runstate: Optional[str]) -> bool: while True: event = self.event_wait('MIGRATION') log(event, filters=[filter_qmp_event]) - if event['data']['status'] == 'completed': + if event['data']['status'] in ('completed', 'failed'): break - # The event may occur in finish-migrate, so wait for the expected - # post-migration runstate - while self.qmp('query-status')['return']['status'] != expect_runstate: - pass + + if event['data']['status'] == 'completed': + # The event may occur in finish-migrate, so wait for the expected + # post-migration runstate + runstate = None + while runstate != expect_runstate: + runstate = self.qmp('query-status')['return']['status'] + return True + else: + return False def node_info(self, node_name): nodes = self.qmp('query-named-block-nodes') diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py index 0910e6ac07..77ca59cc66 100755 --- a/tests/qemu-iotests/qcow2.py +++ b/tests/qemu-iotests/qcow2.py @@ -26,16 +26,19 @@ from qcow2_format import ( ) +is_json = False + + def cmd_dump_header(fd): h = QcowHeader(fd) - h.dump() + h.dump(is_json) print() - h.dump_extensions() + h.dump_extensions(is_json) def cmd_dump_header_exts(fd): h = QcowHeader(fd) - h.dump_extensions() + h.dump_extensions(is_json) def cmd_set_header(fd, name, value): @@ -151,11 +154,14 @@ def main(filename, cmd, args): def usage(): - print("Usage: %s <file> <cmd> [<arg>, ...]" % sys.argv[0]) + print("Usage: %s <file> <cmd> [<arg>, ...] [<key>, ...]" % sys.argv[0]) print("") print("Supported commands:") for name, handler, num_args, desc in cmds: print(" %-20s - %s" % (name, desc)) + print("") + print("Supported keys:") + print(" %-20s - %s" % ('-j', 'Dump in JSON format')) if __name__ == '__main__': @@ -163,4 +169,8 @@ if __name__ == '__main__': usage() sys.exit(1) + is_json = '-j' in sys.argv + if is_json: + sys.argv.remove('-j') + main(sys.argv[1], sys.argv[2], sys.argv[3:]) diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py index cc432e7ae0..8adc9959e1 100644 --- a/tests/qemu-iotests/qcow2_format.py +++ b/tests/qemu-iotests/qcow2_format.py @@ -19,6 +19,15 @@ import struct import string +import json + + +class ComplexEncoder(json.JSONEncoder): + def default(self, obj): + if hasattr(obj, 'to_json'): + return obj.to_json() + else: + return json.JSONEncoder.default(self, obj) class Qcow2Field: @@ -40,6 +49,22 @@ class Flags64(Qcow2Field): return str(bits) +class BitmapFlags(Qcow2Field): + + flags = { + 0x1: 'in-use', + 0x2: 'auto' + } + + def __str__(self): + bits = [] + for bit in range(64): + flag = self.value & (1 << bit) + if flag: + bits.append(self.flags.get(flag, f'bit-{bit}')) + return f'{self.value:#x} ({bits})' + + class Enum(Qcow2Field): def __str__(self): @@ -93,7 +118,11 @@ class Qcow2Struct(metaclass=Qcow2StructMeta): self.__dict__ = dict((field[2], values[i]) for i, field in enumerate(self.fields)) - def dump(self): + def dump(self, is_json=False): + if is_json: + print(json.dumps(self.to_json(), indent=4, cls=ComplexEncoder)) + return + for f in self.fields: value = self.__dict__[f[2]] if isinstance(f[1], str): @@ -103,6 +132,9 @@ class Qcow2Struct(metaclass=Qcow2StructMeta): print('{:<25} {}'.format(f[2], value_str)) + def to_json(self): + return dict((f[2], self.__dict__[f[2]]) for f in self.fields) + class Qcow2BitmapExt(Qcow2Struct): @@ -113,6 +145,131 @@ class Qcow2BitmapExt(Qcow2Struct): ('u64', '{:#x}', 'bitmap_directory_offset') ) + def __init__(self, fd, cluster_size): + super().__init__(fd=fd) + tail = struct.calcsize(self.fmt) % 8 + if tail: + fd.seek(8 - tail, 1) + position = fd.tell() + self.cluster_size = cluster_size + self.read_bitmap_directory(fd) + fd.seek(position) + + def read_bitmap_directory(self, fd): + fd.seek(self.bitmap_directory_offset) + self.bitmap_directory = \ + [Qcow2BitmapDirEntry(fd, cluster_size=self.cluster_size) + for _ in range(self.nb_bitmaps)] + + def dump(self): + super().dump() + for entry in self.bitmap_directory: + print() + entry.dump() + + def to_json(self): + fields_dict = super().to_json() + fields_dict['bitmap_directory'] = self.bitmap_directory + return fields_dict + + +class Qcow2BitmapDirEntry(Qcow2Struct): + + fields = ( + ('u64', '{:#x}', 'bitmap_table_offset'), + ('u32', '{}', 'bitmap_table_size'), + ('u32', BitmapFlags, 'flags'), + ('u8', '{}', 'type'), + ('u8', '{}', 'granularity_bits'), + ('u16', '{}', 'name_size'), + ('u32', '{}', 'extra_data_size') + ) + + def __init__(self, fd, cluster_size): + super().__init__(fd=fd) + self.cluster_size = cluster_size + # Seek relative to the current position in the file + fd.seek(self.extra_data_size, 1) + bitmap_name = fd.read(self.name_size) + self.name = bitmap_name.decode('ascii') + # Move position to the end of the entry in the directory + entry_raw_size = self.bitmap_dir_entry_raw_size() + padding = ((entry_raw_size + 7) & ~7) - entry_raw_size + fd.seek(padding, 1) + self.bitmap_table = Qcow2BitmapTable(fd=fd, + offset=self.bitmap_table_offset, + nb_entries=self.bitmap_table_size, + cluster_size=self.cluster_size) + + def bitmap_dir_entry_raw_size(self): + return struct.calcsize(self.fmt) + self.name_size + \ + self.extra_data_size + + def dump(self): + print(f'{"Bitmap name":<25} {self.name}') + super(Qcow2BitmapDirEntry, self).dump() + self.bitmap_table.dump() + + def to_json(self): + # Put the name ahead of the dict + return { + 'name': self.name, + **super().to_json(), + 'bitmap_table': self.bitmap_table + } + + +class Qcow2BitmapTableEntry(Qcow2Struct): + + fields = ( + ('u64', '{}', 'entry'), + ) + + BME_TABLE_ENTRY_RESERVED_MASK = 0xff000000000001fe + BME_TABLE_ENTRY_OFFSET_MASK = 0x00fffffffffffe00 + BME_TABLE_ENTRY_FLAG_ALL_ONES = 1 + + def __init__(self, fd): + super().__init__(fd=fd) + self.reserved = self.entry & self.BME_TABLE_ENTRY_RESERVED_MASK + self.offset = self.entry & self.BME_TABLE_ENTRY_OFFSET_MASK + if self.offset: + if self.entry & self.BME_TABLE_ENTRY_FLAG_ALL_ONES: + self.type = 'invalid' + else: + self.type = 'serialized' + elif self.entry & self.BME_TABLE_ENTRY_FLAG_ALL_ONES: + self.type = 'all-ones' + else: + self.type = 'all-zeroes' + + def to_json(self): + return {'type': self.type, 'offset': self.offset, + 'reserved': self.reserved} + + +class Qcow2BitmapTable: + + def __init__(self, fd, offset, nb_entries, cluster_size): + self.cluster_size = cluster_size + position = fd.tell() + fd.seek(offset) + self.entries = [Qcow2BitmapTableEntry(fd) for _ in range(nb_entries)] + fd.seek(position) + + def dump(self): + bitmap_table = enumerate(self.entries) + print(f'{"Bitmap table":<14} {"type":<15} {"size":<12} {"offset"}') + for i, entry in bitmap_table: + if entry.type == 'serialized': + size = self.cluster_size + else: + size = 0 + print(f'{i:<14} {entry.type:<15} {size:<12} {entry.offset}') + + def to_json(self): + return self.entries + QCOW2_EXT_MAGIC_BITMAPS = 0x23852875 @@ -128,6 +285,9 @@ class QcowHeaderExtension(Qcow2Struct): 0x44415441: 'Data file' } + def to_json(self): + return self.mapping.get(self.value, "<unknown>") + fields = ( ('u32', Magic, 'magic'), ('u32', '{}', 'length') @@ -135,11 +295,13 @@ class QcowHeaderExtension(Qcow2Struct): # then padding to next multiply of 8 ) - def __init__(self, magic=None, length=None, data=None, fd=None): + def __init__(self, magic=None, length=None, data=None, fd=None, + cluster_size=None): """ Support both loading from fd and creation from user data. For fd-based creation current position in a file will be used to read the data. + The cluster_size value may be obtained by dependent structures. This should be somehow refactored and functionality should be moved to superclass (to allow creation of any qcow2 struct), but then, fields @@ -161,28 +323,43 @@ class QcowHeaderExtension(Qcow2Struct): else: assert all(v is None for v in (magic, length, data)) super().__init__(fd=fd) - padded = (self.length + 7) & ~7 - self.data = fd.read(padded) - assert self.data is not None + if self.magic == QCOW2_EXT_MAGIC_BITMAPS: + self.obj = Qcow2BitmapExt(fd=fd, cluster_size=cluster_size) + self.data = None + else: + padded = (self.length + 7) & ~7 + self.data = fd.read(padded) + assert self.data is not None + self.obj = None + + if self.data is not None: + data_str = self.data[:self.length] + if all(c in string.printable.encode( + 'ascii') for c in data_str): + data_str = f"'{ data_str.decode('ascii') }'" + else: + data_str = '<binary>' + self.data_str = data_str - if self.magic == QCOW2_EXT_MAGIC_BITMAPS: - self.obj = Qcow2BitmapExt(data=self.data) - else: - self.obj = None def dump(self): super().dump() if self.obj is None: - data = self.data[:self.length] - if all(c in string.printable.encode('ascii') for c in data): - data = f"'{ data.decode('ascii') }'" - else: - data = '<binary>' - print(f'{"data":<25} {data}') + print(f'{"data":<25} {self.data_str}') else: self.obj.dump() + def to_json(self): + # Put the name ahead of the dict + res = {'name': self.Magic(self.magic), **super().to_json()} + if self.obj is not None: + res['data'] = self.obj + else: + res['data_str'] = self.data_str + + return res + @classmethod def create(cls, magic, data): return QcowHeaderExtension(magic, len(data), data) @@ -246,7 +423,7 @@ class QcowHeader(Qcow2Struct): end = self.cluster_size while fd.tell() < end: - ext = QcowHeaderExtension(fd=fd) + ext = QcowHeaderExtension(fd=fd, cluster_size=self.cluster_size) if ext.magic == 0: break else: @@ -280,7 +457,11 @@ class QcowHeader(Qcow2Struct): buf = buf[0:header_bytes-1] fd.write(buf) - def dump_extensions(self): + def dump_extensions(self, is_json=False): + if is_json: + print(json.dumps(self.extensions, indent=4, cls=ComplexEncoder)) + return + for ex in self.extensions: print('Header extension:') ex.dump() diff --git a/util/aio-posix.c b/util/aio-posix.c index 1b2a3af65b..f7f13ebfc2 100644 --- a/util/aio-posix.c +++ b/util/aio-posix.c @@ -464,9 +464,6 @@ static bool remove_idle_poll_handlers(AioContext *ctx, int64_t now) * * Polls for a given time. * - * Note that ctx->notify_me must be non-zero so this function can detect - * aio_notify(). - * * Note that the caller must have incremented ctx->list_lock. * * Returns: true if progress was made, false otherwise @@ -476,7 +473,6 @@ static bool run_poll_handlers(AioContext *ctx, int64_t max_ns, int64_t *timeout) bool progress; int64_t start_time, elapsed_time; - assert(ctx->notify_me); assert(qemu_lockcnt_count(&ctx->list_lock) > 0); trace_run_poll_handlers_begin(ctx, max_ns, *timeout); @@ -520,8 +516,6 @@ static bool run_poll_handlers(AioContext *ctx, int64_t max_ns, int64_t *timeout) * @timeout: timeout for blocking wait, computed by the caller and updated if * polling succeeds. * - * ctx->notify_me must be non-zero so this function can detect aio_notify(). - * * Note that the caller must have incremented ctx->list_lock. * * Returns: true if progress was made, false otherwise @@ -556,6 +550,7 @@ bool aio_poll(AioContext *ctx, bool blocking) AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list); int ret = 0; bool progress; + bool use_notify_me; int64_t timeout; int64_t start = 0; @@ -566,33 +561,39 @@ bool aio_poll(AioContext *ctx, bool blocking) */ assert(in_aio_context_home_thread(ctx)); - /* aio_notify can avoid the expensive event_notifier_set if + qemu_lockcnt_inc(&ctx->list_lock); + + if (ctx->poll_max_ns) { + start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); + } + + timeout = blocking ? aio_compute_timeout(ctx) : 0; + progress = try_poll_mode(ctx, &timeout); + assert(!(timeout && progress)); + + /* + * aio_notify can avoid the expensive event_notifier_set if * everything (file descriptors, bottom halves, timers) will * be re-evaluated before the next blocking poll(). This is * already true when aio_poll is called with blocking == false; * if blocking == true, it is only true after poll() returns, * so disable the optimization now. */ - if (blocking) { + use_notify_me = timeout != 0; + if (use_notify_me) { atomic_set(&ctx->notify_me, atomic_read(&ctx->notify_me) + 2); /* - * Write ctx->notify_me before computing the timeout - * (reading bottom half flags, etc.). Pairs with + * Write ctx->notify_me before reading ctx->notified. Pairs with * smp_mb in aio_notify(). */ smp_mb(); - } - - qemu_lockcnt_inc(&ctx->list_lock); - if (ctx->poll_max_ns) { - start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); + /* Don't block if aio_notify() was called */ + if (atomic_read(&ctx->notified)) { + timeout = 0; + } } - timeout = blocking ? aio_compute_timeout(ctx) : 0; - progress = try_poll_mode(ctx, &timeout); - assert(!(timeout && progress)); - /* If polling is allowed, non-blocking aio_poll does not need the * system call---a single round of run_poll_handlers_once suffices. */ @@ -600,12 +601,14 @@ bool aio_poll(AioContext *ctx, bool blocking) ret = ctx->fdmon_ops->wait(ctx, &ready_list, timeout); } - if (blocking) { + if (use_notify_me) { /* Finish the poll before clearing the flag. */ - atomic_store_release(&ctx->notify_me, atomic_read(&ctx->notify_me) - 2); - aio_notify_accept(ctx); + atomic_store_release(&ctx->notify_me, + atomic_read(&ctx->notify_me) - 2); } + aio_notify_accept(ctx); + /* Adjust polling time */ if (ctx->poll_max_ns) { int64_t block_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start; diff --git a/util/async.c b/util/async.c index 1319eee3bc..4266745dee 100644 --- a/util/async.c +++ b/util/async.c @@ -419,25 +419,32 @@ LuringState *aio_get_linux_io_uring(AioContext *ctx) void aio_notify(AioContext *ctx) { - /* Write e.g. bh->scheduled before reading ctx->notify_me. Pairs + /* + * Write e.g. bh->flags before writing ctx->notified. Pairs with smp_mb in + * aio_notify_accept. + */ + smp_wmb(); + atomic_set(&ctx->notified, true); + + /* + * Write ctx->notified before reading ctx->notify_me. Pairs * with smp_mb in aio_ctx_prepare or aio_poll. */ smp_mb(); if (atomic_read(&ctx->notify_me)) { event_notifier_set(&ctx->notifier); - atomic_mb_set(&ctx->notified, true); } } void aio_notify_accept(AioContext *ctx) { - if (atomic_xchg(&ctx->notified, false) -#ifdef WIN32 - || true -#endif - ) { - event_notifier_test_and_clear(&ctx->notifier); - } + atomic_set(&ctx->notified, false); + + /* + * Write ctx->notified before reading e.g. bh->flags. Pairs with smp_wmb + * in aio_notify. + */ + smp_mb(); } static void aio_timerlist_notify(void *opaque, QEMUClockType type) @@ -445,12 +452,15 @@ static void aio_timerlist_notify(void *opaque, QEMUClockType type) aio_notify(opaque); } -static void event_notifier_dummy_cb(EventNotifier *e) +static void aio_context_notifier_cb(EventNotifier *e) { + AioContext *ctx = container_of(e, AioContext, notifier); + + event_notifier_test_and_clear(&ctx->notifier); } /* Returns true if aio_notify() was called (e.g. a BH was scheduled) */ -static bool event_notifier_poll(void *opaque) +static bool aio_context_notifier_poll(void *opaque) { EventNotifier *e = opaque; AioContext *ctx = container_of(e, AioContext, notifier); @@ -508,8 +518,8 @@ AioContext *aio_context_new(Error **errp) aio_set_event_notifier(ctx, &ctx->notifier, false, - event_notifier_dummy_cb, - event_notifier_poll); + aio_context_notifier_cb, + aio_context_notifier_poll); #ifdef CONFIG_LINUX_AIO ctx->linux_aio = NULL; #endif |