diff options
Diffstat (limited to 'tests/qemu-iotests')
| -rwxr-xr-x | tests/qemu-iotests/300 | 593 | ||||
| -rw-r--r-- | tests/qemu-iotests/300.out | 5 | ||||
| -rwxr-xr-x | tests/qemu-iotests/303 | 63 | ||||
| -rw-r--r-- | tests/qemu-iotests/303.out | 158 | ||||
| -rwxr-xr-x | tests/qemu-iotests/304 | 60 | ||||
| -rw-r--r-- | tests/qemu-iotests/304.out | 2 | ||||
| -rwxr-xr-x | tests/qemu-iotests/check | 6 | ||||
| -rw-r--r-- | tests/qemu-iotests/group | 3 | ||||
| -rw-r--r-- | tests/qemu-iotests/iotests.py | 18 | ||||
| -rw-r--r-- | tests/qemu-iotests/meson.build | 10 | ||||
| -rwxr-xr-x | tests/qemu-iotests/qcow2.py | 18 | ||||
| -rw-r--r-- | tests/qemu-iotests/qcow2_format.py | 215 |
12 files changed, 1121 insertions, 30 deletions
diff --git a/tests/qemu-iotests/300 b/tests/qemu-iotests/300 new file mode 100755 index 0000000000..5b75121b84 --- /dev/null +++ b/tests/qemu-iotests/300 @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 Red Hat, Inc. +# +# Tests for dirty bitmaps migration with node aliases +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import os +import random +import re +from typing import Dict, List, Optional, Union +import iotests +import qemu + +BlockBitmapMapping = List[Dict[str, Union[str, List[Dict[str, str]]]]] + +assert iotests.sock_dir is not None +mig_sock = os.path.join(iotests.sock_dir, 'mig_sock') + + +class TestDirtyBitmapMigration(iotests.QMPTestCase): + src_node_name: str = '' + dst_node_name: str = '' + src_bmap_name: str = '' + dst_bmap_name: str = '' + + def setUp(self) -> None: + self.vm_a = iotests.VM(path_suffix='-a') + self.vm_a.add_blockdev(f'node-name={self.src_node_name},' + 'driver=null-co') + self.vm_a.launch() + + self.vm_b = iotests.VM(path_suffix='-b') + self.vm_b.add_blockdev(f'node-name={self.dst_node_name},' + 'driver=null-co') + self.vm_b.add_incoming(f'unix:{mig_sock}') + self.vm_b.launch() + + result = self.vm_a.qmp('block-dirty-bitmap-add', + node=self.src_node_name, + name=self.src_bmap_name) + self.assert_qmp(result, 'return', {}) + + # Dirty some random megabytes + for _ in range(9): + mb_ofs = random.randrange(1024) + self.vm_a.hmp_qemu_io(self.src_node_name, f'discard {mb_ofs}M 1M') + + result = self.vm_a.qmp('x-debug-block-dirty-bitmap-sha256', + node=self.src_node_name, + name=self.src_bmap_name) + self.bitmap_hash_reference = result['return']['sha256'] + + caps = [{'capability': name, 'state': True} + for name in ('dirty-bitmaps', 'events')] + + for vm in (self.vm_a, self.vm_b): + result = vm.qmp('migrate-set-capabilities', capabilities=caps) + self.assert_qmp(result, 'return', {}) + + def tearDown(self) -> None: + self.vm_a.shutdown() + self.vm_b.shutdown() + try: + os.remove(mig_sock) + except OSError: + pass + + def check_bitmap(self, bitmap_name_valid: bool) -> None: + result = self.vm_b.qmp('x-debug-block-dirty-bitmap-sha256', + node=self.dst_node_name, + name=self.dst_bmap_name) + if bitmap_name_valid: + self.assert_qmp(result, 'return/sha256', + self.bitmap_hash_reference) + else: + self.assert_qmp(result, 'error/desc', + f"Dirty bitmap '{self.dst_bmap_name}' not found") + + def migrate(self, bitmap_name_valid: bool = True, + migration_success: bool = True) -> None: + result = self.vm_a.qmp('migrate', uri=f'unix:{mig_sock}') + self.assert_qmp(result, 'return', {}) + + with iotests.Timeout(5, 'Timeout waiting for migration to complete'): + self.assertEqual(self.vm_a.wait_migration('postmigrate'), + migration_success) + self.assertEqual(self.vm_b.wait_migration('running'), + migration_success) + + if migration_success: + self.check_bitmap(bitmap_name_valid) + + def verify_dest_error(self, msg: Optional[str]) -> None: + """ + Check whether the given error message is present in vm_b's log. + (vm_b is shut down to do so.) + If @msg is None, check that there has not been any error. + """ + self.vm_b.shutdown() + if msg is None: + self.assertNotIn('qemu-system-', self.vm_b.get_log()) + else: + self.assertIn(msg, self.vm_b.get_log()) + + @staticmethod + def mapping(node_name: str, node_alias: str, + bitmap_name: str, bitmap_alias: str) -> BlockBitmapMapping: + return [{ + 'node-name': node_name, + 'alias': node_alias, + 'bitmaps': [{ + 'name': bitmap_name, + 'alias': bitmap_alias + }] + }] + + def set_mapping(self, vm: iotests.VM, mapping: BlockBitmapMapping, + error: Optional[str] = None) -> None: + """ + Invoke migrate-set-parameters on @vm to set the given @mapping. + Check for success if @error is None, or verify the error message + if it is not. + On success, verify that "info migrate_parameters" on HMP returns + our mapping. (Just to check its formatting code.) + """ + result = vm.qmp('migrate-set-parameters', + block_bitmap_mapping=mapping) + + if error is None: + self.assert_qmp(result, 'return', {}) + + result = vm.qmp('human-monitor-command', + command_line='info migrate_parameters') + + m = re.search(r'^block-bitmap-mapping:\r?(\n .*)*\n', + result['return'], flags=re.MULTILINE) + hmp_mapping = m.group(0).replace('\r', '') if m else None + + self.assertEqual(hmp_mapping, self.to_hmp_mapping(mapping)) + else: + self.assert_qmp(result, 'error/desc', error) + + @staticmethod + def to_hmp_mapping(mapping: BlockBitmapMapping) -> str: + result = 'block-bitmap-mapping:\n' + + for node in mapping: + result += f" '{node['node-name']}' -> '{node['alias']}'\n" + + assert isinstance(node['bitmaps'], list) + for bitmap in node['bitmaps']: + result += f" '{bitmap['name']}' -> '{bitmap['alias']}'\n" + + return result + + +class TestAliasMigration(TestDirtyBitmapMigration): + src_node_name = 'node0' + dst_node_name = 'node0' + src_bmap_name = 'bmap0' + dst_bmap_name = 'bmap0' + + def test_migration_without_alias(self) -> None: + self.migrate(self.src_node_name == self.dst_node_name and + self.src_bmap_name == self.dst_bmap_name) + + # Check for error message on the destination + if self.src_node_name != self.dst_node_name: + self.verify_dest_error(f"Cannot find " + f"device={self.src_node_name} nor " + f"node_name={self.src_node_name}") + else: + self.verify_dest_error(None) + + def test_alias_on_src_migration(self) -> None: + mapping = self.mapping(self.src_node_name, self.dst_node_name, + self.src_bmap_name, self.dst_bmap_name) + + self.set_mapping(self.vm_a, mapping) + self.migrate() + self.verify_dest_error(None) + + def test_alias_on_dst_migration(self) -> None: + mapping = self.mapping(self.dst_node_name, self.src_node_name, + self.dst_bmap_name, self.src_bmap_name) + + self.set_mapping(self.vm_b, mapping) + self.migrate() + self.verify_dest_error(None) + + def test_alias_on_both_migration(self) -> None: + src_map = self.mapping(self.src_node_name, 'node-alias', + self.src_bmap_name, 'bmap-alias') + + dst_map = self.mapping(self.dst_node_name, 'node-alias', + self.dst_bmap_name, 'bmap-alias') + + self.set_mapping(self.vm_a, src_map) + self.set_mapping(self.vm_b, dst_map) + self.migrate() + self.verify_dest_error(None) + + +class TestNodeAliasMigration(TestAliasMigration): + src_node_name = 'node-src' + dst_node_name = 'node-dst' + + +class TestBitmapAliasMigration(TestAliasMigration): + src_bmap_name = 'bmap-src' + dst_bmap_name = 'bmap-dst' + + +class TestFullAliasMigration(TestAliasMigration): + src_node_name = 'node-src' + dst_node_name = 'node-dst' + src_bmap_name = 'bmap-src' + dst_bmap_name = 'bmap-dst' + + +class TestLongBitmapNames(TestAliasMigration): + # Giving long bitmap names is OK, as long as there is a short alias for + # migration + src_bmap_name = 'a' * 512 + dst_bmap_name = 'b' * 512 + + # Skip all tests that do not use the intermediate alias + def test_migration_without_alias(self) -> None: + pass + + def test_alias_on_src_migration(self) -> None: + pass + + def test_alias_on_dst_migration(self) -> None: + pass + + +class TestBlockBitmapMappingErrors(TestDirtyBitmapMigration): + src_node_name = 'node0' + dst_node_name = 'node0' + src_bmap_name = 'bmap0' + dst_bmap_name = 'bmap0' + + """ + Note that mapping nodes or bitmaps that do not exist is not an error. + """ + + def test_non_injective_node_mapping(self) -> None: + mapping: BlockBitmapMapping = [ + { + 'node-name': 'node0', + 'alias': 'common-alias', + 'bitmaps': [{ + 'name': 'bmap0', + 'alias': 'bmap-alias0' + }] + }, + { + 'node-name': 'node1', + 'alias': 'common-alias', + 'bitmaps': [{ + 'name': 'bmap1', + 'alias': 'bmap-alias1' + }] + } + ] + + self.set_mapping(self.vm_a, mapping, + "Invalid mapping given for block-bitmap-mapping: " + "The node alias 'common-alias' is used twice") + + def test_non_injective_bitmap_mapping(self) -> None: + mapping: BlockBitmapMapping = [{ + 'node-name': 'node0', + 'alias': 'node-alias0', + 'bitmaps': [ + { + 'name': 'bmap0', + 'alias': 'common-alias' + }, + { + 'name': 'bmap1', + 'alias': 'common-alias' + } + ] + }] + + self.set_mapping(self.vm_a, mapping, + "Invalid mapping given for block-bitmap-mapping: " + "The bitmap alias 'node-alias0'/'common-alias' is " + "used twice") + + def test_ambiguous_node_mapping(self) -> None: + mapping: BlockBitmapMapping = [ + { + 'node-name': 'node0', + 'alias': 'node-alias0', + 'bitmaps': [{ + 'name': 'bmap0', + 'alias': 'bmap-alias0' + }] + }, + { + 'node-name': 'node0', + 'alias': 'node-alias1', + 'bitmaps': [{ + 'name': 'bmap0', + 'alias': 'bmap-alias0' + }] + } + ] + + self.set_mapping(self.vm_a, mapping, + "Invalid mapping given for block-bitmap-mapping: " + "The node name 'node0' is mapped twice") + + def test_ambiguous_bitmap_mapping(self) -> None: + mapping: BlockBitmapMapping = [{ + 'node-name': 'node0', + 'alias': 'node-alias0', + 'bitmaps': [ + { + 'name': 'bmap0', + 'alias': 'bmap-alias0' + }, + { + 'name': 'bmap0', + 'alias': 'bmap-alias1' + } + ] + }] + + self.set_mapping(self.vm_a, mapping, + "Invalid mapping given for block-bitmap-mapping: " + "The bitmap 'node0'/'bmap0' is mapped twice") + + def test_migratee_node_is_not_mapped_on_src(self) -> None: + self.set_mapping(self.vm_a, []) + # Should just ignore all bitmaps on unmapped nodes + self.migrate(False) + self.verify_dest_error(None) + + def test_migratee_node_is_not_mapped_on_dst(self) -> None: + self.set_mapping(self.vm_b, []) + self.migrate(False) + self.verify_dest_error(f"Unknown node alias '{self.src_node_name}'") + + def test_migratee_bitmap_is_not_mapped_on_src(self) -> None: + mapping: BlockBitmapMapping = [{ + 'node-name': self.src_node_name, + 'alias': self.dst_node_name, + 'bitmaps': [] + }] + + self.set_mapping(self.vm_a, mapping) + # Should just ignore all unmapped bitmaps + self.migrate(False) + self.verify_dest_error(None) + + def test_migratee_bitmap_is_not_mapped_on_dst(self) -> None: + mapping: BlockBitmapMapping = [{ + 'node-name': self.dst_node_name, + 'alias': self.src_node_name, + 'bitmaps': [] + }] + + self.set_mapping(self.vm_b, mapping) + self.migrate(False) + self.verify_dest_error(f"Unknown bitmap alias " + f"'{self.src_bmap_name}' " + f"on node '{self.dst_node_name}' " + f"(alias '{self.src_node_name}')") + + def test_unused_mapping_on_dst(self) -> None: + # Let the source not send any bitmaps + self.set_mapping(self.vm_a, []) + + # Establish some mapping on the destination + self.set_mapping(self.vm_b, []) + + # The fact that there is a mapping on B without any bitmaps + # being received should be fine, not fatal + self.migrate(False) + self.verify_dest_error(None) + + def test_non_wellformed_node_alias(self) -> None: + alias = '123-foo' + + mapping: BlockBitmapMapping = [{ + 'node-name': self.src_node_name, + 'alias': alias, + 'bitmaps': [] + }] + + self.set_mapping(self.vm_a, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The node alias '{alias}' is not well-formed") + + def test_node_alias_too_long(self) -> None: + alias = 'a' * 256 + + mapping: BlockBitmapMapping = [{ + 'node-name': self.src_node_name, + 'alias': alias, + 'bitmaps': [] + }] + + self.set_mapping(self.vm_a, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The node alias '{alias}' is longer than 255 bytes") + + def test_bitmap_alias_too_long(self) -> None: + alias = 'a' * 256 + + mapping = self.mapping(self.src_node_name, self.dst_node_name, + self.src_bmap_name, alias) + + self.set_mapping(self.vm_a, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The bitmap alias '{alias}' is longer than 255 " + f"bytes") + + def test_bitmap_name_too_long(self) -> None: + name = 'a' * 256 + + result = self.vm_a.qmp('block-dirty-bitmap-add', + node=self.src_node_name, + name=name) + self.assert_qmp(result, 'return', {}) + + self.migrate(False, False) + + # Check for the error in the source's log + self.vm_a.shutdown() + self.assertIn(f"Cannot migrate bitmap '{name}' on node " + f"'{self.src_node_name}': Name is longer than 255 bytes", + self.vm_a.get_log()) + + # Expect abnormal shutdown of the destination VM because of + # the failed migration + try: + self.vm_b.shutdown() + except qemu.machine.AbnormalShutdown: + pass + + def test_aliased_bitmap_name_too_long(self) -> None: + # Longer than the maximum for bitmap names + self.dst_bmap_name = 'a' * 1024 + + mapping = self.mapping(self.dst_node_name, self.src_node_name, + self.dst_bmap_name, self.src_bmap_name) + + # We would have to create this bitmap during migration, and + # that would fail, because the name is too long. Better to + # catch it early. + self.set_mapping(self.vm_b, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The bitmap name '{self.dst_bmap_name}' is longer " + f"than 1023 bytes") + + def test_node_name_too_long(self) -> None: + # Longer than the maximum for node names + self.dst_node_name = 'a' * 32 + + mapping = self.mapping(self.dst_node_name, self.src_node_name, + self.dst_bmap_name, self.src_bmap_name) + + # During migration, this would appear simply as a node that + # cannot be found. Still better to catch impossible node + # names early (similar to test_non_wellformed_node_alias). + self.set_mapping(self.vm_b, mapping, + f"Invalid mapping given for block-bitmap-mapping: " + f"The node name '{self.dst_node_name}' is longer " + f"than 31 bytes") + + +class TestCrossAliasMigration(TestDirtyBitmapMigration): + """ + Swap aliases, both to see that qemu does not get confused, and + that we can migrate multiple things at once. + + So we migrate this: + node-a.bmap-a -> node-b.bmap-b + node-a.bmap-b -> node-b.bmap-a + node-b.bmap-a -> node-a.bmap-b + node-b.bmap-b -> node-a.bmap-a + """ + + src_node_name = 'node-a' + dst_node_name = 'node-b' + src_bmap_name = 'bmap-a' + dst_bmap_name = 'bmap-b' + + def setUp(self) -> None: + TestDirtyBitmapMigration.setUp(self) + + # Now create another block device and let both have two bitmaps each + result = self.vm_a.qmp('blockdev-add', + node_name='node-b', driver='null-co') + self.assert_qmp(result, 'return', {}) + + result = self.vm_b.qmp('blockdev-add', + node_name='node-a', driver='null-co') + self.assert_qmp(result, 'return', {}) + + bmaps_to_add = (('node-a', 'bmap-b'), + ('node-b', 'bmap-a'), + ('node-b', 'bmap-b')) + + for (node, bmap) in bmaps_to_add: + result = self.vm_a.qmp('block-dirty-bitmap-add', + node=node, name=bmap) + self.assert_qmp(result, 'return', {}) + + @staticmethod + def cross_mapping() -> BlockBitmapMapping: + return [ + { + 'node-name': 'node-a', + 'alias': 'node-b', + 'bitmaps': [ + { + 'name': 'bmap-a', + 'alias': 'bmap-b' + }, + { + 'name': 'bmap-b', + 'alias': 'bmap-a' + } + ] + }, + { + 'node-name': 'node-b', + 'alias': 'node-a', + 'bitmaps': [ + { + 'name': 'bmap-b', + 'alias': 'bmap-a' + }, + { + 'name': 'bmap-a', + 'alias': 'bmap-b' + } + ] + } + ] + + def verify_dest_has_all_bitmaps(self) -> None: + bitmaps = self.vm_b.query_bitmaps() + + # Extract and sort bitmap names + for node in bitmaps: + bitmaps[node] = sorted((bmap['name'] for bmap in bitmaps[node])) + + self.assertEqual(bitmaps, + {'node-a': ['bmap-a', 'bmap-b'], + 'node-b': ['bmap-a', 'bmap-b']}) + + def test_alias_on_src(self) -> None: + self.set_mapping(self.vm_a, self.cross_mapping()) + + # Checks that node-a.bmap-a was migrated to node-b.bmap-b, and + # that is enough + self.migrate() + self.verify_dest_has_all_bitmaps() + self.verify_dest_error(None) + + def test_alias_on_dst(self) -> None: + self.set_mapping(self.vm_b, self.cross_mapping()) + + # Checks that node-a.bmap-a was migrated to node-b.bmap-b, and + # that is enough + self.migrate() + self.verify_dest_has_all_bitmaps() + self.verify_dest_error(None) + + +if __name__ == '__main__': + iotests.main(supported_protocols=['file']) diff --git a/tests/qemu-iotests/300.out b/tests/qemu-iotests/300.out new file mode 100644 index 0000000000..cafb8161f7 --- /dev/null +++ b/tests/qemu-iotests/300.out @@ -0,0 +1,5 @@ +..................................... +---------------------------------------------------------------------- +Ran 37 tests + +OK diff --git a/tests/qemu-iotests/303 b/tests/qemu-iotests/303 new file mode 100755 index 0000000000..6c21774483 --- /dev/null +++ b/tests/qemu-iotests/303 @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Test for dumping of qcow2 image metadata +# +# Copyright (c) 2020 Virtuozzo International GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import iotests +import subprocess +from iotests import qemu_img_create, qemu_io, file_path, log, filter_qemu_io + +iotests.script_initialize(supported_fmts=['qcow2']) + +disk = file_path('disk') +chunk = 1024 * 1024 + + +def create_bitmap(bitmap_number, disabled): + granularity = 1 << (14 + bitmap_number) + bitmap_name = 'bitmap-' + str(bitmap_number) + args = ['bitmap', '--add', '-g', f'{granularity}', '-f', iotests.imgfmt, + disk, bitmap_name] + if disabled: + args.append('--disable') + + iotests.qemu_img_pipe(*args) + + +def write_to_disk(offset, size): + write = f'write {offset} {size}' + log(qemu_io('-c', write, disk), filters=[filter_qemu_io]) + + +def add_bitmap(num, begin, end, disabled): + log(f'Add bitmap {num}') + create_bitmap(num, disabled) + for i in range(begin, end): + write_to_disk((i) * chunk, chunk) + log('') + + +qemu_img_create('-f', iotests.imgfmt, disk, '10M') + +add_bitmap(1, 0, 6, False) +add_bitmap(2, 6, 8, True) +dump = ['qcow2.py', disk, 'dump-header'] +subprocess.run(dump) +# Dump the metadata in JSON format +dump.append('-j') +subprocess.run(dump) diff --git a/tests/qemu-iotests/303.out b/tests/qemu-iotests/303.out new file mode 100644 index 0000000000..7fa1edef0d --- /dev/null +++ b/tests/qemu-iotests/303.out @@ -0,0 +1,158 @@ +Add bitmap 1 +wrote 1048576/1048576 bytes at offset 0 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 1048576 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 2097152 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 3145728 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 4194304 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 5242880 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + + +Add bitmap 2 +wrote 1048576/1048576 bytes at offset 6291456 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +wrote 1048576/1048576 bytes at offset 7340032 +1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + + +magic 0x514649fb +version 3 +backing_file_offset 0x0 +backing_file_size 0x0 +cluster_bits 16 +size 10485760 +crypt_method 0 +l1_size 1 +l1_table_offset 0x30000 +refcount_table_offset 0x10000 +refcount_table_clusters 1 +nb_snapshots 0 +snapshot_offset 0x0 +incompatible_features [] +compatible_features [] +autoclear_features [0] +refcount_order 4 +header_length 112 + +Header extension: +magic 0x6803f857 (Feature table) +length 336 +data <binary> + +Header extension: +magic 0x23852875 (Bitmaps) +length 24 +nb_bitmaps 2 +reserved32 0 +bitmap_directory_size 0x40 +bitmap_directory_offset 0x9d0000 + +Bitmap name bitmap-1 +bitmap_table_offset 0x9b0000 +bitmap_table_size 1 +flags 0x2 (['auto']) +type 1 +granularity_bits 15 +name_size 8 +extra_data_size 0 +Bitmap table type size offset +0 serialized 65536 10092544 + +Bitmap name bitmap-2 +bitmap_table_offset 0x9c0000 +bitmap_table_size 1 +flags 0x0 ([]) +type 1 +granularity_bits 16 +name_size 8 +extra_data_size 0 +Bitmap table type size offset +0 all-zeroes 0 0 + +{ + "magic": 1363560955, + "version": 3, + "backing_file_offset": 0, + "backing_file_size": 0, + "cluster_bits": 16, + "size": 10485760, + "crypt_method": 0, + "l1_size": 1, + "l1_table_offset": 196608, + "refcount_table_offset": 65536, + "refcount_table_clusters": 1, + "nb_snapshots": 0, + "snapshot_offset": 0, + "incompatible_features": 0, + "compatible_features": 0, + "autoclear_features": 1, + "refcount_order": 4, + "header_length": 112 +} + +[ + { + "name": "Feature table", + "magic": 1745090647, + "length": 336, + "data_str": "<binary>" + }, + { + "name": "Bitmaps", + "magic": 595929205, + "length": 24, + "data": { + "nb_bitmaps": 2, + "reserved32": 0, + "bitmap_directory_size": 64, + "bitmap_directory_offset": 10289152, + "bitmap_directory": [ + { + "name": "bitmap-1", + "bitmap_table_offset": 10158080, + "bitmap_table_size": 1, + "flags": 2, + "type": 1, + "granularity_bits": 15, + "name_size": 8, + "extra_data_size": 0, + "bitmap_table": [ + { + "type": "serialized", + "offset": 10092544, + "reserved": 0 + } + ] + }, + { + "name": "bitmap-2", + "bitmap_table_offset": 10223616, + "bitmap_table_size": 1, + "flags": 0, + "type": 1, + "granularity_bits": 16, + "name_size": 8, + "extra_data_size": 0, + "bitmap_table": [ + { + "type": "all-zeroes", + "offset": 0, + "reserved": 0 + } + ] + } + ] + } + } +] diff --git a/tests/qemu-iotests/304 b/tests/qemu-iotests/304 new file mode 100755 index 0000000000..aaf9e14617 --- /dev/null +++ b/tests/qemu-iotests/304 @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# +# Tests dirty-bitmap backup with unaligned bitmap granularity +# +# Copyright (c) 2020 Proxmox Server Solutions +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# owner=s.reiter@proxmox.com + +import iotests +from iotests import qemu_img_create, qemu_img_log, file_path + +iotests.script_initialize(supported_fmts=['qcow2'], + supported_protocols=['file']) + +test_img = file_path('test.qcow2') +target_img = file_path('target.qcow2') + +# unaligned by one byte +image_len = 4097 +bitmap_granularity = 4096 + +qemu_img_create('-f', iotests.imgfmt, test_img, str(image_len)) + +# create VM +vm = iotests.VM().add_drive(test_img) +vm.launch() + +# write to the entire image +vm.hmp_qemu_io('drive0', 'write -P0x16 0 4096'); +vm.hmp_qemu_io('drive0', 'write -P0x17 4096 1'); + +# do backup and wait for completion +vm.qmp('drive-backup', **{ + 'device': 'drive0', + 'sync': 'full', + 'target': target_img +}) + +event = vm.event_wait(name='BLOCK_JOB_COMPLETED', + match={'data': {'device': 'drive0'}}, + timeout=5.0) + +# shutdown to sync images +vm.shutdown() + +# backup succeeded, check if image is correct +qemu_img_log('compare', test_img, target_img) diff --git a/tests/qemu-iotests/304.out b/tests/qemu-iotests/304.out new file mode 100644 index 0000000000..381cc056f7 --- /dev/null +++ b/tests/qemu-iotests/304.out @@ -0,0 +1,2 @@ +Images are identical. + diff --git a/tests/qemu-iotests/check b/tests/qemu-iotests/check index 0657f7286c..3ab859ac1a 100755 --- a/tests/qemu-iotests/check +++ b/tests/qemu-iotests/check @@ -577,11 +577,11 @@ if [ -z "$QEMU_PROG" ] then if [ -x "$build_iotests/qemu" ]; then export QEMU_PROG="$build_iotests/qemu" - elif [ -x "$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}" ]; then - export QEMU_PROG="$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}" + elif [ -x "$build_root/qemu-system-${qemu_arch}" ]; then + export QEMU_PROG="$build_root/qemu-system-${qemu_arch}" else pushd "$build_root" > /dev/null - for binary in *-softmmu/qemu-system-* + for binary in qemu-system-* do if [ -x "$binary" ] then diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index 025ed5238d..a53ea7f78b 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -307,5 +307,8 @@ 296 rw 297 meta 299 auto quick +300 migration 301 backing quick 302 quick +303 rw quick +304 rw quick diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 717b5b652c..e197c73ca5 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -729,16 +729,22 @@ class VM(qtest.QEMUQtestMachine): } ])) - def wait_migration(self, expect_runstate): + def wait_migration(self, expect_runstate: Optional[str]) -> bool: while True: event = self.event_wait('MIGRATION') log(event, filters=[filter_qmp_event]) - if event['data']['status'] == 'completed': + if event['data']['status'] in ('completed', 'failed'): break - # The event may occur in finish-migrate, so wait for the expected - # post-migration runstate - while self.qmp('query-status')['return']['status'] != expect_runstate: - pass + + if event['data']['status'] == 'completed': + # The event may occur in finish-migrate, so wait for the expected + # post-migration runstate + runstate = None + while runstate != expect_runstate: + runstate = self.qmp('query-status')['return']['status'] + return True + else: + return False def node_info(self, node_name): nodes = self.qmp('query-named-block-nodes') diff --git a/tests/qemu-iotests/meson.build b/tests/qemu-iotests/meson.build new file mode 100644 index 0000000000..3de09fb8fa --- /dev/null +++ b/tests/qemu-iotests/meson.build @@ -0,0 +1,10 @@ +if 'CONFIG_LINUX' in config_host + socket_scm_helper = executable('socket_scm_helper', 'socket_scm_helper.c', + build_by_default: false) +else + socket_scm_helper = [] +endif +test('qemu-iotests', sh, args: [files('../check-block.sh')], + depends: [qemu_block_tools, emulators, socket_scm_helper], + suite: 'block', timeout: 10000) + diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py index 0910e6ac07..77ca59cc66 100755 --- a/tests/qemu-iotests/qcow2.py +++ b/tests/qemu-iotests/qcow2.py @@ -26,16 +26,19 @@ from qcow2_format import ( ) +is_json = False + + def cmd_dump_header(fd): h = QcowHeader(fd) - h.dump() + h.dump(is_json) print() - h.dump_extensions() + h.dump_extensions(is_json) def cmd_dump_header_exts(fd): h = QcowHeader(fd) - h.dump_extensions() + h.dump_extensions(is_json) def cmd_set_header(fd, name, value): @@ -151,11 +154,14 @@ def main(filename, cmd, args): def usage(): - print("Usage: %s <file> <cmd> [<arg>, ...]" % sys.argv[0]) + print("Usage: %s <file> <cmd> [<arg>, ...] [<key>, ...]" % sys.argv[0]) print("") print("Supported commands:") for name, handler, num_args, desc in cmds: print(" %-20s - %s" % (name, desc)) + print("") + print("Supported keys:") + print(" %-20s - %s" % ('-j', 'Dump in JSON format')) if __name__ == '__main__': @@ -163,4 +169,8 @@ if __name__ == '__main__': usage() sys.exit(1) + is_json = '-j' in sys.argv + if is_json: + sys.argv.remove('-j') + main(sys.argv[1], sys.argv[2], sys.argv[3:]) diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py index cc432e7ae0..8adc9959e1 100644 --- a/tests/qemu-iotests/qcow2_format.py +++ b/tests/qemu-iotests/qcow2_format.py @@ -19,6 +19,15 @@ import struct import string +import json + + +class ComplexEncoder(json.JSONEncoder): + def default(self, obj): + if hasattr(obj, 'to_json'): + return obj.to_json() + else: + return json.JSONEncoder.default(self, obj) class Qcow2Field: @@ -40,6 +49,22 @@ class Flags64(Qcow2Field): return str(bits) +class BitmapFlags(Qcow2Field): + + flags = { + 0x1: 'in-use', + 0x2: 'auto' + } + + def __str__(self): + bits = [] + for bit in range(64): + flag = self.value & (1 << bit) + if flag: + bits.append(self.flags.get(flag, f'bit-{bit}')) + return f'{self.value:#x} ({bits})' + + class Enum(Qcow2Field): def __str__(self): @@ -93,7 +118,11 @@ class Qcow2Struct(metaclass=Qcow2StructMeta): self.__dict__ = dict((field[2], values[i]) for i, field in enumerate(self.fields)) - def dump(self): + def dump(self, is_json=False): + if is_json: + print(json.dumps(self.to_json(), indent=4, cls=ComplexEncoder)) + return + for f in self.fields: value = self.__dict__[f[2]] if isinstance(f[1], str): @@ -103,6 +132,9 @@ class Qcow2Struct(metaclass=Qcow2StructMeta): print('{:<25} {}'.format(f[2], value_str)) + def to_json(self): + return dict((f[2], self.__dict__[f[2]]) for f in self.fields) + class Qcow2BitmapExt(Qcow2Struct): @@ -113,6 +145,131 @@ class Qcow2BitmapExt(Qcow2Struct): ('u64', '{:#x}', 'bitmap_directory_offset') ) + def __init__(self, fd, cluster_size): + super().__init__(fd=fd) + tail = struct.calcsize(self.fmt) % 8 + if tail: + fd.seek(8 - tail, 1) + position = fd.tell() + self.cluster_size = cluster_size + self.read_bitmap_directory(fd) + fd.seek(position) + + def read_bitmap_directory(self, fd): + fd.seek(self.bitmap_directory_offset) + self.bitmap_directory = \ + [Qcow2BitmapDirEntry(fd, cluster_size=self.cluster_size) + for _ in range(self.nb_bitmaps)] + + def dump(self): + super().dump() + for entry in self.bitmap_directory: + print() + entry.dump() + + def to_json(self): + fields_dict = super().to_json() + fields_dict['bitmap_directory'] = self.bitmap_directory + return fields_dict + + +class Qcow2BitmapDirEntry(Qcow2Struct): + + fields = ( + ('u64', '{:#x}', 'bitmap_table_offset'), + ('u32', '{}', 'bitmap_table_size'), + ('u32', BitmapFlags, 'flags'), + ('u8', '{}', 'type'), + ('u8', '{}', 'granularity_bits'), + ('u16', '{}', 'name_size'), + ('u32', '{}', 'extra_data_size') + ) + + def __init__(self, fd, cluster_size): + super().__init__(fd=fd) + self.cluster_size = cluster_size + # Seek relative to the current position in the file + fd.seek(self.extra_data_size, 1) + bitmap_name = fd.read(self.name_size) + self.name = bitmap_name.decode('ascii') + # Move position to the end of the entry in the directory + entry_raw_size = self.bitmap_dir_entry_raw_size() + padding = ((entry_raw_size + 7) & ~7) - entry_raw_size + fd.seek(padding, 1) + self.bitmap_table = Qcow2BitmapTable(fd=fd, + offset=self.bitmap_table_offset, + nb_entries=self.bitmap_table_size, + cluster_size=self.cluster_size) + + def bitmap_dir_entry_raw_size(self): + return struct.calcsize(self.fmt) + self.name_size + \ + self.extra_data_size + + def dump(self): + print(f'{"Bitmap name":<25} {self.name}') + super(Qcow2BitmapDirEntry, self).dump() + self.bitmap_table.dump() + + def to_json(self): + # Put the name ahead of the dict + return { + 'name': self.name, + **super().to_json(), + 'bitmap_table': self.bitmap_table + } + + +class Qcow2BitmapTableEntry(Qcow2Struct): + + fields = ( + ('u64', '{}', 'entry'), + ) + + BME_TABLE_ENTRY_RESERVED_MASK = 0xff000000000001fe + BME_TABLE_ENTRY_OFFSET_MASK = 0x00fffffffffffe00 + BME_TABLE_ENTRY_FLAG_ALL_ONES = 1 + + def __init__(self, fd): + super().__init__(fd=fd) + self.reserved = self.entry & self.BME_TABLE_ENTRY_RESERVED_MASK + self.offset = self.entry & self.BME_TABLE_ENTRY_OFFSET_MASK + if self.offset: + if self.entry & self.BME_TABLE_ENTRY_FLAG_ALL_ONES: + self.type = 'invalid' + else: + self.type = 'serialized' + elif self.entry & self.BME_TABLE_ENTRY_FLAG_ALL_ONES: + self.type = 'all-ones' + else: + self.type = 'all-zeroes' + + def to_json(self): + return {'type': self.type, 'offset': self.offset, + 'reserved': self.reserved} + + +class Qcow2BitmapTable: + + def __init__(self, fd, offset, nb_entries, cluster_size): + self.cluster_size = cluster_size + position = fd.tell() + fd.seek(offset) + self.entries = [Qcow2BitmapTableEntry(fd) for _ in range(nb_entries)] + fd.seek(position) + + def dump(self): + bitmap_table = enumerate(self.entries) + print(f'{"Bitmap table":<14} {"type":<15} {"size":<12} {"offset"}') + for i, entry in bitmap_table: + if entry.type == 'serialized': + size = self.cluster_size + else: + size = 0 + print(f'{i:<14} {entry.type:<15} {size:<12} {entry.offset}') + + def to_json(self): + return self.entries + QCOW2_EXT_MAGIC_BITMAPS = 0x23852875 @@ -128,6 +285,9 @@ class QcowHeaderExtension(Qcow2Struct): 0x44415441: 'Data file' } + def to_json(self): + return self.mapping.get(self.value, "<unknown>") + fields = ( ('u32', Magic, 'magic'), ('u32', '{}', 'length') @@ -135,11 +295,13 @@ class QcowHeaderExtension(Qcow2Struct): # then padding to next multiply of 8 ) - def __init__(self, magic=None, length=None, data=None, fd=None): + def __init__(self, magic=None, length=None, data=None, fd=None, + cluster_size=None): """ Support both loading from fd and creation from user data. For fd-based creation current position in a file will be used to read the data. + The cluster_size value may be obtained by dependent structures. This should be somehow refactored and functionality should be moved to superclass (to allow creation of any qcow2 struct), but then, fields @@ -161,28 +323,43 @@ class QcowHeaderExtension(Qcow2Struct): else: assert all(v is None for v in (magic, length, data)) super().__init__(fd=fd) - padded = (self.length + 7) & ~7 - self.data = fd.read(padded) - assert self.data is not None + if self.magic == QCOW2_EXT_MAGIC_BITMAPS: + self.obj = Qcow2BitmapExt(fd=fd, cluster_size=cluster_size) + self.data = None + else: + padded = (self.length + 7) & ~7 + self.data = fd.read(padded) + assert self.data is not None + self.obj = None + + if self.data is not None: + data_str = self.data[:self.length] + if all(c in string.printable.encode( + 'ascii') for c in data_str): + data_str = f"'{ data_str.decode('ascii') }'" + else: + data_str = '<binary>' + self.data_str = data_str - if self.magic == QCOW2_EXT_MAGIC_BITMAPS: - self.obj = Qcow2BitmapExt(data=self.data) - else: - self.obj = None def dump(self): super().dump() if self.obj is None: - data = self.data[:self.length] - if all(c in string.printable.encode('ascii') for c in data): - data = f"'{ data.decode('ascii') }'" - else: - data = '<binary>' - print(f'{"data":<25} {data}') + print(f'{"data":<25} {self.data_str}') else: self.obj.dump() + def to_json(self): + # Put the name ahead of the dict + res = {'name': self.Magic(self.magic), **super().to_json()} + if self.obj is not None: + res['data'] = self.obj + else: + res['data_str'] = self.data_str + + return res + @classmethod def create(cls, magic, data): return QcowHeaderExtension(magic, len(data), data) @@ -246,7 +423,7 @@ class QcowHeader(Qcow2Struct): end = self.cluster_size while fd.tell() < end: - ext = QcowHeaderExtension(fd=fd) + ext = QcowHeaderExtension(fd=fd, cluster_size=self.cluster_size) if ext.magic == 0: break else: @@ -280,7 +457,11 @@ class QcowHeader(Qcow2Struct): buf = buf[0:header_bytes-1] fd.write(buf) - def dump_extensions(self): + def dump_extensions(self, is_json=False): + if is_json: + print(json.dumps(self.extensions, indent=4, cls=ComplexEncoder)) + return + for ex in self.extensions: print('Header extension:') ex.dump() |