diff options
| -rw-r--r-- | block/crypto.c | 2 | ||||
| -rw-r--r-- | block/qcow2-bitmap.c | 36 | ||||
| -rw-r--r-- | block/qcow2.c | 14 | ||||
| -rw-r--r-- | block/qcow2.h | 2 | ||||
| -rw-r--r-- | block/raw-format.c | 2 | ||||
| -rw-r--r-- | docs/tools/qemu-img.rst | 7 | ||||
| -rw-r--r-- | qapi/block-core.json | 16 | ||||
| -rw-r--r-- | qemu-img.c | 3 | ||||
| -rw-r--r-- | tests/qemu-iotests/178.out.qcow2 | 16 | ||||
| -rwxr-xr-x | tests/qemu-iotests/190 | 47 | ||||
| -rw-r--r-- | tests/qemu-iotests/190.out | 27 |
11 files changed, 159 insertions, 13 deletions
diff --git a/block/crypto.c b/block/crypto.c index b216e12c31..973b57b3eb 100644 --- a/block/crypto.c +++ b/block/crypto.c @@ -552,7 +552,7 @@ static BlockMeasureInfo *block_crypto_measure(QemuOpts *opts, * Unallocated blocks are still encrypted so allocation status makes no * difference to the file size. */ - info = g_new(BlockMeasureInfo, 1); + info = g_new0(BlockMeasureInfo, 1); info->fully_allocated = luks_payload_size + size; info->required = luks_payload_size + size; return info; diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c index 1cf6d2ab77..7bf12502da 100644 --- a/block/qcow2-bitmap.c +++ b/block/qcow2-bitmap.c @@ -1755,3 +1755,39 @@ bool qcow2_supports_persistent_dirty_bitmap(BlockDriverState *bs) return s->qcow_version >= 3; } + +/* + * Compute the space required for bitmaps in @bs. + * + * The computation is based as if copying to a new image with the + * given @cluster_size, which may differ from the cluster size in @bs. + */ +uint64_t qcow2_get_persistent_dirty_bitmap_size(BlockDriverState *bs, + uint32_t cluster_size) +{ + uint64_t bitmaps_size = 0; + BdrvDirtyBitmap *bm; + size_t bitmap_dir_size = 0; + + FOR_EACH_DIRTY_BITMAP(bs, bm) { + if (bdrv_dirty_bitmap_get_persistence(bm)) { + const char *name = bdrv_dirty_bitmap_name(bm); + uint32_t granularity = bdrv_dirty_bitmap_granularity(bm); + uint64_t bmbytes = + get_bitmap_bytes_needed(bdrv_dirty_bitmap_size(bm), + granularity); + uint64_t bmclusters = DIV_ROUND_UP(bmbytes, cluster_size); + + /* Assume the entire bitmap is allocated */ + bitmaps_size += bmclusters * cluster_size; + /* Also reserve space for the bitmap table entries */ + bitmaps_size += ROUND_UP(bmclusters * sizeof(uint64_t), + cluster_size); + /* And space for contribution to bitmap directory size */ + bitmap_dir_size += calc_dir_entry_size(strlen(name), 0); + } + } + bitmaps_size += ROUND_UP(bitmap_dir_size, cluster_size); + + return bitmaps_size; +} diff --git a/block/qcow2.c b/block/qcow2.c index dfab8d2f6c..0cd2e6757e 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -4953,16 +4953,24 @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs, required = virtual_size; } - info = g_new(BlockMeasureInfo, 1); + info = g_new0(BlockMeasureInfo, 1); info->fully_allocated = qcow2_calc_prealloc_size(virtual_size, cluster_size, ctz32(refcount_bits)) + luks_payload_size; - /* Remove data clusters that are not required. This overestimates the + /* + * Remove data clusters that are not required. This overestimates the * required size because metadata needed for the fully allocated file is - * still counted. + * still counted. Show bitmaps only if both source and destination + * would support them. */ info->required = info->fully_allocated - virtual_size + required; + info->has_bitmaps = version >= 3 && in_bs && + bdrv_supports_persistent_dirty_bitmap(in_bs); + if (info->has_bitmaps) { + info->bitmaps = qcow2_get_persistent_dirty_bitmap_size(in_bs, + cluster_size); + } return info; err: diff --git a/block/qcow2.h b/block/qcow2.h index 402e8acb1c..7ce2c23bdb 100644 --- a/block/qcow2.h +++ b/block/qcow2.h @@ -783,6 +783,8 @@ int qcow2_co_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name, Error **errp); bool qcow2_supports_persistent_dirty_bitmap(BlockDriverState *bs); +uint64_t qcow2_get_persistent_dirty_bitmap_size(BlockDriverState *bs, + uint32_t cluster_size); ssize_t coroutine_fn qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size, diff --git a/block/raw-format.c b/block/raw-format.c index 018441bddf..233d019ca3 100644 --- a/block/raw-format.c +++ b/block/raw-format.c @@ -359,7 +359,7 @@ static BlockMeasureInfo *raw_measure(QemuOpts *opts, BlockDriverState *in_bs, BDRV_SECTOR_SIZE); } - info = g_new(BlockMeasureInfo, 1); + info = g_new0(BlockMeasureInfo, 1); info->required = required; /* Unallocated sectors count towards the file size in raw images */ diff --git a/docs/tools/qemu-img.rst b/docs/tools/qemu-img.rst index 38d464ea3f..320cb52b9f 100644 --- a/docs/tools/qemu-img.rst +++ b/docs/tools/qemu-img.rst @@ -616,6 +616,7 @@ Command description: required size: 524288 fully allocated size: 1074069504 + bitmaps size: 0 The ``required size`` is the file size of the new image. It may be smaller than the virtual disk size if the image format supports compact representation. @@ -625,6 +626,12 @@ Command description: occupy with the exception of internal snapshots, dirty bitmaps, vmstate data, and other advanced image format features. + The ``bitmaps size`` is the additional size required in order to + copy bitmaps from a source image in addition to the guest-visible + data; the line is omitted if either source or destination lacks + bitmap support, or 0 if bitmaps are supported but there is nothing + to copy. + .. option:: snapshot [--object OBJECTDEF] [--image-opts] [-U] [-q] [-l | -a SNAPSHOT | -c SNAPSHOT | -d SNAPSHOT] FILENAME List, apply, create or delete snapshots in image *FILENAME*. diff --git a/qapi/block-core.json b/qapi/block-core.json index 6fbacddab2..0e1c6a59f2 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -636,18 +636,24 @@ # efficiently so file size may be smaller than virtual disk size. # # The values are upper bounds that are guaranteed to fit the new image file. -# Subsequent modification, such as internal snapshot or bitmap creation, may -# require additional space and is not covered here. +# Subsequent modification, such as internal snapshot or further bitmap +# creation, may require additional space and is not covered here. # -# @required: Size required for a new image file, in bytes. +# @required: Size required for a new image file, in bytes, when copying just +# allocated guest-visible contents. # # @fully-allocated: Image file size, in bytes, once data has been written -# to all sectors. +# to all sectors, when copying just guest-visible contents. +# +# @bitmaps: Additional size required if all the top-level bitmap metadata +# in the source image were to be copied to the destination, +# present only when source and destination both support +# persistent bitmaps. (since 5.1) # # Since: 2.10 ## { 'struct': 'BlockMeasureInfo', - 'data': {'required': 'int', 'fully-allocated': 'int'} } + 'data': {'required': 'int', 'fully-allocated': 'int', '*bitmaps': 'int'} } ## # @query-block: diff --git a/qemu-img.c b/qemu-img.c index 2d30682f12..b2311bd3f6 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -5302,6 +5302,9 @@ static int img_measure(int argc, char **argv) if (output_format == OFORMAT_HUMAN) { printf("required size: %" PRIu64 "\n", info->required); printf("fully allocated size: %" PRIu64 "\n", info->fully_allocated); + if (info->has_bitmaps) { + printf("bitmaps size: %" PRIu64 "\n", info->bitmaps); + } } else { dump_json_block_measure_info(info); } diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2 index 4b69524c80..c7997760fd 100644 --- a/tests/qemu-iotests/178.out.qcow2 +++ b/tests/qemu-iotests/178.out.qcow2 @@ -37,6 +37,7 @@ qemu-img: The image size is too large (try using a larger cluster size) Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0 required size: 196608 fully allocated size: 196608 +bitmaps size: 0 converted image file size in bytes: 196608 @@ -45,6 +46,7 @@ converted image file size in bytes: 196608 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 required size: 393216 fully allocated size: 1074135040 +bitmaps size: 0 wrote 512/512 bytes at offset 512 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 65536/65536 bytes at offset 65536 @@ -53,6 +55,7 @@ wrote 64512/64512 bytes at offset 134217728 63 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) required size: 589824 fully allocated size: 1074135040 +bitmaps size: 0 converted image file size in bytes: 524288 @@ -60,6 +63,7 @@ converted image file size in bytes: 524288 required size: 524288 fully allocated size: 1074135040 +bitmaps size: 0 converted image file size in bytes: 458752 @@ -67,16 +71,19 @@ converted image file size in bytes: 458752 required size: 1074135040 fully allocated size: 1074135040 +bitmaps size: 0 == qcow2 input image and LUKS encryption == required size: 2686976 fully allocated size: 1076232192 +bitmaps size: 0 == qcow2 input image and preallocation (human) == required size: 1074135040 fully allocated size: 1074135040 +bitmaps size: 0 converted image file size in bytes: 1074135040 @@ -87,6 +94,7 @@ wrote 8388608/8388608 bytes at offset 0 8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) required size: 8716288 fully allocated size: 8716288 +bitmaps size: 0 converted image file size in bytes: 8716288 @@ -173,6 +181,7 @@ qemu-img: The image size is too large (try using a larger cluster size) Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0 { + "bitmaps": 0, "required": 196608, "fully-allocated": 196608 } @@ -183,6 +192,7 @@ converted image file size in bytes: 196608 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 { + "bitmaps": 0, "required": 393216, "fully-allocated": 1074135040 } @@ -193,6 +203,7 @@ wrote 65536/65536 bytes at offset 65536 wrote 64512/64512 bytes at offset 134217728 63 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) { + "bitmaps": 0, "required": 589824, "fully-allocated": 1074135040 } @@ -202,6 +213,7 @@ converted image file size in bytes: 524288 == qcow2 input image with internal snapshot (json) == { + "bitmaps": 0, "required": 524288, "fully-allocated": 1074135040 } @@ -211,6 +223,7 @@ converted image file size in bytes: 458752 == qcow2 input image and a backing file (json) == { + "bitmaps": 0, "required": 1074135040, "fully-allocated": 1074135040 } @@ -218,6 +231,7 @@ converted image file size in bytes: 458752 == qcow2 input image and LUKS encryption == { + "bitmaps": 0, "required": 2686976, "fully-allocated": 1076232192 } @@ -225,6 +239,7 @@ converted image file size in bytes: 458752 == qcow2 input image and preallocation (json) == { + "bitmaps": 0, "required": 1074135040, "fully-allocated": 1074135040 } @@ -237,6 +252,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8388608 wrote 8388608/8388608 bytes at offset 0 8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) { + "bitmaps": 0, "required": 8716288, "fully-allocated": 8716288 } diff --git a/tests/qemu-iotests/190 b/tests/qemu-iotests/190 index 6d41650438..fe630918e9 100755 --- a/tests/qemu-iotests/190 +++ b/tests/qemu-iotests/190 @@ -2,7 +2,7 @@ # # qemu-img measure sub-command tests on huge qcow2 files # -# Copyright (C) 2017 Red Hat, Inc. +# Copyright (C) 2017-2020 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -42,7 +42,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt qcow2 _supported_proto file -echo "== Huge file ==" +echo "== Huge file without bitmaps ==" echo _make_test_img -o 'cluster_size=2M' 2T @@ -51,6 +51,49 @@ $QEMU_IMG measure -O raw -f qcow2 "$TEST_IMG" $QEMU_IMG measure -O qcow2 -o cluster_size=64k -f qcow2 "$TEST_IMG" $QEMU_IMG measure -O qcow2 -o cluster_size=2M -f qcow2 "$TEST_IMG" +echo +echo "== Huge file with bitmaps ==" +echo + +$QEMU_IMG bitmap --add --granularity 512 -f qcow2 "$TEST_IMG" b1 +$QEMU_IMG bitmap --add -g 2M -f qcow2 "$TEST_IMG" b2 + +# No bitmap without a source +$QEMU_IMG measure -O qcow2 --size 10M +# No bitmap output, since raw does not support it +$QEMU_IMG measure -O raw -f qcow2 "$TEST_IMG" +# No bitmap output, since no bitmaps on raw source. Munge required size, as +# some filesystems store the qcow2 file with less sparseness than others +$QEMU_IMG measure -O qcow2 -f raw "$TEST_IMG" | + sed '/^required size:/ s/[0-9][0-9]*/SIZE/' +# No bitmap output, since v2 does not support it +$QEMU_IMG measure -O qcow2 -o compat=0.10 -f qcow2 "$TEST_IMG" + +# Compute expected output: bitmap clusters + bitmap tables + bitmaps directory +echo +val2T=$((2*1024*1024*1024*1024)) +cluster=$((64*1024)) +b1clusters=$(( (val2T/512/8 + cluster - 1) / cluster )) +b2clusters=$(( (val2T/2/1024/1024/8 + cluster - 1) / cluster )) +echo expected bitmap $((b1clusters * cluster + + (b1clusters * 8 + cluster - 1) / cluster * cluster + + b2clusters * cluster + + (b2clusters * 8 + cluster - 1) / cluster * cluster + + cluster)) +$QEMU_IMG measure -O qcow2 -o cluster_size=64k -f qcow2 "$TEST_IMG" + +# Compute expected output: bitmap clusters + bitmap tables + bitmaps directory +echo +cluster=$((2*1024*1024)) +b1clusters=$(( (val2T/512/8 + cluster - 1) / cluster )) +b2clusters=$(( (val2T/2/1024/1024/8 + cluster - 1) / cluster )) +echo expected bitmap $((b1clusters * cluster + + (b1clusters * 8 + cluster - 1) / cluster * cluster + + b2clusters * cluster + + (b2clusters * 8 + cluster - 1) / cluster * cluster + + cluster)) +$QEMU_IMG measure --output=json -O qcow2 -o cluster_size=2M -f qcow2 "$TEST_IMG" + # success, all done echo "*** done" rm -f $seq.full diff --git a/tests/qemu-iotests/190.out b/tests/qemu-iotests/190.out index d001942002..ed9d8214eb 100644 --- a/tests/qemu-iotests/190.out +++ b/tests/qemu-iotests/190.out @@ -1,11 +1,36 @@ QA output created by 190 -== Huge file == +== Huge file without bitmaps == Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2199023255552 required size: 2199023255552 fully allocated size: 2199023255552 required size: 335806464 fully allocated size: 2199359062016 +bitmaps size: 0 required size: 18874368 fully allocated size: 2199042129920 +bitmaps size: 0 + +== Huge file with bitmaps == + +required size: 327680 +fully allocated size: 10813440 +required size: 2199023255552 +fully allocated size: 2199023255552 +required size: SIZE +fully allocated size: 17170432 +required size: 335806464 +fully allocated size: 2199359062016 + +expected bitmap 537198592 +required size: 335806464 +fully allocated size: 2199359062016 +bitmaps size: 537198592 + +expected bitmap 545259520 +{ + "bitmaps": 545259520, + "required": 18874368, + "fully-allocated": 2199042129920 +} *** done |