diff options
Diffstat (limited to 'tests')
30 files changed, 529 insertions, 219 deletions
diff --git a/tests/decode/err_pattern_group_nest1.decode b/tests/decode/err_pattern_group_nest1.decode index 92e971c3c5..7d09891a1c 100644 --- a/tests/decode/err_pattern_group_nest1.decode +++ b/tests/decode/err_pattern_group_nest1.decode @@ -3,11 +3,12 @@ %sub1 0:8 %sub2 8:8 -%sub3 16:8 -%sub4 24:8 -# Groups with no overlap are supposed to fail +# Make sure braces are matched { - top 00000000 00000000 00000000 00000000 - sub4 ........ ........ ........ ........ %sub1 %sub2 %sub3 %sub4 + top 00000000 00000000 00000000 00000000 + [ + sub1 00000000 00000000 00000000 ........ %sub1 + sub2 00000000 00000000 ........ ........ %sub1 %sub2 + } } diff --git a/tests/decode/err_pattern_group_nest2.decode b/tests/decode/err_pattern_group_nest2.decode new file mode 100644 index 0000000000..c172239e9b --- /dev/null +++ b/tests/decode/err_pattern_group_nest2.decode @@ -0,0 +1,6 @@ +# This work is licensed under the terms of the GNU LGPL, version 2 or later. +# See the COPYING.LIB file in the top-level directory. + +# Make sure braces are matched +{ + [ diff --git a/tests/decode/err_pattern_group_nest3.decode b/tests/decode/err_pattern_group_nest3.decode new file mode 100644 index 0000000000..b085d01410 --- /dev/null +++ b/tests/decode/err_pattern_group_nest3.decode @@ -0,0 +1,14 @@ +# This work is licensed under the terms of the GNU LGPL, version 2 or later. +# See the COPYING.LIB file in the top-level directory. + +%sub1 0:8 +%sub2 8:8 + +# The exclusive group should error for overlap. +{ + top 00000000 00000000 00000000 00000000 + [ + sub1 00000000 00000000 00000000 ........ %sub1 + sub2 00000000 00000000 ........ ........ %sub1 %sub2 + ] +} diff --git a/tests/decode/succ_pattern_group_nest2.decode b/tests/decode/succ_pattern_group_nest2.decode new file mode 100644 index 0000000000..8d5ab4b2d3 --- /dev/null +++ b/tests/decode/succ_pattern_group_nest2.decode @@ -0,0 +1,13 @@ +# This work is licensed under the terms of the GNU LGPL, version 2 or later. +# See the COPYING.LIB file in the top-level directory. + +%sub1 0:8 +%sub2 8:8 +%sub3 16:8 +%sub4 24:8 + +# Group with complete overlap of the two patterns +{ + top 00000000 00000000 00000000 00000000 + sub4 ........ ........ ........ ........ %sub1 %sub2 %sub3 %sub4 +} diff --git a/tests/decode/succ_pattern_group_nest3.decode b/tests/decode/succ_pattern_group_nest3.decode new file mode 100644 index 0000000000..156249f090 --- /dev/null +++ b/tests/decode/succ_pattern_group_nest3.decode @@ -0,0 +1,11 @@ +# This work is licensed under the terms of the GNU LGPL, version 2 or later. +# See the COPYING.LIB file in the top-level directory. + +{ + [ + sub1 00000000 a:8 b:8 c:8 + sub2 00000001 a:8 b:8 c:8 + sub3 00000010 a:8 b:8 c:8 + ] + sub4 000000 d:2 a:8 b:8 c:8 +} diff --git a/tests/decode/succ_pattern_group_nest4.decode b/tests/decode/succ_pattern_group_nest4.decode new file mode 100644 index 0000000000..dc54a1d285 --- /dev/null +++ b/tests/decode/succ_pattern_group_nest4.decode @@ -0,0 +1,13 @@ +# This work is licensed under the terms of the GNU LGPL, version 2 or later. +# See the COPYING.LIB file in the top-level directory. + +# Verify deeper nesting, and a single element in the groups. +{ + [ + { + [ + sub1 00000000 a:8 b:8 c:8 + ] + } + ] +} diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index ed46bd98eb..981b7fcf2a 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -130,7 +130,7 @@ docker-image-debian-sparc64-cross: docker-image-debian10 docker-image-travis: NOUSER=1 # Specialist build images, sometimes very limited tools -docker-image-tricore-cross: docker-image-debian9 +docker-image-debian-tricore-cross: docker-image-debian9 docker-image-debian-arm64-test-cross: docker-image-debian11 # These images may be good enough for building tests but not for test builds diff --git a/tests/docker/dockerfiles/centos7.docker b/tests/docker/dockerfiles/centos7.docker index 9a2a2e515d..e197acdc3c 100644 --- a/tests/docker/dockerfiles/centos7.docker +++ b/tests/docker/dockerfiles/centos7.docker @@ -5,13 +5,11 @@ RUN yum -y update # Please keep this list sorted alphabetically ENV PACKAGES \ - bison \ bzip2 \ bzip2-devel \ ccache \ csnappy-devel \ dbus-daemon \ - flex \ gcc-c++ \ gcc \ gettext \ diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index bfa0d33c9c..9852c5b9ee 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -3,11 +3,9 @@ FROM centos:8.1.1911 RUN dnf -y update ENV PACKAGES \ SDL-devel \ - bison \ bzip2 \ bzip2-devel \ dbus-daemon \ - flex \ gcc \ gcc-c++ \ gettext \ diff --git a/tests/docker/dockerfiles/debian-xtensa-cross.docker b/tests/docker/dockerfiles/debian-xtensa-cross.docker index e6f93f65ee..beb73f46ba 100644 --- a/tests/docker/dockerfiles/debian-xtensa-cross.docker +++ b/tests/docker/dockerfiles/debian-xtensa-cross.docker @@ -11,11 +11,9 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt-get install -y --no-install-recommends \ - bison \ build-essential \ ca-certificates \ curl \ - flex \ gettext \ git \ python3-minimal diff --git a/tests/docker/dockerfiles/debian10.docker b/tests/docker/dockerfiles/debian10.docker index 0769700a41..bcdff04ddf 100644 --- a/tests/docker/dockerfiles/debian10.docker +++ b/tests/docker/dockerfiles/debian10.docker @@ -18,12 +18,10 @@ RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends \ bc \ - bison \ build-essential \ ca-certificates \ clang \ dbus \ - flex \ gdb-multiarch \ gettext \ git \ diff --git a/tests/docker/dockerfiles/debian9.docker b/tests/docker/dockerfiles/debian9.docker index 08cc970feb..0f0ebe530a 100644 --- a/tests/docker/dockerfiles/debian9.docker +++ b/tests/docker/dockerfiles/debian9.docker @@ -18,11 +18,9 @@ RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends \ bc \ - bison \ build-essential \ ca-certificates \ clang \ - flex \ gdb-multiarch \ gettext \ git \ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index 179575ecaa..92b6e11c8a 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -3,7 +3,6 @@ FROM fedora:30 # Please keep this list sorted alphabetically ENV PACKAGES \ bc \ - bison \ brlapi-devel \ bzip2 \ bzip2-devel \ @@ -13,7 +12,6 @@ ENV PACKAGES \ dbus-daemon \ device-mapper-multipath-devel \ findutils \ - flex \ gcc \ gcc-c++ \ gettext \ diff --git a/tests/docker/dockerfiles/ubuntu.docker b/tests/docker/dockerfiles/ubuntu.docker index eeb3b22bf2..161806e6b8 100644 --- a/tests/docker/dockerfiles/ubuntu.docker +++ b/tests/docker/dockerfiles/ubuntu.docker @@ -9,8 +9,8 @@ # system won't pick up that it has changed. # -FROM ubuntu:19.04 -ENV PACKAGES flex bison \ +FROM ubuntu:20.04 +ENV PACKAGES \ ccache \ clang \ dbus \ diff --git a/tests/docker/dockerfiles/ubuntu1804.docker b/tests/docker/dockerfiles/ubuntu1804.docker index f66b06f4cf..a10ea2850b 100644 --- a/tests/docker/dockerfiles/ubuntu1804.docker +++ b/tests/docker/dockerfiles/ubuntu1804.docker @@ -1,5 +1,5 @@ FROM ubuntu:18.04 -ENV PACKAGES flex bison \ +ENV PACKAGES \ ccache \ clang \ gcc \ diff --git a/tests/plugin/mem.c b/tests/plugin/mem.c index 878abf09d1..4725bd851d 100644 --- a/tests/plugin/mem.c +++ b/tests/plugin/mem.c @@ -28,7 +28,7 @@ static void plugin_exit(qemu_plugin_id_t id, void *p) g_string_printf(out, "mem accesses: %" PRIu64 "\n", mem_count); if (do_haddr) { - g_string_append_printf(out, "io accesses: %" PRIu64 "\n", mem_count); + g_string_append_printf(out, "io accesses: %" PRIu64 "\n", io_count); } qemu_plugin_outs(out->str); } diff --git a/tests/qemu-iotests/031.out b/tests/qemu-iotests/031.out index 5a4beda6a2..4b21d6a9ba 100644 --- a/tests/qemu-iotests/031.out +++ b/tests/qemu-iotests/031.out @@ -25,7 +25,7 @@ refcount_order 4 header_length 72 Header extension: -magic 0x12345678 +magic 0x12345678 (<unknown>) length 31 data 'This is a test header extension' @@ -53,7 +53,7 @@ refcount_order 4 header_length 72 Header extension: -magic 0x12345678 +magic 0x12345678 (<unknown>) length 31 data 'This is a test header extension' @@ -81,12 +81,12 @@ refcount_order 4 header_length 72 Header extension: -magic 0xe2792aca +magic 0xe2792aca (Backing format) length 11 data 'host_device' Header extension: -magic 0x12345678 +magic 0x12345678 (<unknown>) length 31 data 'This is a test header extension' @@ -116,12 +116,12 @@ refcount_order 4 header_length 112 Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> Header extension: -magic 0x12345678 +magic 0x12345678 (<unknown>) length 31 data 'This is a test header extension' @@ -149,12 +149,12 @@ refcount_order 4 header_length 112 Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> Header extension: -magic 0x12345678 +magic 0x12345678 (<unknown>) length 31 data 'This is a test header extension' @@ -182,17 +182,17 @@ refcount_order 4 header_length 112 Header extension: -magic 0xe2792aca +magic 0xe2792aca (Backing format) length 11 data 'host_device' Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> Header extension: -magic 0x12345678 +magic 0x12345678 (<unknown>) length 31 data 'This is a test header extension' diff --git a/tests/qemu-iotests/036.out b/tests/qemu-iotests/036.out index e409acf60e..a9bed828e5 100644 --- a/tests/qemu-iotests/036.out +++ b/tests/qemu-iotests/036.out @@ -25,7 +25,7 @@ incompatible_features [] compatible_features [] autoclear_features [63] Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> @@ -37,7 +37,7 @@ incompatible_features [] compatible_features [] autoclear_features [] Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out index a51ad1b5ba..2f03cf045c 100644 --- a/tests/qemu-iotests/061.out +++ b/tests/qemu-iotests/061.out @@ -25,7 +25,7 @@ refcount_order 4 header_length 112 Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> @@ -83,7 +83,7 @@ refcount_order 4 header_length 112 Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> @@ -139,7 +139,7 @@ refcount_order 4 header_length 112 Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> @@ -194,7 +194,7 @@ refcount_order 4 header_length 112 Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> @@ -263,7 +263,7 @@ refcount_order 4 header_length 112 Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> @@ -325,7 +325,7 @@ refcount_order 4 header_length 112 Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> @@ -354,7 +354,7 @@ refcount_order 4 header_length 112 Header extension: -magic 0x6803f857 +magic 0x6803f857 (Feature table) length 336 data <binary> diff --git a/tests/qemu-iotests/291 b/tests/qemu-iotests/291 index 3ca83b9cd1..404f8521f7 100755 --- a/tests/qemu-iotests/291 +++ b/tests/qemu-iotests/291 @@ -62,6 +62,8 @@ $QEMU_IO -c 'w 1M 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io $QEMU_IMG bitmap --disable -f $IMGFMT "$TEST_IMG" b1 $QEMU_IMG bitmap --enable -f $IMGFMT "$TEST_IMG" b2 $QEMU_IO -c 'w 2M 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io +echo "Check resulting qcow2 header extensions:" +$PYTHON qcow2.py "$TEST_IMG" dump-header-exts echo echo "=== Bitmap preservation not possible to non-qcow2 ===" @@ -77,7 +79,7 @@ echo # Only bitmaps from the active layer are copied $QEMU_IMG convert --bitmaps -O qcow2 "$TEST_IMG.orig" "$TEST_IMG" -$QEMU_IMG info "$TEST_IMG" | _filter_img_info --format-specific +_img_info --format-specific # But we can also merge in bitmaps from other layers. This test is a bit # contrived to cover more code paths, in reality, you could merge directly # into b0 without going through tmp @@ -87,7 +89,9 @@ $QEMU_IMG bitmap --add --merge b0 -b "$TEST_IMG.base" -F $IMGFMT \ $QEMU_IMG bitmap --merge tmp -f $IMGFMT "$TEST_IMG" b0 $QEMU_IMG bitmap --remove --image-opts \ driver=$IMGFMT,file.driver=file,file.filename="$TEST_IMG" tmp -$QEMU_IMG info "$TEST_IMG" | _filter_img_info --format-specific +_img_info --format-specific +echo "Check resulting qcow2 header extensions:" +$PYTHON qcow2.py "$TEST_IMG" dump-header-exts echo echo "=== Check bitmap contents ===" diff --git a/tests/qemu-iotests/291.out b/tests/qemu-iotests/291.out index 8c62017567..08bfaaaa6b 100644 --- a/tests/qemu-iotests/291.out +++ b/tests/qemu-iotests/291.out @@ -14,6 +14,25 @@ wrote 1048576/1048576 bytes at offset 1048576 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 1048576/1048576 bytes at offset 2097152 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Check resulting qcow2 header extensions: +Header extension: +magic 0xe2792aca (Backing format) +length 5 +data 'qcow2' + +Header extension: +magic 0x6803f857 (Feature table) +length 336 +data <binary> + +Header extension: +magic 0x23852875 (Bitmaps) +length 24 +nb_bitmaps 2 +reserved32 0 +bitmap_directory_size 0x40 +bitmap_directory_offset 0x510000 + === Bitmap preservation not possible to non-qcow2 === @@ -24,7 +43,7 @@ qemu-img: Format driver 'raw' does not support bitmaps image: TEST_DIR/t.IMGFMT file format: IMGFMT virtual size: 10 MiB (10485760 bytes) -disk size: 4.39 MiB +cluster_size: 65536 Format specific information: compat: 1.1 compression type: zlib @@ -44,7 +63,7 @@ Format specific information: image: TEST_DIR/t.IMGFMT file format: IMGFMT virtual size: 10 MiB (10485760 bytes) -disk size: 4.48 MiB +cluster_size: 65536 Format specific information: compat: 1.1 compression type: zlib @@ -65,6 +84,20 @@ Format specific information: granularity: 65536 refcount bits: 16 corrupt: false +Check resulting qcow2 header extensions: +Header extension: +magic 0x6803f857 (Feature table) +length 336 +data <binary> + +Header extension: +magic 0x23852875 (Bitmaps) +length 24 +nb_bitmaps 3 +reserved32 0 +bitmap_directory_size 0x60 +bitmap_directory_offset 0x520000 + === Check bitmap contents === diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py index 94a07b2f6f..8c187e9a72 100755 --- a/tests/qemu-iotests/qcow2.py +++ b/tests/qemu-iotests/qcow2.py @@ -1,181 +1,50 @@ #!/usr/bin/env python3 +# +# Manipulations with qcow2 image +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# import sys -import struct -import string - -class QcowHeaderExtension: - - def __init__(self, magic, length, data): - if length % 8 != 0: - padding = 8 - (length % 8) - data += b"\0" * padding - - self.magic = magic - self.length = length - self.data = data - - @classmethod - def create(cls, magic, data): - return QcowHeaderExtension(magic, len(data), data) - -class QcowHeader: - - uint32_t = 'I' - uint64_t = 'Q' - - fields = [ - # Version 2 header fields - [ uint32_t, '%#x', 'magic' ], - [ uint32_t, '%d', 'version' ], - [ uint64_t, '%#x', 'backing_file_offset' ], - [ uint32_t, '%#x', 'backing_file_size' ], - [ uint32_t, '%d', 'cluster_bits' ], - [ uint64_t, '%d', 'size' ], - [ uint32_t, '%d', 'crypt_method' ], - [ uint32_t, '%d', 'l1_size' ], - [ uint64_t, '%#x', 'l1_table_offset' ], - [ uint64_t, '%#x', 'refcount_table_offset' ], - [ uint32_t, '%d', 'refcount_table_clusters' ], - [ uint32_t, '%d', 'nb_snapshots' ], - [ uint64_t, '%#x', 'snapshot_offset' ], - - # Version 3 header fields - [ uint64_t, 'mask', 'incompatible_features' ], - [ uint64_t, 'mask', 'compatible_features' ], - [ uint64_t, 'mask', 'autoclear_features' ], - [ uint32_t, '%d', 'refcount_order' ], - [ uint32_t, '%d', 'header_length' ], - ]; - - fmt = '>' + ''.join(field[0] for field in fields) - - def __init__(self, fd): - - buf_size = struct.calcsize(QcowHeader.fmt) - - fd.seek(0) - buf = fd.read(buf_size) - - header = struct.unpack(QcowHeader.fmt, buf) - self.__dict__ = dict((field[2], header[i]) - for i, field in enumerate(QcowHeader.fields)) - - self.set_defaults() - self.cluster_size = 1 << self.cluster_bits - - fd.seek(self.header_length) - self.load_extensions(fd) - - if self.backing_file_offset: - fd.seek(self.backing_file_offset) - self.backing_file = fd.read(self.backing_file_size) - else: - self.backing_file = None - - def set_defaults(self): - if self.version == 2: - self.incompatible_features = 0 - self.compatible_features = 0 - self.autoclear_features = 0 - self.refcount_order = 4 - self.header_length = 72 - - def load_extensions(self, fd): - self.extensions = [] - - if self.backing_file_offset != 0: - end = min(self.cluster_size, self.backing_file_offset) - else: - end = self.cluster_size - - while fd.tell() < end: - (magic, length) = struct.unpack('>II', fd.read(8)) - if magic == 0: - break - else: - padded = (length + 7) & ~7 - data = fd.read(padded) - self.extensions.append(QcowHeaderExtension(magic, length, data)) - - def update_extensions(self, fd): - - fd.seek(self.header_length) - extensions = self.extensions - extensions.append(QcowHeaderExtension(0, 0, b"")) - for ex in extensions: - buf = struct.pack('>II', ex.magic, ex.length) - fd.write(buf) - fd.write(ex.data) - - if self.backing_file != None: - self.backing_file_offset = fd.tell() - fd.write(self.backing_file) - - if fd.tell() > self.cluster_size: - raise Exception("I think I just broke the image...") - - - def update(self, fd): - header_bytes = self.header_length - - self.update_extensions(fd) - - fd.seek(0) - header = tuple(self.__dict__[f] for t, p, f in QcowHeader.fields) - buf = struct.pack(QcowHeader.fmt, *header) - buf = buf[0:header_bytes-1] - fd.write(buf) - - def dump(self): - for f in QcowHeader.fields: - value = self.__dict__[f[2]] - if f[1] == 'mask': - bits = [] - for bit in range(64): - if value & (1 << bit): - bits.append(bit) - value_str = str(bits) - else: - value_str = f[1] % value - - print("%-25s" % f[2], value_str) - print("") - - def dump_extensions(self): - for ex in self.extensions: - data = ex.data[:ex.length] - if all(c in string.printable.encode('ascii') for c in data): - data = "'%s'" % data.decode('ascii') - else: - data = "<binary>" - - print("Header extension:") - print("%-25s %#x" % ("magic", ex.magic)) - print("%-25s %d" % ("length", ex.length)) - print("%-25s %s" % ("data", data)) - print("") +from qcow2_format import ( + QcowHeader, + QcowHeaderExtension +) def cmd_dump_header(fd): h = QcowHeader(fd) h.dump() + print() h.dump_extensions() + def cmd_dump_header_exts(fd): h = QcowHeader(fd) h.dump_extensions() + def cmd_set_header(fd, name, value): try: value = int(value, 0) - except: + except ValueError: print("'%s' is not a valid number" % value) sys.exit(1) fields = (field[2] for field in QcowHeader.fields) - if not name in fields: + if name not in fields: print("'%s' is not a known header field" % name) sys.exit(1) @@ -183,25 +52,29 @@ def cmd_set_header(fd, name, value): h.__dict__[name] = value h.update(fd) + def cmd_add_header_ext(fd, magic, data): try: magic = int(magic, 0) - except: + except ValueError: print("'%s' is not a valid magic number" % magic) sys.exit(1) h = QcowHeader(fd) - h.extensions.append(QcowHeaderExtension.create(magic, data.encode('ascii'))) + h.extensions.append(QcowHeaderExtension.create(magic, + data.encode('ascii'))) h.update(fd) + def cmd_add_header_ext_stdio(fd, magic): data = sys.stdin.read() cmd_add_header_ext(fd, magic, data) + def cmd_del_header_ext(fd, magic): try: magic = int(magic, 0) - except: + except ValueError: print("'%s' is not a valid magic number" % magic) sys.exit(1) @@ -219,12 +92,13 @@ def cmd_del_header_ext(fd, magic): h.update(fd) + def cmd_set_feature_bit(fd, group, bit): try: bit = int(bit, 0) if bit < 0 or bit >= 64: raise ValueError - except: + except ValueError: print("'%s' is not a valid bit number in range [0, 64)" % bit) sys.exit(1) @@ -236,21 +110,27 @@ def cmd_set_feature_bit(fd, group, bit): elif group == 'autoclear': h.autoclear_features |= 1 << bit else: - print("'%s' is not a valid group, try 'incompatible', 'compatible', or 'autoclear'" % group) + print("'%s' is not a valid group, try " + "'incompatible', 'compatible', or 'autoclear'" % group) sys.exit(1) h.update(fd) + cmds = [ - [ 'dump-header', cmd_dump_header, 0, 'Dump image header and header extensions' ], - [ 'dump-header-exts', cmd_dump_header_exts, 0, 'Dump image header extensions' ], - [ 'set-header', cmd_set_header, 2, 'Set a field in the header'], - [ 'add-header-ext', cmd_add_header_ext, 2, 'Add a header extension' ], - [ 'add-header-ext-stdio', cmd_add_header_ext_stdio, 1, 'Add a header extension, data from stdin' ], - [ 'del-header-ext', cmd_del_header_ext, 1, 'Delete a header extension' ], - [ 'set-feature-bit', cmd_set_feature_bit, 2, 'Set a feature bit'], + ['dump-header', cmd_dump_header, 0, + 'Dump image header and header extensions'], + ['dump-header-exts', cmd_dump_header_exts, 0, + 'Dump image header extensions'], + ['set-header', cmd_set_header, 2, 'Set a field in the header'], + ['add-header-ext', cmd_add_header_ext, 2, 'Add a header extension'], + ['add-header-ext-stdio', cmd_add_header_ext_stdio, 1, + 'Add a header extension, data from stdin'], + ['del-header-ext', cmd_del_header_ext, 1, 'Delete a header extension'], + ['set-feature-bit', cmd_set_feature_bit, 2, 'Set a feature bit'], ] + def main(filename, cmd, args): fd = open(filename, "r+b") try: @@ -267,6 +147,7 @@ def main(filename, cmd, args): finally: fd.close() + def usage(): print("Usage: %s <file> <cmd> [<arg>, ...]" % sys.argv[0]) print("") @@ -274,6 +155,7 @@ def usage(): for name, handler, num_args, desc in cmds: print(" %-20s - %s" % (name, desc)) + if __name__ == '__main__': if len(sys.argv) < 3: usage() diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py new file mode 100644 index 0000000000..0f65fd161d --- /dev/null +++ b/tests/qemu-iotests/qcow2_format.py @@ -0,0 +1,286 @@ +# Library for manipulations with qcow2 image +# +# Copyright (c) 2020 Virtuozzo International GmbH. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import struct +import string + + +class Qcow2Field: + + def __init__(self, value): + self.value = value + + def __str__(self): + return str(self.value) + + +class Flags64(Qcow2Field): + + def __str__(self): + bits = [] + for bit in range(64): + if self.value & (1 << bit): + bits.append(bit) + return str(bits) + + +class Enum(Qcow2Field): + + def __str__(self): + return f'{self.value:#x} ({self.mapping.get(self.value, "<unknown>")})' + + +class Qcow2StructMeta(type): + + # Mapping from c types to python struct format + ctypes = { + 'u8': 'B', + 'u16': 'H', + 'u32': 'I', + 'u64': 'Q' + } + + def __init__(self, name, bases, attrs): + if 'fields' in attrs: + self.fmt = '>' + ''.join(self.ctypes[f[0]] for f in self.fields) + + +class Qcow2Struct(metaclass=Qcow2StructMeta): + + """Qcow2Struct: base class for qcow2 data structures + + Successors should define fields class variable, which is: list of tuples, + each of three elements: + - c-type (one of 'u8', 'u16', 'u32', 'u64') + - format (format_spec to use with .format() when dump or 'mask' to dump + bitmasks) + - field name + """ + + def __init__(self, fd=None, offset=None, data=None): + """ + Two variants: + 1. Specify data. fd and offset must be None. + 2. Specify fd and offset, data must be None. offset may be omitted + in this case, than current position of fd is used. + """ + if data is None: + assert fd is not None + buf_size = struct.calcsize(self.fmt) + if offset is not None: + fd.seek(offset) + data = fd.read(buf_size) + else: + assert fd is None and offset is None + + values = struct.unpack(self.fmt, data) + self.__dict__ = dict((field[2], values[i]) + for i, field in enumerate(self.fields)) + + def dump(self): + for f in self.fields: + value = self.__dict__[f[2]] + if isinstance(f[1], str): + value_str = f[1].format(value) + else: + value_str = str(f[1](value)) + + print('{:<25} {}'.format(f[2], value_str)) + + +class Qcow2BitmapExt(Qcow2Struct): + + fields = ( + ('u32', '{}', 'nb_bitmaps'), + ('u32', '{}', 'reserved32'), + ('u64', '{:#x}', 'bitmap_directory_size'), + ('u64', '{:#x}', 'bitmap_directory_offset') + ) + + +QCOW2_EXT_MAGIC_BITMAPS = 0x23852875 + + +class QcowHeaderExtension(Qcow2Struct): + + class Magic(Enum): + mapping = { + 0xe2792aca: 'Backing format', + 0x6803f857: 'Feature table', + 0x0537be77: 'Crypto header', + QCOW2_EXT_MAGIC_BITMAPS: 'Bitmaps', + 0x44415441: 'Data file' + } + + fields = ( + ('u32', Magic, 'magic'), + ('u32', '{}', 'length') + # length bytes of data follows + # then padding to next multiply of 8 + ) + + def __init__(self, magic=None, length=None, data=None, fd=None): + """ + Support both loading from fd and creation from user data. + For fd-based creation current position in a file will be used to read + the data. + + This should be somehow refactored and functionality should be moved to + superclass (to allow creation of any qcow2 struct), but then, fields + of variable length (data here) should be supported in base class + somehow. Note also, that we probably want to parse different + extensions. Should they be subclasses of this class, or how to do it + better? Should it be something like QAPI union with discriminator field + (magic here). So, it's a TODO. We'll see how to properly refactor this + when we have more qcow2 structures. + """ + if fd is None: + assert all(v is not None for v in (magic, length, data)) + self.magic = magic + self.length = length + if length % 8 != 0: + padding = 8 - (length % 8) + data += b'\0' * padding + self.data = data + else: + assert all(v is None for v in (magic, length, data)) + super().__init__(fd=fd) + padded = (self.length + 7) & ~7 + self.data = fd.read(padded) + assert self.data is not None + + if self.magic == QCOW2_EXT_MAGIC_BITMAPS: + self.obj = Qcow2BitmapExt(data=self.data) + else: + self.obj = None + + def dump(self): + super().dump() + + if self.obj is None: + data = self.data[:self.length] + if all(c in string.printable.encode('ascii') for c in data): + data = f"'{ data.decode('ascii') }'" + else: + data = '<binary>' + print(f'{"data":<25} {data}') + else: + self.obj.dump() + + @classmethod + def create(cls, magic, data): + return QcowHeaderExtension(magic, len(data), data) + + +class QcowHeader(Qcow2Struct): + + fields = ( + # Version 2 header fields + ('u32', '{:#x}', 'magic'), + ('u32', '{}', 'version'), + ('u64', '{:#x}', 'backing_file_offset'), + ('u32', '{:#x}', 'backing_file_size'), + ('u32', '{}', 'cluster_bits'), + ('u64', '{}', 'size'), + ('u32', '{}', 'crypt_method'), + ('u32', '{}', 'l1_size'), + ('u64', '{:#x}', 'l1_table_offset'), + ('u64', '{:#x}', 'refcount_table_offset'), + ('u32', '{}', 'refcount_table_clusters'), + ('u32', '{}', 'nb_snapshots'), + ('u64', '{:#x}', 'snapshot_offset'), + + # Version 3 header fields + ('u64', Flags64, 'incompatible_features'), + ('u64', Flags64, 'compatible_features'), + ('u64', Flags64, 'autoclear_features'), + ('u32', '{}', 'refcount_order'), + ('u32', '{}', 'header_length'), + ) + + def __init__(self, fd): + super().__init__(fd=fd, offset=0) + + self.set_defaults() + self.cluster_size = 1 << self.cluster_bits + + fd.seek(self.header_length) + self.load_extensions(fd) + + if self.backing_file_offset: + fd.seek(self.backing_file_offset) + self.backing_file = fd.read(self.backing_file_size) + else: + self.backing_file = None + + def set_defaults(self): + if self.version == 2: + self.incompatible_features = 0 + self.compatible_features = 0 + self.autoclear_features = 0 + self.refcount_order = 4 + self.header_length = 72 + + def load_extensions(self, fd): + self.extensions = [] + + if self.backing_file_offset != 0: + end = min(self.cluster_size, self.backing_file_offset) + else: + end = self.cluster_size + + while fd.tell() < end: + ext = QcowHeaderExtension(fd=fd) + if ext.magic == 0: + break + else: + self.extensions.append(ext) + + def update_extensions(self, fd): + + fd.seek(self.header_length) + extensions = self.extensions + extensions.append(QcowHeaderExtension(0, 0, b'')) + for ex in extensions: + buf = struct.pack('>II', ex.magic, ex.length) + fd.write(buf) + fd.write(ex.data) + + if self.backing_file is not None: + self.backing_file_offset = fd.tell() + fd.write(self.backing_file) + + if fd.tell() > self.cluster_size: + raise Exception('I think I just broke the image...') + + def update(self, fd): + header_bytes = self.header_length + + self.update_extensions(fd) + + fd.seek(0) + header = tuple(self.__dict__[f] for t, p, f in QcowHeader.fields) + buf = struct.pack(QcowHeader.fmt, *header) + buf = buf[0:header_bytes-1] + fd.write(buf) + + def dump_extensions(self): + for ex in self.extensions: + print('Header extension:') + ex.dump() + print() diff --git a/tests/tcg/arm/Makefile.target b/tests/tcg/arm/Makefile.target index 11c39c601e..3da09a38be 100644 --- a/tests/tcg/arm/Makefile.target +++ b/tests/tcg/arm/Makefile.target @@ -68,6 +68,8 @@ run-semiconsole-arm: semiconsole-arm run-plugin-semiconsole-arm-with-%: $(call skip-test, $<, "MANUAL ONLY") +ARM_TESTS += commpage + TESTS += $(ARM_TESTS) # On ARM Linux only supports 4k pages diff --git a/tests/tcg/arm/commpage.c b/tests/tcg/arm/commpage.c new file mode 100644 index 0000000000..c76e70cb8b --- /dev/null +++ b/tests/tcg/arm/commpage.c @@ -0,0 +1,61 @@ +/* + * Verify the COMMPAGE emulation + * + * The ARM commpage is a set of user space helper functions provided + * by the kernel in an effort to ease portability of user space code + * between different CPUs with potentially different capabilities. It + * is a 32 bit invention and similar to the vdso segment in many ways. + * + * The ABI is documented in the Linux kernel: + * Documentation/arm/kernel_userspace_helpers.rst + * + * Copyright (c) 2020 Linaro Ltd + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> + +#define ARM_COMMPAGE (0xffff0f00u) +#define ARM_KUSER_VERSION (*(int32_t *)(ARM_COMMPAGE + 0xfc)) +typedef void * (get_tls_fn)(void); +#define ARM_KUSER_GET_TLS (*(get_tls_fn *)(ARM_COMMPAGE + 0xe0)) +typedef int (cmpxchg_fn)(int oldval, int newval, volatile int *ptr); +#define ARM_KUSER_CMPXCHG (*(cmpxchg_fn *)(ARM_COMMPAGE + 0xc0)) +typedef void (dmb_fn)(void); +#define ARM_KUSER_DMB (*(dmb_fn *)(ARM_COMMPAGE + 0xa0)) +typedef int (cmpxchg64_fn)(const int64_t *oldval, + const int64_t *newval, + volatile int64_t *ptr); +#define ARM_KUSER_CMPXCHG64 (*(cmpxchg64_fn *)(ARM_COMMPAGE + 0x60)) + +#define fail_unless(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "FAILED at %s:%d\n", __FILE__, __LINE__); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + + +int main(int argc, char *argv[argc]) +{ + void *kuser_tls; + int val = 1; + const int64_t oldval = 1, newval = 2; + int64_t val64 = 1; + + fail_unless(ARM_KUSER_VERSION == 0x5); + kuser_tls = ARM_KUSER_GET_TLS(); + printf("TLS = %p\n", kuser_tls); + fail_unless(kuser_tls != 0); + fail_unless(ARM_KUSER_CMPXCHG(1, 2, &val) == 0); + printf("val = %d\n", val); + /* this is a crash test, not checking an actual barrier occurs */ + ARM_KUSER_DMB(); + fail_unless(ARM_KUSER_CMPXCHG64(&oldval, &newval, &val64) == 0); + printf("val64 = %lld\n", val64); + return 0; +} diff --git a/tests/vm/fedora b/tests/vm/fedora index bd9c6cf295..a9195670f4 100755 --- a/tests/vm/fedora +++ b/tests/vm/fedora @@ -32,7 +32,6 @@ class FedoraVM(basevm.BaseVM): pkgs = [ # tools 'git-core', - 'flex', 'bison', 'gcc', 'binutils', 'make', # perl diff --git a/tests/vm/freebsd b/tests/vm/freebsd index 298967fe9c..f87db2b126 100755 --- a/tests/vm/freebsd +++ b/tests/vm/freebsd @@ -38,7 +38,6 @@ class FreeBSDVM(basevm.BaseVM): "bash", "gmake", "gsed", - "flex", "bison", # libs: crypto "gnutls", diff --git a/tests/vm/netbsd b/tests/vm/netbsd index b10c9d429d..cdac502dad 100755 --- a/tests/vm/netbsd +++ b/tests/vm/netbsd @@ -36,7 +36,6 @@ class NetBSDVM(basevm.BaseVM): "bash", "gmake", "gsed", - "flex", "bison", # libs: crypto "gnutls", diff --git a/tests/vm/openbsd b/tests/vm/openbsd index 0b705f4945..13e7f9a6d5 100755 --- a/tests/vm/openbsd +++ b/tests/vm/openbsd @@ -35,7 +35,6 @@ class OpenBSDVM(basevm.BaseVM): "bash", "gmake", "gsed", - "bison", # libs: usb "libusb1", diff --git a/tests/vm/ubuntu.i386 b/tests/vm/ubuntu.i386 index 1570775335..24527cc78c 100755 --- a/tests/vm/ubuntu.i386 +++ b/tests/vm/ubuntu.i386 @@ -52,7 +52,7 @@ class UbuntuX86VM(basevm.BaseVM): self.ssh_root_check("sed -ie s/^#\ deb-src/deb-src/g /etc/apt/sources.list") self.ssh_root_check("apt-get update") self.ssh_root_check("apt-get build-dep -y qemu") - self.ssh_root_check("apt-get install -y libfdt-dev flex bison language-pack-en") + self.ssh_root_check("apt-get install -y libfdt-dev language-pack-en") self.ssh_root("poweroff") self.wait() os.rename(img_tmp, img) |