diff options
48 files changed, 962 insertions, 432 deletions
diff --git a/.gitlab-ci.d/cirrus/freebsd-12.vars b/.gitlab-ci.d/cirrus/freebsd-12.vars index 07f313aa3a..b4842271b2 100644 --- a/.gitlab-ci.d/cirrus/freebsd-12.vars +++ b/.gitlab-ci.d/cirrus/freebsd-12.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage ctags curl cyrus-sasl dbus diffutils dtc fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 libepoxy libffi libgcrypt libjpeg-turbo libnfs libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv p5-Test-Harness perl5 pixman pkgconf png py38-numpy py38-pillow py38-pip py38-sphinx py38-sphinx_rtd_theme py38-virtualenv py38-yaml python3 rpm2cpio sdl2 sdl2_image snappy spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage ctags curl cyrus-sasl dbus diffutils dtc fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 libepoxy libffi libgcrypt libjpeg-turbo libnfs libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py38-numpy py38-pillow py38-pip py38-sphinx py38-sphinx_rtd_theme py38-virtualenv py38-yaml python3 rpm2cpio sdl2 sdl2_image snappy spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/freebsd-13.vars b/.gitlab-ci.d/cirrus/freebsd-13.vars index 8a648dda1e..546a82dd75 100644 --- a/.gitlab-ci.d/cirrus/freebsd-13.vars +++ b/.gitlab-ci.d/cirrus/freebsd-13.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage ctags curl cyrus-sasl dbus diffutils dtc fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 libepoxy libffi libgcrypt libjpeg-turbo libnfs libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv p5-Test-Harness perl5 pixman pkgconf png py38-numpy py38-pillow py38-pip py38-sphinx py38-sphinx_rtd_theme py38-virtualenv py38-yaml python3 rpm2cpio sdl2 sdl2_image snappy spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage ctags curl cyrus-sasl dbus diffutils dtc fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 libepoxy libffi libgcrypt libjpeg-turbo libnfs libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py38-numpy py38-pillow py38-pip py38-sphinx py38-sphinx_rtd_theme py38-virtualenv py38-yaml python3 rpm2cpio sdl2 sdl2_image snappy spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/macos-11.vars b/.gitlab-ci.d/cirrus/macos-11.vars index 08183f8793..cfe9181fd4 100644 --- a/.gitlab-ci.d/cirrus/macos-11.vars +++ b/.gitlab-ci.d/cirrus/macos-11.vars @@ -5,12 +5,12 @@ # https://gitlab.com/libvirt/libvirt-ci CCACHE='/usr/local/bin/ccache' -CPAN_PKGS='Test::Harness' +CPAN_PKGS='' CROSS_PKGS='' MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='brew' PIP3='/usr/local/bin/pip3' -PKGS='bash bc bzip2 capstone ccache cpanminus ctags curl dbus diffutils dtc gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson ncurses nettle ninja perl pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy sparse spice-protocol tesseract texinfo usbredir vde vte3 zlib zstd' +PKGS='bash bc bzip2 capstone ccache ctags curl dbus diffutils dtc gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson ncurses nettle ninja perl pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy sparse spice-protocol tesseract texinfo usbredir vde vte3 zlib zstd' PYPI_PKGS='PyYAML numpy pillow sphinx sphinx-rtd-theme virtualenv' PYTHON='/usr/local/bin/python3' diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index b91499410c..ed0ecfaa89 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refcount.c @@ -2438,188 +2438,329 @@ static int64_t alloc_clusters_imrt(BlockDriverState *bs, } /* - * Creates a new refcount structure based solely on the in-memory information - * given through *refcount_table. All necessary allocations will be reflected - * in that array. + * Helper function for rebuild_refcount_structure(). * - * On success, the old refcount structure is leaked (it will be covered by the - * new refcount structure). + * Scan the range of clusters [first_cluster, end_cluster) for allocated + * clusters and write all corresponding refblocks to disk. The refblock + * and allocation data is taken from the in-memory refcount table + * *refcount_table[] (of size *nb_clusters), which is basically one big + * (unlimited size) refblock for the whole image. + * + * For these refblocks, clusters are allocated using said in-memory + * refcount table. Care is taken that these allocations are reflected + * in the refblocks written to disk. + * + * The refblocks' offsets are written into a reftable, which is + * *on_disk_reftable_ptr[] (of size *on_disk_reftable_entries_ptr). If + * that reftable is of insufficient size, it will be resized to fit. + * This reftable is not written to disk. + * + * (If *on_disk_reftable_ptr is not NULL, the entries within are assumed + * to point to existing valid refblocks that do not need to be allocated + * again.) + * + * Return whether the on-disk reftable array was resized (true/false), + * or -errno on error. */ -static int rebuild_refcount_structure(BlockDriverState *bs, - BdrvCheckResult *res, - void **refcount_table, - int64_t *nb_clusters) +static int rebuild_refcounts_write_refblocks( + BlockDriverState *bs, void **refcount_table, int64_t *nb_clusters, + int64_t first_cluster, int64_t end_cluster, + uint64_t **on_disk_reftable_ptr, uint32_t *on_disk_reftable_entries_ptr, + Error **errp + ) { BDRVQcow2State *s = bs->opaque; - int64_t first_free_cluster = 0, reftable_offset = -1, cluster = 0; + int64_t cluster; int64_t refblock_offset, refblock_start, refblock_index; - uint32_t reftable_size = 0; - uint64_t *on_disk_reftable = NULL; + int64_t first_free_cluster = 0; + uint64_t *on_disk_reftable = *on_disk_reftable_ptr; + uint32_t on_disk_reftable_entries = *on_disk_reftable_entries_ptr; void *on_disk_refblock; - int ret = 0; - struct { - uint64_t reftable_offset; - uint32_t reftable_clusters; - } QEMU_PACKED reftable_offset_and_clusters; - - qcow2_cache_empty(bs, s->refcount_block_cache); + bool reftable_grown = false; + int ret; -write_refblocks: - for (; cluster < *nb_clusters; cluster++) { + for (cluster = first_cluster; cluster < end_cluster; cluster++) { + /* Check all clusters to find refblocks that contain non-zero entries */ if (!s->get_refcount(*refcount_table, cluster)) { continue; } + /* + * This cluster is allocated, so we need to create a refblock + * for it. The data we will write to disk is just the + * respective slice from *refcount_table, so it will contain + * accurate refcounts for all clusters belonging to this + * refblock. After we have written it, we will therefore skip + * all remaining clusters in this refblock. + */ + refblock_index = cluster >> s->refcount_block_bits; refblock_start = refblock_index << s->refcount_block_bits; - /* Don't allocate a cluster in a refblock already written to disk */ - if (first_free_cluster < refblock_start) { - first_free_cluster = refblock_start; - } - refblock_offset = alloc_clusters_imrt(bs, 1, refcount_table, - nb_clusters, &first_free_cluster); - if (refblock_offset < 0) { - fprintf(stderr, "ERROR allocating refblock: %s\n", - strerror(-refblock_offset)); - res->check_errors++; - ret = refblock_offset; - goto fail; - } + if (on_disk_reftable_entries > refblock_index && + on_disk_reftable[refblock_index]) + { + /* + * We can get here after a `goto write_refblocks`: We have a + * reftable from a previous run, and the refblock is already + * allocated. No need to allocate it again. + */ + refblock_offset = on_disk_reftable[refblock_index]; + } else { + int64_t refblock_cluster_index; - if (reftable_size <= refblock_index) { - uint32_t old_reftable_size = reftable_size; - uint64_t *new_on_disk_reftable; + /* Don't allocate a cluster in a refblock already written to disk */ + if (first_free_cluster < refblock_start) { + first_free_cluster = refblock_start; + } + refblock_offset = alloc_clusters_imrt(bs, 1, refcount_table, + nb_clusters, + &first_free_cluster); + if (refblock_offset < 0) { + error_setg_errno(errp, -refblock_offset, + "ERROR allocating refblock"); + return refblock_offset; + } - reftable_size = ROUND_UP((refblock_index + 1) * REFTABLE_ENTRY_SIZE, - s->cluster_size) / REFTABLE_ENTRY_SIZE; - new_on_disk_reftable = g_try_realloc(on_disk_reftable, - reftable_size * - REFTABLE_ENTRY_SIZE); - if (!new_on_disk_reftable) { - res->check_errors++; - ret = -ENOMEM; - goto fail; + refblock_cluster_index = refblock_offset / s->cluster_size; + if (refblock_cluster_index >= end_cluster) { + /* + * We must write the refblock that holds this refblock's + * refcount + */ + end_cluster = refblock_cluster_index + 1; } - on_disk_reftable = new_on_disk_reftable; - memset(on_disk_reftable + old_reftable_size, 0, - (reftable_size - old_reftable_size) * REFTABLE_ENTRY_SIZE); + if (on_disk_reftable_entries <= refblock_index) { + on_disk_reftable_entries = + ROUND_UP((refblock_index + 1) * REFTABLE_ENTRY_SIZE, + s->cluster_size) / REFTABLE_ENTRY_SIZE; + on_disk_reftable = + g_try_realloc(on_disk_reftable, + on_disk_reftable_entries * + REFTABLE_ENTRY_SIZE); + if (!on_disk_reftable) { + error_setg(errp, "ERROR allocating reftable memory"); + return -ENOMEM; + } - /* The offset we have for the reftable is now no longer valid; - * this will leak that range, but we can easily fix that by running - * a leak-fixing check after this rebuild operation */ - reftable_offset = -1; - } else { - assert(on_disk_reftable); - } - on_disk_reftable[refblock_index] = refblock_offset; + memset(on_disk_reftable + *on_disk_reftable_entries_ptr, 0, + (on_disk_reftable_entries - + *on_disk_reftable_entries_ptr) * + REFTABLE_ENTRY_SIZE); - /* If this is apparently the last refblock (for now), try to squeeze the - * reftable in */ - if (refblock_index == (*nb_clusters - 1) >> s->refcount_block_bits && - reftable_offset < 0) - { - uint64_t reftable_clusters = size_to_clusters(s, reftable_size * - REFTABLE_ENTRY_SIZE); - reftable_offset = alloc_clusters_imrt(bs, reftable_clusters, - refcount_table, nb_clusters, - &first_free_cluster); - if (reftable_offset < 0) { - fprintf(stderr, "ERROR allocating reftable: %s\n", - strerror(-reftable_offset)); - res->check_errors++; - ret = reftable_offset; - goto fail; + *on_disk_reftable_ptr = on_disk_reftable; + *on_disk_reftable_entries_ptr = on_disk_reftable_entries; + + reftable_grown = true; + } else { + assert(on_disk_reftable); } + on_disk_reftable[refblock_index] = refblock_offset; } + /* Refblock is allocated, write it to disk */ + ret = qcow2_pre_write_overlap_check(bs, 0, refblock_offset, s->cluster_size, false); if (ret < 0) { - fprintf(stderr, "ERROR writing refblock: %s\n", strerror(-ret)); - goto fail; + error_setg_errno(errp, -ret, "ERROR writing refblock"); + return ret; } - /* The size of *refcount_table is always cluster-aligned, therefore the - * write operation will not overflow */ + /* + * The refblock is simply a slice of *refcount_table. + * Note that the size of *refcount_table is always aligned to + * whole clusters, so the write operation will not result in + * out-of-bounds accesses. + */ on_disk_refblock = (void *)((char *) *refcount_table + refblock_index * s->cluster_size); ret = bdrv_pwrite(bs->file, refblock_offset, on_disk_refblock, s->cluster_size); if (ret < 0) { - fprintf(stderr, "ERROR writing refblock: %s\n", strerror(-ret)); - goto fail; + error_setg_errno(errp, -ret, "ERROR writing refblock"); + return ret; } - /* Go to the end of this refblock */ + /* This refblock is done, skip to its end */ cluster = refblock_start + s->refcount_block_size - 1; } - if (reftable_offset < 0) { - uint64_t post_refblock_start, reftable_clusters; + return reftable_grown; +} + +/* + * Creates a new refcount structure based solely on the in-memory information + * given through *refcount_table (this in-memory information is basically just + * the concatenation of all refblocks). All necessary allocations will be + * reflected in that array. + * + * On success, the old refcount structure is leaked (it will be covered by the + * new refcount structure). + */ +static int rebuild_refcount_structure(BlockDriverState *bs, + BdrvCheckResult *res, + void **refcount_table, + int64_t *nb_clusters, + Error **errp) +{ + BDRVQcow2State *s = bs->opaque; + int64_t reftable_offset = -1; + int64_t reftable_length = 0; + int64_t reftable_clusters; + int64_t refblock_index; + uint32_t on_disk_reftable_entries = 0; + uint64_t *on_disk_reftable = NULL; + int ret = 0; + int reftable_size_changed = 0; + struct { + uint64_t reftable_offset; + uint32_t reftable_clusters; + } QEMU_PACKED reftable_offset_and_clusters; + + qcow2_cache_empty(bs, s->refcount_block_cache); + + /* + * For each refblock containing entries, we try to allocate a + * cluster (in the in-memory refcount table) and write its offset + * into on_disk_reftable[]. We then write the whole refblock to + * disk (as a slice of the in-memory refcount table). + * This is done by rebuild_refcounts_write_refblocks(). + * + * Once we have scanned all clusters, we try to find space for the + * reftable. This will dirty the in-memory refcount table (i.e. + * make it differ from the refblocks we have already written), so we + * need to run rebuild_refcounts_write_refblocks() again for the + * range of clusters where the reftable has been allocated. + * + * This second run might make the reftable grow again, in which case + * we will need to allocate another space for it, which is why we + * repeat all this until the reftable stops growing. + * + * (This loop will terminate, because with every cluster the + * reftable grows, it can accomodate a multitude of more refcounts, + * so that at some point this must be able to cover the reftable + * and all refblocks describing it.) + * + * We then convert the reftable to big-endian and write it to disk. + * + * Note that we never free any reftable allocations. Doing so would + * needlessly complicate the algorithm: The eventual second check + * run we do will clean up all leaks we have caused. + */ + + reftable_size_changed = + rebuild_refcounts_write_refblocks(bs, refcount_table, nb_clusters, + 0, *nb_clusters, + &on_disk_reftable, + &on_disk_reftable_entries, errp); + if (reftable_size_changed < 0) { + res->check_errors++; + ret = reftable_size_changed; + goto fail; + } + + /* + * There was no reftable before, so rebuild_refcounts_write_refblocks() + * must have increased its size (from 0 to something). + */ + assert(reftable_size_changed); + + do { + int64_t reftable_start_cluster, reftable_end_cluster; + int64_t first_free_cluster = 0; + + reftable_length = on_disk_reftable_entries * REFTABLE_ENTRY_SIZE; + reftable_clusters = size_to_clusters(s, reftable_length); - post_refblock_start = ROUND_UP(*nb_clusters, s->refcount_block_size); - reftable_clusters = - size_to_clusters(s, reftable_size * REFTABLE_ENTRY_SIZE); - /* Not pretty but simple */ - if (first_free_cluster < post_refblock_start) { - first_free_cluster = post_refblock_start; - } reftable_offset = alloc_clusters_imrt(bs, reftable_clusters, refcount_table, nb_clusters, &first_free_cluster); if (reftable_offset < 0) { - fprintf(stderr, "ERROR allocating reftable: %s\n", - strerror(-reftable_offset)); + error_setg_errno(errp, -reftable_offset, + "ERROR allocating reftable"); res->check_errors++; ret = reftable_offset; goto fail; } - goto write_refblocks; - } + /* + * We need to update the affected refblocks, so re-run the + * write_refblocks loop for the reftable's range of clusters. + */ + assert(offset_into_cluster(s, reftable_offset) == 0); + reftable_start_cluster = reftable_offset / s->cluster_size; + reftable_end_cluster = reftable_start_cluster + reftable_clusters; + reftable_size_changed = + rebuild_refcounts_write_refblocks(bs, refcount_table, nb_clusters, + reftable_start_cluster, + reftable_end_cluster, + &on_disk_reftable, + &on_disk_reftable_entries, errp); + if (reftable_size_changed < 0) { + res->check_errors++; + ret = reftable_size_changed; + goto fail; + } + + /* + * If the reftable size has changed, we will need to find a new + * allocation, repeating the loop. + */ + } while (reftable_size_changed); - for (refblock_index = 0; refblock_index < reftable_size; refblock_index++) { + /* The above loop must have run at least once */ + assert(reftable_offset >= 0); + + /* + * All allocations are done, all refblocks are written, convert the + * reftable to big-endian and write it to disk. + */ + + for (refblock_index = 0; refblock_index < on_disk_reftable_entries; + refblock_index++) + { cpu_to_be64s(&on_disk_reftable[refblock_index]); } - ret = qcow2_pre_write_overlap_check(bs, 0, reftable_offset, - reftable_size * REFTABLE_ENTRY_SIZE, + ret = qcow2_pre_write_overlap_check(bs, 0, reftable_offset, reftable_length, false); if (ret < 0) { - fprintf(stderr, "ERROR writing reftable: %s\n", strerror(-ret)); + error_setg_errno(errp, -ret, "ERROR writing reftable"); goto fail; } - assert(reftable_size < INT_MAX / REFTABLE_ENTRY_SIZE); + assert(reftable_length < INT_MAX); ret = bdrv_pwrite(bs->file, reftable_offset, on_disk_reftable, - reftable_size * REFTABLE_ENTRY_SIZE); + reftable_length); if (ret < 0) { - fprintf(stderr, "ERROR writing reftable: %s\n", strerror(-ret)); + error_setg_errno(errp, -ret, "ERROR writing reftable"); goto fail; } /* Enter new reftable into the image header */ reftable_offset_and_clusters.reftable_offset = cpu_to_be64(reftable_offset); reftable_offset_and_clusters.reftable_clusters = - cpu_to_be32(size_to_clusters(s, reftable_size * REFTABLE_ENTRY_SIZE)); + cpu_to_be32(reftable_clusters); ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, refcount_table_offset), &reftable_offset_and_clusters, sizeof(reftable_offset_and_clusters)); if (ret < 0) { - fprintf(stderr, "ERROR setting reftable: %s\n", strerror(-ret)); + error_setg_errno(errp, -ret, "ERROR setting reftable"); goto fail; } - for (refblock_index = 0; refblock_index < reftable_size; refblock_index++) { + for (refblock_index = 0; refblock_index < on_disk_reftable_entries; + refblock_index++) + { be64_to_cpus(&on_disk_reftable[refblock_index]); } s->refcount_table = on_disk_reftable; s->refcount_table_offset = reftable_offset; - s->refcount_table_size = reftable_size; + s->refcount_table_size = on_disk_reftable_entries; update_max_refcount_table_index(s); return 0; @@ -2676,11 +2817,13 @@ int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res, if (rebuild && (fix & BDRV_FIX_ERRORS)) { BdrvCheckResult old_res = *res; int fresh_leaks = 0; + Error *local_err = NULL; fprintf(stderr, "Rebuilding refcount structure\n"); ret = rebuild_refcount_structure(bs, res, &refcount_table, - &nb_clusters); + &nb_clusters, &local_err); if (ret < 0) { + error_report_err(local_err); goto fail; } diff --git a/docs/ccid.txt b/docs/ccid.txt deleted file mode 100644 index 2b85b1bd42..0000000000 --- a/docs/ccid.txt +++ /dev/null @@ -1,182 +0,0 @@ -QEMU CCID Device Documentation. - -Contents -1. USB CCID device -2. Building -3. Using ccid-card-emulated with hardware -4. Using ccid-card-emulated with certificates -5. Using ccid-card-passthru with client side hardware -6. Using ccid-card-passthru with client side certificates -7. Passthrough protocol scenario -8. libcacard - -1. USB CCID device - -The USB CCID device is a USB device implementing the CCID specification, which -lets one connect smart card readers that implement the same spec. For more -information see the specification: - - Universal Serial Bus - Device Class: Smart Card - CCID - Specification for - Integrated Circuit(s) Cards Interface Devices - Revision 1.1 - April 22rd, 2005 - -Smartcards are used for authentication, single sign on, decryption in -public/private schemes and digital signatures. A smartcard reader on the client -cannot be used on a guest with simple usb passthrough since it will then not be -available on the client, possibly locking the computer when it is "removed". On -the other hand this device can let you use the smartcard on both the client and -the guest machine. It is also possible to have a completely virtual smart card -reader and smart card (i.e. not backed by a physical device) using this device. - -2. Building - -The cryptographic functions and access to the physical card is done via the -libcacard library, whose development package must be installed prior to -building QEMU: - -In redhat/fedora: - yum install libcacard-devel -In ubuntu: - apt-get install libcacard-dev - -Configuring and building: - ./configure --enable-smartcard && make - - -3. Using ccid-card-emulated with hardware - -Assuming you have a working smartcard on the host with the current -user, using libcacard, QEMU acts as another client using ccid-card-emulated: - - qemu -usb -device usb-ccid -device ccid-card-emulated - - -4. Using ccid-card-emulated with certificates stored in files - -You must create the CA and card certificates. This is a one time process. -We use NSS certificates: - - mkdir fake-smartcard - cd fake-smartcard - certutil -N -d sql:$PWD - certutil -S -d sql:$PWD -s "CN=Fake Smart Card CA" -x -t TC,TC,TC -n fake-smartcard-ca - certutil -S -d sql:$PWD -t ,, -s "CN=John Doe" -n id-cert -c fake-smartcard-ca - certutil -S -d sql:$PWD -t ,, -s "CN=John Doe (signing)" --nsCertType smime -n signing-cert -c fake-smartcard-ca - certutil -S -d sql:$PWD -t ,, -s "CN=John Doe (encryption)" --nsCertType sslClient -n encryption-cert -c fake-smartcard-ca - -Note: you must have exactly three certificates. - -You can use the emulated card type with the certificates backend: - - qemu -usb -device usb-ccid -device ccid-card-emulated,backend=certificates,db=sql:$PWD,cert1=id-cert,cert2=signing-cert,cert3=encryption-cert - -To use the certificates in the guest, export the CA certificate: - - certutil -L -r -d sql:$PWD -o fake-smartcard-ca.cer -n fake-smartcard-ca - -and import it in the guest: - - certutil -A -d /etc/pki/nssdb -i fake-smartcard-ca.cer -t TC,TC,TC -n fake-smartcard-ca - -In a Linux guest you can then use the CoolKey PKCS #11 module to access -the card: - - certutil -d /etc/pki/nssdb -L -h all - -It will prompt you for the PIN (which is the password you assigned to the -certificate database early on), and then show you all three certificates -together with the manually imported CA cert: - - Certificate Nickname Trust Attributes - fake-smartcard-ca CT,C,C - John Doe:CAC ID Certificate u,u,u - John Doe:CAC Email Signature Certificate u,u,u - John Doe:CAC Email Encryption Certificate u,u,u - -If this does not happen, CoolKey is not installed or not registered with -NSS. Registration can be done from Firefox or the command line: - - modutil -dbdir /etc/pki/nssdb -add "CAC Module" -libfile /usr/lib64/pkcs11/libcoolkeypk11.so - modutil -dbdir /etc/pki/nssdb -list - - -5. Using ccid-card-passthru with client side hardware - -on the host specify the ccid-card-passthru device with a suitable chardev: - - qemu -chardev socket,server=on,host=0.0.0.0,port=2001,id=ccid,wait=off \ - -usb -device usb-ccid -device ccid-card-passthru,chardev=ccid - -on the client run vscclient, built when you built QEMU: - - vscclient <qemu-host> 2001 - - -6. Using ccid-card-passthru with client side certificates - -This case is not particularly useful, but you can use it to debug -your setup if #4 works but #5 does not. - -Follow instructions as per #4, except run QEMU and vscclient as follows: -Run qemu as per #5, and run vscclient from the "fake-smartcard" -directory as follows: - - qemu -chardev socket,server=on,host=0.0.0.0,port=2001,id=ccid,wait=off \ - -usb -device usb-ccid -device ccid-card-passthru,chardev=ccid - vscclient -e "db=\"sql:$PWD\" use_hw=no soft=(,Test,CAC,,id-cert,signing-cert,encryption-cert)" <qemu-host> 2001 - - -7. Passthrough protocol scenario - -This is a typical interchange of messages when using the passthru card device. -usb-ccid is a usb device. It defaults to an unattached usb device on startup. -usb-ccid expects a chardev and expects the protocol defined in -cac_card/vscard_common.h to be passed over that. -The usb-ccid device can be in one of three modes: - * detached - * attached with no card - * attached with card - -A typical interchange is: (the arrow shows who started each exchange, it can be client -originated or guest originated) - -client event | vscclient | passthru | usb-ccid | guest event ----------------------------------------------------------------------------------------------- - | VSC_Init | | | - | VSC_ReaderAdd | | attach | - | | | | sees new usb device. -card inserted -> | | | | - | VSC_ATR | insert | insert | see new card - | | | | - | VSC_APDU | VSC_APDU | | <- guest sends APDU -client<->physical | | | | -card APDU exchange| | | | -client response ->| VSC_APDU | VSC_APDU | | receive APDU response - ... - [APDU<->APDU repeats several times] - ... -card removed -> | | | | - | VSC_CardRemove | remove | remove | card removed - ... - [(card insert, apdu's, card remove) repeat] - ... -kill/quit | | | | - vscclient | | | | - | VSC_ReaderRemove | | detach | - | | | | usb device removed. - - -8. libcacard - -Both ccid-card-emulated and vscclient use libcacard as the card emulator. -libcacard implements a completely virtual CAC (DoD standard for smart -cards) compliant card and uses NSS to retrieve certificates and do -any encryption. The backend can then be a real reader and card, or -certificates stored in files. - -For documentation of the library see docs/libcacard.txt. - diff --git a/docs/system/device-emulation.rst b/docs/system/device-emulation.rst index 0b3a3d73ad..ae8dd233e8 100644 --- a/docs/system/device-emulation.rst +++ b/docs/system/device-emulation.rst @@ -83,6 +83,7 @@ Emulated Devices :maxdepth: 1 devices/can.rst + devices/ccid.rst devices/ivshmem.rst devices/net.rst devices/nvme.rst diff --git a/docs/system/devices/ccid.rst b/docs/system/devices/ccid.rst new file mode 100644 index 0000000000..3b8c2ab46a --- /dev/null +++ b/docs/system/devices/ccid.rst @@ -0,0 +1,171 @@ +Chip Card Interface Device (CCID) +================================= + +USB CCID device +--------------- +The USB CCID device is a USB device implementing the CCID specification, which +lets one connect smart card readers that implement the same spec. For more +information see the specification:: + + Universal Serial Bus + Device Class: Smart Card + CCID + Specification for + Integrated Circuit(s) Cards Interface Devices + Revision 1.1 + April 22rd, 2005 + +Smartcards are used for authentication, single sign on, decryption in +public/private schemes and digital signatures. A smartcard reader on the client +cannot be used on a guest with simple usb passthrough since it will then not be +available on the client, possibly locking the computer when it is "removed". On +the other hand this device can let you use the smartcard on both the client and +the guest machine. It is also possible to have a completely virtual smart card +reader and smart card (i.e. not backed by a physical device) using this device. + +Building +-------- +The cryptographic functions and access to the physical card is done via the +libcacard library, whose development package must be installed prior to +building QEMU: + +In redhat/fedora:: + + yum install libcacard-devel + +In ubuntu:: + + apt-get install libcacard-dev + +Configuring and building:: + + ./configure --enable-smartcard && make + +Using ccid-card-emulated with hardware +-------------------------------------- +Assuming you have a working smartcard on the host with the current +user, using libcacard, QEMU acts as another client using ccid-card-emulated:: + + qemu -usb -device usb-ccid -device ccid-card-emulated + +Using ccid-card-emulated with certificates stored in files +---------------------------------------------------------- +You must create the CA and card certificates. This is a one time process. +We use NSS certificates:: + + mkdir fake-smartcard + cd fake-smartcard + certutil -N -d sql:$PWD + certutil -S -d sql:$PWD -s "CN=Fake Smart Card CA" -x -t TC,TC,TC -n fake-smartcard-ca + certutil -S -d sql:$PWD -t ,, -s "CN=John Doe" -n id-cert -c fake-smartcard-ca + certutil -S -d sql:$PWD -t ,, -s "CN=John Doe (signing)" --nsCertType smime -n signing-cert -c fake-smartcard-ca + certutil -S -d sql:$PWD -t ,, -s "CN=John Doe (encryption)" --nsCertType sslClient -n encryption-cert -c fake-smartcard-ca + +Note: you must have exactly three certificates. + +You can use the emulated card type with the certificates backend:: + + qemu -usb -device usb-ccid -device ccid-card-emulated,backend=certificates,db=sql:$PWD,cert1=id-cert,cert2=signing-cert,cert3=encryption-cert + +To use the certificates in the guest, export the CA certificate:: + + certutil -L -r -d sql:$PWD -o fake-smartcard-ca.cer -n fake-smartcard-ca + +and import it in the guest:: + + certutil -A -d /etc/pki/nssdb -i fake-smartcard-ca.cer -t TC,TC,TC -n fake-smartcard-ca + +In a Linux guest you can then use the CoolKey PKCS #11 module to access +the card:: + + certutil -d /etc/pki/nssdb -L -h all + +It will prompt you for the PIN (which is the password you assigned to the +certificate database early on), and then show you all three certificates +together with the manually imported CA cert:: + + Certificate Nickname Trust Attributes + fake-smartcard-ca CT,C,C + John Doe:CAC ID Certificate u,u,u + John Doe:CAC Email Signature Certificate u,u,u + John Doe:CAC Email Encryption Certificate u,u,u + +If this does not happen, CoolKey is not installed or not registered with +NSS. Registration can be done from Firefox or the command line:: + + modutil -dbdir /etc/pki/nssdb -add "CAC Module" -libfile /usr/lib64/pkcs11/libcoolkeypk11.so + modutil -dbdir /etc/pki/nssdb -list + +Using ccid-card-passthru with client side hardware +-------------------------------------------------- +On the host specify the ccid-card-passthru device with a suitable chardev:: + + qemu -chardev socket,server=on,host=0.0.0.0,port=2001,id=ccid,wait=off \ + -usb -device usb-ccid -device ccid-card-passthru,chardev=ccid + +On the client run vscclient, built when you built QEMU:: + + vscclient <qemu-host> 2001 + +Using ccid-card-passthru with client side certificates +------------------------------------------------------ +This case is not particularly useful, but you can use it to debug +your setup. + +Follow instructions above, except run QEMU and vscclient as follows. + +Run qemu as per above, and run vscclient from the "fake-smartcard" +directory as follows:: + + qemu -chardev socket,server=on,host=0.0.0.0,port=2001,id=ccid,wait=off \ + -usb -device usb-ccid -device ccid-card-passthru,chardev=ccid + vscclient -e "db=\"sql:$PWD\" use_hw=no soft=(,Test,CAC,,id-cert,signing-cert,encryption-cert)" <qemu-host> 2001 + + +Passthrough protocol scenario +----------------------------- +This is a typical interchange of messages when using the passthru card device. +usb-ccid is a usb device. It defaults to an unattached usb device on startup. +usb-ccid expects a chardev and expects the protocol defined in +cac_card/vscard_common.h to be passed over that. +The usb-ccid device can be in one of three modes: + +* detached +* attached with no card +* attached with card + +A typical interchange is (the arrow shows who started each exchange, it can be client +originated or guest originated):: + + client event | vscclient | passthru | usb-ccid | guest event + ------------------------------------------------------------------------------------------------ + | VSC_Init | | | + | VSC_ReaderAdd | | attach | + | | | | sees new usb device. + card inserted -> | | | | + | VSC_ATR | insert | insert | see new card + | | | | + | VSC_APDU | VSC_APDU | | <- guest sends APDU + client <-> physical | | | | + card APDU exchange | | | | + client response -> | VSC_APDU | VSC_APDU | | receive APDU response + ... + [APDU<->APDU repeats several times] + ... + card removed -> | | | | + | VSC_CardRemove | remove | remove | card removed + ... + [(card insert, apdu's, card remove) repeat] + ... + kill/quit | | | | + vscclient | | | | + | VSC_ReaderRemove | | detach | + | | | | usb device removed. + +libcacard +--------- +Both ccid-card-emulated and vscclient use libcacard as the card emulator. +libcacard implements a completely virtual CAC (DoD standard for smart +cards) compliant card and uses NSS to retrieve certificates and do +any encryption. The backend can then be a real reader and card, or +certificates stored in files. diff --git a/hw/arm/virt.c b/hw/arm/virt.c index d2e5ecd234..9284f7d28e 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -3022,10 +3022,17 @@ static void machvirt_machine_init(void) } type_init(machvirt_machine_init); +static void virt_machine_7_1_options(MachineClass *mc) +{ +} +DEFINE_VIRT_MACHINE_AS_LATEST(7, 1) + static void virt_machine_7_0_options(MachineClass *mc) { + virt_machine_7_1_options(mc); + compat_props_add(mc->compat_props, hw_compat_7_0, hw_compat_7_0_len); } -DEFINE_VIRT_MACHINE_AS_LATEST(7, 0) +DEFINE_VIRT_MACHINE(7, 0) static void virt_machine_6_2_options(MachineClass *mc) { diff --git a/hw/core/machine.c b/hw/core/machine.c index 1e23fdc14b..cb9bbc844d 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -37,6 +37,9 @@ #include "hw/virtio/virtio.h" #include "hw/virtio/virtio-pci.h" +GlobalProperty hw_compat_7_0[] = {}; +const size_t hw_compat_7_0_len = G_N_ELEMENTS(hw_compat_7_0); + GlobalProperty hw_compat_6_2[] = { { "PIIX4_PM", "x-not-migrate-acpi-index", "on"}, }; diff --git a/hw/i386/pc.c b/hw/i386/pc.c index fd55fc725c..23bba9d82c 100644 --- a/hw/i386/pc.c +++ b/hw/i386/pc.c @@ -95,6 +95,9 @@ #include "trace.h" #include CONFIG_DEVICES +GlobalProperty pc_compat_7_0[] = {}; +const size_t pc_compat_7_0_len = G_N_ELEMENTS(pc_compat_7_0); + GlobalProperty pc_compat_6_2[] = { { "virtio-mem", "unplugged-inaccessible", "off" }, }; diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c index b72c03d0a6..4c185c72d0 100644 --- a/hw/i386/pc_piix.c +++ b/hw/i386/pc_piix.c @@ -415,7 +415,7 @@ static void pc_i440fx_machine_options(MachineClass *m) machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE); } -static void pc_i440fx_7_0_machine_options(MachineClass *m) +static void pc_i440fx_7_1_machine_options(MachineClass *m) { PCMachineClass *pcmc = PC_MACHINE_CLASS(m); pc_i440fx_machine_options(m); @@ -424,6 +424,18 @@ static void pc_i440fx_7_0_machine_options(MachineClass *m) pcmc->default_cpu_version = 1; } +DEFINE_I440FX_MACHINE(v7_1, "pc-i440fx-7.1", NULL, + pc_i440fx_7_1_machine_options); + +static void pc_i440fx_7_0_machine_options(MachineClass *m) +{ + pc_i440fx_7_1_machine_options(m); + m->alias = NULL; + m->is_default = false; + compat_props_add(m->compat_props, hw_compat_7_0, hw_compat_7_0_len); + compat_props_add(m->compat_props, pc_compat_7_0, pc_compat_7_0_len); +} + DEFINE_I440FX_MACHINE(v7_0, "pc-i440fx-7.0", NULL, pc_i440fx_7_0_machine_options); diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c index 1780f79bc1..302288342a 100644 --- a/hw/i386/pc_q35.c +++ b/hw/i386/pc_q35.c @@ -360,7 +360,7 @@ static void pc_q35_machine_options(MachineClass *m) m->max_cpus = 288; } -static void pc_q35_7_0_machine_options(MachineClass *m) +static void pc_q35_7_1_machine_options(MachineClass *m) { PCMachineClass *pcmc = PC_MACHINE_CLASS(m); pc_q35_machine_options(m); @@ -368,6 +368,17 @@ static void pc_q35_7_0_machine_options(MachineClass *m) pcmc->default_cpu_version = 1; } +DEFINE_Q35_MACHINE(v7_1, "pc-q35-7.1", NULL, + pc_q35_7_1_machine_options); + +static void pc_q35_7_0_machine_options(MachineClass *m) +{ + pc_q35_7_1_machine_options(m); + m->alias = NULL; + compat_props_add(m->compat_props, hw_compat_7_0, hw_compat_7_0_len); + compat_props_add(m->compat_props, pc_compat_7_0, pc_compat_7_0_len); +} + DEFINE_Q35_MACHINE(v7_0, "pc-q35-7.0", NULL, pc_q35_7_0_machine_options); diff --git a/hw/m68k/virt.c b/hw/m68k/virt.c index 7de8b5f833..e215aa3d42 100644 --- a/hw/m68k/virt.c +++ b/hw/m68k/virt.c @@ -315,10 +315,17 @@ type_init(virt_machine_register_types) } \ type_init(machvirt_machine_##major##_##minor##_init); +static void virt_machine_7_1_options(MachineClass *mc) +{ +} +DEFINE_VIRT_MACHINE(7, 1, true) + static void virt_machine_7_0_options(MachineClass *mc) { + virt_machine_7_1_options(mc); + compat_props_add(mc->compat_props, hw_compat_7_0, hw_compat_7_0_len); } -DEFINE_VIRT_MACHINE(7, 0, true) +DEFINE_VIRT_MACHINE(7, 0, false) static void virt_machine_6_2_options(MachineClass *mc) { diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index cc11fcc8b2..22569305d2 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -4703,14 +4703,25 @@ static void spapr_machine_latest_class_options(MachineClass *mc) type_init(spapr_machine_register_##suffix) /* + * pseries-7.1 + */ +static void spapr_machine_7_1_class_options(MachineClass *mc) +{ + /* Defaults for the latest behaviour inherited from the base class */ +} + +DEFINE_SPAPR_MACHINE(7_1, "7.1", true); + +/* * pseries-7.0 */ static void spapr_machine_7_0_class_options(MachineClass *mc) { - /* Defaults for the latest behaviour inherited from the base class */ + spapr_machine_7_1_class_options(mc); + compat_props_add(mc->compat_props, hw_compat_7_0, hw_compat_7_0_len); } -DEFINE_SPAPR_MACHINE(7_0, "7.0", true); +DEFINE_SPAPR_MACHINE(7_0, "7.0", false); /* * pseries-6.2 diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index 2d32647d08..8fa488d13a 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -792,14 +792,26 @@ bool css_migration_enabled(void) } \ type_init(ccw_machine_register_##suffix) +static void ccw_machine_7_1_instance_options(MachineState *machine) +{ +} + +static void ccw_machine_7_1_class_options(MachineClass *mc) +{ +} +DEFINE_CCW_MACHINE(7_1, "7.1", true); + static void ccw_machine_7_0_instance_options(MachineState *machine) { + ccw_machine_7_1_instance_options(machine); } static void ccw_machine_7_0_class_options(MachineClass *mc) { + ccw_machine_7_1_class_options(mc); + compat_props_add(mc->compat_props, hw_compat_7_0, hw_compat_7_0_len); } -DEFINE_CCW_MACHINE(7_0, "7.0", true); +DEFINE_CCW_MACHINE(7_0, "7.0", false); static void ccw_machine_6_2_instance_options(MachineState *machine) { diff --git a/include/hw/boards.h b/include/hw/boards.h index c92ac8815c..d64b5481e8 100644 --- a/include/hw/boards.h +++ b/include/hw/boards.h @@ -380,6 +380,9 @@ struct MachineState { } \ type_init(machine_initfn##_register_types) +extern GlobalProperty hw_compat_7_0[]; +extern const size_t hw_compat_7_0_len; + extern GlobalProperty hw_compat_6_2[]; extern const size_t hw_compat_6_2_len; diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h index 1a27de9c8b..637367dc5f 100644 --- a/include/hw/i386/pc.h +++ b/include/hw/i386/pc.h @@ -202,6 +202,9 @@ void pc_madt_cpu_entry(AcpiDeviceIf *adev, int uid, /* sgx.c */ void pc_machine_init_sgx_epc(PCMachineState *pcms); +extern GlobalProperty pc_compat_7_0[]; +extern const size_t pc_compat_7_0_len; + extern GlobalProperty pc_compat_6_2[]; extern const size_t pc_compat_6_2_len; diff --git a/tests/avocado/avocado_qemu/__init__.py b/tests/avocado/avocado_qemu/__init__.py index ac85e36a4d..39f15c1d51 100644 --- a/tests/avocado/avocado_qemu/__init__.py +++ b/tests/avocado/avocado_qemu/__init__.py @@ -516,6 +516,8 @@ class LinuxTest(LinuxSSHMixIn, QemuSystemTest): distro = None username = 'root' password = 'password' + smp = '2' + memory = '1024' def _set_distro(self): distro_name = self.params.get( @@ -546,8 +548,8 @@ class LinuxTest(LinuxSSHMixIn, QemuSystemTest): def setUp(self, ssh_pubkey=None, network_device_type='virtio-net'): super().setUp() self._set_distro() - self.vm.add_args('-smp', '2') - self.vm.add_args('-m', '1024') + self.vm.add_args('-smp', self.smp) + self.vm.add_args('-m', self.memory) # The following network device allows for SSH connections self.vm.add_args('-netdev', 'user,id=vnet,hostfwd=:127.0.0.1:0-:22', '-device', '%s,netdev=vnet' % network_device_type) diff --git a/tests/check-block.sh b/tests/check-block.sh index f59496396c..5de2c1ba0b 100755 --- a/tests/check-block.sh +++ b/tests/check-block.sh @@ -18,36 +18,10 @@ skip() { exit 0 } -# Disable tests with any sanitizer except for specific ones -SANITIZE_FLAGS=$( grep "CFLAGS.*-fsanitize" config-host.mak 2>/dev/null ) -ALLOWED_SANITIZE_FLAGS="safe-stack cfi-icall" -#Remove all occurrencies of allowed Sanitize flags -for j in ${ALLOWED_SANITIZE_FLAGS}; do - TMP_FLAGS=${SANITIZE_FLAGS} - SANITIZE_FLAGS="" - for i in ${TMP_FLAGS}; do - if ! echo ${i} | grep -q "${j}" 2>/dev/null; then - SANITIZE_FLAGS="${SANITIZE_FLAGS} ${i}" - fi - done -done -if echo ${SANITIZE_FLAGS} | grep -q "\-fsanitize" 2>/dev/null; then - # Have a sanitize flag that is not allowed, stop - skip "Sanitizers are enabled ==> Not running the qemu-iotests." -fi - if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then skip "No qemu-system binary available ==> Not running the qemu-iotests." fi -if ! command -v bash >/dev/null 2>&1 ; then - skip "bash not available ==> Not running the qemu-iotests." -fi - -if LANG=C bash --version | grep -q 'GNU bash, version [123]' ; then - skip "bash version too old ==> Not running the qemu-iotests." -fi - cd tests/qemu-iotests # QEMU_CHECK_BLOCK_AUTO is used to disable some unstable sub-tests diff --git a/tests/docker/dockerfiles/alpine.docker b/tests/docker/dockerfiles/alpine.docker index 591af43d60..3f4c0f95cb 100644 --- a/tests/docker/dockerfiles/alpine.docker +++ b/tests/docker/dockerfiles/alpine.docker @@ -73,7 +73,6 @@ RUN apk update && \ openssh-client \ pcre-dev \ perl \ - perl-test-harness \ pixman-dev \ pkgconf \ pulseaudio-dev \ diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index 3ede55d09b..4b20925bbf 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -87,7 +87,6 @@ RUN dnf update -y && \ pam-devel \ pcre-static \ perl \ - perl-Test-Harness \ pixman-devel \ pkgconfig \ pulseaudio-libs-devel \ diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker index 589510a7be..b60426834c 100644 --- a/tests/docker/dockerfiles/debian-arm64-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -29,7 +29,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ hostname \ libpcre2-dev \ libspice-protocol-dev \ - libtest-harness-perl \ llvm \ locales \ make \ diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker index aa1bd6eb4c..ff79a2cc4f 100644 --- a/tests/docker/dockerfiles/debian-s390x-cross.docker +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -29,7 +29,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ hostname \ libpcre2-dev \ libspice-protocol-dev \ - libtest-harness-perl \ llvm \ locales \ make \ diff --git a/tests/docker/dockerfiles/debian-tricore-cross.docker b/tests/docker/dockerfiles/debian-tricore-cross.docker index 3f6b55562c..b573b9ded2 100644 --- a/tests/docker/dockerfiles/debian-tricore-cross.docker +++ b/tests/docker/dockerfiles/debian-tricore-cross.docker @@ -25,7 +25,6 @@ RUN apt update && \ git \ libglib2.0-dev \ libpixman-1-dev \ - libtest-harness-perl \ locales \ make \ ninja-build \ diff --git a/tests/docker/dockerfiles/fedora-i386-cross.docker b/tests/docker/dockerfiles/fedora-i386-cross.docker index 13328e6081..0a3ec346e6 100644 --- a/tests/docker/dockerfiles/fedora-i386-cross.docker +++ b/tests/docker/dockerfiles/fedora-i386-cross.docker @@ -20,7 +20,6 @@ ENV PACKAGES \ gnutls-devel.i686 \ nettle-devel.i686 \ pcre-devel.i686 \ - perl-Test-Harness \ pixman-devel.i686 \ sysprof-capture-devel.i686 \ zlib-devel.i686 diff --git a/tests/docker/dockerfiles/fedora-win32-cross.docker b/tests/docker/dockerfiles/fedora-win32-cross.docker index d80e66c651..84a8f5524d 100644 --- a/tests/docker/dockerfiles/fedora-win32-cross.docker +++ b/tests/docker/dockerfiles/fedora-win32-cross.docker @@ -31,7 +31,6 @@ ENV PACKAGES \ mingw32-SDL2 \ msitools \ perl \ - perl-Test-Harness \ python3 \ python3-PyYAML \ tar \ diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index 2b12b94ccf..d7ed8eb1cf 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -28,7 +28,6 @@ ENV PACKAGES \ mingw64-pkg-config \ msitools \ perl \ - perl-Test-Harness \ python3 \ python3-PyYAML \ tar \ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index 1d01cd9440..2e6a84abfd 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -94,7 +94,6 @@ exec "$@"' > /usr/bin/nosync && \ openssh-clients \ pam-devel \ pcre-static \ - perl-Test-Harness \ perl-base \ pixman-devel \ pkgconfig \ diff --git a/tests/docker/dockerfiles/opensuse-leap.docker b/tests/docker/dockerfiles/opensuse-leap.docker index e1ad9434a3..6f5993d602 100644 --- a/tests/docker/dockerfiles/opensuse-leap.docker +++ b/tests/docker/dockerfiles/opensuse-leap.docker @@ -83,7 +83,6 @@ RUN zypper update -y && \ openssh \ pam-devel \ pcre-devel-static \ - perl-Test-Harness \ perl-base \ pkgconfig \ python3-Pillow \ diff --git a/tests/docker/dockerfiles/ubuntu1804.docker b/tests/docker/dockerfiles/ubuntu1804.docker index 0a622b467c..b3f2156580 100644 --- a/tests/docker/dockerfiles/ubuntu1804.docker +++ b/tests/docker/dockerfiles/ubuntu1804.docker @@ -81,7 +81,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libssh-dev \ libsystemd-dev \ libtasn1-6-dev \ - libtest-harness-perl \ libubsan1 \ libudev-dev \ libusb-1.0-0-dev \ diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index b9d06cb040..a3b38884e3 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -83,7 +83,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libssh-dev \ libsystemd-dev \ libtasn1-6-dev \ - libtest-harness-perl \ libubsan1 \ libudev-dev \ libusb-1.0-0-dev \ diff --git a/tests/lcitool/projects/qemu.yml b/tests/lcitool/projects/qemu.yml index 958868a6ee..d068a7a8de 100644 --- a/tests/lcitool/projects/qemu.yml +++ b/tests/lcitool/projects/qemu.yml @@ -79,7 +79,6 @@ packages: - pam - pcre-static - perl - - perl-Test-Harness - pixman - pkg-config - pulseaudio diff --git a/tests/qemu-iotests/065 b/tests/qemu-iotests/065 index ba94e19349..b724c89c7c 100755 --- a/tests/qemu-iotests/065 +++ b/tests/qemu-iotests/065 @@ -24,7 +24,7 @@ import os import re import json import iotests -from iotests import qemu_img, qemu_img_info +from iotests import qemu_img, qemu_img_info, supports_qcow2_zstd_compression import unittest test_img = os.path.join(iotests.test_dir, 'test.img') @@ -95,11 +95,17 @@ class TestQCow2(TestQemuImgInfo): class TestQCow3NotLazy(TestQemuImgInfo): '''Testing a qcow2 version 3 image with lazy refcounts disabled''' - img_options = 'compat=1.1,lazy_refcounts=off,compression_type=zstd' + if supports_qcow2_zstd_compression(): + compression_type = 'zstd' + else: + compression_type = 'zlib' + + img_options = 'compat=1.1,lazy_refcounts=off' + img_options += f',compression_type={compression_type}' json_compare = { 'compat': '1.1', 'lazy-refcounts': False, 'refcount-bits': 16, 'corrupt': False, - 'compression-type': 'zstd', 'extended-l2': False } - human_compare = [ 'compat: 1.1', 'compression type: zstd', + 'compression-type': compression_type, 'extended-l2': False } + human_compare = [ 'compat: 1.1', f'compression type: {compression_type}', 'lazy refcounts: false', 'refcount bits: 16', 'corrupt: false', 'extended l2: false' ] @@ -126,11 +132,17 @@ class TestQCow3NotLazyQMP(TestQMP): class TestQCow3LazyQMP(TestQMP): '''Testing a qcow2 version 3 image with lazy refcounts enabled, opening with lazy refcounts disabled''' - img_options = 'compat=1.1,lazy_refcounts=on,compression_type=zstd' + if supports_qcow2_zstd_compression(): + compression_type = 'zstd' + else: + compression_type = 'zlib' + + img_options = 'compat=1.1,lazy_refcounts=on' + img_options += f',compression_type={compression_type}' qemu_options = 'lazy-refcounts=off' compare = { 'compat': '1.1', 'lazy-refcounts': True, 'refcount-bits': 16, 'corrupt': False, - 'compression-type': 'zstd', 'extended-l2': False } + 'compression-type': compression_type, 'extended-l2': False } TestImageInfoSpecific = None TestQemuImgInfo = None diff --git a/tests/qemu-iotests/108 b/tests/qemu-iotests/108 index 56339ab2c5..688d3ae8f6 100755 --- a/tests/qemu-iotests/108 +++ b/tests/qemu-iotests/108 @@ -30,13 +30,20 @@ status=1 # failure is the default! _cleanup() { - _cleanup_test_img + _cleanup_test_img + if [ -f "$TEST_DIR/qsd.pid" ]; then + qsd_pid=$(cat "$TEST_DIR/qsd.pid") + kill -KILL "$qsd_pid" + fusermount -u "$TEST_DIR/fuse-export" &>/dev/null + fi + rm -f "$TEST_DIR/fuse-export" } trap "_cleanup; exit \$status" 0 1 2 3 15 # get standard environment, filters and checks . ./common.rc . ./common.filter +. ./common.qemu # This tests qcow2-specific low-level functionality _supported_fmt qcow2 @@ -47,6 +54,22 @@ _supported_os Linux # files _unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' data_file +# This test either needs sudo -n losetup or FUSE exports to work +if sudo -n losetup &>/dev/null; then + loopdev=true +else + loopdev=false + + # QSD --export fuse will either yield "Parameter 'id' is missing" + # or "Invalid parameter 'fuse'", depending on whether there is + # FUSE support or not. + error=$($QSD --export fuse 2>&1) + if [[ $error = *"'fuse'"* ]]; then + _notrun 'Passwordless sudo for losetup or FUSE support required, but' \ + 'neither is available' + fi +fi + echo echo '=== Repairing an image without any refcount table ===' echo @@ -138,6 +161,240 @@ _make_test_img 64M poke_file "$TEST_IMG" $((0x10008)) "\xff\xff\xff\xff\xff\xff\x00\x00" _check_test_img -r all +echo +echo '=== Check rebuilt reftable location ===' + +# In an earlier version of the refcount rebuild algorithm, the +# reftable was generally placed at the image end (unless something was +# allocated in the area covered by the refblock right before the image +# file end, then we would try to place the reftable in that refblock). +# This was later changed so the reftable would be placed in the +# earliest possible location. Test this. + +echo +echo '--- Does the image size increase? ---' +echo + +# First test: Just create some image, write some data to it, and +# resize it so there is free space at the end of the image (enough +# that it spans at least one full refblock, which for cluster_size=512 +# images, spans 128k). With the old algorithm, the reftable would +# have then been placed at the end of the image file, but with the new +# one, it will be put in that free space. +# We want to check whether the size of the image file increases due to +# rebuilding the refcount structures (it should not). + +_make_test_img -o 'cluster_size=512' 1M +# Write something +$QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io + +# Add free space +file_len=$(stat -c '%s' "$TEST_IMG") +truncate -s $((file_len + 256 * 1024)) "$TEST_IMG" + +# Corrupt the image by saying the image header was not allocated +rt_offset=$(peek_file_be "$TEST_IMG" 48 8) +rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8) +poke_file "$TEST_IMG" $rb_offset "\x00\x00" + +# Check whether rebuilding the refcount structures increases the image +# file size +file_len=$(stat -c '%s' "$TEST_IMG") +echo +# The only leaks there can be are the old refcount structures that are +# leaked during rebuilding, no need to clutter the output with them +_check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0' +echo +post_repair_file_len=$(stat -c '%s' "$TEST_IMG") + +if [[ $file_len -eq $post_repair_file_len ]]; then + echo 'OK: Image size did not change' +else + echo 'ERROR: Image size differs' \ + "($file_len before, $post_repair_file_len after)" +fi + +echo +echo '--- Will the reftable occupy a hole specifically left for it? ---' +echo + +# Note: With cluster_size=512, every refblock covers 128k. +# The reftable covers 8M per reftable cluster. + +# Create an image that requires two reftable clusters (just because +# this is more interesting than a single-clustered reftable). +_make_test_img -o 'cluster_size=512' 9M +$QEMU_IO -c 'write 0 8M' "$TEST_IMG" | _filter_qemu_io + +# Writing 8M will have resized the reftable. Unfortunately, doing so +# will leave holes in the file, so we need to fill them up so we can +# be sure the whole file is allocated. Do that by writing +# consecutively smaller chunks starting from 8 MB, until the file +# length increases even with a chunk size of 512. Then we must have +# filled all holes. +ofs=$((8 * 1024 * 1024)) +block_len=$((16 * 1024)) +while [[ $block_len -ge 512 ]]; do + file_len=$(stat -c '%s' "$TEST_IMG") + while [[ $(stat -c '%s' "$TEST_IMG") -eq $file_len ]]; do + # Do not include this in the reference output, it does not + # really matter which qemu-io calls we do here exactly + $QEMU_IO -c "write $ofs $block_len" "$TEST_IMG" >/dev/null + ofs=$((ofs + block_len)) + done + block_len=$((block_len / 2)) +done + +# Fill up to 9M (do not include this in the reference output either, +# $ofs is random for all we know) +$QEMU_IO -c "write $ofs $((9 * 1024 * 1024 - ofs))" "$TEST_IMG" >/dev/null + +# Make space as follows: +# - For the first refblock: Right at the beginning of the image (this +# refblock is placed in the first place possible), +# - For the reftable somewhere soon afterwards, still near the +# beginning of the image (i.e. covered by the first refblock); the +# reftable too is placed in the first place possible, but only after +# all refblocks have been placed) +# No space is needed for the other refblocks, because no refblock is +# put before the space it covers. In this test case, we do not mind +# if they are placed at the image file's end. + +# Before we make that space, we have to find out the host offset of +# the area that belonged to the two data clusters at guest offset 4k, +# because we expect the reftable to be placed there, and we will have +# to verify that it is. + +l1_offset=$(peek_file_be "$TEST_IMG" 40 8) +l2_offset=$(peek_file_be "$TEST_IMG" $l1_offset 8) +l2_offset=$((l2_offset & 0x00fffffffffffe00)) +data_4k_offset=$(peek_file_be "$TEST_IMG" \ + $((l2_offset + 4096 / 512 * 8)) 8) +data_4k_offset=$((data_4k_offset & 0x00fffffffffffe00)) + +$QEMU_IO -c "discard 0 512" -c "discard 4k 1k" "$TEST_IMG" | _filter_qemu_io + +# Corrupt the image by saying the image header was not allocated +rt_offset=$(peek_file_be "$TEST_IMG" 48 8) +rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8) +poke_file "$TEST_IMG" $rb_offset "\x00\x00" + +echo +# The only leaks there can be are the old refcount structures that are +# leaked during rebuilding, no need to clutter the output with them +_check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0' +echo + +# Check whether the reftable was put where we expected +rt_offset=$(peek_file_be "$TEST_IMG" 48 8) +if [[ $rt_offset -eq $data_4k_offset ]]; then + echo 'OK: Reftable is where we expect it' +else + echo "ERROR: Reftable is at $rt_offset, but was expected at $data_4k_offset" +fi + +echo +echo '--- Rebuilding refcount structures on block devices ---' +echo + +# A block device cannot really grow, at least not during qemu-img +# check. As mentioned in the above cases, rebuilding the refcount +# structure may lead to new refcount structures being written after +# the end of the image, and in the past that happened even if there +# was more than sufficient space in the image. Such post-EOF writes +# will not work on block devices, so test that the new algorithm +# avoids it. + +# If we have passwordless sudo and losetup, we can use those to create +# a block device. Otherwise, we can resort to qemu's FUSE export to +# create a file that isn't growable, which effectively tests the same +# thing. + +_cleanup_test_img +truncate -s $((64 * 1024 * 1024)) "$TEST_IMG" + +if $loopdev; then + export_mp=$(sudo -n losetup --show -f "$TEST_IMG") + export_mp_driver=host_device + sudo -n chmod go+rw "$export_mp" +else + # Create non-growable FUSE export that is a bit like an empty + # block device + export_mp="$TEST_DIR/fuse-export" + export_mp_driver=file + touch "$export_mp" + + $QSD \ + --blockdev file,node-name=export-node,filename="$TEST_IMG" \ + --export fuse,id=fuse-export,node-name=export-node,mountpoint="$export_mp",writable=on,growable=off \ + --pidfile "$TEST_DIR/qsd.pid" \ + --daemonize +fi + +# Now create a qcow2 image on the device -- unfortunately, qemu-img +# create force-creates the file, so we have to resort to the +# blockdev-create job. +_launch_qemu \ + --blockdev $export_mp_driver,node-name=file,filename="$export_mp" + +_send_qemu_cmd \ + $QEMU_HANDLE \ + '{ "execute": "qmp_capabilities" }' \ + 'return' + +# Small cluster size again, so the image needs multiple refblocks +_send_qemu_cmd \ + $QEMU_HANDLE \ + '{ "execute": "blockdev-create", + "arguments": { + "job-id": "create", + "options": { + "driver": "qcow2", + "file": "file", + "size": '$((64 * 1024 * 1024))', + "cluster-size": 512 + } } }' \ + '"concluded"' + +_send_qemu_cmd \ + $QEMU_HANDLE \ + '{ "execute": "job-dismiss", "arguments": { "id": "create" } }' \ + 'return' + +_send_qemu_cmd \ + $QEMU_HANDLE \ + '{ "execute": "quit" }' \ + 'return' + +wait=y _cleanup_qemu +echo + +# Write some data +$QEMU_IO -c 'write 0 64k' "$export_mp" | _filter_qemu_io + +# Corrupt the image by saying the image header was not allocated +rt_offset=$(peek_file_be "$export_mp" 48 8) +rb_offset=$(peek_file_be "$export_mp" $rt_offset 8) +poke_file "$export_mp" $rb_offset "\x00\x00" + +# Repairing such a simple case should just work +# (We used to put the reftable at the end of the image file, which can +# never work for non-growable devices.) +echo +TEST_IMG="$export_mp" _check_test_img -r all \ + | grep -v '^Repairing cluster.*refcount=1 reference=0' + +if $loopdev; then + sudo -n losetup -d "$export_mp" +else + qsd_pid=$(cat "$TEST_DIR/qsd.pid") + kill -TERM "$qsd_pid" + # Wait for process to exit (cannot `wait` because the QSD is daemonized) + while [ -f "$TEST_DIR/qsd.pid" ]; do + true + done +fi + # success, all done echo '*** done' rm -f $seq.full diff --git a/tests/qemu-iotests/108.out b/tests/qemu-iotests/108.out index 75bab8dc84..b5401d788d 100644 --- a/tests/qemu-iotests/108.out +++ b/tests/qemu-iotests/108.out @@ -107,4 +107,85 @@ The following inconsistencies were found and repaired: Double checking the fixed image now... No errors were found on the image. + +=== Check rebuilt reftable location === + +--- Does the image size increase? --- + +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 +wrote 65536/65536 bytes at offset 0 +64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +ERROR cluster 0 refcount=0 reference=1 +Rebuilding refcount structure +The following inconsistencies were found and repaired: + + 0 leaked clusters + 1 corruptions + +Double checking the fixed image now... +No errors were found on the image. + +OK: Image size did not change + +--- Will the reftable occupy a hole specifically left for it? --- + +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=9437184 +wrote 8388608/8388608 bytes at offset 0 +8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +discard 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +discard 1024/1024 bytes at offset 4096 +1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +ERROR cluster 0 refcount=0 reference=1 +Rebuilding refcount structure +The following inconsistencies were found and repaired: + + 0 leaked clusters + 1 corruptions + +Double checking the fixed image now... +No errors were found on the image. + +OK: Reftable is where we expect it + +--- Rebuilding refcount structures on block devices --- + +{ "execute": "qmp_capabilities" } +{"return": {}} +{ "execute": "blockdev-create", + "arguments": { + "job-id": "create", + "options": { + "driver": "IMGFMT", + "file": "file", + "size": 67108864, + "cluster-size": 512 + } } } +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "create"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "create"}} +{"return": {}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "create"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "create"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "create"}} +{ "execute": "job-dismiss", "arguments": { "id": "create" } } +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "create"}} +{"return": {}} +{ "execute": "quit" } +{"return": {}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} + +wrote 65536/65536 bytes at offset 0 +64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +ERROR cluster 0 refcount=0 reference=1 +Rebuilding refcount structure +The following inconsistencies were found and repaired: + + 0 leaked clusters + 1 corruptions + +Double checking the fixed image now... +No errors were found on the image. *** done diff --git a/tests/qemu-iotests/303 b/tests/qemu-iotests/303 index 93aa5ce9b7..40e947f26c 100755 --- a/tests/qemu-iotests/303 +++ b/tests/qemu-iotests/303 @@ -21,10 +21,12 @@ import iotests import subprocess -from iotests import qemu_img_create, qemu_io, file_path, log, filter_qemu_io +from iotests import qemu_img_create, qemu_io, file_path, log, filter_qemu_io, \ + verify_qcow2_zstd_compression iotests.script_initialize(supported_fmts=['qcow2'], unsupported_imgopts=['refcount_bits', 'compat']) +verify_qcow2_zstd_compression() disk = file_path('disk') chunk = 1024 * 1024 diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index fcec3e51e5..fe10a6cf05 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -1471,6 +1471,26 @@ def verify_working_luks(): if not working: notrun(reason) +def supports_qcow2_zstd_compression() -> bool: + img_file = f'{test_dir}/qcow2-zstd-test.qcow2' + res = qemu_img('create', '-f', 'qcow2', '-o', 'compression_type=zstd', + img_file, '0', + check=False) + try: + os.remove(img_file) + except OSError: + pass + + if res.returncode == 1 and \ + "'compression-type' does not accept value 'zstd'" in res.stdout: + return False + else: + return True + +def verify_qcow2_zstd_compression(): + if not supports_qcow2_zstd_compression(): + notrun('zstd compression not supported') + def qemu_pipe(*args: str) -> str: """ Run qemu with an option to print something and exit (e.g. a help option). diff --git a/tests/qemu-iotests/meson.build b/tests/qemu-iotests/meson.build index 9747bb68a5..323a4acb6a 100644 --- a/tests/qemu-iotests/meson.build +++ b/tests/qemu-iotests/meson.build @@ -1,30 +1,47 @@ -if have_tools and targetos != 'windows' and not get_option('gprof') - qemu_iotests_binaries = [qemu_img, qemu_io, qemu_nbd, qsd] - qemu_iotests_env = {'PYTHON': python.full_path()} - qemu_iotests_formats = { - 'qcow2': 'quick', - 'raw': 'slow', - 'qed': 'thorough', - 'vmdk': 'thorough', - 'vpc': 'thorough' - } +if not have_tools or targetos == 'windows' or get_option('gprof') + subdir_done() +endif + +foreach cflag: config_host['QEMU_CFLAGS'].split() + if cflag.startswith('-fsanitize') and \ + not cflag.contains('safe-stack') and not cflag.contains('cfi-icall') + message('Sanitizers are enabled ==> Disabled the qemu-iotests.') + subdir_done() + endif +endforeach - foreach k, v : emulators - if k.startswith('qemu-system-') - qemu_iotests_binaries += v - endif - endforeach - foreach format, speed: qemu_iotests_formats - if speed == 'quick' - suites = 'block' - else - suites = ['block-' + speed, speed] - endif - test('qemu-iotests ' + format, sh, args: [files('../check-block.sh'), format], - depends: qemu_iotests_binaries, env: qemu_iotests_env, - protocol: 'tap', - suite: suites, - timeout: 0, - is_parallel: false) - endforeach +bash = find_program('bash', required: false, version: '>= 4.0') +if not bash.found() + message('bash >= v4.0 not available ==> Disabled the qemu-iotests.') + subdir_done() endif + +qemu_iotests_binaries = [qemu_img, qemu_io, qemu_nbd, qsd] +qemu_iotests_env = {'PYTHON': python.full_path()} +qemu_iotests_formats = { + 'qcow2': 'quick', + 'raw': 'slow', + 'qed': 'thorough', + 'vmdk': 'thorough', + 'vpc': 'thorough' +} + +foreach k, v : emulators + if k.startswith('qemu-system-') + qemu_iotests_binaries += v + endif +endforeach + +foreach format, speed: qemu_iotests_formats + if speed == 'quick' + suites = 'block' + else + suites = ['block-' + speed, speed] + endif + test('qemu-iotests ' + format, sh, args: [files('../check-block.sh'), format], + depends: qemu_iotests_binaries, env: qemu_iotests_env, + protocol: 'tap', + suite: suites, + timeout: 0, + is_parallel: false) +endforeach diff --git a/tests/qtest/endianness-test.c b/tests/qtest/endianness-test.c index 9c03b72dc9..2f5a88bf4c 100644 --- a/tests/qtest/endianness-test.c +++ b/tests/qtest/endianness-test.c @@ -28,6 +28,7 @@ struct TestCase { static const TestCase test_cases[] = { { "i386", "pc", -1 }, { "mips", "malta", 0x10000000, .bswap = true }, + { "mipsel", "malta", 0x10000000 }, { "mips64", "magnum", 0x90000000, .bswap = true }, { "mips64", "pica61", 0x90000000, .bswap = true }, { "mips64", "malta", 0x10000000, .bswap = true }, diff --git a/tests/qtest/fuzz-lsi53c895a-test.c b/tests/qtest/fuzz-lsi53c895a-test.c index ba5d468970..031d9de241 100644 --- a/tests/qtest/fuzz-lsi53c895a-test.c +++ b/tests/qtest/fuzz-lsi53c895a-test.c @@ -39,14 +39,10 @@ static void test_lsi_do_dma_empty_queue(void) int main(int argc, char **argv) { - const char *arch = qtest_get_arch(); - g_test_init(&argc, &argv, NULL); - if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) { - qtest_add_func("fuzz/lsi53c895a/lsi_do_dma_empty_queue", - test_lsi_do_dma_empty_queue); - } + qtest_add_func("fuzz/lsi53c895a/lsi_do_dma_empty_queue", + test_lsi_do_dma_empty_queue); return g_test_run(); } diff --git a/tests/qtest/fuzz-megasas-test.c b/tests/qtest/fuzz-megasas-test.c index e1141c58a4..129b182f83 100644 --- a/tests/qtest/fuzz-megasas-test.c +++ b/tests/qtest/fuzz-megasas-test.c @@ -64,16 +64,12 @@ static void test_gitlab_issue521_megasas_sgl_ovf(void) int main(int argc, char **argv) { - const char *arch = qtest_get_arch(); - g_test_init(&argc, &argv, NULL); - if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) { - qtest_add_func("fuzz/test_lp1878263_megasas_zero_iov_cnt", - test_lp1878263_megasas_zero_iov_cnt); - qtest_add_func("fuzz/gitlab_issue521_megasas_sgl_ovf", - test_gitlab_issue521_megasas_sgl_ovf); - } + qtest_add_func("fuzz/test_lp1878263_megasas_zero_iov_cnt", + test_lp1878263_megasas_zero_iov_cnt); + qtest_add_func("fuzz/gitlab_issue521_megasas_sgl_ovf", + test_gitlab_issue521_megasas_sgl_ovf); return g_test_run(); } diff --git a/tests/qtest/fuzz-sb16-test.c b/tests/qtest/fuzz-sb16-test.c index f47a8bcdbd..91fdcd1e8a 100644 --- a/tests/qtest/fuzz-sb16-test.c +++ b/tests/qtest/fuzz-sb16-test.c @@ -55,15 +55,11 @@ static void test_fuzz_sb16_0xd4(void) int main(int argc, char **argv) { - const char *arch = qtest_get_arch(); - g_test_init(&argc, &argv, NULL); - if (strcmp(arch, "i386") == 0) { - qtest_add_func("fuzz/test_fuzz_sb16/1c", test_fuzz_sb16_0x1c); - qtest_add_func("fuzz/test_fuzz_sb16/91", test_fuzz_sb16_0x91); - qtest_add_func("fuzz/test_fuzz_sb16/d4", test_fuzz_sb16_0xd4); - } + qtest_add_func("fuzz/test_fuzz_sb16/1c", test_fuzz_sb16_0x1c); + qtest_add_func("fuzz/test_fuzz_sb16/91", test_fuzz_sb16_0x91); + qtest_add_func("fuzz/test_fuzz_sb16/d4", test_fuzz_sb16_0xd4); - return g_test_run(); + return g_test_run(); } diff --git a/tests/qtest/fuzz-sdcard-test.c b/tests/qtest/fuzz-sdcard-test.c index 0f94965a66..d0f4e0e93c 100644 --- a/tests/qtest/fuzz-sdcard-test.c +++ b/tests/qtest/fuzz-sdcard-test.c @@ -164,15 +164,11 @@ static void oss_fuzz_36391(void) int main(int argc, char **argv) { - const char *arch = qtest_get_arch(); - g_test_init(&argc, &argv, NULL); - if (strcmp(arch, "i386") == 0) { - qtest_add_func("fuzz/sdcard/oss_fuzz_29225", oss_fuzz_29225); - qtest_add_func("fuzz/sdcard/oss_fuzz_36217", oss_fuzz_36217); - qtest_add_func("fuzz/sdcard/oss_fuzz_36391", oss_fuzz_36391); - } + qtest_add_func("fuzz/sdcard/oss_fuzz_29225", oss_fuzz_29225); + qtest_add_func("fuzz/sdcard/oss_fuzz_36217", oss_fuzz_36217); + qtest_add_func("fuzz/sdcard/oss_fuzz_36391", oss_fuzz_36391); - return g_test_run(); + return g_test_run(); } diff --git a/tests/qtest/fuzz-virtio-scsi-test.c b/tests/qtest/fuzz-virtio-scsi-test.c index aaf6d10e18..c9b6fe2123 100644 --- a/tests/qtest/fuzz-virtio-scsi-test.c +++ b/tests/qtest/fuzz-virtio-scsi-test.c @@ -62,14 +62,10 @@ static void test_mmio_oob_from_memory_region_cache(void) int main(int argc, char **argv) { - const char *arch = qtest_get_arch(); - g_test_init(&argc, &argv, NULL); - if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) { - qtest_add_func("fuzz/test_mmio_oob_from_memory_region_cache", - test_mmio_oob_from_memory_region_cache); - } + qtest_add_func("fuzz/test_mmio_oob_from_memory_region_cache", + test_mmio_oob_from_memory_region_cache); return g_test_run(); } diff --git a/tests/qtest/libqos/virtio.c b/tests/qtest/libqos/virtio.c index 6fe7bf9555..fba9186659 100644 --- a/tests/qtest/libqos/virtio.c +++ b/tests/qtest/libqos/virtio.c @@ -260,6 +260,8 @@ void qvring_init(QTestState *qts, const QGuestAllocator *alloc, QVirtQueue *vq, /* vq->used->flags */ qvirtio_writew(vq->vdev, qts, vq->used, 0); + /* vq->used->idx */ + qvirtio_writew(vq->vdev, qts, vq->used + 2, 0); /* vq->used->avail_event */ qvirtio_writew(vq->vdev, qts, vq->used + 2 + sizeof(struct vring_used_elem) * vq->size, 0); diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build index d25f82bb5a..22e1361210 100644 --- a/tests/qtest/meson.build +++ b/tests/qtest/meson.build @@ -17,13 +17,7 @@ slow_qtests = { 'test-hmp' : 120, } -qtests_generic = \ - (config_all_devices.has_key('CONFIG_MEGASAS_SCSI_PCI') ? ['fuzz-megasas-test'] : []) + \ - (config_all_devices.has_key('CONFIG_LSI_SCSI_PCI') ? ['fuzz-lsi53c895a-test'] : []) + \ - (config_all_devices.has_key('CONFIG_VIRTIO_SCSI') ? ['fuzz-virtio-scsi-test'] : []) + \ - (config_all_devices.has_key('CONFIG_SB16') ? ['fuzz-sb16-test'] : []) + \ - (config_all_devices.has_key('CONFIG_SDHCI_PCI') ? ['fuzz-sdcard-test'] : []) + \ - [ +qtests_generic = [ 'cdrom-test', 'device-introspect-test', 'machine-none-test', @@ -67,6 +61,11 @@ qtests_i386 = \ (config_all_devices.has_key('CONFIG_TPM_TIS_ISA') ? ['tpm-tis-swtpm-test'] : []) + \ (config_all_devices.has_key('CONFIG_RTL8139_PCI') ? ['rtl8139-test'] : []) + \ (config_all_devices.has_key('CONFIG_E1000E_PCI_EXPRESS') ? ['fuzz-e1000e-test'] : []) + \ + (config_all_devices.has_key('CONFIG_MEGASAS_SCSI_PCI') ? ['fuzz-megasas-test'] : []) + \ + (config_all_devices.has_key('CONFIG_LSI_SCSI_PCI') ? ['fuzz-lsi53c895a-test'] : []) + \ + (config_all_devices.has_key('CONFIG_VIRTIO_SCSI') ? ['fuzz-virtio-scsi-test'] : []) + \ + (config_all_devices.has_key('CONFIG_SB16') ? ['fuzz-sb16-test'] : []) + \ + (config_all_devices.has_key('CONFIG_SDHCI_PCI') ? ['fuzz-sdcard-test'] : []) + \ (config_all_devices.has_key('CONFIG_ESP_PCI') ? ['am53c974-test'] : []) + \ (config_all_devices.has_key('CONFIG_ACPI_ERST') ? ['erst-test'] : []) + \ (config_all_devices.has_key('CONFIG_VIRTIO_NET') and \ @@ -143,17 +142,9 @@ qtests_mips = \ (config_all_devices.has_key('CONFIG_ISA_TESTDEV') ? ['endianness-test'] : []) + \ (config_all_devices.has_key('CONFIG_VGA') ? ['display-vga-test'] : []) -qtests_mips64 = \ - ['test-filter-mirror', 'test-filter-redirector'] + \ - (slirp.found() ? ['test-netfilter'] : []) + \ - (config_all_devices.has_key('CONFIG_ISA_TESTDEV') ? ['endianness-test'] : []) + \ - (config_all_devices.has_key('CONFIG_VGA') ? ['display-vga-test'] : []) - -qtests_mips64el = \ - ['test-filter-mirror', 'test-filter-redirector'] + \ - (slirp.found() ? ['test-netfilter'] : []) + \ - (config_all_devices.has_key('CONFIG_ISA_TESTDEV') ? ['endianness-test'] : []) + \ - (config_all_devices.has_key('CONFIG_VGA') ? ['display-vga-test'] : []) +qtests_mipsel = qtests_mips +qtests_mips64 = qtests_mips +qtests_mips64el = qtests_mips qtests_ppc = \ ['test-filter-mirror', 'test-filter-redirector'] + \ diff --git a/tests/vm/centos.aarch64 b/tests/vm/centos.aarch64 index 81c3004c3c..96c450f8be 100755 --- a/tests/vm/centos.aarch64 +++ b/tests/vm/centos.aarch64 @@ -24,8 +24,7 @@ DEFAULT_CONFIG = { 'cpu' : "max", 'machine' : "virt,gic-version=max", 'install_cmds' : "yum install -y make ninja-build git python3 gcc gcc-c++ flex bison, "\ - "yum install -y glib2-devel pixman-devel zlib-devel, "\ - "yum install -y perl-Test-Harness, "\ + "yum install -y glib2-devel perl pixman-devel zlib-devel, "\ "alternatives --set python /usr/bin/python3, "\ "sudo dnf config-manager "\ "--add-repo=https://download.docker.com/linux/centos/docker-ce.repo,"\ diff --git a/tests/vm/fedora b/tests/vm/fedora index b977efe4a2..92b78d6e2c 100755 --- a/tests/vm/fedora +++ b/tests/vm/fedora @@ -35,7 +35,7 @@ class FedoraVM(basevm.BaseVM): 'gcc', 'binutils', 'make', 'ninja-build', # perl - 'perl-Test-Harness', + 'perl', # libs: usb '"pkgconfig(libusb-1.0)"', |