From fa32a634329f4b2cdab8e380d5ccf263b1491daa Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Fri, 30 Aug 2024 15:38:02 +0200 Subject: tests/functional: Add base classes for the upcoming pytest-based tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The files are mostly a copy of the tests/avocado/avocado_qemu/__init__.py file with some adjustments to get rid of the Avocado dependencies (i.e. we also have to drop the LinuxSSHMixIn and LinuxTest for now). The emulator binary and build directory are now passed via environment variables that will be set via meson.build later. Signed-off-by: Daniel P. Berrangé Message-ID: <20240830133841.142644-9-thuth@redhat.com> Signed-off-by: Thomas Huth --- tests/functional/qemu_test/testcase.py | 153 +++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 tests/functional/qemu_test/testcase.py (limited to 'tests/functional/qemu_test/testcase.py') diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py new file mode 100644 index 0000000000..6905675778 --- /dev/null +++ b/tests/functional/qemu_test/testcase.py @@ -0,0 +1,153 @@ +# Test class and utilities for functional tests +# +# Copyright 2018, 2024 Red Hat, Inc. +# +# Original Author (Avocado-based tests): +# Cleber Rosa +# +# Adaption for standalone version: +# Thomas Huth +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import logging +import os +import pycotap +import sys +import unittest +import uuid + +from qemu.machine import QEMUMachine +from qemu.utils import kvm_available, tcg_available + +from .cmd import run_cmd +from .config import BUILD_DIR + + +class QemuBaseTest(unittest.TestCase): + + qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY') + arch = None + + workdir = None + log = logging.getLogger('qemu-test') + + def setUp(self, bin_prefix): + self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set') + self.arch = self.qemu_bin.split('-')[-1] + + self.workdir = os.path.join(BUILD_DIR, 'tests/functional', self.arch, + self.id()) + os.makedirs(self.workdir, exist_ok=True) + + def main(): + path = os.path.basename(sys.argv[0])[:-3] + tr = pycotap.TAPTestRunner(message_log = pycotap.LogMode.LogToError, + test_output_log = pycotap.LogMode.LogToError) + unittest.main(module = None, testRunner = tr, argv=["__dummy__", path]) + + +class QemuSystemTest(QemuBaseTest): + """Facilitates system emulation tests.""" + + cpu = None + machine = None + _machinehelp = None + + def setUp(self): + self._vms = {} + + super().setUp('qemu-system-') + + def set_machine(self, machinename): + # TODO: We should use QMP to get the list of available machines + if not self._machinehelp: + self._machinehelp = run_cmd([self.qemu_bin, '-M', 'help'])[0]; + if self._machinehelp.find(machinename) < 0: + self.skipTest('no support for machine ' + machinename) + self.machine = machinename + + def require_accelerator(self, accelerator): + """ + Requires an accelerator to be available for the test to continue + + It takes into account the currently set qemu binary. + + If the check fails, the test is canceled. If the check itself + for the given accelerator is not available, the test is also + canceled. + + :param accelerator: name of the accelerator, such as "kvm" or "tcg" + :type accelerator: str + """ + checker = {'tcg': tcg_available, + 'kvm': kvm_available}.get(accelerator) + if checker is None: + self.skipTest("Don't know how to check for the presence " + "of accelerator %s" % accelerator) + if not checker(qemu_bin=self.qemu_bin): + self.skipTest("%s accelerator does not seem to be " + "available" % accelerator) + + def require_netdev(self, netdevname): + netdevhelp = run_cmd([self.qemu_bin, + '-M', 'none', '-netdev', 'help'])[0]; + if netdevhelp.find('\n' + netdevname + '\n') < 0: + self.skipTest('no support for " + netdevname + " networking') + + def require_device(self, devicename): + devhelp = run_cmd([self.qemu_bin, + '-M', 'none', '-device', 'help'])[0]; + if devhelp.find(devicename) < 0: + self.skipTest('no support for device ' + devicename) + + def _new_vm(self, name, *args): + vm = QEMUMachine(self.qemu_bin, base_temp_dir=self.workdir) + self.log.debug('QEMUMachine "%s" created', name) + self.log.debug('QEMUMachine "%s" temp_dir: %s', name, vm.temp_dir) + self.log.debug('QEMUMachine "%s" log_dir: %s', name, vm.log_dir) + if args: + vm.add_args(*args) + return vm + + @property + def vm(self): + return self.get_vm(name='default') + + def get_vm(self, *args, name=None): + if not name: + name = str(uuid.uuid4()) + if self._vms.get(name) is None: + self._vms[name] = self._new_vm(name, *args) + if self.cpu is not None: + self._vms[name].add_args('-cpu', self.cpu) + if self.machine is not None: + self._vms[name].set_machine(self.machine) + return self._vms[name] + + def set_vm_arg(self, arg, value): + """ + Set an argument to list of extra arguments to be given to the QEMU + binary. If the argument already exists then its value is replaced. + + :param arg: the QEMU argument, such as "-cpu" in "-cpu host" + :type arg: str + :param value: the argument value, such as "host" in "-cpu host" + :type value: str + """ + if not arg or not value: + return + if arg not in self.vm.args: + self.vm.args.extend([arg, value]) + else: + idx = self.vm.args.index(arg) + 1 + if idx < len(self.vm.args): + self.vm.args[idx] = value + else: + self.vm.args.append(value) + + def tearDown(self): + for vm in self._vms.values(): + vm.shutdown() + super().tearDown() -- cgit 1.4.1 From 84e4a27feda50323361af775ffe21fd26db3a051 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Fri, 30 Aug 2024 15:38:03 +0200 Subject: tests/functional: Set up logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create log files for each test separately, one file that contains the basic logging and one that contains the console output. Reviewed-by: Daniel P. Berrangé Message-ID: <20240830133841.142644-10-thuth@redhat.com> Signed-off-by: Thomas Huth --- tests/functional/qemu_test/testcase.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'tests/functional/qemu_test/testcase.py') diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index 6905675778..b2dd863c6e 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -31,7 +31,8 @@ class QemuBaseTest(unittest.TestCase): arch = None workdir = None - log = logging.getLogger('qemu-test') + log = None + logdir = None def setUp(self, bin_prefix): self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set') @@ -41,6 +42,20 @@ class QemuBaseTest(unittest.TestCase): self.id()) os.makedirs(self.workdir, exist_ok=True) + self.logdir = self.workdir + self.log = logging.getLogger('qemu-test') + self.log.setLevel(logging.DEBUG) + self._log_fh = logging.FileHandler(os.path.join(self.logdir, + 'base.log'), mode='w') + self._log_fh.setLevel(logging.DEBUG) + fileFormatter = logging.Formatter( + '%(asctime)s - %(levelname)s: %(message)s') + self._log_fh.setFormatter(fileFormatter) + self.log.addHandler(self._log_fh) + + def tearDown(self): + self.log.removeHandler(self._log_fh) + def main(): path = os.path.basename(sys.argv[0])[:-3] tr = pycotap.TAPTestRunner(message_log = pycotap.LogMode.LogToError, @@ -60,6 +75,15 @@ class QemuSystemTest(QemuBaseTest): super().setUp('qemu-system-') + console_log = logging.getLogger('console') + console_log.setLevel(logging.DEBUG) + self._console_log_fh = logging.FileHandler(os.path.join(self.workdir, + 'console.log'), mode='w') + self._console_log_fh.setLevel(logging.DEBUG) + fileFormatter = logging.Formatter('%(asctime)s: %(message)s') + self._console_log_fh.setFormatter(fileFormatter) + console_log.addHandler(self._console_log_fh) + def set_machine(self, machinename): # TODO: We should use QMP to get the list of available machines if not self._machinehelp: @@ -150,4 +174,5 @@ class QemuSystemTest(QemuBaseTest): def tearDown(self): for vm in self._vms.values(): vm.shutdown() + logging.getLogger('console').removeHandler(self._console_log_fh) super().tearDown() -- cgit 1.4.1 From f57213f85b49f2271d2a9bba354a160de326eeb9 Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrangé" Date: Fri, 30 Aug 2024 15:38:09 +0200 Subject: tests/functional: enable pre-emptive caching of assets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many tests need to access assets stored on remote sites. We don't want to download these during test execution when run by meson, since this risks hitting test timeouts when data transfers are slow. Add support for pre-emptive caching of assets by setting the env var QEMU_TEST_PRECACHE to point to a timestamp file. When this is set, instead of running the test, the assets will be downloaded and saved to the cache, then the timestamp file created. A meson custom target is created as a dependency of each test suite to trigger the pre-emptive caching logic before the test runs. When run in caching mode, it will locate assets by looking for class level variables with a name prefix "ASSET_", and type "Asset". At the ninja level ninja test --suite functional will speculatively download any assets that are not already cached, so it is advisable to set a timeout multiplier. QEMU_TEST_NO_DOWNLOAD=1 ninja test --suite functional will fail the test if a required asset is not already cached ninja precache-functional will download and cache all assets required by the functional tests At the make level, precaching is always done by make check-functional Signed-off-by: Daniel P. Berrangé Tested-by: Richard Henderson [thuth: Remove the duplicated "path = os.path.basename(...)" line] Message-ID: <20240830133841.142644-16-thuth@redhat.com> Signed-off-by: Thomas Huth --- tests/Makefile.include | 3 ++- tests/functional/meson.build | 33 +++++++++++++++++++++++++++++++-- tests/functional/qemu_test/asset.py | 34 ++++++++++++++++++++++++++++++++++ tests/functional/qemu_test/testcase.py | 7 +++++++ 4 files changed, 74 insertions(+), 3 deletions(-) (limited to 'tests/functional/qemu_test/testcase.py') diff --git a/tests/Makefile.include b/tests/Makefile.include index 66c8cc3123..010369bd3a 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -161,7 +161,8 @@ $(FUNCTIONAL_TARGETS): .PHONY: check-functional check-functional: - @$(MAKE) SPEED=thorough check-func check-func-quick + @$(NINJA) precache-functional + @QEMU_TEST_NO_DOWNLOAD=1 $(MAKE) SPEED=thorough check-func check-func-quick # Consolidated targets diff --git a/tests/functional/meson.build b/tests/functional/meson.build index f1f344f860..df79775df3 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -39,6 +39,7 @@ tests_x86_64_system_quick = [ tests_x86_64_system_thorough = [ ] +precache_all = [] foreach speed : ['quick', 'thorough'] foreach dir : target_dirs @@ -78,11 +79,35 @@ foreach speed : ['quick', 'thorough'] meson.current_source_dir()) foreach test : target_tests - test('func-@0@/@1@'.format(target_base, test), + testname = '@0@-@1@'.format(target_base, test) + testfile = 'test_' + test + '.py' + testpath = meson.current_source_dir() / testfile + teststamp = testname + '.tstamp' + test_precache_env = environment() + test_precache_env.set('QEMU_TEST_PRECACHE', meson.current_build_dir() / teststamp) + test_precache_env.set('PYTHONPATH', meson.project_source_root() / 'python:' + + meson.current_source_dir()) + precache = custom_target('func-precache-' + testname, + output: teststamp, + command: [python, testpath], + depend_files: files(testpath), + build_by_default: false, + env: test_precache_env) + precache_all += precache + + # Ideally we would add 'precache' to 'depends' here, such that + # 'build_by_default: false' lets the pre-caching automatically + # run immediately before the test runs. In practice this is + # broken in meson, with it running the pre-caching in the normal + # compile phase https://github.com/mesonbuild/meson/issues/2518 + # If the above bug ever gets fixed, when QEMU changes the min + # meson version, add the 'depends' and remove the custom + # 'run_target' logic below & in Makefile.include + test('func-' + testname, python, depends: [test_deps, test_emulator, emulator_modules], env: test_env, - args: [meson.current_source_dir() / 'test_' + test + '.py'], + args: [testpath], protocol: 'tap', timeout: test_timeouts.get(test, 60), priority: test_timeouts.get(test, 60), @@ -90,3 +115,7 @@ foreach speed : ['quick', 'thorough'] endforeach endforeach endforeach + +run_target('precache-functional', + depends: precache_all, + command: ['true']) diff --git a/tests/functional/qemu_test/asset.py b/tests/functional/qemu_test/asset.py index c0e675d847..b329ab7dbe 100644 --- a/tests/functional/qemu_test/asset.py +++ b/tests/functional/qemu_test/asset.py @@ -9,6 +9,8 @@ import hashlib import logging import os import subprocess +import sys +import unittest import urllib.request from pathlib import Path from shutil import copyfileobj @@ -62,6 +64,9 @@ class Asset: self.cache_file, self.url) return str(self.cache_file) + if os.environ.get("QEMU_TEST_NO_DOWNLOAD", False): + raise Exception("Asset cache is invalid and downloads disabled") + self.log.info("Downloading %s to %s...", self.url, self.cache_file) tmp_cache_file = self.cache_file.with_suffix(".download") @@ -95,3 +100,32 @@ class Asset: self.log.info("Cached %s at %s" % (self.url, self.cache_file)) return str(self.cache_file) + + def precache_test(test): + log = logging.getLogger('qemu-test') + log.setLevel(logging.DEBUG) + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.DEBUG) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + log.addHandler(handler) + for name, asset in vars(test.__class__).items(): + if name.startswith("ASSET_") and type(asset) == Asset: + log.info("Attempting to cache '%s'" % asset) + asset.fetch() + log.removeHandler(handler) + + def precache_suite(suite): + for test in suite: + if isinstance(test, unittest.TestSuite): + Asset.precache_suite(test) + elif isinstance(test, unittest.TestCase): + Asset.precache_test(test) + + def precache_suites(path, cacheTstamp): + loader = unittest.loader.defaultTestLoader + tests = loader.loadTestsFromNames([path], None) + + with open(cacheTstamp, "w") as fh: + Asset.precache_suite(tests) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index b2dd863c6e..18314be9d1 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -21,6 +21,7 @@ import uuid from qemu.machine import QEMUMachine from qemu.utils import kvm_available, tcg_available +from .asset import Asset from .cmd import run_cmd from .config import BUILD_DIR @@ -58,6 +59,12 @@ class QemuBaseTest(unittest.TestCase): def main(): path = os.path.basename(sys.argv[0])[:-3] + + cache = os.environ.get("QEMU_TEST_PRECACHE", None) + if cache is not None: + Asset.precache_suites(path, cache) + return + tr = pycotap.TAPTestRunner(message_log = pycotap.LogMode.LogToError, test_output_log = pycotap.LogMode.LogToError) unittest.main(module = None, testRunner = tr, argv=["__dummy__", path]) -- cgit 1.4.1 From 99465d3fe451964f1bb3c355054cd7ced05dcf88 Mon Sep 17 00:00:00 2001 From: Philippe Mathieu-Daudé Date: Fri, 30 Aug 2024 15:38:31 +0200 Subject: tests/functional: Add QemuUserTest class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per commit 5334df4822 ("tests/avocado: Introduce QemuUserTest base class"): Similarly to the 'System' Test base class with methods for testing system emulation, the QemuUserTest class contains methods useful to test user-mode emulation. Signed-off-by: Philippe Mathieu-Daudé Message-ID: <20240822104238.75045-2-philmd@linaro.org> Reviewed-by: Thomas Huth Message-ID: <20240830133841.142644-38-thuth@redhat.com> Signed-off-by: Thomas Huth --- tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/testcase.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) (limited to 'tests/functional/qemu_test/testcase.py') diff --git a/tests/functional/qemu_test/__init__.py b/tests/functional/qemu_test/__init__.py index 1d87d7122f..f33282efe8 100644 --- a/tests/functional/qemu_test/__init__.py +++ b/tests/functional/qemu_test/__init__.py @@ -11,4 +11,4 @@ from .config import BUILD_DIR from .cmd import has_cmd, has_cmds, run_cmd, is_readable_executable_file, \ interrupt_interactive_console_until_pattern, wait_for_console_pattern, \ exec_command, exec_command_and_wait_for_pattern, get_qemu_img -from .testcase import QemuSystemTest, QemuBaseTest +from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index 18314be9d1..aa0146265a 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -13,6 +13,7 @@ import logging import os +import subprocess import pycotap import sys import unittest @@ -70,6 +71,22 @@ class QemuBaseTest(unittest.TestCase): unittest.main(module = None, testRunner = tr, argv=["__dummy__", path]) +class QemuUserTest(QemuBaseTest): + + def setUp(self): + super().setUp('qemu-') + self._ldpath = [] + + def add_ldpath(self, ldpath): + self._ldpath.append(os.path.abspath(ldpath)) + + def run_cmd(self, bin_path, args=[]): + return subprocess.run([self.qemu_bin] + + ["-L %s" % ldpath for ldpath in self._ldpath] + + [bin_path] + + args, + text=True, capture_output=True) + class QemuSystemTest(QemuBaseTest): """Facilitates system emulation tests.""" -- cgit 1.4.1