From 9903217a4ed013228d95d8b1876b6053b2bc5e95 Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrangé" Date: Fri, 30 Aug 2024 15:38:08 +0200 Subject: tests/functional: add a module for handling asset download & caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'Asset' class is a simple module that declares a downloadable asset that can be cached locally. Downloads are stored in the user's home dir at ~/.cache/qemu/download, using a sha256 sum of the URL. [thuth: Drop sha1 support, use hash on file content for naming instead of URL, add the possibility to specify the cache dir via environment variable] Signed-off-by: Daniel P. Berrangé Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Message-ID: <20240830133841.142644-15-thuth@redhat.com> Signed-off-by: Thomas Huth --- tests/functional/qemu_test/asset.py | 97 +++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tests/functional/qemu_test/asset.py (limited to 'tests/functional/qemu_test/asset.py') diff --git a/tests/functional/qemu_test/asset.py b/tests/functional/qemu_test/asset.py new file mode 100644 index 0000000000..c0e675d847 --- /dev/null +++ b/tests/functional/qemu_test/asset.py @@ -0,0 +1,97 @@ +# Test utilities for fetching & caching assets +# +# Copyright 2024 Red Hat, Inc. +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import hashlib +import logging +import os +import subprocess +import urllib.request +from pathlib import Path +from shutil import copyfileobj + + +# Instances of this class must be declared as class level variables +# starting with a name "ASSET_". This enables the pre-caching logic +# to easily find all referenced assets and download them prior to +# execution of the tests. +class Asset: + + def __init__(self, url, hashsum): + self.url = url + self.hash = hashsum + cache_dir_env = os.getenv('QEMU_TEST_CACHE_DIR') + if cache_dir_env: + self.cache_dir = Path(cache_dir_env, "download") + else: + self.cache_dir = Path(Path("~").expanduser(), + ".cache", "qemu", "download") + self.cache_file = Path(self.cache_dir, hashsum) + self.log = logging.getLogger('qemu-test') + + def __repr__(self): + return "Asset: url=%s hash=%s cache=%s" % ( + self.url, self.hash, self.cache_file) + + def _check(self, cache_file): + if self.hash is None: + return True + if len(self.hash) == 64: + sum_prog = 'sha256sum' + elif len(self.hash) == 128: + sum_prog = 'sha512sum' + else: + raise Exception("unknown hash type") + + checksum = subprocess.check_output( + [sum_prog, str(cache_file)]).split()[0] + return self.hash == checksum.decode("utf-8") + + def valid(self): + return self.cache_file.exists() and self._check(self.cache_file) + + def fetch(self): + if not self.cache_dir.exists(): + self.cache_dir.mkdir(parents=True, exist_ok=True) + + if self.valid(): + self.log.debug("Using cached asset %s for %s", + self.cache_file, self.url) + return str(self.cache_file) + + self.log.info("Downloading %s to %s...", self.url, self.cache_file) + tmp_cache_file = self.cache_file.with_suffix(".download") + + try: + resp = urllib.request.urlopen(self.url) + except Exception as e: + self.log.error("Unable to download %s: %s", self.url, e) + raise + + try: + with tmp_cache_file.open("wb+") as dst: + copyfileobj(resp, dst) + except: + tmp_cache_file.unlink() + raise + try: + # Set these just for informational purposes + os.setxattr(str(tmp_cache_file), "user.qemu-asset-url", + self.url.encode('utf8')) + os.setxattr(str(tmp_cache_file), "user.qemu-asset-hash", + self.hash.encode('utf8')) + except Exception as e: + self.log.debug("Unable to set xattr on %s: %s", tmp_cache_file, e) + pass + + if not self._check(tmp_cache_file): + tmp_cache_file.unlink() + raise Exception("Hash of %s does not match %s" % + (self.url, self.hash)) + tmp_cache_file.replace(self.cache_file) + + self.log.info("Cached %s at %s" % (self.url, self.cache_file)) + return str(self.cache_file) -- cgit 1.4.1 From f57213f85b49f2271d2a9bba354a160de326eeb9 Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrangé" Date: Fri, 30 Aug 2024 15:38:09 +0200 Subject: tests/functional: enable pre-emptive caching of assets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many tests need to access assets stored on remote sites. We don't want to download these during test execution when run by meson, since this risks hitting test timeouts when data transfers are slow. Add support for pre-emptive caching of assets by setting the env var QEMU_TEST_PRECACHE to point to a timestamp file. When this is set, instead of running the test, the assets will be downloaded and saved to the cache, then the timestamp file created. A meson custom target is created as a dependency of each test suite to trigger the pre-emptive caching logic before the test runs. When run in caching mode, it will locate assets by looking for class level variables with a name prefix "ASSET_", and type "Asset". At the ninja level ninja test --suite functional will speculatively download any assets that are not already cached, so it is advisable to set a timeout multiplier. QEMU_TEST_NO_DOWNLOAD=1 ninja test --suite functional will fail the test if a required asset is not already cached ninja precache-functional will download and cache all assets required by the functional tests At the make level, precaching is always done by make check-functional Signed-off-by: Daniel P. Berrangé Tested-by: Richard Henderson [thuth: Remove the duplicated "path = os.path.basename(...)" line] Message-ID: <20240830133841.142644-16-thuth@redhat.com> Signed-off-by: Thomas Huth --- tests/Makefile.include | 3 ++- tests/functional/meson.build | 33 +++++++++++++++++++++++++++++++-- tests/functional/qemu_test/asset.py | 34 ++++++++++++++++++++++++++++++++++ tests/functional/qemu_test/testcase.py | 7 +++++++ 4 files changed, 74 insertions(+), 3 deletions(-) (limited to 'tests/functional/qemu_test/asset.py') diff --git a/tests/Makefile.include b/tests/Makefile.include index 66c8cc3123..010369bd3a 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -161,7 +161,8 @@ $(FUNCTIONAL_TARGETS): .PHONY: check-functional check-functional: - @$(MAKE) SPEED=thorough check-func check-func-quick + @$(NINJA) precache-functional + @QEMU_TEST_NO_DOWNLOAD=1 $(MAKE) SPEED=thorough check-func check-func-quick # Consolidated targets diff --git a/tests/functional/meson.build b/tests/functional/meson.build index f1f344f860..df79775df3 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -39,6 +39,7 @@ tests_x86_64_system_quick = [ tests_x86_64_system_thorough = [ ] +precache_all = [] foreach speed : ['quick', 'thorough'] foreach dir : target_dirs @@ -78,11 +79,35 @@ foreach speed : ['quick', 'thorough'] meson.current_source_dir()) foreach test : target_tests - test('func-@0@/@1@'.format(target_base, test), + testname = '@0@-@1@'.format(target_base, test) + testfile = 'test_' + test + '.py' + testpath = meson.current_source_dir() / testfile + teststamp = testname + '.tstamp' + test_precache_env = environment() + test_precache_env.set('QEMU_TEST_PRECACHE', meson.current_build_dir() / teststamp) + test_precache_env.set('PYTHONPATH', meson.project_source_root() / 'python:' + + meson.current_source_dir()) + precache = custom_target('func-precache-' + testname, + output: teststamp, + command: [python, testpath], + depend_files: files(testpath), + build_by_default: false, + env: test_precache_env) + precache_all += precache + + # Ideally we would add 'precache' to 'depends' here, such that + # 'build_by_default: false' lets the pre-caching automatically + # run immediately before the test runs. In practice this is + # broken in meson, with it running the pre-caching in the normal + # compile phase https://github.com/mesonbuild/meson/issues/2518 + # If the above bug ever gets fixed, when QEMU changes the min + # meson version, add the 'depends' and remove the custom + # 'run_target' logic below & in Makefile.include + test('func-' + testname, python, depends: [test_deps, test_emulator, emulator_modules], env: test_env, - args: [meson.current_source_dir() / 'test_' + test + '.py'], + args: [testpath], protocol: 'tap', timeout: test_timeouts.get(test, 60), priority: test_timeouts.get(test, 60), @@ -90,3 +115,7 @@ foreach speed : ['quick', 'thorough'] endforeach endforeach endforeach + +run_target('precache-functional', + depends: precache_all, + command: ['true']) diff --git a/tests/functional/qemu_test/asset.py b/tests/functional/qemu_test/asset.py index c0e675d847..b329ab7dbe 100644 --- a/tests/functional/qemu_test/asset.py +++ b/tests/functional/qemu_test/asset.py @@ -9,6 +9,8 @@ import hashlib import logging import os import subprocess +import sys +import unittest import urllib.request from pathlib import Path from shutil import copyfileobj @@ -62,6 +64,9 @@ class Asset: self.cache_file, self.url) return str(self.cache_file) + if os.environ.get("QEMU_TEST_NO_DOWNLOAD", False): + raise Exception("Asset cache is invalid and downloads disabled") + self.log.info("Downloading %s to %s...", self.url, self.cache_file) tmp_cache_file = self.cache_file.with_suffix(".download") @@ -95,3 +100,32 @@ class Asset: self.log.info("Cached %s at %s" % (self.url, self.cache_file)) return str(self.cache_file) + + def precache_test(test): + log = logging.getLogger('qemu-test') + log.setLevel(logging.DEBUG) + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.DEBUG) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + log.addHandler(handler) + for name, asset in vars(test.__class__).items(): + if name.startswith("ASSET_") and type(asset) == Asset: + log.info("Attempting to cache '%s'" % asset) + asset.fetch() + log.removeHandler(handler) + + def precache_suite(suite): + for test in suite: + if isinstance(test, unittest.TestSuite): + Asset.precache_suite(test) + elif isinstance(test, unittest.TestCase): + Asset.precache_test(test) + + def precache_suites(path, cacheTstamp): + loader = unittest.loader.defaultTestLoader + tests = loader.loadTestsFromNames([path], None) + + with open(cacheTstamp, "w") as fh: + Asset.precache_suite(tests) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index b2dd863c6e..18314be9d1 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -21,6 +21,7 @@ import uuid from qemu.machine import QEMUMachine from qemu.utils import kvm_available, tcg_available +from .asset import Asset from .cmd import run_cmd from .config import BUILD_DIR @@ -58,6 +59,12 @@ class QemuBaseTest(unittest.TestCase): def main(): path = os.path.basename(sys.argv[0])[:-3] + + cache = os.environ.get("QEMU_TEST_PRECACHE", None) + if cache is not None: + Asset.precache_suites(path, cache) + return + tr = pycotap.TAPTestRunner(message_log = pycotap.LogMode.LogToError, test_output_log = pycotap.LogMode.LogToError) unittest.main(module = None, testRunner = tr, argv=["__dummy__", path]) -- cgit 1.4.1 From 34b17c0a6564833ae8eb9846ec9faedb212d80ac Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Fri, 30 Aug 2024 15:38:10 +0200 Subject: tests/functional: Allow asset downloading with concurrent threads When running "make -j$(nproc) check-functional", tests that use the same asset might be running in parallel. Improve the downloading to detect this situation and wait for the other thread to finish the download. Message-ID: <20240830133841.142644-17-thuth@redhat.com> Signed-off-by: Thomas Huth --- tests/functional/qemu_test/asset.py | 62 ++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 11 deletions(-) (limited to 'tests/functional/qemu_test/asset.py') diff --git a/tests/functional/qemu_test/asset.py b/tests/functional/qemu_test/asset.py index b329ab7dbe..d3be2aff82 100644 --- a/tests/functional/qemu_test/asset.py +++ b/tests/functional/qemu_test/asset.py @@ -12,6 +12,7 @@ import subprocess import sys import unittest import urllib.request +from time import sleep from pathlib import Path from shutil import copyfileobj @@ -55,6 +56,35 @@ class Asset: def valid(self): return self.cache_file.exists() and self._check(self.cache_file) + def _wait_for_other_download(self, tmp_cache_file): + # Another thread already seems to download the asset, so wait until + # it is done, while also checking the size to see whether it is stuck + try: + current_size = tmp_cache_file.stat().st_size + new_size = current_size + except: + if os.path.exists(self.cache_file): + return True + raise + waittime = lastchange = 600 + while waittime > 0: + sleep(1) + waittime -= 1 + try: + new_size = tmp_cache_file.stat().st_size + except: + if os.path.exists(self.cache_file): + return True + raise + if new_size != current_size: + lastchange = waittime + current_size = new_size + elif lastchange - waittime > 90: + return False + + self.log.debug("Time out while waiting for %s!", tmp_cache_file) + raise + def fetch(self): if not self.cache_dir.exists(): self.cache_dir.mkdir(parents=True, exist_ok=True) @@ -70,18 +100,28 @@ class Asset: self.log.info("Downloading %s to %s...", self.url, self.cache_file) tmp_cache_file = self.cache_file.with_suffix(".download") - try: - resp = urllib.request.urlopen(self.url) - except Exception as e: - self.log.error("Unable to download %s: %s", self.url, e) - raise + for retries in range(3): + try: + with tmp_cache_file.open("xb") as dst: + with urllib.request.urlopen(self.url) as resp: + copyfileobj(resp, dst) + break + except FileExistsError: + self.log.debug("%s already exists, " + "waiting for other thread to finish...", + tmp_cache_file) + if self._wait_for_other_download(tmp_cache_file): + return str(self.cache_file) + self.log.debug("%s seems to be stale, " + "deleting and retrying download...", + tmp_cache_file) + tmp_cache_file.unlink() + continue + except Exception as e: + self.log.error("Unable to download %s: %s", self.url, e) + tmp_cache_file.unlink() + raise - try: - with tmp_cache_file.open("wb+") as dst: - copyfileobj(resp, dst) - except: - tmp_cache_file.unlink() - raise try: # Set these just for informational purposes os.setxattr(str(tmp_cache_file), "user.qemu-asset-url", -- cgit 1.4.1