From 67b9a83daf384f3dc24e83f22da40e34da49021d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 8 Aug 2023 13:25:09 +0200 Subject: python: mkvenv: tweak the matching of --diagnose to depspecs Move the matching between the "absent" array and dep_specs[0] inside the loop, preparing for the possibility of having multiple canaries among the installed packages. Signed-off-by: Paolo Bonzini --- python/scripts/mkvenv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'python/scripts/mkvenv.py') diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index a47f1eaf5d..399659b22f 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -806,6 +806,7 @@ def _do_ensure( """ absent = [] present = [] + canary = None for spec in dep_specs: matcher = distlib.version.LegacyMatcher(spec) ver = _get_version(matcher.name) @@ -817,6 +818,8 @@ def _do_ensure( or not matcher.match(distlib.version.LegacyVersion(ver)) ): absent.append(spec) + if spec == dep_specs[0]: + canary = prog else: logger.info("found %s %s", matcher.name, ver) present.append(matcher.name) @@ -839,7 +842,7 @@ def _do_ensure( absent[0], online, wheels_dir, - prog if absent[0] == dep_specs[0] else None, + canary, ) return None -- cgit 1.4.1 From 0f1ec0705b92b79e5e5f69bed236639dae67d312 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 8 Aug 2023 09:47:25 +0200 Subject: python: mkvenv: introduce TOML-like representation of dependencies We would like to place all Python dependencies in the same file, so that we can add more information without having long and complex command lines. The plan is to have a TOML file with one entry per package, for example [avocado] avocado-framework = { accepted = "(>=88.1, <93.0)", installed = "88.1", canary = "avocado" } Each TOML section will thus be a dictionary of dictionaries. Modify mkvenv.py's workhorse function, _do_ensure, to already operate on such a data structure. The "ensure" subcommand is modified to separate the depspec into a name and a version part, and use the result (plus the --diagnose argument) to build a dictionary for each command line argument. Signed-off-by: Paolo Bonzini --- python/scripts/mkvenv.py | 77 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 16 deletions(-) (limited to 'python/scripts/mkvenv.py') diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 399659b22f..96f506d7e2 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -46,6 +46,9 @@ options: """ +# The duplication between importlib and pkg_resources does not help +# pylint: disable=too-many-lines + # Copyright (C) 2022-2023 Red Hat, Inc. # # Authors: @@ -69,6 +72,7 @@ import sysconfig from types import SimpleNamespace from typing import ( Any, + Dict, Iterator, Optional, Sequence, @@ -786,43 +790,67 @@ def pip_install( ) +def _make_version_constraint(info: Dict[str, str], install: bool) -> str: + """ + Construct the version constraint part of a PEP 508 dependency + specification (for example '>=0.61.5') from the accepted and + installed keys of the provided dictionary. + + :param info: A dictionary corresponding to a TOML key-value list. + :param install: True generates install constraints, False generates + presence constraints + """ + if install and "installed" in info: + return "==" + info["installed"] + + dep_spec = info.get("accepted", "") + dep_spec = dep_spec.strip() + # Double check that they didn't just use a version number + if dep_spec and dep_spec[0] not in "!~><=(": + raise Ouch( + "invalid dependency specifier " + dep_spec + " in dependency file" + ) + + return dep_spec + + def _do_ensure( - dep_specs: Sequence[str], + group: Dict[str, Dict[str, str]], online: bool = False, wheels_dir: Optional[Union[str, Path]] = None, - prog: Optional[str] = None, ) -> Optional[Tuple[str, bool]]: """ - Use pip to ensure we have the package specified by @dep_specs. + Use pip to ensure we have the packages specified in @group. - If the package is already installed, do nothing. If online and + If the packages are already installed, do nothing. If online and wheels_dir are both provided, prefer packages found in wheels_dir first before connecting to PyPI. - :param dep_specs: - PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param group: A dictionary of dictionaries, corresponding to a + section in a pythondeps.toml file. :param online: If True, fall back to PyPI. :param wheels_dir: If specified, search this path for packages. """ absent = [] present = [] canary = None - for spec in dep_specs: - matcher = distlib.version.LegacyMatcher(spec) - ver = _get_version(matcher.name) + for name, info in group.items(): + constraint = _make_version_constraint(info, False) + matcher = distlib.version.LegacyMatcher(name + constraint) + ver = _get_version(name) if ( ver is None # Always pass installed package to pip, so that they can be # updated if the requested version changes - or not _is_system_package(matcher.name) + or not _is_system_package(name) or not matcher.match(distlib.version.LegacyVersion(ver)) ): - absent.append(spec) - if spec == dep_specs[0]: - canary = prog + absent.append(name + _make_version_constraint(info, True)) + if len(absent) == 1: + canary = info.get("canary", None) else: - logger.info("found %s %s", matcher.name, ver) - present.append(matcher.name) + logger.info("found %s %s", name, ver) + present.append(name) if present: generate_console_scripts(present) @@ -875,7 +903,24 @@ def ensure( if not HAVE_DISTLIB: raise Ouch("a usable distlib could not be found, please install it") - result = _do_ensure(dep_specs, online, wheels_dir, prog) + # Convert the depspecs to a dictionary, as if they came + # from a section in a pythondeps.toml file + group: Dict[str, Dict[str, str]] = {} + for spec in dep_specs: + name = distlib.version.LegacyMatcher(spec).name + group[name] = {} + + spec = spec.strip() + pos = len(name) + ver = spec[pos:].strip() + if ver: + group[name]["accepted"] = ver + + if prog: + group[name]["canary"] = prog + prog = None + + result = _do_ensure(group, online, wheels_dir) if result: # Well, that's not good. if result[1]: -- cgit 1.4.1 From 71ed611cd47d961e1897721d7d002ffb498221d4 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 8 Aug 2023 10:03:42 +0200 Subject: python: mkvenv: add ensuregroup command Introduce a new subcommand that retrieves the packages to be installed from a TOML file. This allows being more flexible in using the system version of a package, while at the same time using a known-good version when installing the package. This is important for packages that sometimes have backwards-incompatible changes or that depend on specific versions of their dependencies. Compared to JSON, TOML is more human readable and easier to edit. A parser is available in 3.11 but also available as a small (12k) package for older versions, tomli. While tomli is bundled with pip, this is only true of recent versions of pip. Of all the supported OSes pretty much only FreeBSD has a recent enough version of pip while staying on Python <3.11. So we cannot use the same trick that is in place for distlib. Signed-off-by: Paolo Bonzini --- python/scripts/mkvenv.py | 126 ++++++++++++++++++++++++++++++++++++++++++++++- python/setup.cfg | 6 +++ pythondeps.toml | 17 +++++++ 3 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 pythondeps.toml (limited to 'python/scripts/mkvenv.py') diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 96f506d7e2..02bcd9a8c9 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -14,6 +14,8 @@ Commands: post_init post-venv initialization ensure Ensure that the specified package is installed. + ensuregroup + Ensure that the specified package group is installed. -------------------------------------------------- @@ -44,6 +46,19 @@ options: --online Install packages from PyPI, if necessary. --dir DIR Path to vendored packages where we may install from. +-------------------------------------------------- + +usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group... + +positional arguments: + file pointer to a TOML file + group section name in the TOML file + +options: + -h, --help show this help message and exit + --online Install packages from PyPI, if necessary. + --dir DIR Path to vendored packages where we may install from. + """ # The duplication between importlib and pkg_resources does not help @@ -99,6 +114,18 @@ except ImportError: except ImportError: HAVE_DISTLIB = False +# Try to load tomllib, with a fallback to tomli. +# HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail +# outside the venv or before a potential call to ensurepip in checkpip(). +HAVE_TOMLLIB = True +try: + import tomllib +except ImportError: + try: + import tomli as tomllib + except ImportError: + HAVE_TOMLLIB = False + # Do not add any mandatory dependencies from outside the stdlib: # This script *must* be usable standalone! @@ -837,6 +864,7 @@ def _do_ensure( for name, info in group.items(): constraint = _make_version_constraint(info, False) matcher = distlib.version.LegacyMatcher(name + constraint) + print(f"mkvenv: checking for {matcher}", file=sys.stderr) ver = _get_version(name) if ( ver is None @@ -898,7 +926,6 @@ def ensure( be presented to the user. e.g., 'sphinx-build' can be used as a bellwether for the presence of 'sphinx'. """ - print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) if not HAVE_DISTLIB: raise Ouch("a usable distlib could not be found, please install it") @@ -928,6 +955,64 @@ def ensure( raise SystemExit(f"\n{result[0]}\n\n") +def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: + if not HAVE_TOMLLIB: + if sys.version_info < (3, 11): + raise Ouch("found no usable tomli, please install it") + + raise Ouch( + "Python >=3.11 does not have tomllib... what have you done!?" + ) + + try: + # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, + # Debian bullseye-backports) and v2.0.x + with open(file, "r", encoding="ascii") as depfile: + contents = depfile.read() + return tomllib.loads(contents) # type: ignore + except tomllib.TOMLDecodeError as exc: + raise Ouch(f"parsing {file} failed: {exc}") from exc + + +def ensure_group( + file: str, + groups: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, +) -> None: + """ + Use pip to ensure we have the package specified by @dep_specs. + + If the package is already installed, do nothing. If online and + wheels_dir are both provided, prefer packages found in wheels_dir + first before connecting to PyPI. + + :param dep_specs: + PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param online: If True, fall back to PyPI. + :param wheels_dir: If specified, search this path for packages. + """ + + if not HAVE_DISTLIB: + raise Ouch("found no usable distlib, please install it") + + parsed_deps = _parse_groups(file) + + to_install: Dict[str, Dict[str, str]] = {} + for group in groups: + try: + to_install.update(parsed_deps[group]) + except KeyError as exc: + raise Ouch(f"group {group} not defined") from exc + + result = _do_ensure(to_install, online, wheels_dir) + if result: + # Well, that's not good. + if result[1]: + raise Ouch(result[0]) + raise SystemExit(f"\n{result[0]}\n\n") + + def post_venv_setup() -> None: """ This is intended to be run *inside the venv* after it is created. @@ -955,6 +1040,37 @@ def _add_post_init_subcommand(subparsers: Any) -> None: subparsers.add_parser("post_init", help="post-venv initialization") +def _add_ensuregroup_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser( + "ensuregroup", + help="Ensure that the specified package group is installed.", + ) + subparser.add_argument( + "--online", + action="store_true", + help="Install packages from PyPI, if necessary.", + ) + subparser.add_argument( + "--dir", + type=str, + action="store", + help="Path to vendored packages where we may install from.", + ) + subparser.add_argument( + "file", + type=str, + action="store", + help=("Path to a TOML file describing package groups"), + ) + subparser.add_argument( + "group", + type=str, + action="store", + help="One or more package group names", + nargs="+", + ) + + def _add_ensure_subcommand(subparsers: Any) -> None: subparser = subparsers.add_parser( "ensure", help="Ensure that the specified package is installed." @@ -1012,6 +1128,7 @@ def main() -> int: _add_create_subcommand(subparsers) _add_post_init_subcommand(subparsers) _add_ensure_subcommand(subparsers) + _add_ensuregroup_subcommand(subparsers) args = parser.parse_args() try: @@ -1030,6 +1147,13 @@ def main() -> int: wheels_dir=args.dir, prog=args.diagnose, ) + if args.command == "ensuregroup": + ensure_group( + file=args.file, + groups=args.group, + online=args.online, + wheels_dir=args.dir, + ) logger.debug("mkvenv.py %s: exiting", args.command) except Ouch as exc: print("\n*** Ouch! ***\n", file=sys.stderr) diff --git a/python/setup.cfg b/python/setup.cfg index 5d7e95f5d2..e74b58a8c2 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -94,6 +94,12 @@ allow_subclassing_any = True [mypy-fuse] ignore_missing_imports = True +[mypy-tomli] +ignore_missing_imports = True + +[mypy-tomllib] +ignore_missing_imports = True + [mypy-urwid] ignore_missing_imports = True diff --git a/pythondeps.toml b/pythondeps.toml new file mode 100644 index 0000000000..362f63ff2c --- /dev/null +++ b/pythondeps.toml @@ -0,0 +1,17 @@ +# This file describes Python package requirements to be +# installed in the pyvenv Python virtual environment. +# +# Packages are placed in groups, which are installed using +# the ensuregroup subcommand of python/scripts/mkvenv.py. +# Each group forms a TOML section and each entry in the +# section is a TOML key-value list describing a package. +# All fields are optional; valid fields are: +# +# - accepted: accepted versions when using a system package +# - installed: fixed version to install in the virtual environment +# if a system package is not found; if not specified, +# the minimum and maximum +# - canary: if specified, use this program name to present more +# precise error diagnostics to the user. For example, +# 'sphinx-build' can be used as a bellwether for the +# presence of 'sphinx' in the system. -- cgit 1.4.1 From c03f57fd5bf72588a05750f14202f63be7ddbd0c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 8 Aug 2023 11:28:08 +0200 Subject: Revert "tests: Use separate virtual environment for avocado" This reverts commit e8e4298feadae7924cf7600bb3bcc5b0a8d7cbe9. ensuregroup allows to specify both the acceptable versions of avocado, and a locked version to be used when avocado is not installed as a system pacakge. This lets us install avocado in pyvenv/ using "mkvenv.py" and reuse the distro package on Fedora and CentOS Stream (the only distros where it's available). ensuregroup's usage of "(>=..., <=...)" constraints when evaluating the distro package, and "==" constraints when installing it from PyPI, makes it possible to avoid conflicts between the known-good version and a package plugins included in the distro. This is because package plugins have "==" constraints on the version that is included in the distro, and, using "pip install avocado==88.1" on a venv that includes system packages will result in an error: avocado-framework-plugin-varianter-yaml-to-mux 98.0 requires avocado-framework==98.0, but you have avocado-framework 88.1 which is incompatible. avocado-framework-plugin-result-html 98.0 requires avocado-framework==98.0, but you have avocado-framework 88.1 which is incompatible. But at the same time, if the venv does not include a system distribution of avocado then we can install a known-good version and stick to LTS releases. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1663 Signed-off-by: Paolo Bonzini --- .gitlab-ci.d/buildtest.yml | 6 +++--- docs/devel/acpi-bits.rst | 6 +++--- docs/devel/testing.rst | 14 +++++++------- python/scripts/mkvenv.py | 13 +++++-------- pythondeps.toml | 7 +++++++ scripts/ci/org.centos/stream/8/x86_64/test-avocado | 4 ++-- scripts/device-crash-test | 2 +- tests/Makefile.include | 19 ++++++++----------- tests/requirements.txt | 6 ------ tests/vm/Makefile.include | 2 +- 10 files changed, 37 insertions(+), 42 deletions(-) delete mode 100644 tests/requirements.txt (limited to 'python/scripts/mkvenv.py') diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index 77dc83a6be..aee9101507 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -103,7 +103,7 @@ crash-test-debian: script: - cd build - make NINJA=":" check-venv - - tests/venv/bin/python3 scripts/device-crash-test -q --tcg-only ./qemu-system-i386 + - pyvenv/bin/python3 scripts/device-crash-test -q --tcg-only ./qemu-system-i386 build-system-fedora: extends: @@ -146,8 +146,8 @@ crash-test-fedora: script: - cd build - make NINJA=":" check-venv - - tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-ppc - - tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-riscv32 + - pyvenv/bin/python3 scripts/device-crash-test -q ./qemu-system-ppc + - pyvenv/bin/python3 scripts/device-crash-test -q ./qemu-system-riscv32 build-system-centos: extends: diff --git a/docs/devel/acpi-bits.rst b/docs/devel/acpi-bits.rst index 22e2580200..9677b0098f 100644 --- a/docs/devel/acpi-bits.rst +++ b/docs/devel/acpi-bits.rst @@ -61,19 +61,19 @@ Under ``tests/avocado/`` as the root we have: :: $ make check-venv (needed only the first time to create the venv) - $ ./tests/venv/bin/avocado run -t acpi tests/avocado + $ ./pyvenv/bin/avocado run -t acpi tests/avocado The above will run all acpi avocado tests including this one. In order to run the individual tests, perform the following: :: - $ ./tests/venv/bin/avocado run tests/avocado/acpi-bits.py --tap - + $ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py --tap - The above will produce output in tap format. You can omit "--tap -" in the end and it will produce output like the following: :: - $ ./tests/venv/bin/avocado run tests/avocado/acpi-bits.py + $ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py Fetching asset from tests/avocado/acpi-bits.py:AcpiBitsTest.test_acpi_smbios_bits JOB ID : eab225724da7b64c012c65705dc2fa14ab1defef JOB LOG : /home/anisinha/avocado/job-results/job-2022-10-10T17.58-eab2257/job.log diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst index b6ad21bed1..5d1fc0aa95 100644 --- a/docs/devel/testing.rst +++ b/docs/devel/testing.rst @@ -894,9 +894,9 @@ You can run the avocado tests simply by executing: make check-avocado -This involves the automatic creation of Python virtual environment -within the build tree (at ``tests/venv``) which will have all the -right dependencies, and will save tests results also within the +This involves the automatic installation, from PyPI, of all the +necessary avocado-framework dependencies into the QEMU venv within the +build tree (at ``./pyvenv``). Test results are also saved within the build tree (at ``tests/results``). Note: the build environment must be using a Python 3 stack, and have @@ -953,7 +953,7 @@ may be invoked by running: .. code:: - tests/venv/bin/avocado run $OPTION1 $OPTION2 tests/avocado/ + pyvenv/bin/avocado run $OPTION1 $OPTION2 tests/avocado/ Note that if ``make check-avocado`` was not executed before, it is possible to create the Python virtual environment with the dependencies @@ -968,20 +968,20 @@ a test file. To run tests from a single file within the build tree, use: .. code:: - tests/venv/bin/avocado run tests/avocado/$TESTFILE + pyvenv/bin/avocado run tests/avocado/$TESTFILE To run a single test within a test file, use: .. code:: - tests/venv/bin/avocado run tests/avocado/$TESTFILE:$TESTCLASS.$TESTNAME + pyvenv/bin/avocado run tests/avocado/$TESTFILE:$TESTCLASS.$TESTNAME Valid test names are visible in the output from any previous execution of Avocado or ``make check-avocado``, and can also be queried using: .. code:: - tests/venv/bin/avocado list tests/avocado + pyvenv/bin/avocado list tests/avocado Manual Installation ~~~~~~~~~~~~~~~~~~~ diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 02bcd9a8c9..4f2349fbb6 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -964,14 +964,11 @@ def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: "Python >=3.11 does not have tomllib... what have you done!?" ) - try: - # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, - # Debian bullseye-backports) and v2.0.x - with open(file, "r", encoding="ascii") as depfile: - contents = depfile.read() - return tomllib.loads(contents) # type: ignore - except tomllib.TOMLDecodeError as exc: - raise Ouch(f"parsing {file} failed: {exc}") from exc + # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, + # Debian bullseye-backports) and v2.0.x + with open(file, "r", encoding="ascii") as depfile: + contents = depfile.read() + return tomllib.loads(contents) # type: ignore def ensure_group( diff --git a/pythondeps.toml b/pythondeps.toml index 6be31dba30..0a35ebcf9f 100644 --- a/pythondeps.toml +++ b/pythondeps.toml @@ -23,3 +23,10 @@ meson = { accepted = ">=0.63.0", installed = "0.63.3", canary = "meson" } [docs] sphinx = { accepted = ">=1.6", installed = "5.3.0", canary = "sphinx-build" } sphinx_rtd_theme = { accepted = ">=0.5", installed = "1.1.1" } + +[avocado] +# Note that qemu.git/python/ is always implicitly installed. +# Prefer an LTS version when updating the accepted versions of +# avocado-framework, for example right now the limit is 92.x. +avocado-framework = { accepted = "(>=88.1, <93.0)", installed = "88.1", canary = "avocado" } +pycdlib = { accepted = ">=1.11.0" } diff --git a/scripts/ci/org.centos/stream/8/x86_64/test-avocado b/scripts/ci/org.centos/stream/8/x86_64/test-avocado index e0443fc8ae..73e7a1a312 100755 --- a/scripts/ci/org.centos/stream/8/x86_64/test-avocado +++ b/scripts/ci/org.centos/stream/8/x86_64/test-avocado @@ -4,7 +4,7 @@ # KVM and x86_64, or tests that are generic enough to be valid for all # targets. Such a test list can be generated with: # -# ./tests/venv/bin/avocado list --filter-by-tags-include-empty \ +# ./pyvenv/bin/avocado list --filter-by-tags-include-empty \ # --filter-by-tags-include-empty-key -t accel:kvm,arch:x86_64 \ # tests/avocado/ # @@ -22,7 +22,7 @@ # - tests/avocado/virtio_check_params.py:VirtioMaxSegSettingsCheck.test_machine_types # make get-vm-images -./tests/venv/bin/avocado run \ +./pyvenv/bin/avocado run \ --job-results-dir=tests/results/ \ tests/avocado/boot_linux.py:BootLinuxX8664.test_pc_i440fx_kvm \ tests/avocado/boot_linux.py:BootLinuxX8664.test_pc_q35_kvm \ diff --git a/scripts/device-crash-test b/scripts/device-crash-test index b74d887331..353aa575d7 100755 --- a/scripts/device-crash-test +++ b/scripts/device-crash-test @@ -43,7 +43,7 @@ except ModuleNotFoundError as exc: print(f"Module '{exc.name}' not found.") print(" Try 'make check-venv' from your build directory,") print(" and then one way to run this script is like so:") - print(f' > $builddir/tests/venv/bin/python3 "{path}"') + print(f' > $builddir/pyvenv/bin/python3 "{path}"') sys.exit(1) logger = logging.getLogger('device-crash-test') diff --git a/tests/Makefile.include b/tests/Makefile.include index 9422ddaece..985cda7a94 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -89,10 +89,8 @@ distclean-tcg: $(DISTCLEAN_TCG_TARGET_RULES) # Build up our target list from the filtered list of ninja targets TARGETS=$(patsubst libqemu-%.fa, %, $(filter libqemu-%.fa, $(ninja-targets))) -TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv -TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt +TESTS_VENV_TOKEN=$(BUILD_DIR)/pyvenv/tests.group TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results -TESTS_PYTHON=$(TESTS_VENV_DIR)/bin/python3 ifndef AVOCADO_TESTS AVOCADO_TESTS=tests/avocado endif @@ -108,20 +106,19 @@ else endif quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \ - $(TESTS_PYTHON) -m pip -q --disable-pip-version-check $1, \ + $(PYTHON) -m pip -q --disable-pip-version-check $1, \ "VENVPIP","$1") -$(TESTS_VENV_DIR): $(TESTS_VENV_REQ) - $(call quiet-command, $(PYTHON) -m venv $@, VENV, $@) +$(TESTS_VENV_TOKEN): $(SRC_PATH)/pythondeps.toml $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/") - $(call quiet-venv-pip,install -r $(TESTS_VENV_REQ)) + $(PYTHON) python/scripts/mkvenv.py ensuregroup --online $< avocado $(call quiet-command, touch $@) $(TESTS_RESULTS_DIR): $(call quiet-command, mkdir -p $@, \ MKDIR, $@) -check-venv: $(TESTS_VENV_DIR) +check-venv: $(TESTS_VENV_TOKEN) FEDORA_31_ARCHES_TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGETS))) FEDORA_31_ARCHES_CANDIDATES=$(patsubst ppc64,ppc64le,$(FEDORA_31_ARCHES_TARGETS)) @@ -131,7 +128,7 @@ FEDORA_31_DOWNLOAD=$(filter $(FEDORA_31_ARCHES),$(FEDORA_31_ARCHES_CANDIDATES)) # download one specific Fedora 31 image get-vm-image-fedora-31-%: check-venv $(call quiet-command, \ - $(TESTS_PYTHON) -m avocado vmimage get \ + $(PYTHON) -m avocado vmimage get \ --distro=fedora --distro-version=31 --arch=$*, \ "AVOCADO", "Downloading avocado tests VM image for $*") @@ -140,7 +137,7 @@ get-vm-images: check-venv $(patsubst %,get-vm-image-fedora-31-%, $(FEDORA_31_DOW check-avocado: check-venv $(TESTS_RESULTS_DIR) get-vm-images $(call quiet-command, \ - $(TESTS_PYTHON) -m avocado \ + $(PYTHON) -m avocado \ --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \ $(if $(AVOCADO_TAGS),, --filter-by-tags-include-empty \ --filter-by-tags-include-empty-key) \ @@ -163,7 +160,7 @@ check: check-build: run-ninja check-clean: - rm -rf $(TESTS_VENV_DIR) $(TESTS_RESULTS_DIR) + rm -rf $(TESTS_RESULTS_DIR) clean: check-clean clean-tcg distclean: distclean-tcg diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 0ba561b6bd..0000000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Add Python module requirements, one per line, to be installed -# in the tests/venv Python virtual environment. For more info, -# refer to: https://pip.pypa.io/en/stable/user_guide/#id1 -# Note that qemu.git/python/ is always implicitly installed. -avocado-framework==88.1 -pycdlib==1.11.0 diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index c2a8ca1c17..f0f5d32fb0 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -5,7 +5,7 @@ ifeq ($(realpath $(SRC_PATH)),$(realpath .)) VM_PYTHON = PYTHONPATH=$(SRC_PATH)/python /usr/bin/env python3 VM_VENV = else -VM_PYTHON = $(TESTS_PYTHON) +VM_PYTHON = $(PYTHON) VM_VENV = check-venv endif -- cgit 1.4.1