From 6c2537d35bf121d8924b66a795a58dd761d459b4 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 10 May 2023 23:54:10 -0400 Subject: python: update pylint configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pylint 2.17.x decided that SocketAddrT was a bad name for a Type Alias for some reason. Sure, fine, whatever. Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé Message-Id: <20230511035435.734312-3-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- python/setup.cfg | 1 + 1 file changed, 1 insertion(+) (limited to 'python/setup.cfg') diff --git a/python/setup.cfg b/python/setup.cfg index 9e923d9762..3f36502954 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -132,6 +132,7 @@ good-names=i, fd, # fd = os.open(...) c, # for c in string: ... T, # for TypeVars. See pylint#3401 + SocketAddrT, # Not sure why this is invalid. [pylint.similarities] # Ignore imports when computing similarities. -- cgit 1.4.1 From dd84028ff911e75028dea0ac1e578c2a7bcc3ffd Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 10 May 2023 23:54:11 -0400 Subject: python: add mkvenv.py This script will be responsible for building a lightweight Python virtual environment at configure time. It works with Python 3.6 or newer. It has been designed to: - work *offline*, no PyPI required. - work *quickly*, The fast path is only ~65ms on my machine. - work *robustly*, with multiple fallbacks to keep things working. - work *cooperatively*, using system packages where possible. (You can use your distro's meson, no problem.) Due to its unique position in the build chain, it exists outside of the installable python packages in-tree and *must* be runnable without any third party dependencies. Under normal circumstances, the only dependency required to execute this script is Python 3.6+ itself. The script is *faster* by several seconds when setuptools and pip are installed in the host environment, which is probably the case for a typical multi-purpose developer workstation. In the event that pip/setuptools are missing or not usable, additional dependencies may be required on some distributions which remove certain Python stdlib modules to package them separately: - Debian may require python3-venv to provide "ensurepip" - NetBSD may require py310-expat to provide "pyexpat" * (* Or whichever version is current for NetBSD.) Signed-off-by: Paolo Bonzini Signed-off-by: John Snow Message-Id: <20230511035435.734312-4-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- python/scripts/mkvenv.py | 247 +++++++++++++++++++++++++++++++++++++++++++++++ python/setup.cfg | 9 ++ python/tests/flake8.sh | 1 + python/tests/isort.sh | 1 + python/tests/mypy.sh | 1 + python/tests/pylint.sh | 1 + 6 files changed, 260 insertions(+) create mode 100644 python/scripts/mkvenv.py (limited to 'python/setup.cfg') diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py new file mode 100644 index 0000000000..a4534e41b5 --- /dev/null +++ b/python/scripts/mkvenv.py @@ -0,0 +1,247 @@ +""" +mkvenv - QEMU pyvenv bootstrapping utility + +usage: mkvenv [-h] command ... + +QEMU pyvenv bootstrapping utility + +options: + -h, --help show this help message and exit + +Commands: + command Description + create create a venv + +-------------------------------------------------- + +usage: mkvenv create [-h] target + +positional arguments: + target Target directory to install virtual environment into. + +options: + -h, --help show this help message and exit + +""" + +# Copyright (C) 2022-2023 Red Hat, Inc. +# +# Authors: +# John Snow +# Paolo Bonzini +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import argparse +import logging +import os +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace +from typing import Any, Optional, Union +import venv + + +# Do not add any mandatory dependencies from outside the stdlib: +# This script *must* be usable standalone! + +DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] +logger = logging.getLogger("mkvenv") + + +class Ouch(RuntimeError): + """An Exception class we can't confuse with a builtin.""" + + +class QemuEnvBuilder(venv.EnvBuilder): + """ + An extension of venv.EnvBuilder for building QEMU's configure-time venv. + + As of this commit, it does not yet do anything particularly + different than the standard venv-creation utility. The next several + commits will gradually change that in small commits that highlight + each feature individually. + + Parameters for base class init: + - system_site_packages: bool = False + - clear: bool = False + - symlinks: bool = False + - upgrade: bool = False + - with_pip: bool = False + - prompt: Optional[str] = None + - upgrade_deps: bool = False (Since 3.9) + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + logger.debug("QemuEnvBuilder.__init__(...)") + super().__init__(*args, **kwargs) + + # Make the context available post-creation: + self._context: Optional[SimpleNamespace] = None + + def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: + logger.debug("ensure_directories(env_dir=%s)", env_dir) + self._context = super().ensure_directories(env_dir) + return self._context + + def get_value(self, field: str) -> str: + """ + Get a string value from the context namespace after a call to build. + + For valid field names, see: + https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories + """ + ret = getattr(self._context, field) + assert isinstance(ret, str) + return ret + + +def make_venv( # pylint: disable=too-many-arguments + env_dir: Union[str, Path], + system_site_packages: bool = False, + clear: bool = True, + symlinks: Optional[bool] = None, + with_pip: bool = True, +) -> None: + """ + Create a venv using `QemuEnvBuilder`. + + This is analogous to the `venv.create` module-level convenience + function that is part of the Python stdblib, except it uses + `QemuEnvBuilder` instead. + + :param env_dir: The directory to create/install to. + :param system_site_packages: + Allow inheriting packages from the system installation. + :param clear: When True, fully remove any prior venv and files. + :param symlinks: + Whether to use symlinks to the target interpreter or not. If + left unspecified, it will use symlinks except on Windows to + match behavior with the "venv" CLI tool. + :param with_pip: + Whether to install "pip" binaries or not. + """ + logger.debug( + "%s: make_venv(env_dir=%s, system_site_packages=%s, " + "clear=%s, symlinks=%s, with_pip=%s)", + __file__, + str(env_dir), + system_site_packages, + clear, + symlinks, + with_pip, + ) + + if symlinks is None: + # Default behavior of standard venv CLI + symlinks = os.name != "nt" + + builder = QemuEnvBuilder( + system_site_packages=system_site_packages, + clear=clear, + symlinks=symlinks, + with_pip=with_pip, + ) + + style = "non-isolated" if builder.system_site_packages else "isolated" + print( + f"mkvenv: Creating {style} virtual environment" + f" at '{str(env_dir)}'", + file=sys.stderr, + ) + + try: + logger.debug("Invoking builder.create()") + try: + builder.create(str(env_dir)) + except SystemExit as exc: + # Some versions of the venv module raise SystemExit; *nasty*! + # We want the exception that prompted it. It might be a subprocess + # error that has output we *really* want to see. + logger.debug("Intercepted SystemExit from EnvBuilder.create()") + raise exc.__cause__ or exc.__context__ or exc + logger.debug("builder.create() finished") + except subprocess.CalledProcessError as exc: + logger.error("mkvenv subprocess failed:") + logger.error("cmd: %s", exc.cmd) + logger.error("returncode: %d", exc.returncode) + + def _stringify(data: Union[str, bytes]) -> str: + if isinstance(data, bytes): + return data.decode() + return data + + lines = [] + if exc.stdout: + lines.append("========== stdout ==========") + lines.append(_stringify(exc.stdout)) + lines.append("============================") + if exc.stderr: + lines.append("========== stderr ==========") + lines.append(_stringify(exc.stderr)) + lines.append("============================") + if lines: + logger.error(os.linesep.join(lines)) + + raise Ouch("VENV creation subprocess failed.") from exc + + # print the python executable to stdout for configure. + print(builder.get_value("env_exe")) + + +def _add_create_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser("create", help="create a venv") + subparser.add_argument( + "target", + type=str, + action="store", + help="Target directory to install virtual environment into.", + ) + + +def main() -> int: + """CLI interface to make_qemu_venv. See module docstring.""" + if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): + # You're welcome. + logging.basicConfig(level=logging.DEBUG) + elif os.environ.get("V"): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + prog="mkvenv", + description="QEMU pyvenv bootstrapping utility", + ) + subparsers = parser.add_subparsers( + title="Commands", + dest="command", + metavar="command", + help="Description", + ) + + _add_create_subcommand(subparsers) + + args = parser.parse_args() + try: + if args.command == "create": + make_venv( + args.target, + system_site_packages=True, + clear=True, + ) + logger.debug("mkvenv.py %s: exiting", args.command) + except Ouch as exc: + print("\n*** Ouch! ***\n", file=sys.stderr) + print(str(exc), "\n\n", file=sys.stderr) + return 1 + except SystemExit: + raise + except: # pylint: disable=bare-except + logger.exception("mkvenv did not complete successfully:") + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/setup.cfg b/python/setup.cfg index 3f36502954..5b25f810fa 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -103,6 +103,15 @@ ignore_missing_imports = True [mypy-pygments] ignore_missing_imports = True +[mypy-importlib.metadata] +ignore_missing_imports = True + +[mypy-importlib_metadata] +ignore_missing_imports = True + +[mypy-pkg_resources] +ignore_missing_imports = True + [pylint.messages control] # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this diff --git a/python/tests/flake8.sh b/python/tests/flake8.sh index 1cd7d40fad..e013699645 100755 --- a/python/tests/flake8.sh +++ b/python/tests/flake8.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m flake8 qemu/ +python3 -m flake8 scripts/ diff --git a/python/tests/isort.sh b/python/tests/isort.sh index 4480405bfb..66c2f7df0f 100755 --- a/python/tests/isort.sh +++ b/python/tests/isort.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m isort -c qemu/ +python3 -m isort -c scripts/ diff --git a/python/tests/mypy.sh b/python/tests/mypy.sh index 5f980f563b..a33a3f58ab 100755 --- a/python/tests/mypy.sh +++ b/python/tests/mypy.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m mypy -p qemu +python3 -m mypy scripts/ diff --git a/python/tests/pylint.sh b/python/tests/pylint.sh index 03d64705a1..2b68da90df 100755 --- a/python/tests/pylint.sh +++ b/python/tests/pylint.sh @@ -1,3 +1,4 @@ #!/bin/sh -e # See commit message for environment variable explainer. SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint qemu/ +SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint scripts/ -- cgit 1.4.1 From c5538eed12e2e427e3715b39cca46c00afcde78d Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 10 May 2023 23:54:14 -0400 Subject: mkvenv: add ensure subcommand This command is to be used to add various packages (or ensure they're already present) into the configure-provided venv in a modular fashion. Examples: mkvenv ensure --online --dir "${source_dir}/python/wheels/" "meson>=0.61.5" mkvenv ensure --online "sphinx>=1.6.0" mkvenv ensure "qemu.qmp==0.0.2" It's designed to look for packages in three places, in order: (1) In system packages, if the version installed is already good enough. This way your distribution-provided meson, sphinx, etc are always used as first preference. (2) In a vendored packages directory. Here I am suggesting qemu.git/python/wheels/ as that directory. This is intended to serve as a replacement for vendoring the meson source for QEMU tarballs. It is also highly likely to be extremely useful for packaging the "qemu.qmp" package in source distributions for platforms that do not yet package qemu.qmp separately. (3) Online, via PyPI, ***only when "--online" is passed***. This is only ever used as a fallback if the first two sources do not have an appropriate package that meets the requirement. The ability to build QEMU and run tests *completely offline* is not impinged. Signed-off-by: Paolo Bonzini Signed-off-by: John Snow Message-Id: <20230511035435.734312-7-jsnow@redhat.com> [Use distlib to lookup distributions. - Paolo] Signed-off-by: Paolo Bonzini --- python/scripts/mkvenv.py | 135 +++++++++++++++++++++++++++++++++++++++++++++-- python/setup.cfg | 10 ++++ python/tests/minreqs.txt | 3 ++ 3 files changed, 145 insertions(+), 3 deletions(-) (limited to 'python/setup.cfg') diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 1c1dd89a9a..f6c4f37f84 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -11,6 +11,7 @@ options: Commands: command Description create create a venv + ensure Ensure that the specified package is installed. -------------------------------------------------- @@ -22,6 +23,18 @@ positional arguments: options: -h, --help show this help message and exit +-------------------------------------------------- + +usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... + +positional arguments: + dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' + +options: + -h, --help show this help message and exit + --online Install packages from PyPI, if necessary. + --dir DIR Path to vendored packages where we may install from. + """ # Copyright (C) 2022-2023 Red Hat, Inc. @@ -43,8 +56,17 @@ import subprocess import sys import sysconfig from types import SimpleNamespace -from typing import Any, Optional, Union +from typing import ( + Any, + Optional, + Sequence, + Union, +) import venv +import warnings + +import distlib.database +import distlib.version # Do not add any mandatory dependencies from outside the stdlib: @@ -309,6 +331,77 @@ def make_venv( # pylint: disable=too-many-arguments print(builder.get_value("env_exe")) +def pip_install( + args: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, +) -> None: + """ + Use pip to install a package or package(s) as specified in @args. + """ + loud = bool( + os.environ.get("DEBUG") + or os.environ.get("GITLAB_CI") + or os.environ.get("V") + ) + + full_args = [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "-v" if loud else "-q", + ] + if not online: + full_args += ["--no-index"] + if wheels_dir: + full_args += ["--find-links", f"file://{str(wheels_dir)}"] + full_args += list(args) + subprocess.run( + full_args, + check=True, + ) + + +def ensure( + dep_specs: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, +) -> None: + """ + Use pip to ensure we have the package specified by @dep_specs. + + If the package is already installed, do nothing. If online and + wheels_dir are both provided, prefer packages found in wheels_dir + first before connecting to PyPI. + + :param dep_specs: + PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param online: If True, fall back to PyPI. + :param wheels_dir: If specified, search this path for packages. + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=UserWarning, module="distlib" + ) + dist_path = distlib.database.DistributionPath(include_egg=True) + absent = [] + for spec in dep_specs: + matcher = distlib.version.LegacyMatcher(spec) + dist = dist_path.get_distribution(matcher.name) + if dist is None or not matcher.match(dist.version): + absent.append(spec) + else: + logger.info("found %s", dist) + + if absent: + # Some packages are missing or aren't a suitable version, + # install a suitable (possibly vendored) package. + print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) + pip_install(args=absent, online=online, wheels_dir=wheels_dir) + + def _add_create_subcommand(subparsers: Any) -> None: subparser = subparsers.add_parser("create", help="create a venv") subparser.add_argument( @@ -319,13 +412,42 @@ def _add_create_subcommand(subparsers: Any) -> None: ) +def _add_ensure_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser( + "ensure", help="Ensure that the specified package is installed." + ) + subparser.add_argument( + "--online", + action="store_true", + help="Install packages from PyPI, if necessary.", + ) + subparser.add_argument( + "--dir", + type=str, + action="store", + help="Path to vendored packages where we may install from.", + ) + subparser.add_argument( + "dep_specs", + type=str, + action="store", + help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", + nargs="+", + ) + + def main() -> int: """CLI interface to make_qemu_venv. See module docstring.""" if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): # You're welcome. logging.basicConfig(level=logging.DEBUG) - elif os.environ.get("V"): - logging.basicConfig(level=logging.INFO) + else: + if os.environ.get("V"): + logging.basicConfig(level=logging.INFO) + + # These are incredibly noisy even for V=1 + logging.getLogger("distlib.metadata").addFilter(lambda record: False) + logging.getLogger("distlib.database").addFilter(lambda record: False) parser = argparse.ArgumentParser( prog="mkvenv", @@ -339,6 +461,7 @@ def main() -> int: ) _add_create_subcommand(subparsers) + _add_ensure_subcommand(subparsers) args = parser.parse_args() try: @@ -348,6 +471,12 @@ def main() -> int: system_site_packages=True, clear=True, ) + if args.command == "ensure": + ensure( + dep_specs=args.dep_specs, + online=args.online, + wheels_dir=args.dir, + ) logger.debug("mkvenv.py %s: exiting", args.command) except Ouch as exc: print("\n*** Ouch! ***\n", file=sys.stderr) diff --git a/python/setup.cfg b/python/setup.cfg index 5b25f810fa..d680374b29 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -36,6 +36,7 @@ packages = # Remember to update tests/minreqs.txt if changing anything below: devel = avocado-framework >= 90.0 + distlib >= 0.3.6 flake8 >= 3.6.0 fusepy >= 2.0.4 isort >= 5.1.2 @@ -112,6 +113,15 @@ ignore_missing_imports = True [mypy-pkg_resources] ignore_missing_imports = True +[mypy-distlib] +ignore_missing_imports = True + +[mypy-distlib.database] +ignore_missing_imports = True + +[mypy-distlib.version] +ignore_missing_imports = True + [pylint.messages control] # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this diff --git a/python/tests/minreqs.txt b/python/tests/minreqs.txt index dfb8abb155..7ecf5e7fe4 100644 --- a/python/tests/minreqs.txt +++ b/python/tests/minreqs.txt @@ -16,6 +16,9 @@ urwid==2.1.2 urwid-readline==0.13 Pygments==2.9.0 +# Dependencies for mkvenv +distlib==0.3.6 + # Dependencies for FUSE support for qom-fuse fusepy==2.0.4 -- cgit 1.4.1 From 928348949d1d04f67715fa7125e7e1fa3ff40f7c Mon Sep 17 00:00:00 2001 From: John Snow Date: Tue, 16 May 2023 12:08:11 +0200 Subject: mkvenv: add console script entry point generation When creating a virtual environment that inherits system packages, script entry points (like "meson", "sphinx-build", etc) are not re-generated with the correct shebang. When you are *inside* of the venv, this is not a problem, but if you are *outside* of it, you will not have a script that engages the virtual environment appropriately. Add a mechanism that generates new entry points for pre-existing packages so that we can use these scripts to run "meson", "sphinx-build", "pip", unambiguously inside the venv. Signed-off-by: Paolo Bonzini Signed-off-by: John Snow Message-Id: <20230511035435.734312-9-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- python/scripts/mkvenv.py | 114 +++++++++++++++++++++++++++++++++++++++++++++++ python/setup.cfg | 3 ++ 2 files changed, 117 insertions(+) (limited to 'python/setup.cfg') diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index 2dbbc7020b..f17c3d3606 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -60,6 +60,7 @@ import sysconfig from types import SimpleNamespace from typing import ( Any, + Iterator, Optional, Sequence, Tuple, @@ -69,6 +70,7 @@ import venv import warnings import distlib.database +import distlib.scripts import distlib.version @@ -334,6 +336,113 @@ def make_venv( # pylint: disable=too-many-arguments print(builder.get_value("env_exe")) +def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: + # pylint: disable=import-outside-toplevel + # pylint: disable=no-name-in-module + # pylint: disable=import-error + try: + # First preference: Python 3.8+ stdlib + from importlib.metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + except ImportError as exc: + logger.debug("%s", str(exc)) + # Second preference: Commonly available PyPI backport + from importlib_metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + + def _generator() -> Iterator[str]: + for package in packages: + try: + entry_points = distribution(package).entry_points + except PackageNotFoundError: + continue + + # The EntryPoints type is only available in 3.10+, + # treat this as a vanilla list and filter it ourselves. + entry_points = filter( + lambda ep: ep.group == "console_scripts", entry_points + ) + + for entry_point in entry_points: + yield f"{entry_point.name} = {entry_point.value}" + + return _generator() + + +def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: + # pylint: disable=import-outside-toplevel + # Bundled with setuptools; has a good chance of being available. + import pkg_resources + + def _generator() -> Iterator[str]: + for package in packages: + try: + eps = pkg_resources.get_entry_map(package, "console_scripts") + except pkg_resources.DistributionNotFound: + continue + + for entry_point in eps.values(): + yield str(entry_point) + + return _generator() + + +def generate_console_scripts( + packages: Sequence[str], + python_path: Optional[str] = None, + bin_path: Optional[str] = None, +) -> None: + """ + Generate script shims for console_script entry points in @packages. + """ + if python_path is None: + python_path = sys.executable + if bin_path is None: + bin_path = sysconfig.get_path("scripts") + assert bin_path is not None + + logger.debug( + "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", + packages, + python_path, + bin_path, + ) + + if not packages: + return + + def _get_entry_points() -> Iterator[str]: + """Python 3.7 compatibility shim for iterating entry points.""" + # Python 3.8+, or Python 3.7 with importlib_metadata installed. + try: + return _gen_importlib(packages) + except ImportError as exc: + logger.debug("%s", str(exc)) + + # Python 3.7 with setuptools installed. + try: + return _gen_pkg_resources(packages) + except ImportError as exc: + logger.debug("%s", str(exc)) + raise Ouch( + "Neither importlib.metadata nor pkg_resources found, " + "can't generate console script shims.\n" + "Use Python 3.8+, or install importlib-metadata or setuptools." + ) from exc + + maker = distlib.scripts.ScriptMaker(None, bin_path) + maker.variants = {""} + maker.clobber = False + + for entry_point in _get_entry_points(): + for filename in maker.make(entry_point): + logger.debug("wrote console_script '%s'", filename) + + def pkgname_from_depspec(dep_spec: str) -> str: """ Parse package name out of a PEP-508 depspec. @@ -512,6 +621,7 @@ def _do_ensure( ) dist_path = distlib.database.DistributionPath(include_egg=True) absent = [] + present = [] for spec in dep_specs: matcher = distlib.version.LegacyMatcher(spec) dist = dist_path.get_distribution(matcher.name) @@ -519,6 +629,10 @@ def _do_ensure( absent.append(spec) else: logger.info("found %s", dist) + present.append(matcher.name) + + if present: + generate_console_scripts(present) if absent: # Some packages are missing or aren't a suitable version, diff --git a/python/setup.cfg b/python/setup.cfg index d680374b29..826a2771ba 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -119,6 +119,9 @@ ignore_missing_imports = True [mypy-distlib.database] ignore_missing_imports = True +[mypy-distlib.scripts] +ignore_missing_imports = True + [mypy-distlib.version] ignore_missing_imports = True -- cgit 1.4.1 From 68ea6d17fe531e383394573251359ab4f99f7091 Mon Sep 17 00:00:00 2001 From: John Snow Date: Tue, 16 May 2023 09:05:51 +0200 Subject: mkvenv: use pip's vendored distlib as a fallback distlib is usually not installed on Linux distribution, but it is vendored into pip. Because the virtual environment has pip via ensurepip, we can piggy-back on pip's vendored version. This could break if they move our cheese in the future, but the fix would be simply to require distlib. If it is debundled, as it is on msys, it is simply available directly. Signed-off-by: John Snow [Move to toplevel. - Paolo] Signed-off-by: Paolo Bonzini --- python/scripts/mkvenv.py | 25 ++++++++++++++++++++++--- python/setup.cfg | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) (limited to 'python/setup.cfg') diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py index f17c3d3606..fb91f922d2 100644 --- a/python/scripts/mkvenv.py +++ b/python/scripts/mkvenv.py @@ -69,10 +69,25 @@ from typing import ( import venv import warnings -import distlib.database -import distlib.scripts -import distlib.version +# Try to load distlib, with a fallback to pip's vendored version. +# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail +# outside the venv or before a potential call to ensurepip in checkpip(). +HAVE_DISTLIB = True +try: + import distlib.database + import distlib.scripts + import distlib.version +except ImportError: + try: + # Reach into pip's cookie jar. pylint and flake8 don't understand + # that these imports will be used via distlib.xxx. + from pip._vendor import distlib + import pip._vendor.distlib.database # noqa, pylint: disable=unused-import + import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import + import pip._vendor.distlib.version # noqa, pylint: disable=unused-import + except ImportError: + HAVE_DISTLIB = False # Do not add any mandatory dependencies from outside the stdlib: # This script *must* be usable standalone! @@ -664,6 +679,10 @@ def ensure( bellwether for the presence of 'sphinx'. """ print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) + + if not HAVE_DISTLIB: + raise Ouch("a usable distlib could not be found, please install it") + try: _do_ensure(dep_specs, online, wheels_dir) except subprocess.CalledProcessError as exc: diff --git a/python/setup.cfg b/python/setup.cfg index 826a2771ba..fc3fae5b10 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -125,6 +125,24 @@ ignore_missing_imports = True [mypy-distlib.version] ignore_missing_imports = True +[mypy-pip] +ignore_missing_imports = True + +[mypy-pip._vendor] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.database] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.scripts] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.version] +ignore_missing_imports = True + [pylint.messages control] # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this -- cgit 1.4.1 From 5591b74511ab370b2b7705c5ce97116030038990 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 10 May 2023 23:54:31 -0400 Subject: Python: Drop support for Python 3.6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 3.6 was EOL 2021-12-31. Newer versions of upstream libraries have begun dropping support for this version and it is becoming more cumbersome to support. Avocado-framework and qemu.qmp each have their own reasons for wanting to drop Python 3.6, but won't until QEMU does. Versions of Python available in our supported build platforms as of today, with optional versions available in parentheses: openSUSE Leap 15.4: 3.6.15 (3.9.10, 3.10.2) CentOS Stream 8: 3.6.8 (3.8.13, 3.9.16) CentOS Stream 9: 3.9.13 Fedora 36: 3.10 Fedora 37: 3.11 Debian 11: 3.9.2 Alpine 3.14, 3.15: 3.9.16 Alpine 3.16, 3.17: 3.10.10 Ubuntu 20.04 LTS: 3.8.10 Ubuntu 22.04 LTS: 3.10.4 NetBSD 9.3: 3.9.13* FreeBSD 12.4: 3.9.16 FreeBSD 13.1: 3.9.16 OpenBSD 7.2: 3.9.16 Note: Our VM tests install 3.9 explicitly for FreeBSD and 3.10 for NetBSD; the default for "python" or "python3" in FreeBSD is 3.9.16. NetBSD does not appear to have a default meta-package, but offers several options, the lowest of which is 3.7.15. "python39" appears to be a pre-requisite to one of the other packages we request in tests/vm/netbsd. pip, ensurepip and other Python essentials are currently only available for Python 3.10 for NetBSD. CentOS and OpenSUSE support parallel installation of multiple Python interpreters, and binaries in /usr/bin will always use Python 3.6. However, the newly introduced support for virtual environments ensures that all build steps that execute QEMU Python code use a single interpreter. Since it is safe to under our supported platform policy, bump our minimum supported version of Python to 3.7. Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé Signed-off-by: Paolo Bonzini Message-Id: <20230511035435.734312-24-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- configure | 10 +++++----- docs/about/build-platforms.rst | 2 +- python/Makefile | 10 +++++----- python/setup.cfg | 7 +++---- python/tests/minreqs.txt | 2 +- scripts/qapi/mypy.ini | 2 +- 6 files changed, 16 insertions(+), 17 deletions(-) (limited to 'python/setup.cfg') diff --git a/configure b/configure index 0c9b8421aa..28d5ced335 100755 --- a/configure +++ b/configure @@ -617,9 +617,9 @@ esac check_py_version() { - # We require python >= 3.6. + # We require python >= 3.7. # NB: a True python conditional creates a non-zero return code (Failure) - "$1" -c 'import sys; sys.exit(sys.version_info < (3,6))' + "$1" -c 'import sys; sys.exit(sys.version_info < (3,7))' } python= @@ -628,7 +628,7 @@ first_python= if test -z "${PYTHON}"; then # A bare 'python' is traditionally python 2.x, but some distros # have it as python 3.x, so check in both places. - for binary in python3 python python3.11 python3.10 python3.9 python3.8 python3.7 python3.6; do + for binary in python3 python python3.11 python3.10 python3.9 python3.8 python3.7; do if has "$binary"; then python=$(command -v "$binary") if check_py_version "$python"; then @@ -1077,7 +1077,7 @@ then # If first_python is set, there was a binary somewhere even though # it was not suitable. Use it for the error message. if test -n "$first_python"; then - error_exit "Cannot use '$first_python', Python >= 3.6 is required." \ + error_exit "Cannot use '$first_python', Python >= 3.7 is required." \ "Use --python=/path/to/python to specify a supported Python." else error_exit "Python not found. Use --python=/path/to/python" @@ -1090,7 +1090,7 @@ then fi if ! check_py_version "$python"; then - error_exit "Cannot use '$python', Python >= 3.6 is required." \ + error_exit "Cannot use '$python', Python >= 3.7 is required." \ "Use --python=/path/to/python to specify a supported Python." fi diff --git a/docs/about/build-platforms.rst b/docs/about/build-platforms.rst index 89cae5a6bb..0e2cb9e770 100644 --- a/docs/about/build-platforms.rst +++ b/docs/about/build-platforms.rst @@ -98,7 +98,7 @@ Python runtime option of the ``configure`` script to point QEMU to a supported version of the Python runtime. - As of QEMU |version|, the minimum supported version of Python is 3.6. + As of QEMU |version|, the minimum supported version of Python is 3.7. Python build dependencies Some of QEMU's build dependencies are written in Python. Usually these diff --git a/python/Makefile b/python/Makefile index 47560657d2..7c70dcc8d1 100644 --- a/python/Makefile +++ b/python/Makefile @@ -9,14 +9,14 @@ help: @echo "make check-minreqs:" @echo " Run tests in the minreqs virtual environment." @echo " These tests use the oldest dependencies." - @echo " Requires: Python 3.6" - @echo " Hint (Fedora): 'sudo dnf install python3.6'" + @echo " Requires: Python 3.7" + @echo " Hint (Fedora): 'sudo dnf install python3.7'" @echo "" @echo "make check-tox:" @echo " Run tests against multiple python versions." @echo " These tests use the newest dependencies." - @echo " Requires: Python 3.6 - 3.10, and tox." - @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.10'" + @echo " Requires: Python 3.7 - 3.11, and tox." + @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.11'" @echo " The variable QEMU_TOX_EXTRA_ARGS can be use to pass extra" @echo " arguments to tox". @echo "" @@ -59,7 +59,7 @@ PIP_INSTALL = pip install --disable-pip-version-check min-venv: $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate: setup.cfg tests/minreqs.txt @echo "VENV $(QEMU_MINVENV_DIR)" - @python3.6 -m venv $(QEMU_MINVENV_DIR) + @python3.7 -m venv $(QEMU_MINVENV_DIR) @( \ echo "ACTIVATE $(QEMU_MINVENV_DIR)"; \ . $(QEMU_MINVENV_DIR)/bin/activate; \ diff --git a/python/setup.cfg b/python/setup.cfg index fc3fae5b10..55c0993e70 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -14,7 +14,6 @@ classifiers = Natural Language :: English Operating System :: OS Independent Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -23,7 +22,7 @@ classifiers = Typing :: Typed [options] -python_requires = >= 3.6 +python_requires = >= 3.7 packages = qemu.qmp qemu.machine @@ -77,7 +76,7 @@ exclude = __pycache__, [mypy] strict = True -python_version = 3.6 +python_version = 3.7 warn_unused_configs = True namespace_packages = True warn_unused_ignores = False @@ -199,7 +198,7 @@ multi_line_output=3 # of python available on your system to run this test. [tox:tox] -envlist = py36, py37, py38, py39, py310, py311 +envlist = py37, py38, py39, py310, py311 skip_missing_interpreters = true [testenv] diff --git a/python/tests/minreqs.txt b/python/tests/minreqs.txt index 7ecf5e7fe4..10b181d43a 100644 --- a/python/tests/minreqs.txt +++ b/python/tests/minreqs.txt @@ -1,5 +1,5 @@ # This file lists the ***oldest possible dependencies*** needed to run -# "make check" successfully under ***Python 3.6***. It is used primarily +# "make check" successfully under ***Python 3.7***. It is used primarily # by GitLab CI to ensure that our stated minimum versions in setup.cfg # are truthful and regularly validated. # diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini index 6625356429..3463307ddc 100644 --- a/scripts/qapi/mypy.ini +++ b/scripts/qapi/mypy.ini @@ -1,7 +1,7 @@ [mypy] strict = True disallow_untyped_calls = False -python_version = 3.6 +python_version = 3.7 [mypy-qapi.schema] disallow_untyped_defs = False -- cgit 1.4.1 From 7b4b98c46cb47771aba3469435fe81beda99de6d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 16 May 2023 11:59:36 +0200 Subject: python: bump some of the dependencies The version of pyflakes that is listed in python/tests/minreqs.txt breaks on Python 3.8 with the following message: AttributeError: 'FlakesChecker' object has no attribute 'CONSTANT' Now that we do not support EOL'd Python versions anymore, we can update to newer, fixed versions. It is a good time to do so, before Python packages start dropping support for Python 3.7 as well! The new mypy is also a bit smarter about which packages are actually being used, so remove the now-unnecessary sections from setup.cfg. Signed-off-by: Paolo Bonzini Signed-off-by: John Snow Message-Id: <20230511035435.734312-27-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- python/setup.cfg | 10 ++-------- python/tests/minreqs.txt | 14 +++++++------- 2 files changed, 9 insertions(+), 15 deletions(-) (limited to 'python/setup.cfg') diff --git a/python/setup.cfg b/python/setup.cfg index 55c0993e70..5abb7d30ad 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -36,11 +36,11 @@ packages = devel = avocado-framework >= 90.0 distlib >= 0.3.6 - flake8 >= 3.6.0 + flake8 >= 5.0.4 fusepy >= 2.0.4 isort >= 5.1.2 mypy >= 0.780 - pylint >= 2.8.0 + pylint >= 2.17.3 tox >= 3.18.0 urwid >= 2.1.2 urwid-readline >= 0.13 @@ -124,12 +124,6 @@ ignore_missing_imports = True [mypy-distlib.version] ignore_missing_imports = True -[mypy-pip] -ignore_missing_imports = True - -[mypy-pip._vendor] -ignore_missing_imports = True - [mypy-pip._vendor.distlib] ignore_missing_imports = True diff --git a/python/tests/minreqs.txt b/python/tests/minreqs.txt index 10b181d43a..1ce72cef6d 100644 --- a/python/tests/minreqs.txt +++ b/python/tests/minreqs.txt @@ -26,23 +26,23 @@ fusepy==2.0.4 avocado-framework==90.0 # Linters -flake8==3.6.0 +flake8==5.0.4 isort==5.1.2 mypy==0.780 -pylint==2.8.0 +pylint==2.17.3 # Transitive flake8 dependencies -mccabe==0.6.0 -pycodestyle==2.4.0 -pyflakes==2.0.0 +mccabe==0.7.0 +pycodestyle==2.9.1 +pyflakes==2.5.0 # Transitive mypy dependencies mypy-extensions==0.4.3 typed-ast==1.4.0 -typing-extensions==3.7.4 +typing-extensions==4.5.0 # Transitive pylint dependencies -astroid==2.5.4 +astroid==2.15.4 lazy-object-proxy==1.4.0 toml==0.10.0 wrapt==1.12.1 -- cgit 1.4.1