diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/Makefile | 19 | ||||
| -rw-r--r-- | python/qemu/machine/machine.py | 17 | ||||
| -rw-r--r-- | python/qemu/qmp/legacy.py | 26 | ||||
| -rw-r--r-- | python/qemu/qmp/models.py | 8 | ||||
| -rw-r--r-- | python/qemu/qmp/protocol.py | 45 | ||||
| -rw-r--r-- | python/qemu/qmp/qmp_client.py | 4 | ||||
| -rw-r--r-- | python/scripts/mkvenv.py | 999 | ||||
| -rwxr-xr-x | python/scripts/vendor.py | 74 | ||||
| -rw-r--r-- | python/setup.cfg | 40 | ||||
| -rwxr-xr-x | python/tests/flake8.sh | 1 | ||||
| -rwxr-xr-x | python/tests/isort.sh | 1 | ||||
| -rw-r--r-- | python/tests/minreqs.txt | 19 | ||||
| -rwxr-xr-x | python/tests/mypy.sh | 1 | ||||
| -rwxr-xr-x | python/tests/pylint.sh | 1 | ||||
| -rw-r--r-- | python/wheels/meson-0.63.3-py3-none-any.whl | bin | 0 -> 926526 bytes |
15 files changed, 1180 insertions, 75 deletions
diff --git a/python/Makefile b/python/Makefile index c5bd6ff83a..7c70dcc8d1 100644 --- a/python/Makefile +++ b/python/Makefile @@ -9,14 +9,14 @@ help: @echo "make check-minreqs:" @echo " Run tests in the minreqs virtual environment." @echo " These tests use the oldest dependencies." - @echo " Requires: Python 3.6" - @echo " Hint (Fedora): 'sudo dnf install python3.6'" + @echo " Requires: Python 3.7" + @echo " Hint (Fedora): 'sudo dnf install python3.7'" @echo "" @echo "make check-tox:" @echo " Run tests against multiple python versions." @echo " These tests use the newest dependencies." - @echo " Requires: Python 3.6 - 3.10, and tox." - @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.10'" + @echo " Requires: Python 3.7 - 3.11, and tox." + @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.11'" @echo " The variable QEMU_TOX_EXTRA_ARGS can be use to pass extra" @echo " arguments to tox". @echo "" @@ -54,18 +54,21 @@ pipenv check-pipenv: @echo "pipenv was dropped; try 'make check-minreqs' or 'make min-venv'" @exit 1 +PIP_INSTALL = pip install --disable-pip-version-check .PHONY: min-venv min-venv: $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate: setup.cfg tests/minreqs.txt @echo "VENV $(QEMU_MINVENV_DIR)" - @python3.6 -m venv $(QEMU_MINVENV_DIR) + @python3.7 -m venv $(QEMU_MINVENV_DIR) @( \ echo "ACTIVATE $(QEMU_MINVENV_DIR)"; \ . $(QEMU_MINVENV_DIR)/bin/activate; \ + echo "INSTALL wheel $(QEMU_MINVENV_DIR)"; \ + $(PIP_INSTALL) wheel 1>/dev/null; \ echo "INSTALL -r tests/minreqs.txt $(QEMU_MINVENV_DIR)";\ - pip install -r tests/minreqs.txt 1>/dev/null; \ + $(PIP_INSTALL) -r tests/minreqs.txt 1>/dev/null; \ echo "INSTALL -e qemu $(QEMU_MINVENV_DIR)"; \ - pip install -e . 1>/dev/null; \ + $(PIP_INSTALL) -e . 1>/dev/null; \ ) @touch $(QEMU_MINVENV_DIR) @@ -100,7 +103,7 @@ check-dev: dev-venv .PHONY: develop develop: - pip3 install --disable-pip-version-check -e .[devel] + $(PIP_INSTALL) -e .[devel] .PHONY: check check: diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py index e57c254484..c16a0b6fed 100644 --- a/python/qemu/machine/machine.py +++ b/python/qemu/machine/machine.py @@ -337,18 +337,18 @@ class QEMUMachine: self._remove_files.append(self._console_address) if self._qmp_set: - monitor_address = None - sock = None if self._monitor_address is None: self._sock_pair = socket.socketpair() sock = self._sock_pair[1] if isinstance(self._monitor_address, str): self._remove_files.append(self._monitor_address) - monitor_address = self._monitor_address + + sock_or_addr = self._monitor_address or sock + assert sock_or_addr is not None + self._qmp_connection = QEMUMonitorProtocol( - address=monitor_address, - sock=sock, - server=True, + sock_or_addr, + server=bool(self._monitor_address), nickname=self._name ) @@ -370,7 +370,10 @@ class QEMUMachine: if self._sock_pair: self._sock_pair[0].close() if self._qmp_connection: - self._qmp.accept(self._qmp_timer) + if self._sock_pair: + self._qmp.connect() + else: + self._qmp.accept(self._qmp_timer) def _close_qemu_log_file(self) -> None: if self._qemu_log_file is not None: diff --git a/python/qemu/qmp/legacy.py b/python/qemu/qmp/legacy.py index 8b09ee7dbb..e1e9383978 100644 --- a/python/qemu/qmp/legacy.py +++ b/python/qemu/qmp/legacy.py @@ -68,34 +68,31 @@ class QEMUMonitorProtocol: Provide an API to connect to QEMU via QEMU Monitor Protocol (QMP) and then allow to handle commands and events. - :param address: QEMU address, can be either a unix socket path (string) - or a tuple in the form ( address, port ) for a TCP - connection or None - :param sock: a socket or None + :param address: QEMU address, can be a unix socket path (string), a tuple + in the form ( address, port ) for a TCP connection, or an + existing `socket.socket` object. :param server: Act as the socket server. (See 'accept') + Not applicable when passing a socket directly. :param nickname: Optional nickname used for logging. """ def __init__(self, - address: Optional[SocketAddrT] = None, - sock: Optional[socket.socket] = None, + address: Union[SocketAddrT, socket.socket], server: bool = False, nickname: Optional[str] = None): - assert address or sock + if server and isinstance(address, socket.socket): + raise ValueError( + "server argument should be False when passing a socket") + self._qmp = QMPClient(nickname) self._aloop = asyncio.get_event_loop() self._address = address - self._sock = sock self._timeout: Optional[float] = None if server: - if sock: - assert self._sock is not None - self._sync(self._qmp.open_with_socket(self._sock)) - else: - assert self._address is not None - self._sync(self._qmp.start_server(self._address)) + assert not isinstance(self._address, socket.socket) + self._sync(self._qmp.start_server(self._address)) _T = TypeVar('_T') @@ -150,7 +147,6 @@ class QEMUMonitorProtocol: :return: QMP greeting dict, or None if negotiate is false :raise ConnectError: on connection errors """ - assert self._address is not None self._qmp.await_greeting = negotiate self._qmp.negotiate = negotiate diff --git a/python/qemu/qmp/models.py b/python/qemu/qmp/models.py index de87f87804..da52848d5a 100644 --- a/python/qemu/qmp/models.py +++ b/python/qemu/qmp/models.py @@ -54,7 +54,7 @@ class Model: class Greeting(Model): """ - Defined in qmp-spec.txt, section 2.2, "Server Greeting". + Defined in qmp-spec.rst, section "Server Greeting". :param raw: The raw Greeting object. :raise KeyError: If any required fields are absent. @@ -82,7 +82,7 @@ class Greeting(Model): class QMPGreeting(Model): """ - Defined in qmp-spec.txt, section 2.2, "Server Greeting". + Defined in qmp-spec.rst, section "Server Greeting". :param raw: The raw QMPGreeting object. :raise KeyError: If any required fields are absent. @@ -104,7 +104,7 @@ class QMPGreeting(Model): class ErrorResponse(Model): """ - Defined in qmp-spec.txt, section 2.4.2, "error". + Defined in qmp-spec.rst, section "Error". :param raw: The raw ErrorResponse object. :raise KeyError: If any required fields are absent. @@ -126,7 +126,7 @@ class ErrorResponse(Model): class ErrorInfo(Model): """ - Defined in qmp-spec.txt, section 2.4.2, "error". + Defined in qmp-spec.rst, section "Error". :param raw: The raw ErrorInfo object. :raise KeyError: If any required fields are absent. diff --git a/python/qemu/qmp/protocol.py b/python/qemu/qmp/protocol.py index 22e60298d2..753182131f 100644 --- a/python/qemu/qmp/protocol.py +++ b/python/qemu/qmp/protocol.py @@ -299,19 +299,6 @@ class AsyncProtocol(Generic[T]): @upper_half @require(Runstate.IDLE) - async def open_with_socket(self, sock: socket.socket) -> None: - """ - Start connection with given socket. - - :param sock: A socket. - - :raise StateError: When the `Runstate` is not `IDLE`. - """ - self._reader, self._writer = await asyncio.open_connection(sock=sock) - self._set_state(Runstate.CONNECTING) - - @upper_half - @require(Runstate.IDLE) async def start_server(self, address: SocketAddrT, ssl: Optional[SSLContext] = None) -> None: """ @@ -357,12 +344,11 @@ class AsyncProtocol(Generic[T]): protocol-level failure occurs while establishing a new session, the wrapped error may also be an `QMPError`. """ - if not self._reader: - if self._accepted is None: - raise QMPError("Cannot call accept() before start_server().") - await self._session_guard( - self._do_accept(), - 'Failed to establish connection') + if self._accepted is None: + raise QMPError("Cannot call accept() before start_server().") + await self._session_guard( + self._do_accept(), + 'Failed to establish connection') await self._session_guard( self._establish_session(), 'Failed to establish session') @@ -370,7 +356,7 @@ class AsyncProtocol(Generic[T]): @upper_half @require(Runstate.IDLE) - async def connect(self, address: SocketAddrT, + async def connect(self, address: Union[SocketAddrT, socket.socket], ssl: Optional[SSLContext] = None) -> None: """ Connect to the server and begin processing message queues. @@ -615,7 +601,7 @@ class AsyncProtocol(Generic[T]): self.logger.debug("Connection accepted.") @upper_half - async def _do_connect(self, address: SocketAddrT, + async def _do_connect(self, address: Union[SocketAddrT, socket.socket], ssl: Optional[SSLContext] = None) -> None: """ Acting as the transport client, initiate a connection to a server. @@ -634,9 +620,17 @@ class AsyncProtocol(Generic[T]): # otherwise yield. await asyncio.sleep(0) - self.logger.debug("Connecting to %s ...", address) - - if isinstance(address, tuple): + if isinstance(address, socket.socket): + self.logger.debug("Connecting with existing socket: " + "fd=%d, family=%r, type=%r", + address.fileno(), address.family, address.type) + connect = asyncio.open_connection( + limit=self._limit, + ssl=ssl, + sock=address, + ) + elif isinstance(address, tuple): + self.logger.debug("Connecting to %s ...", address) connect = asyncio.open_connection( address[0], address[1], @@ -644,13 +638,14 @@ class AsyncProtocol(Generic[T]): limit=self._limit, ) else: + self.logger.debug("Connecting to file://%s ...", address) connect = asyncio.open_unix_connection( path=address, ssl=ssl, limit=self._limit, ) - self._reader, self._writer = await connect + self._reader, self._writer = await connect self.logger.debug("Connected.") @upper_half diff --git a/python/qemu/qmp/qmp_client.py b/python/qemu/qmp/qmp_client.py index 9d73ae6e7a..2a817f9db3 100644 --- a/python/qemu/qmp/qmp_client.py +++ b/python/qemu/qmp/qmp_client.py @@ -369,7 +369,7 @@ class QMPClient(AsyncProtocol[Message], Events): # This is very likely a server parsing error. # It doesn't inherently belong to any pending execution. # Instead of performing clever recovery, just terminate. - # See "NOTE" in qmp-spec.txt, section 2.4.2 + # See "NOTE" in qmp-spec.rst, section "Error". raise ServerParseError( ("Server sent an error response without an ID, " "but there are no ID-less executions pending. " @@ -377,7 +377,7 @@ class QMPClient(AsyncProtocol[Message], Events): msg ) - # qmp-spec.txt, section 2.4: + # qmp-spec.rst, section "Commands Responses": # 'Clients should drop all the responses # that have an unknown "id" field.' self.logger.log( diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py new file mode 100644 index 0000000000..a47f1eaf5d --- /dev/null +++ b/python/scripts/mkvenv.py @@ -0,0 +1,999 @@ +""" +mkvenv - QEMU pyvenv bootstrapping utility + +usage: mkvenv [-h] command ... + +QEMU pyvenv bootstrapping utility + +options: + -h, --help show this help message and exit + +Commands: + command Description + create create a venv + post_init + post-venv initialization + ensure Ensure that the specified package is installed. + +-------------------------------------------------- + +usage: mkvenv create [-h] target + +positional arguments: + target Target directory to install virtual environment into. + +options: + -h, --help show this help message and exit + +-------------------------------------------------- + +usage: mkvenv post_init [-h] + +options: + -h, --help show this help message and exit + +-------------------------------------------------- + +usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... + +positional arguments: + dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' + +options: + -h, --help show this help message and exit + --online Install packages from PyPI, if necessary. + --dir DIR Path to vendored packages where we may install from. + +""" + +# Copyright (C) 2022-2023 Red Hat, Inc. +# +# Authors: +# John Snow <jsnow@redhat.com> +# Paolo Bonzini <pbonzini@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import argparse +from importlib.util import find_spec +import logging +import os +from pathlib import Path +import re +import shutil +import site +import subprocess +import sys +import sysconfig +from types import SimpleNamespace +from typing import ( + Any, + Iterator, + Optional, + Sequence, + Tuple, + Union, +) +import venv + + +# Try to load distlib, with a fallback to pip's vendored version. +# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail +# outside the venv or before a potential call to ensurepip in checkpip(). +HAVE_DISTLIB = True +try: + import distlib.scripts + import distlib.version +except ImportError: + try: + # Reach into pip's cookie jar. pylint and flake8 don't understand + # that these imports will be used via distlib.xxx. + from pip._vendor import distlib + import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import + import pip._vendor.distlib.version # noqa, pylint: disable=unused-import + except ImportError: + HAVE_DISTLIB = False + +# Do not add any mandatory dependencies from outside the stdlib: +# This script *must* be usable standalone! + +DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] +logger = logging.getLogger("mkvenv") + + +def inside_a_venv() -> bool: + """Returns True if it is executed inside of a virtual environment.""" + return sys.prefix != sys.base_prefix + + +class Ouch(RuntimeError): + """An Exception class we can't confuse with a builtin.""" + + +class QemuEnvBuilder(venv.EnvBuilder): + """ + An extension of venv.EnvBuilder for building QEMU's configure-time venv. + + The primary difference is that it emulates a "nested" virtual + environment when invoked from inside of an existing virtual + environment by including packages from the parent. Also, + "ensurepip" is replaced if possible with just recreating pip's + console_scripts inside the virtual environment. + + Parameters for base class init: + - system_site_packages: bool = False + - clear: bool = False + - symlinks: bool = False + - upgrade: bool = False + - with_pip: bool = False + - prompt: Optional[str] = None + - upgrade_deps: bool = False (Since 3.9) + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + logger.debug("QemuEnvBuilder.__init__(...)") + + # For nested venv emulation: + self.use_parent_packages = False + if inside_a_venv(): + # Include parent packages only if we're in a venv and + # system_site_packages was True. + self.use_parent_packages = kwargs.pop( + "system_site_packages", False + ) + # Include system_site_packages only when the parent, + # The venv we are currently in, also does so. + kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES + + # ensurepip is slow: venv creation can be very fast for cases where + # we allow the use of system_site_packages. Therefore, ensurepip is + # replaced with our own script generation once the virtual environment + # is setup. + self.want_pip = kwargs.get("with_pip", False) + if self.want_pip: + if ( + kwargs.get("system_site_packages", False) + and not need_ensurepip() + ): + kwargs["with_pip"] = False + else: + check_ensurepip(suggest_remedy=True) + + super().__init__(*args, **kwargs) + + # Make the context available post-creation: + self._context: Optional[SimpleNamespace] = None + + def get_parent_libpath(self) -> Optional[str]: + """Return the libpath of the parent venv, if applicable.""" + if self.use_parent_packages: + return sysconfig.get_path("purelib") + return None + + @staticmethod + def compute_venv_libpath(context: SimpleNamespace) -> str: + """ + Compatibility wrapper for context.lib_path for Python < 3.12 + """ + # Python 3.12+, not strictly necessary because it's documented + # to be the same as 3.10 code below: + if sys.version_info >= (3, 12): + return context.lib_path + + # Python 3.10+ + if "venv" in sysconfig.get_scheme_names(): + lib_path = sysconfig.get_path( + "purelib", scheme="venv", vars={"base": context.env_dir} + ) + assert lib_path is not None + return lib_path + + # For Python <= 3.9 we need to hardcode this. Fortunately the + # code below was the same in Python 3.6-3.10, so there is only + # one case. + if sys.platform == "win32": + return os.path.join(context.env_dir, "Lib", "site-packages") + return os.path.join( + context.env_dir, + "lib", + "python%d.%d" % sys.version_info[:2], + "site-packages", + ) + + def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: + logger.debug("ensure_directories(env_dir=%s)", env_dir) + self._context = super().ensure_directories(env_dir) + return self._context + + def create(self, env_dir: DirType) -> None: + logger.debug("create(env_dir=%s)", env_dir) + super().create(env_dir) + assert self._context is not None + self.post_post_setup(self._context) + + def post_post_setup(self, context: SimpleNamespace) -> None: + """ + The final, final hook. Enter the venv and run commands inside of it. + """ + if self.use_parent_packages: + # We're inside of a venv and we want to include the parent + # venv's packages. + parent_libpath = self.get_parent_libpath() + assert parent_libpath is not None + logger.debug("parent_libpath: %s", parent_libpath) + + our_libpath = self.compute_venv_libpath(context) + logger.debug("our_libpath: %s", our_libpath) + + pth_file = os.path.join(our_libpath, "nested.pth") + with open(pth_file, "w", encoding="UTF-8") as file: + file.write(parent_libpath + os.linesep) + + if self.want_pip: + args = [ + context.env_exe, + __file__, + "post_init", + ] + subprocess.run(args, check=True) + + def get_value(self, field: str) -> str: + """ + Get a string value from the context namespace after a call to build. + + For valid field names, see: + https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories + """ + ret = getattr(self._context, field) + assert isinstance(ret, str) + return ret + + +def need_ensurepip() -> bool: + """ + Tests for the presence of setuptools and pip. + + :return: `True` if we do not detect both packages. + """ + # Don't try to actually import them, it's fraught with danger: + # https://github.com/pypa/setuptools/issues/2993 + if find_spec("setuptools") and find_spec("pip"): + return False + return True + + +def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: + """ + Check that we have ensurepip. + + Raise a fatal exception with a helpful hint if it isn't available. + """ + if not find_spec("ensurepip"): + msg = ( + "Python's ensurepip module is not found.\n" + "It's normally part of the Python standard library, " + "maybe your distribution packages it separately?\n" + "(Debian puts ensurepip in its python3-venv package.)\n" + ) + if suggest_remedy: + msg += ( + "Either install ensurepip, or alleviate the need for it in the" + " first place by installing pip and setuptools for " + f"'{sys.executable}'.\n" + ) + raise Ouch(prefix + msg) + + # ensurepip uses pyexpat, which can also go missing on us: + if not find_spec("pyexpat"): + msg = ( + "Python's pyexpat module is not found.\n" + "It's normally part of the Python standard library, " + "maybe your distribution packages it separately?\n" + "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n" + ) + if suggest_remedy: + msg += ( + "Either install pyexpat, or alleviate the need for it in the " + "first place by installing pip and setuptools for " + f"'{sys.executable}'.\n" + ) + raise Ouch(prefix + msg) + + +def make_venv( # pylint: disable=too-many-arguments + env_dir: Union[str, Path], + system_site_packages: bool = False, + clear: bool = True, + symlinks: Optional[bool] = None, + with_pip: bool = True, +) -> None: + """ + Create a venv using `QemuEnvBuilder`. + + This is analogous to the `venv.create` module-level convenience + function that is part of the Python stdblib, except it uses + `QemuEnvBuilder` instead. + + :param env_dir: The directory to create/install to. + :param system_site_packages: + Allow inheriting packages from the system installation. + :param clear: When True, fully remove any prior venv and files. + :param symlinks: + Whether to use symlinks to the target interpreter or not. If + left unspecified, it will use symlinks except on Windows to + match behavior with the "venv" CLI tool. + :param with_pip: + Whether to install "pip" binaries or not. + """ + logger.debug( + "%s: make_venv(env_dir=%s, system_site_packages=%s, " + "clear=%s, symlinks=%s, with_pip=%s)", + __file__, + str(env_dir), + system_site_packages, + clear, + symlinks, + with_pip, + ) + + if symlinks is None: + # Default behavior of standard venv CLI + symlinks = os.name != "nt" + + builder = QemuEnvBuilder( + system_site_packages=system_site_packages, + clear=clear, + symlinks=symlinks, + with_pip=with_pip, + ) + + style = "non-isolated" if builder.system_site_packages else "isolated" + nested = "" + if builder.use_parent_packages: + nested = f"(with packages from '{builder.get_parent_libpath()}') " + print( + f"mkvenv: Creating {style} virtual environment" + f" {nested}at '{str(env_dir)}'", + file=sys.stderr, + ) + + try: + logger.debug("Invoking builder.create()") + try: + builder.create(str(env_dir)) + except SystemExit as exc: + # Some versions of the venv module raise SystemExit; *nasty*! + # We want the exception that prompted it. It might be a subprocess + # error that has output we *really* want to see. + logger.debug("Intercepted SystemExit from EnvBuilder.create()") + raise exc.__cause__ or exc.__context__ or exc + logger.debug("builder.create() finished") + except subprocess.CalledProcessError as exc: + logger.error("mkvenv subprocess failed:") + logger.error("cmd: %s", exc.cmd) + logger.error("returncode: %d", exc.returncode) + + def _stringify(data: Union[str, bytes]) -> str: + if isinstance(data, bytes): + return data.decode() + return data + + lines = [] + if exc.stdout: + lines.append("========== stdout ==========") + lines.append(_stringify(exc.stdout)) + lines.append("============================") + if exc.stderr: + lines.append("========== stderr ==========") + lines.append(_stringify(exc.stderr)) + lines.append("============================") + if lines: + logger.error(os.linesep.join(lines)) + + raise Ouch("VENV creation subprocess failed.") from exc + + # print the python executable to stdout for configure. + print(builder.get_value("env_exe")) + + +def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: + # pylint: disable=import-outside-toplevel + # pylint: disable=no-name-in-module + # pylint: disable=import-error + try: + # First preference: Python 3.8+ stdlib + from importlib.metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + except ImportError as exc: + logger.debug("%s", str(exc)) + # Second preference: Commonly available PyPI backport + from importlib_metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + + def _generator() -> Iterator[str]: + for package in packages: + try: + entry_points = distribution(package).entry_points + except PackageNotFoundError: + continue + + # The EntryPoints type is only available in 3.10+, + # treat this as a vanilla list and filter it ourselves. + entry_points = filter( + lambda ep: ep.group == "console_scripts", entry_points + ) + + for entry_point in entry_points: + yield f"{entry_point.name} = {entry_point.value}" + + return _generator() + + +def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: + # pylint: disable=import-outside-toplevel + # Bundled with setuptools; has a good chance of being available. + import pkg_resources + + def _generator() -> Iterator[str]: + for package in packages: + try: + eps = pkg_resources.get_entry_map(package, "console_scripts") + except pkg_resources.DistributionNotFound: + continue + + for entry_point in eps.values(): + yield str(entry_point) + + return _generator() + + +def generate_console_scripts( + packages: Sequence[str], + python_path: Optional[str] = None, + bin_path: Optional[str] = None, +) -> None: + """ + Generate script shims for console_script entry points in @packages. + """ + if python_path is None: + python_path = sys.executable + if bin_path is None: + bin_path = sysconfig.get_path("scripts") + assert bin_path is not None + + logger.debug( + "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", + packages, + python_path, + bin_path, + ) + + if not packages: + return + + def _get_entry_points() -> Iterator[str]: + """Python 3.7 compatibility shim for iterating entry points.""" + # Python 3.8+, or Python 3.7 with importlib_metadata installed. + try: + return _gen_importlib(packages) + except ImportError as exc: + logger.debug("%s", str(exc)) + + # Python 3.7 with setuptools installed. + try: + return _gen_pkg_resources(packages) + except ImportError as exc: + logger.debug("%s", str(exc)) + raise Ouch( + "Neither importlib.metadata nor pkg_resources found, " + "can't generate console script shims.\n" + "Use Python 3.8+, or install importlib-metadata or setuptools." + ) from exc + + maker = distlib.scripts.ScriptMaker(None, bin_path) + maker.variants = {""} + maker.clobber = False + + for entry_point in _get_entry_points(): + for filename in maker.make(entry_point): + logger.debug("wrote console_script '%s'", filename) + + +def checkpip() -> bool: + """ + Debian10 has a pip that's broken when used inside of a virtual environment. + + We try to detect and correct that case here. + """ + try: + # pylint: disable=import-outside-toplevel,unused-import,import-error + # pylint: disable=redefined-outer-name + import pip._internal # type: ignore # noqa: F401 + + logger.debug("pip appears to be working correctly.") + return False + except ModuleNotFoundError as exc: + if exc.name == "pip._internal": + # Uh, fair enough. They did say "internal". + # Let's just assume it's fine. + return False + logger.warning("pip appears to be malfunctioning: %s", str(exc)) + + check_ensurepip("pip appears to be non-functional, and ") + + logger.debug("Attempting to repair pip ...") + subprocess.run( + (sys.executable, "-m", "ensurepip"), + stdout=subprocess.DEVNULL, + check=True, + ) + logger.debug("Pip is now (hopefully) repaired!") + return True + + +def pkgname_from_depspec(dep_spec: str) -> str: + """ + Parse package name out of a PEP-508 depspec. + + See https://peps.python.org/pep-0508/#names + """ + match = re.match( + r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE + ) + if not match: + raise ValueError( + f"dep_spec '{dep_spec}'" + " does not appear to contain a valid package name" + ) + return match.group(0) + + +def _get_path_importlib(package: str) -> Optional[str]: + # pylint: disable=import-outside-toplevel + # pylint: disable=no-name-in-module + # pylint: disable=import-error + try: + # First preference: Python 3.8+ stdlib + from importlib.metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + except ImportError as exc: + logger.debug("%s", str(exc)) + # Second preference: Commonly available PyPI backport + from importlib_metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + + try: + return str(distribution(package).locate_file(".")) + except PackageNotFoundError: + return None + + +def _get_path_pkg_resources(package: str) -> Optional[str]: + # pylint: disable=import-outside-toplevel + # Bundled with setuptools; has a good chance of being available. + import pkg_resources + + try: + return str(pkg_resources.get_distribution(package).location) + except pkg_resources.DistributionNotFound: + return None + + +def _get_path(package: str) -> Optional[str]: + try: + return _get_path_importlib(package) + except ImportError as exc: + logger.debug("%s", str(exc)) + + try: + return _get_path_pkg_resources(package) + except ImportError as exc: + logger.debug("%s", str(exc)) + raise Ouch( + "Neither importlib.metadata nor pkg_resources found. " + "Use Python 3.8+, or install importlib-metadata or setuptools." + ) from exc + + +def _path_is_prefix(prefix: Optional[str], path: str) -> bool: + try: + return ( + prefix is not None and os.path.commonpath([prefix, path]) == prefix + ) + except ValueError: + return False + + +def _is_system_package(package: str) -> bool: + path = _get_path(package) + return path is not None and not ( + _path_is_prefix(sysconfig.get_path("purelib"), path) + or _path_is_prefix(sysconfig.get_path("platlib"), path) + ) + + +def _get_version_importlib(package: str) -> Optional[str]: + # pylint: disable=import-outside-toplevel + # pylint: disable=no-name-in-module + # pylint: disable=import-error + try: + # First preference: Python 3.8+ stdlib + from importlib.metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + except ImportError as exc: + logger.debug("%s", str(exc)) + # Second preference: Commonly available PyPI backport + from importlib_metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + + try: + return str(distribution(package).version) + except PackageNotFoundError: + return None + + +def _get_version_pkg_resources(package: str) -> Optional[str]: + # pylint: disable=import-outside-toplevel + # Bundled with setuptools; has a good chance of being available. + import pkg_resources + + try: + return str(pkg_resources.get_distribution(package).version) + except pkg_resources.DistributionNotFound: + return None + + +def _get_version(package: str) -> Optional[str]: + try: + return _get_version_importlib(package) + except ImportError as exc: + logger.debug("%s", str(exc)) + + try: + return _get_version_pkg_resources(package) + except ImportError as exc: + logger.debug("%s", str(exc)) + raise Ouch( + "Neither importlib.metadata nor pkg_resources found. " + "Use Python 3.8+, or install importlib-metadata or setuptools." + ) from exc + + +def diagnose( + dep_spec: str, + online: bool, + wheels_dir: Optional[Union[str, Path]], + prog: Optional[str], +) -> Tuple[str, bool]: + """ + Offer a summary to the user as to why a package failed to be installed. + + :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' + :param online: Did we allow PyPI access? + :param prog: + Optionally, a shell program name that can be used as a + bellwether to detect if this program is installed elsewhere on + the system. This is used to offer advice when a program is + detected for a different python version. + :param wheels_dir: + Optionally, a directory that was searched for vendored packages. + """ + # pylint: disable=too-many-branches + + # Some errors are not particularly serious + bad = False + + pkg_name = pkgname_from_depspec(dep_spec) + pkg_version = _get_version(pkg_name) + + lines = [] + + if pkg_version: + lines.append( + f"Python package '{pkg_name}' version '{pkg_version}' was found," + " but isn't suitable." + ) + else: + lines.append( + f"Python package '{pkg_name}' was not found nor installed." + ) + + if wheels_dir: + lines.append( + "No suitable version found in, or failed to install from" + f" '{wheels_dir}'." + ) + bad = True + + if online: + lines.append("A suitable version could not be obtained from PyPI.") + bad = True + else: + lines.append( + "mkvenv was configured to operate offline and did not check PyPI." + ) + + if prog and not pkg_version: + which = shutil.which(prog) + if which: + if sys.base_prefix in site.PREFIXES: + pypath = Path(sys.executable).resolve() + lines.append( + f"'{prog}' was detected on your system at '{which}', " + f"but the Python package '{pkg_name}' was not found by " + f"this Python interpreter ('{pypath}'). " + f"Typically this means that '{prog}' has been installed " + "against a different Python interpreter on your system." + ) + else: + lines.append( + f"'{prog}' was detected on your system at '{which}', " + "but the build is using an isolated virtual environment." + ) + bad = True + + lines = [f" • {line}" for line in lines] + if bad: + lines.insert(0, f"Could not provide build dependency '{dep_spec}':") + else: + lines.insert(0, f"'{dep_spec}' not found:") + return os.linesep.join(lines), bad + + +def pip_install( + args: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, +) -> None: + """ + Use pip to install a package or package(s) as specified in @args. + """ + loud = bool( + os.environ.get("DEBUG") + or os.environ.get("GITLAB_CI") + or os.environ.get("V") + ) + + full_args = [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "-v" if loud else "-q", + ] + if not online: + full_args += ["--no-index"] + if wheels_dir: + full_args += ["--find-links", f"file://{str(wheels_dir)}"] + full_args += list(args) + subprocess.run( + full_args, + check=True, + ) + + +def _do_ensure( + dep_specs: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, + prog: Optional[str] = None, +) -> Optional[Tuple[str, bool]]: + """ + Use pip to ensure we have the package specified by @dep_specs. + + If the package is already installed, do nothing. If online and + wheels_dir are both provided, prefer packages found in wheels_dir + first before connecting to PyPI. + + :param dep_specs: + PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param online: If True, fall back to PyPI. + :param wheels_dir: If specified, search this path for packages. + """ + absent = [] + present = [] + for spec in dep_specs: + matcher = distlib.version.LegacyMatcher(spec) + ver = _get_version(matcher.name) + if ( + ver is None + # Always pass installed package to pip, so that they can be + # updated if the requested version changes + or not _is_system_package(matcher.name) + or not matcher.match(distlib.version.LegacyVersion(ver)) + ): + absent.append(spec) + else: + logger.info("found %s %s", matcher.name, ver) + present.append(matcher.name) + + if present: + generate_console_scripts(present) + + if absent: + if online or wheels_dir: + # Some packages are missing or aren't a suitable version, + # install a suitable (possibly vendored) package. + print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) + try: + pip_install(args=absent, online=online, wheels_dir=wheels_dir) + return None + except subprocess.CalledProcessError: + pass + + return diagnose( + absent[0], + online, + wheels_dir, + prog if absent[0] == dep_specs[0] else None, + ) + + return None + + +def ensure( + dep_specs: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, + prog: Optional[str] = None, +) -> None: + """ + Use pip to ensure we have the package specified by @dep_specs. + + If the package is already installed, do nothing. If online and + wheels_dir are both provided, prefer packages found in wheels_dir + first before connecting to PyPI. + + :param dep_specs: + PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param online: If True, fall back to PyPI. + :param wheels_dir: If specified, search this path for packages. + :param prog: + If specified, use this program name for error diagnostics that will + be presented to the user. e.g., 'sphinx-build' can be used as a + bellwether for the presence of 'sphinx'. + """ + print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) + + if not HAVE_DISTLIB: + raise Ouch("a usable distlib could not be found, please install it") + + result = _do_ensure(dep_specs, online, wheels_dir, prog) + if result: + # Well, that's not good. + if result[1]: + raise Ouch(result[0]) + raise SystemExit(f"\n{result[0]}\n\n") + + +def post_venv_setup() -> None: + """ + This is intended to be run *inside the venv* after it is created. + """ + logger.debug("post_venv_setup()") + # Test for a broken pip (Debian 10 or derivative?) and fix it if needed + if not checkpip(): + # Finally, generate a 'pip' script so the venv is usable in a normal + # way from the CLI. This only happens when we inherited pip from a + # parent/system-site and haven't run ensurepip in some way. + generate_console_scripts(["pip"]) + + +def _add_create_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser("create", help="create a venv") + subparser.add_argument( + "target", + type=str, + action="store", + help="Target directory to install virtual environment into.", + ) + + +def _add_post_init_subcommand(subparsers: Any) -> None: + subparsers.add_parser("post_init", help="post-venv initialization") + + +def _add_ensure_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser( + "ensure", help="Ensure that the specified package is installed." + ) + subparser.add_argument( + "--online", + action="store_true", + help="Install packages from PyPI, if necessary.", + ) + subparser.add_argument( + "--dir", + type=str, + action="store", + help="Path to vendored packages where we may install from.", + ) + subparser.add_argument( + "--diagnose", + type=str, + action="store", + help=( + "Name of a shell utility to use for " + "diagnostics if this command fails." + ), + ) + subparser.add_argument( + "dep_specs", + type=str, + action="store", + help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", + nargs="+", + ) + + +def main() -> int: + """CLI interface to make_qemu_venv. See module docstring.""" + if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): + # You're welcome. + logging.basicConfig(level=logging.DEBUG) + else: + if os.environ.get("V"): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + prog="mkvenv", + description="QEMU pyvenv bootstrapping utility", + ) + subparsers = parser.add_subparsers( + title="Commands", + dest="command", + required=True, + metavar="command", + help="Description", + ) + + _add_create_subcommand(subparsers) + _add_post_init_subcommand(subparsers) + _add_ensure_subcommand(subparsers) + + args = parser.parse_args() + try: + if args.command == "create": + make_venv( + args.target, + system_site_packages=True, + clear=True, + ) + if args.command == "post_init": + post_venv_setup() + if args.command == "ensure": + ensure( + dep_specs=args.dep_specs, + online=args.online, + wheels_dir=args.dir, + prog=args.diagnose, + ) + logger.debug("mkvenv.py %s: exiting", args.command) + except Ouch as exc: + print("\n*** Ouch! ***\n", file=sys.stderr) + print(str(exc), "\n\n", file=sys.stderr) + return 1 + except SystemExit: + raise + except: # pylint: disable=bare-except + logger.exception("mkvenv did not complete successfully:") + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/scripts/vendor.py b/python/scripts/vendor.py new file mode 100755 index 0000000000..34486a51f4 --- /dev/null +++ b/python/scripts/vendor.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +vendor - QEMU python vendoring utility + +usage: vendor [-h] + +QEMU python vendoring utility + +options: + -h, --help show this help message and exit +""" + +# Copyright (C) 2023 Red Hat, Inc. +# +# Authors: +# John Snow <jsnow@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import argparse +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +def main() -> int: + """Run the vendoring utility. See module-level docstring.""" + loud = False + if os.environ.get("DEBUG") or os.environ.get("V"): + loud = True + + # No options or anything for now, but I guess + # you'll figure that out when you run --help. + parser = argparse.ArgumentParser( + prog="vendor", + description="QEMU python vendoring utility", + ) + parser.parse_args() + + packages = { + "meson==0.63.3": + "d677b809c4895dcbaac9bf6c43703fcb3609a4b24c6057c78f828590049cf43a", + } + + vendor_dir = Path(__file__, "..", "..", "wheels").resolve() + + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as file: + for dep_spec, checksum in packages.items(): + file.write(f"{dep_spec} --hash=sha256:{checksum}") + file.flush() + + cli_args = [ + "pip", + "download", + "--dest", + str(vendor_dir), + "--require-hashes", + "-r", + file.name, + ] + if loud: + cli_args.append("-v") + + print(" ".join(cli_args)) + subprocess.run(cli_args, check=True) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/setup.cfg b/python/setup.cfg index 9e923d9762..42f0b0be07 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -14,7 +14,6 @@ classifiers = Natural Language :: English Operating System :: OS Independent Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -23,7 +22,7 @@ classifiers = Typing :: Typed [options] -python_requires = >= 3.6 +python_requires = >= 3.7 packages = qemu.qmp qemu.machine @@ -36,11 +35,12 @@ packages = # Remember to update tests/minreqs.txt if changing anything below: devel = avocado-framework >= 90.0 - flake8 >= 3.6.0 + distlib >= 0.3.6 + flake8 >= 5.0.4 fusepy >= 2.0.4 isort >= 5.1.2 mypy >= 0.780 - pylint >= 2.8.0 + pylint >= 2.17.3 tox >= 3.18.0 urwid >= 2.1.2 urwid-readline >= 0.13 @@ -76,7 +76,7 @@ exclude = __pycache__, [mypy] strict = True -python_version = 3.6 +python_version = 3.7 warn_unused_configs = True namespace_packages = True warn_unused_ignores = False @@ -103,6 +103,33 @@ ignore_missing_imports = True [mypy-pygments] ignore_missing_imports = True +[mypy-importlib.metadata] +ignore_missing_imports = True + +[mypy-importlib_metadata] +ignore_missing_imports = True + +[mypy-pkg_resources] +ignore_missing_imports = True + +[mypy-distlib] +ignore_missing_imports = True + +[mypy-distlib.scripts] +ignore_missing_imports = True + +[mypy-distlib.version] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.scripts] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.version] +ignore_missing_imports = True + [pylint.messages control] # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this @@ -132,6 +159,7 @@ good-names=i, fd, # fd = os.open(...) c, # for c in string: ... T, # for TypeVars. See pylint#3401 + SocketAddrT, # Not sure why this is invalid. [pylint.similarities] # Ignore imports when computing similarities. @@ -158,7 +186,7 @@ multi_line_output=3 # of python available on your system to run this test. [tox:tox] -envlist = py36, py37, py38, py39, py310, py311 +envlist = py37, py38, py39, py310, py311 skip_missing_interpreters = true [testenv] diff --git a/python/tests/flake8.sh b/python/tests/flake8.sh index 1cd7d40fad..e013699645 100755 --- a/python/tests/flake8.sh +++ b/python/tests/flake8.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m flake8 qemu/ +python3 -m flake8 scripts/ diff --git a/python/tests/isort.sh b/python/tests/isort.sh index 4480405bfb..66c2f7df0f 100755 --- a/python/tests/isort.sh +++ b/python/tests/isort.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m isort -c qemu/ +python3 -m isort -c scripts/ diff --git a/python/tests/minreqs.txt b/python/tests/minreqs.txt index dfb8abb155..1ce72cef6d 100644 --- a/python/tests/minreqs.txt +++ b/python/tests/minreqs.txt @@ -1,5 +1,5 @@ # This file lists the ***oldest possible dependencies*** needed to run -# "make check" successfully under ***Python 3.6***. It is used primarily +# "make check" successfully under ***Python 3.7***. It is used primarily # by GitLab CI to ensure that our stated minimum versions in setup.cfg # are truthful and regularly validated. # @@ -16,6 +16,9 @@ urwid==2.1.2 urwid-readline==0.13 Pygments==2.9.0 +# Dependencies for mkvenv +distlib==0.3.6 + # Dependencies for FUSE support for qom-fuse fusepy==2.0.4 @@ -23,23 +26,23 @@ fusepy==2.0.4 avocado-framework==90.0 # Linters -flake8==3.6.0 +flake8==5.0.4 isort==5.1.2 mypy==0.780 -pylint==2.8.0 +pylint==2.17.3 # Transitive flake8 dependencies -mccabe==0.6.0 -pycodestyle==2.4.0 -pyflakes==2.0.0 +mccabe==0.7.0 +pycodestyle==2.9.1 +pyflakes==2.5.0 # Transitive mypy dependencies mypy-extensions==0.4.3 typed-ast==1.4.0 -typing-extensions==3.7.4 +typing-extensions==4.5.0 # Transitive pylint dependencies -astroid==2.5.4 +astroid==2.15.4 lazy-object-proxy==1.4.0 toml==0.10.0 wrapt==1.12.1 diff --git a/python/tests/mypy.sh b/python/tests/mypy.sh index 5f980f563b..a33a3f58ab 100755 --- a/python/tests/mypy.sh +++ b/python/tests/mypy.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m mypy -p qemu +python3 -m mypy scripts/ diff --git a/python/tests/pylint.sh b/python/tests/pylint.sh index 03d64705a1..2b68da90df 100755 --- a/python/tests/pylint.sh +++ b/python/tests/pylint.sh @@ -1,3 +1,4 @@ #!/bin/sh -e # See commit message for environment variable explainer. SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint qemu/ +SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint scripts/ diff --git a/python/wheels/meson-0.63.3-py3-none-any.whl b/python/wheels/meson-0.63.3-py3-none-any.whl new file mode 100644 index 0000000000..8a191e3a20 --- /dev/null +++ b/python/wheels/meson-0.63.3-py3-none-any.whl Binary files differ |