From f9d2e0a3bd7ba2a693a892881f91cf53fa90cc71 Mon Sep 17 00:00:00 2001 From: John Snow Date: Tue, 6 Jun 2023 13:19:11 -0400 Subject: python: backport 'drop Python3.6 workarounds' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the minimum version is 3.7, drop some of the 3.6-specific hacks we've been carrying. A single remaining compatibility hack concerning 3.6's lack of @asynccontextmanager is addressed in the following commit. Signed-off-by: John Snow cherry picked from commit python-qemu-qmp@3e8e34e594cfc6b707e6f67959166acde4b421b8 Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé --- python/qemu/qmp/util.py | 107 ++---------------------------------------------- 1 file changed, 4 insertions(+), 103 deletions(-) (limited to 'python/qemu/qmp/util.py') diff --git a/python/qemu/qmp/util.py b/python/qemu/qmp/util.py index ca6225e9cd..0b3e781373 100644 --- a/python/qemu/qmp/util.py +++ b/python/qemu/qmp/util.py @@ -1,25 +1,15 @@ """ Miscellaneous Utilities -This module provides asyncio utilities and compatibility wrappers for -Python 3.6 to provide some features that otherwise become available in -Python 3.7+. - -Various logging and debugging utilities are also provided, such as -`exception_summary()` and `pretty_traceback()`, used primarily for -adding information into the logging stream. +This module provides asyncio and various logging and debugging +utilities, such as `exception_summary()` and `pretty_traceback()`, used +primarily for adding information into the logging stream. """ import asyncio import sys import traceback -from typing import ( - Any, - Coroutine, - Optional, - TypeVar, - cast, -) +from typing import TypeVar, cast T = TypeVar('T') @@ -79,95 +69,6 @@ def bottom_half(func: T) -> T: return func -# ------------------------------- -# Section: Compatibility Wrappers -# ------------------------------- - - -def create_task(coro: Coroutine[Any, Any, T], - loop: Optional[asyncio.AbstractEventLoop] = None - ) -> 'asyncio.Future[T]': - """ - Python 3.6-compatible `asyncio.create_task` wrapper. - - :param coro: The coroutine to execute in a task. - :param loop: Optionally, the loop to create the task in. - - :return: An `asyncio.Future` object. - """ - if sys.version_info >= (3, 7): - if loop is not None: - return loop.create_task(coro) - return asyncio.create_task(coro) # pylint: disable=no-member - - # Python 3.6: - return asyncio.ensure_future(coro, loop=loop) - - -def is_closing(writer: asyncio.StreamWriter) -> bool: - """ - Python 3.6-compatible `asyncio.StreamWriter.is_closing` wrapper. - - :param writer: The `asyncio.StreamWriter` object. - :return: `True` if the writer is closing, or closed. - """ - if sys.version_info >= (3, 7): - return writer.is_closing() - - # Python 3.6: - transport = writer.transport - assert isinstance(transport, asyncio.WriteTransport) - return transport.is_closing() - - -async def wait_closed(writer: asyncio.StreamWriter) -> None: - """ - Python 3.6-compatible `asyncio.StreamWriter.wait_closed` wrapper. - - :param writer: The `asyncio.StreamWriter` to wait on. - """ - if sys.version_info >= (3, 7): - await writer.wait_closed() - return - - # Python 3.6 - transport = writer.transport - assert isinstance(transport, asyncio.WriteTransport) - - while not transport.is_closing(): - await asyncio.sleep(0) - - # This is an ugly workaround, but it's the best I can come up with. - sock = transport.get_extra_info('socket') - - if sock is None: - # Our transport doesn't have a socket? ... - # Nothing we can reasonably do. - return - - while sock.fileno() != -1: - await asyncio.sleep(0) - - -def asyncio_run(coro: Coroutine[Any, Any, T], *, debug: bool = False) -> T: - """ - Python 3.6-compatible `asyncio.run` wrapper. - - :param coro: A coroutine to execute now. - :return: The return value from the coroutine. - """ - if sys.version_info >= (3, 7): - return asyncio.run(coro, debug=debug) - - # Python 3.6 - loop = asyncio.get_event_loop() - loop.set_debug(debug) - ret = loop.run_until_complete(coro) - loop.close() - - return ret - - # ---------------------------- # Section: Logging & Debugging # ---------------------------- -- cgit 1.4.1 From 85f223e5b031eb8ab63fbca314a4fb296a3a2632 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 3 Sep 2025 01:06:30 -0400 Subject: python: backport 'avoid creating additional event loops per thread' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit is two backports squashed into one to avoid regressions. python: *really* remove get_event_loop A prior commit, aa1ff990, switched away from using get_event_loop *by default*, but this is not good enough to avoid deprecation warnings as `asyncio.get_event_loop_policy().get_event_loop()` is *also* deprecated. Replace this mechanism with explicit calls to asyncio.get_new_loop() and revise the cleanup mechanisms in __del__ to match. python: avoid creating additional event loops per thread "Too hasty by far!", commit 21ce2ee4 attempted to avoid deprecated behavior altogether by calling new_event_loop() directly if there was no loop currently running, but this has the unfortunate side effect of potentially creating multiple event loops per thread if tests instantiate multiple QMP connections in a single thread. This behavior is apparently not well-defined and causes problems in some, but not all, combinations of Python interpreter version and platform environment. Partially revert to Daniel Berrange's original patch, which calls get_event_loop and simply suppresses the deprecation warning in Python<=3.13. This time, however, additionally register new loops created with new_event_loop() so that future calls to get_event_loop() will return the loop already created. Reported-by: Richard W.M. Jones Reported-by: Daniel P. Berrangé Signed-off-by: John Snow cherry picked from commit python-qemu-qmp@21ce2ee4f2df87efe84a27b9c5112487f4670622 cherry picked from commit python-qemu-qmp@c08fb82b38212956ccffc03fc6d015c3979f42fe Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé --- python/qemu/qmp/legacy.py | 46 ++++++++++++++++++++++++++++------------------ python/qemu/qmp/qmp_tui.py | 10 ++-------- python/qemu/qmp/util.py | 27 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 26 deletions(-) (limited to 'python/qemu/qmp/util.py') diff --git a/python/qemu/qmp/legacy.py b/python/qemu/qmp/legacy.py index 735d42971e..e46695ae2c 100644 --- a/python/qemu/qmp/legacy.py +++ b/python/qemu/qmp/legacy.py @@ -38,6 +38,7 @@ from typing import ( from .error import QMPError from .protocol import Runstate, SocketAddrT from .qmp_client import QMPClient +from .util import get_or_create_event_loop #: QMPMessage is an entire QMP message of any kind. @@ -86,17 +87,13 @@ class QEMUMonitorProtocol: "server argument should be False when passing a socket") self._qmp = QMPClient(nickname) - - try: - self._aloop = asyncio.get_running_loop() - except RuntimeError: - # No running loop; since this is a sync shim likely to be - # used in fully sync programs, create one if neccessary. - self._aloop = asyncio.get_event_loop_policy().get_event_loop() - self._address = address self._timeout: Optional[float] = None + # This is a sync shim intended for use in fully synchronous + # programs. Create and set an event loop if necessary. + self._aloop = get_or_create_event_loop() + if server: assert not isinstance(self._address, socket.socket) self._sync(self._qmp.start_server(self._address)) @@ -313,17 +310,30 @@ class QEMUMonitorProtocol: self._qmp.send_fd_scm(fd) def __del__(self) -> None: - if self._qmp.runstate == Runstate.IDLE: - return + if self._qmp.runstate != Runstate.IDLE: + self._qmp.logger.warning( + "QEMUMonitorProtocol object garbage collected without a prior " + "call to close()" + ) if not self._aloop.is_running(): - self.close() - else: - # Garbage collection ran while the event loop was running. - # Nothing we can do about it now, but if we don't raise our - # own error, the user will be treated to a lot of traceback - # they might not understand. + if self._qmp.runstate != Runstate.IDLE: + # If the user neglected to close the QMP session and we + # are not currently running in an asyncio context, we + # have the opportunity to close the QMP session. If we + # do not do this, the error messages presented over + # dangling async resources may not make any sense to the + # user. + self.close() + + if self._qmp.runstate != Runstate.IDLE: + # If QMP is still not quiesced, it means that the garbage + # collector ran from a context within the event loop and we + # are simply too late to take any corrective action. Raise + # our own error to give meaningful feedback to the user in + # order to prevent pages of asyncio stacktrace jargon. raise QMPError( - "QEMUMonitorProtocol.close()" - " was not called before object was garbage collected" + "QEMUMonitorProtocol.close() was not called before object was " + "garbage collected, and could not be closed due to GC running " + "in the event loop" ) diff --git a/python/qemu/qmp/qmp_tui.py b/python/qemu/qmp/qmp_tui.py index 12bdc17c99..d946c20513 100644 --- a/python/qemu/qmp/qmp_tui.py +++ b/python/qemu/qmp/qmp_tui.py @@ -51,7 +51,7 @@ from .legacy import QEMUMonitorProtocol, QMPBadPortError from .message import DeserializationError, Message, UnexpectedTypeError from .protocol import ConnectError, Runstate from .qmp_client import ExecInterruptedError, QMPClient -from .util import pretty_traceback +from .util import get_or_create_event_loop, pretty_traceback # The name of the signal that is used to update the history list @@ -387,13 +387,7 @@ class App(QMPClient): """ screen = urwid.raw_display.Screen() screen.set_terminal_properties(256) - - try: - self.aloop = asyncio.get_running_loop() - except RuntimeError: - # No running asyncio event loop. Create one if necessary. - self.aloop = asyncio.get_event_loop_policy().get_event_loop() - + self.aloop = get_or_create_event_loop() self.aloop.set_debug(debug) # Gracefully handle SIGTERM and SIGINT signals diff --git a/python/qemu/qmp/util.py b/python/qemu/qmp/util.py index 0b3e781373..47ec39a8b5 100644 --- a/python/qemu/qmp/util.py +++ b/python/qemu/qmp/util.py @@ -10,6 +10,7 @@ import asyncio import sys import traceback from typing import TypeVar, cast +import warnings T = TypeVar('T') @@ -20,6 +21,32 @@ T = TypeVar('T') # -------------------------- +def get_or_create_event_loop() -> asyncio.AbstractEventLoop: + """ + Return this thread's current event loop, or create a new one. + + This function behaves similarly to asyncio.get_event_loop() in + Python<=3.13, where if there is no event loop currently associated + with the current context, it will create and register one. It should + generally not be used in any asyncio-native applications. + """ + try: + with warnings.catch_warnings(): + # Python <= 3.13 will trigger deprecation warnings if no + # event loop is set, but will create and set a new loop. + warnings.simplefilter("ignore") + loop = asyncio.get_event_loop() + except RuntimeError: + # Python 3.14+: No event loop set for this thread, + # create and set one. + loop = asyncio.new_event_loop() + # Set this loop as the current thread's loop, to be returned + # by calls to get_event_loop() in the future. + asyncio.set_event_loop(loop) + + return loop + + async def flush(writer: asyncio.StreamWriter) -> None: """ Utility function to ensure a StreamWriter is *fully* drained. -- cgit 1.4.1 From f414048e32262830e06d50240b2f15b6e5857efe Mon Sep 17 00:00:00 2001 From: John Snow Date: Tue, 26 Aug 2025 13:04:50 -0400 Subject: python: synchronize qemu.qmp documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch collects comments and documentation changes from many commits in the python-qemu-qmp repository; bringing the qemu.git copy in bit-identical alignment with the standalone library *except* for several copyright messages that reference the "LICENSE" file which is, for QEMU, named "COPYING" instead and are therefore left unchanged. Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé --- python/qemu/qmp/__init__.py | 3 +- python/qemu/qmp/events.py | 35 +++++++++--- python/qemu/qmp/legacy.py | 4 +- python/qemu/qmp/message.py | 10 +++- python/qemu/qmp/models.py | 8 +-- python/qemu/qmp/protocol.py | 37 ++++++++---- python/qemu/qmp/qmp_client.py | 117 +++++++++++++++++++++++++++++--------- python/qemu/qmp/qmp_shell.py | 128 +++++++++++++++++++++++++++++++++--------- python/qemu/qmp/util.py | 9 ++- 9 files changed, 264 insertions(+), 87 deletions(-) (limited to 'python/qemu/qmp/util.py') diff --git a/python/qemu/qmp/__init__.py b/python/qemu/qmp/__init__.py index 69190d057a..058139dc3c 100644 --- a/python/qemu/qmp/__init__.py +++ b/python/qemu/qmp/__init__.py @@ -39,7 +39,8 @@ from .qmp_client import ExecInterruptedError, ExecuteError, QMPClient logging.getLogger('qemu.qmp').addHandler(logging.NullHandler()) -# The order of these fields impact the Sphinx documentation order. +# IMPORTANT: When modifying this list, update the Sphinx overview docs. +# Anything visible in the qemu.qmp namespace should be on the overview page. __all__ = ( # Classes, most to least important 'QMPClient', diff --git a/python/qemu/qmp/events.py b/python/qemu/qmp/events.py index 6658349619..cfb5f0ac62 100644 --- a/python/qemu/qmp/events.py +++ b/python/qemu/qmp/events.py @@ -12,7 +12,14 @@ EventListener Tutorial ---------------------- In all of the following examples, we assume that we have a `QMPClient` -instantiated named ``qmp`` that is already connected. +instantiated named ``qmp`` that is already connected. For example: + +.. code:: python + + from qemu.qmp import QMPClient + + qmp = QMPClient('example-vm') + await qmp.connect('127.0.0.1', 1234) `listener()` context blocks with one name @@ -87,7 +94,9 @@ This is analogous to the following code: event = listener.get() print(f"Event arrived: {event['event']}") -This event stream will never end, so these blocks will never terminate. +This event stream will never end, so these blocks will never +terminate. Even if the QMP connection errors out prematurely, this +listener will go silent without raising an error. Using asyncio.Task to concurrently retrieve events @@ -227,16 +236,20 @@ Clearing listeners .. code:: python await qmp.execute('stop') - qmp.events.clear() + discarded = qmp.events.clear() await qmp.execute('cont') event = await qmp.events.get() assert event['event'] == 'RESUME' + assert discarded[0]['event'] == 'STOP' `EventListener` objects are FIFO queues. If events are not consumed, they will remain in the queue until they are witnessed or discarded via `clear()`. FIFO queues will be drained automatically upon leaving a context block, or when calling `remove_listener()`. +Any events removed from the queue in this fashion will be returned by +the clear call. + Accessing listener history ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -350,6 +363,12 @@ While `listener()` is only capable of creating a single listener, break +Note that in the above example, we explicitly wait on jobA to conclude +first, and then wait for jobB to do the same. All we have guaranteed is +that the code that waits for jobA will not accidentally consume the +event intended for the jobB waiter. + + Extending the `EventListener` class ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -407,13 +426,13 @@ Experimental Interfaces & Design Issues These interfaces are not ones I am sure I will keep or otherwise modify heavily. -qmp.listener()’s type signature +qmp.listen()’s type signature ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -`listener()` does not return anything, because it was assumed the caller +`listen()` does not return anything, because it was assumed the caller already had a handle to the listener. However, for -``qmp.listener(EventListener())`` forms, the caller will not have saved -a handle to the listener. +``qmp.listen(EventListener())`` forms, the caller will not have saved a +handle to the listener. Because this function can accept *many* listeners, I found it hard to accurately type in a way where it could be used in both “one” or “many” @@ -633,7 +652,7 @@ class Events: def __init__(self) -> None: self._listeners: List[EventListener] = [] - #: Default, all-events `EventListener`. + #: Default, all-events `EventListener`. See `qmp.events` for more info. self.events: EventListener = EventListener() self.register_listener(self.events) diff --git a/python/qemu/qmp/legacy.py b/python/qemu/qmp/legacy.py index e46695ae2c..060ed0eb9d 100644 --- a/python/qemu/qmp/legacy.py +++ b/python/qemu/qmp/legacy.py @@ -293,8 +293,8 @@ class QEMUMonitorProtocol: """ Set the timeout for QMP RPC execution. - This timeout affects the `cmd`, `cmd_obj`, and `command` methods. - The `accept`, `pull_event` and `get_event` methods have their + This timeout affects the `cmd`, `cmd_obj`, and `cmd_raw` methods. + The `accept`, `pull_event` and `get_events` methods have their own configurable timeouts. :param timeout: diff --git a/python/qemu/qmp/message.py b/python/qemu/qmp/message.py index c2e9dd0dd5..dabb8ec360 100644 --- a/python/qemu/qmp/message.py +++ b/python/qemu/qmp/message.py @@ -28,7 +28,8 @@ class Message(MutableMapping[str, object]): be instantiated from either another mapping (like a `dict`), or from raw `bytes` that still need to be deserialized. - Once instantiated, it may be treated like any other MutableMapping:: + Once instantiated, it may be treated like any other + :py:obj:`~collections.abc.MutableMapping`:: >>> msg = Message(b'{"hello": "world"}') >>> assert msg['hello'] == 'world' @@ -50,12 +51,19 @@ class Message(MutableMapping[str, object]): >>> dict(msg) {'hello': 'world'} + Or pretty-printed:: + + >>> print(str(msg)) + { + "hello": "world" + } :param value: Initial value, if any. :param eager: When `True`, attempt to serialize or deserialize the initial value immediately, so that conversion exceptions are raised during the call to ``__init__()``. + """ # pylint: disable=too-many-ancestors diff --git a/python/qemu/qmp/models.py b/python/qemu/qmp/models.py index da52848d5a..7e0d0baf03 100644 --- a/python/qemu/qmp/models.py +++ b/python/qemu/qmp/models.py @@ -54,7 +54,7 @@ class Model: class Greeting(Model): """ - Defined in qmp-spec.rst, section "Server Greeting". + Defined in `interop/qmp-spec`, "Server Greeting" section. :param raw: The raw Greeting object. :raise KeyError: If any required fields are absent. @@ -82,7 +82,7 @@ class Greeting(Model): class QMPGreeting(Model): """ - Defined in qmp-spec.rst, section "Server Greeting". + Defined in `interop/qmp-spec`, "Server Greeting" section. :param raw: The raw QMPGreeting object. :raise KeyError: If any required fields are absent. @@ -104,7 +104,7 @@ class QMPGreeting(Model): class ErrorResponse(Model): """ - Defined in qmp-spec.rst, section "Error". + Defined in `interop/qmp-spec`, "Error" section. :param raw: The raw ErrorResponse object. :raise KeyError: If any required fields are absent. @@ -126,7 +126,7 @@ class ErrorResponse(Model): class ErrorInfo(Model): """ - Defined in qmp-spec.rst, section "Error". + Defined in `interop/qmp-spec`, "Error" section. :param raw: The raw ErrorInfo object. :raise KeyError: If any required fields are absent. diff --git a/python/qemu/qmp/protocol.py b/python/qemu/qmp/protocol.py index 4d8a39f014..219d092a79 100644 --- a/python/qemu/qmp/protocol.py +++ b/python/qemu/qmp/protocol.py @@ -79,6 +79,12 @@ class ConnectError(QMPError): This Exception always wraps a "root cause" exception that can be interrogated for additional information. + For example, when connecting to a non-existent socket:: + + await qmp.connect('not_found.sock') + # ConnectError: Failed to establish connection: + # [Errno 2] No such file or directory + :param error_message: Human-readable string describing the error. :param exc: The root-cause exception. """ @@ -102,8 +108,8 @@ class StateError(QMPError): An API command (connect, execute, etc) was issued at an inappropriate time. This error is raised when a command like - :py:meth:`~AsyncProtocol.connect()` is issued at an inappropriate - time. + :py:meth:`~AsyncProtocol.connect()` is called when the client is + already connected. :param error_message: Human-readable string describing the state violation. :param state: The actual `Runstate` seen at the time of the violation. @@ -298,7 +304,7 @@ class AsyncProtocol(Generic[T]): @upper_half async def runstate_changed(self) -> Runstate: """ - Wait for the `runstate` to change, then return that runstate. + Wait for the `runstate` to change, then return that `Runstate`. """ await self._runstate_event.wait() return self.runstate @@ -312,9 +318,9 @@ class AsyncProtocol(Generic[T]): """ Accept a connection and begin processing message queues. - If this call fails, `runstate` is guaranteed to be set back to `IDLE`. - This method is precisely equivalent to calling `start_server()` - followed by `accept()`. + If this call fails, `runstate` is guaranteed to be set back to + `IDLE`. This method is precisely equivalent to calling + `start_server()` followed by :py:meth:`~AsyncProtocol.accept()`. :param address: Address to listen on; UNIX socket path or TCP address/port. @@ -327,7 +333,8 @@ class AsyncProtocol(Generic[T]): This exception will wrap a more concrete one. In most cases, the wrapped exception will be `OSError` or `EOFError`. If a protocol-level failure occurs while establishing a new - session, the wrapped error may also be an `QMPError`. + session, the wrapped error may also be a `QMPError`. + """ await self.start_server(address, ssl) await self.accept() @@ -343,8 +350,8 @@ class AsyncProtocol(Generic[T]): This method starts listening for an incoming connection, but does not block waiting for a peer. This call will return immediately after binding and listening on a socket. A later - call to `accept()` must be made in order to finalize the - incoming connection. + call to :py:meth:`~AsyncProtocol.accept()` must be made in order + to finalize the incoming connection. :param address: Address to listen on; UNIX socket path or TCP address/port. @@ -367,10 +374,12 @@ class AsyncProtocol(Generic[T]): """ Accept an incoming connection and begin processing message queues. - If this call fails, `runstate` is guaranteed to be set back to `IDLE`. + Used after a previous call to `start_server()` to accept an + incoming connection. If this call fails, `runstate` is + guaranteed to be set back to `IDLE`. :raise StateError: When the `Runstate` is not `CONNECTING`. - :raise QMPError: When `start_server()` was not called yet. + :raise QMPError: When `start_server()` was not called first. :raise ConnectError: When a connection or session cannot be established. @@ -423,7 +432,11 @@ class AsyncProtocol(Generic[T]): If there was an exception that caused the reader/writers to terminate prematurely, it will be raised here. - :raise Exception: When the reader or writer terminate unexpectedly. + :raise Exception: + When the reader or writer terminate unexpectedly. You can + expect to see `EOFError` if the server hangs up, or + `OSError` for connection-related issues. If there was a QMP + protocol-level problem, `ProtocolError` will be seen. """ self.logger.debug("disconnect() called.") self._schedule_disconnect() diff --git a/python/qemu/qmp/qmp_client.py b/python/qemu/qmp/qmp_client.py index d826331b6d..8beccfe29d 100644 --- a/python/qemu/qmp/qmp_client.py +++ b/python/qemu/qmp/qmp_client.py @@ -70,6 +70,17 @@ class ExecuteError(QMPError): """ Exception raised by `QMPClient.execute()` on RPC failure. + This exception is raised when the server received, interpreted, and + replied to a command successfully; but the command itself returned a + failure status. + + For example:: + + await qmp.execute('block-dirty-bitmap-add', + {'node': 'foo', 'name': 'my_bitmap'}) + # qemu.qmp.qmp_client.ExecuteError: + # Cannot find device='foo' nor node-name='foo' + :param error_response: The RPC error response object. :param sent: The sent RPC message that caused the failure. :param received: The raw RPC error reply received. @@ -99,9 +110,22 @@ class ExecInterruptedError(QMPError): This error is raised when an `execute()` statement could not be completed. This can occur because the connection itself was - terminated before a reply was received. + terminated before a reply was received. The true cause of the + interruption will be available via `disconnect()`. + + The QMP protocol does not make it possible to know if a command + succeeded or failed after such an event; the client will need to + query the server to determine the state of the server on a + case-by-case basis. + + For example, ECONNRESET might look like this:: - The true cause of the interruption will be available via `disconnect()`. + try: + await qmp.execute('query-block') + # ExecInterruptedError: Disconnected + except ExecInterruptedError: + await qmp.disconnect() + # ConnectionResetError: [Errno 104] Connection reset by peer """ @@ -162,13 +186,14 @@ class BadReplyError(_MsgProtocolError): class QMPClient(AsyncProtocol[Message], Events): - """ - Implements a QMP client connection. + """Implements a QMP client connection. - QMP can be used to establish a connection as either the transport - client or server, though this class always acts as the QMP client. + `QMPClient` can be used to either connect or listen to a QMP server, + but always acts as the QMP client. - :param name: Optional nickname for the connection, used for logging. + :param name: + Optional nickname for the connection, used to differentiate + instances when logging. :param readbuflen: The maximum buffer length for reads and writes to and from the QMP @@ -178,14 +203,21 @@ class QMPClient(AsyncProtocol[Message], Events): Basic script-style usage looks like this:: - qmp = QMPClient('my_virtual_machine_name') - await qmp.connect(('127.0.0.1', 1234)) - ... - res = await qmp.execute('block-query') - ... - await qmp.disconnect() + import asyncio + from qemu.qmp import QMPClient + + async def main(): + qmp = QMPClient('my_virtual_machine_name') + await qmp.connect(('127.0.0.1', 1234)) + ... + res = await qmp.execute('query-block') + ... + await qmp.disconnect() - Basic async client-style usage looks like this:: + asyncio.run(main()) + + A more advanced example that starts to take advantage of asyncio + might look like this:: class Client: def __init__(self, name: str): @@ -205,6 +237,7 @@ class QMPClient(AsyncProtocol[Message], Events): await self.disconnect() See `qmp.events` for more detail on event handling patterns. + """ #: Logger object used for debugging messages. logger = logging.getLogger(__name__) @@ -224,10 +257,12 @@ class QMPClient(AsyncProtocol[Message], Events): Events.__init__(self) #: Whether or not to await a greeting after establishing a connection. + #: Defaults to True; QGA servers expect this to be False. self.await_greeting: bool = True - #: Whether or not to perform capabilities negotiation upon connection. - #: Implies `await_greeting`. + #: Whether or not to perform capabilities negotiation upon + #: connection. Implies `await_greeting`. Defaults to True; QGA + #: servers expect this to be False. self.negotiate: bool = True # Cached Greeting, if one was awaited. @@ -244,7 +279,13 @@ class QMPClient(AsyncProtocol[Message], Events): @property def greeting(self) -> Optional[Greeting]: - """The `Greeting` from the QMP server, if any.""" + """ + The `Greeting` from the QMP server, if any. + + Defaults to ``None``, and will be set after a greeting is + received during the connection process. It is reset at the start + of each connection attempt. + """ return self._greeting @upper_half @@ -385,7 +426,7 @@ class QMPClient(AsyncProtocol[Message], Events): # This is very likely a server parsing error. # It doesn't inherently belong to any pending execution. # Instead of performing clever recovery, just terminate. - # See "NOTE" in qmp-spec.rst, section "Error". + # See "NOTE" in interop/qmp-spec, "Error" section. raise ServerParseError( ("Server sent an error response without an ID, " "but there are no ID-less executions pending. " @@ -393,7 +434,7 @@ class QMPClient(AsyncProtocol[Message], Events): msg ) - # qmp-spec.rst, section "Commands Responses": + # qmp-spec.rst, "Commands Responses" section: # 'Clients should drop all the responses # that have an unknown "id" field.' self.logger.log( @@ -566,7 +607,7 @@ class QMPClient(AsyncProtocol[Message], Events): @require(Runstate.RUNNING) async def execute_msg(self, msg: Message) -> object: """ - Execute a QMP command and return its value. + Execute a QMP command on the server and return its value. :param msg: The QMP `Message` to execute. @@ -578,7 +619,9 @@ class QMPClient(AsyncProtocol[Message], Events): If the QMP `Message` does not have either the 'execute' or 'exec-oob' fields set. :raise ExecuteError: When the server returns an error response. - :raise ExecInterruptedError: if the connection was terminated early. + :raise ExecInterruptedError: + If the connection was disrupted before + receiving a reply from the server. """ if not ('execute' in msg or 'exec-oob' in msg): raise ValueError("Requires 'execute' or 'exec-oob' message") @@ -617,9 +660,11 @@ class QMPClient(AsyncProtocol[Message], Events): :param cmd: QMP command name. :param arguments: Arguments (if any). Must be JSON-serializable. - :param oob: If `True`, execute "out of band". + :param oob: + If `True`, execute "out of band". See `interop/qmp-spec` + section "Out-of-band execution". - :return: An executable QMP `Message`. + :return: A QMP `Message` that can be executed with `execute_msg()`. """ msg = Message({'exec-oob' if oob else 'execute': cmd}) if arguments is not None: @@ -631,18 +676,22 @@ class QMPClient(AsyncProtocol[Message], Events): arguments: Optional[Mapping[str, object]] = None, oob: bool = False) -> object: """ - Execute a QMP command and return its value. + Execute a QMP command on the server and return its value. :param cmd: QMP command name. :param arguments: Arguments (if any). Must be JSON-serializable. - :param oob: If `True`, execute "out of band". + :param oob: + If `True`, execute "out of band". See `interop/qmp-spec` + section "Out-of-band execution". :return: The command execution return value from the server. The type of object returned depends on the command that was issued, though most in QEMU return a `dict`. :raise ExecuteError: When the server returns an error response. - :raise ExecInterruptedError: if the connection was terminated early. + :raise ExecInterruptedError: + If the connection was disrupted before + receiving a reply from the server. """ msg = self.make_execute_msg(cmd, arguments, oob=oob) return await self.execute_msg(msg) @@ -650,8 +699,20 @@ class QMPClient(AsyncProtocol[Message], Events): @upper_half @require(Runstate.RUNNING) def send_fd_scm(self, fd: int) -> None: - """ - Send a file descriptor to the remote via SCM_RIGHTS. + """Send a file descriptor to the remote via SCM_RIGHTS. + + This method does not close the file descriptor. + + :param fd: The file descriptor to send to QEMU. + + This is an advanced feature of QEMU where file descriptors can + be passed from client to server. This is usually used as a + security measure to isolate the QEMU process from being able to + open its own files. See the QMP commands ``getfd`` and + ``add-fd`` for more information. + + See `socket.socket.sendmsg` for more information on the Python + implementation for sending file descriptors over a UNIX socket. """ assert self._writer is not None sock = self._writer.transport.get_extra_info('socket') diff --git a/python/qemu/qmp/qmp_shell.py b/python/qemu/qmp/qmp_shell.py index c923ff09e1..f818800568 100644 --- a/python/qemu/qmp/qmp_shell.py +++ b/python/qemu/qmp/qmp_shell.py @@ -10,9 +10,15 @@ # """ -Low-level QEMU shell on top of QMP. +qmp-shell - An interactive QEMU shell powered by QMP -usage: qmp-shell [-h] [-H] [-N] [-v] [-p] qmp_server +qmp-shell offers a simple shell with a convenient shorthand syntax as an +alternative to typing JSON by hand. This syntax is not standardized and +is not meant to be used as a scriptable interface. This shorthand *may* +change incompatibly in the future, and it is strongly encouraged to use +the QMP library to provide API-stable scripting when needed. + +usage: qmp-shell [-h] [-H] [-v] [-p] [-l LOGFILE] [-N] qmp_server positional arguments: qmp_server < UNIX socket path | TCP address:port > @@ -20,41 +26,52 @@ positional arguments: optional arguments: -h, --help show this help message and exit -H, --hmp Use HMP interface - -N, --skip-negotiation - Skip negotiate (for qemu-ga) -v, --verbose Verbose (echo commands sent and received) -p, --pretty Pretty-print JSON + -l LOGFILE, --logfile LOGFILE + Save log of all QMP messages to PATH + -N, --skip-negotiation + Skip negotiate (for qemu-ga) +Usage +----- -Start QEMU with: +First, start QEMU with:: -# qemu [...] -qmp unix:./qmp-sock,server + > qemu [...] -qmp unix:./qmp-sock,server=on[,wait=off] -Run the shell: +Then run the shell, passing the address of the socket:: -$ qmp-shell ./qmp-sock + > qmp-shell ./qmp-sock -Commands have the following format: +Syntax +------ - < command-name > [ arg-name1=arg1 ] ... [ arg-nameN=argN ] +Commands have the following format:: -For example: + < command-name > [ arg-name1=arg1 ] ... [ arg-nameN=argN ] -(QEMU) device_add driver=e1000 id=net1 -{'return': {}} -(QEMU) +For example, to add a network device:: -key=value pairs also support Python or JSON object literal subset notations, -without spaces. Dictionaries/objects {} are supported as are arrays []. + (QEMU) device_add driver=e1000 id=net1 + {'return': {}} + (QEMU) - example-command arg-name1={'key':'value','obj'={'prop':"value"}} +key=value pairs support either Python or JSON object literal notations, +**without spaces**. Dictionaries/objects ``{}`` are supported, as are +arrays ``[]``:: -Both JSON and Python formatting should work, including both styles of -string literal quotes. Both paradigms of literal values should work, -including null/true/false for JSON and None/True/False for Python. + example-command arg-name1={'key':'value','obj'={'prop':"value"}} +Either JSON or Python formatting for compound values works, including +both styles of string literal quotes (either single or double +quotes). Both paradigms of literal values are accepted, including +``null/true/false`` for JSON and ``None/True/False`` for Python. -Transactions have the following multi-line format: +Transactions +------------ + +Transactions have the following multi-line format:: transaction( action-name1 [ arg-name1=arg1 ] ... [arg-nameN=argN ] @@ -62,11 +79,11 @@ Transactions have the following multi-line format: action-nameN [ arg-name1=arg1 ] ... [arg-nameN=argN ] ) -One line transactions are also supported: +One line transactions are also supported:: transaction( action-name1 ... ) -For example: +For example:: (QEMU) transaction( TRANS> block-dirty-bitmap-add node=drive0 name=bitmap1 @@ -75,9 +92,35 @@ For example: {"return": {}} (QEMU) -Use the -v and -p options to activate the verbose and pretty-print options, -which will echo back the properly formatted JSON-compliant QMP that is being -sent to QEMU, which is useful for debugging and documentation generation. +Commands +-------- + +Autocomplete of command names using is supported. Pressing +at a blank CLI prompt will show you a list of all available commands +that the connected QEMU instance supports. + +For documentation on QMP commands and their arguments, please see +`qmp ref`. + +Events +------ + +qmp-shell will display events received from the server, but this version +does not do so asynchronously. To check for new events from the server, +press on a blank line:: + + (QEMU) ⏎ + {'timestamp': {'seconds': 1660071944, 'microseconds': 184667}, + 'event': 'STOP'} + +Display options +--------------- + +Use the -v and -p options to activate the verbose and pretty-print +options, which will echo back the properly formatted JSON-compliant QMP +that is being sent to QEMU. This is useful for debugging to see the +wire-level QMP data being exchanged, and generating output for use in +writing documentation for QEMU. """ import argparse @@ -525,6 +568,8 @@ def common_parser() -> argparse.ArgumentParser: help='Pretty-print JSON') parser.add_argument('-l', '--logfile', help='Save log of all QMP messages to PATH') + # NOTE: When changing arguments, update both this module docstring + # and the manpage synopsis in docs/man/qmp_shell.rst. return parser @@ -567,8 +612,35 @@ def main() -> None: def main_wrap() -> None: """ - qmp-shell-wrap entry point: parse command line arguments and - start the REPL. + qmp-shell-wrap - QEMU + qmp-shell launcher utility + + Launch QEMU and connect to it with `qmp-shell` in a single command. + CLI arguments will be forwarded to qemu, with additional arguments + added to allow `qmp-shell` to then connect to the recently launched + QEMU instance. + + usage: qmp-shell-wrap [-h] [-H] [-v] [-p] [-l LOGFILE] ... + + positional arguments: + command QEMU command line to invoke + + optional arguments: + -h, --help show this help message and exit + -H, --hmp Use HMP interface + -v, --verbose Verbose (echo commands sent and received) + -p, --pretty Pretty-print JSON + -l LOGFILE, --logfile LOGFILE + Save log of all QMP messages to PATH + + Usage + ----- + + Prepend "qmp-shell-wrap" to your usual QEMU command line:: + + > qmp-shell-wrap qemu-system-x86_64 -M q35 -m 4096 -display none + Welcome to the QMP low-level shell! + Connected + (QEMU) """ parser = common_parser() parser.add_argument('command', nargs=argparse.REMAINDER, diff --git a/python/qemu/qmp/util.py b/python/qemu/qmp/util.py index 47ec39a8b5..a8229e5524 100644 --- a/python/qemu/qmp/util.py +++ b/python/qemu/qmp/util.py @@ -49,7 +49,7 @@ def get_or_create_event_loop() -> asyncio.AbstractEventLoop: async def flush(writer: asyncio.StreamWriter) -> None: """ - Utility function to ensure a StreamWriter is *fully* drained. + Utility function to ensure an `asyncio.StreamWriter` is *fully* drained. `asyncio.StreamWriter.drain` only promises we will return to below the "high-water mark". This function ensures we flush the entire @@ -89,7 +89,7 @@ def bottom_half(func: T) -> T: These methods do not, in general, have the ability to directly report information to a caller’s context and will usually be - collected as a Task result instead. + collected as an `asyncio.Task` result instead. They must not call upper-half functions directly. """ @@ -105,8 +105,11 @@ def exception_summary(exc: BaseException) -> str: """ Return a summary string of an arbitrary exception. - It will be of the form "ExceptionType: Error Message", if the error + It will be of the form "ExceptionType: Error Message" if the error string is non-empty, and just "ExceptionType" otherwise. + + This code is based on CPython's implementation of + `traceback.TracebackException.format_exception_only`. """ name = type(exc).__qualname__ smod = type(exc).__module__ -- cgit 1.4.1