From 323c668934a650673548088c6718c633b57b6ce5 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:31 -0400 Subject: qapi: clean up encoding of section kinds We have several kinds of sections, and to tell them apart, we use Section attribute @tag and also the section object's Python type: type @tag untagged Section None @foo: ArgSection 'foo' Returns: Section 'Returns' Errors: Section 'Errors' Since: Section 'Since' TODO: Section 'TODO' Note: * @foo can be a member or a feature description, depending on context. * tag == 'Since' can be a Since: section or a member or feature description. If it's a Section, it's the former, and if it's an ArgSection, it's the latter. Clean this up as follows. Move the member or feature name to new ArgSection attribute @name, and replace @tag by enum @kind like this: type kind name untagged Section PLAIN @foo: ArgSection MEMBER 'foo' if member or argument ArgSection FEATURE 'foo' if feature Returns: Section RETURNS Errors: Section ERRORS Since: Section SINCE TODO: Section TODO The qapi-schema tests are updated to account for the new section names; "TODO" becomes "Todo" and `None` becomes "Plain" there. Signed-off-by: John Snow Message-ID: <20250311034303.75779-34-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index 61997fd21a..d622398f1d 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -35,6 +35,7 @@ from docutils.parsers.rst import Directive, directives from docutils.statemachine import ViewList from qapi.error import QAPIError, QAPISemError from qapi.gen import QAPISchemaVisitor +from qapi.parser import QAPIDoc from qapi.schema import QAPISchema from sphinx import addnodes @@ -258,11 +259,11 @@ class QAPISchemaGenRSTVisitor(QAPISchemaVisitor): """Return list of doctree nodes for additional sections""" nodelist = [] for section in doc.sections: - if section.tag and section.tag == 'TODO': + if section.kind == QAPIDoc.Kind.TODO: # Hide TODO: sections continue - if not section.tag: + if section.kind == QAPIDoc.Kind.PLAIN: # Sphinx cannot handle sectionless titles; # Instead, just append the results to the prior section. container = nodes.container() @@ -270,7 +271,7 @@ class QAPISchemaGenRSTVisitor(QAPISchemaVisitor): nodelist += container.children continue - snode = self._make_section(section.tag) + snode = self._make_section(section.kind.name.title()) self._parse_text_into_node(dedent(section.text), snode) nodelist.append(snode) return nodelist -- cgit 1.4.1 From 7d125c339d32cbef65404b621d3ca02cc67f3090 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:33 -0400 Subject: docs/qapidoc: add transmogrifier stub This commit adds a stubbed option to the qapi-doc directive that opts-in to the new rST generator; the implementation of which will follow in subsequent commits. Once all QAPI documents have been converted, this option and the old qapidoc implementation can be dropped. Note that moving code outside of the try...except block has no impact because the code moved outside of that block does not ever raise a QAPIError. Signed-off-by: John Snow Message-ID: <20250311034303.75779-36-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index d622398f1d..dc72f3fd3f 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -452,9 +452,9 @@ class QAPISchemaGenRSTVisitor(QAPISchemaVisitor): rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line) self._sphinx_directive.do_parse(rstlist, node) - def get_document_nodes(self): - """Return the list of docutils nodes which make up the document""" - return self._top_node.children + def get_document_node(self): + """Return the root docutils node which makes up the document""" + return self._top_node # Turn the black formatter on for the rest of the file. @@ -503,7 +503,10 @@ class QAPIDocDirective(NestedDirective): required_argument = 1 optional_arguments = 1 - option_spec = {"qapifile": directives.unchanged_required} + option_spec = { + "qapifile": directives.unchanged_required, + "transmogrify": directives.flag, + } has_content = False def new_serialno(self): @@ -511,10 +514,24 @@ class QAPIDocDirective(NestedDirective): env = self.state.document.settings.env return "qapidoc-%d" % env.new_serialno("qapidoc") + def transmogrify(self, schema) -> nodes.Element: + raise NotImplementedError + + def legacy(self, schema) -> nodes.Element: + vis = QAPISchemaGenRSTVisitor(self) + vis.visit_begin(schema) + for doc in schema.docs: + if doc.symbol: + vis.symbol(doc, schema.lookup_entity(doc.symbol)) + else: + vis.freeform(doc) + return vis.get_document_node() + def run(self): env = self.state.document.settings.env qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] qapidir = os.path.dirname(qapifile) + transmogrify = "transmogrify" in self.options try: schema = QAPISchema(qapifile) @@ -522,20 +539,18 @@ class QAPIDocDirective(NestedDirective): # First tell Sphinx about all the schema files that the # output documentation depends on (including 'qapifile' itself) schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) - - vis = QAPISchemaGenRSTVisitor(self) - vis.visit_begin(schema) - for doc in schema.docs: - if doc.symbol: - vis.symbol(doc, schema.lookup_entity(doc.symbol)) - else: - vis.freeform(doc) - return vis.get_document_nodes() except QAPIError as err: # Launder QAPI parse errors into Sphinx extension errors # so they are displayed nicely to the user raise ExtensionError(str(err)) from err + if transmogrify: + contentnode = self.transmogrify(schema) + else: + contentnode = self.legacy(schema) + + return contentnode.children + class QMPExample(CodeBlock, NestedDirective): """ -- cgit 1.4.1 From 7640410d5c2f7a37c8b6409f2d288807c99afd1e Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:34 -0400 Subject: docs/qapidoc: split old implementation into qapidoc_legacy.py This is being done primarily to be able to type check and delint the new implementation without needing to worry about fixing up the old implementation. I'm adding the new implementation into the existing file instead of into a new file so that when the dust settles, qapidoc.py will contain the full history of development on this generative module. This patch *should* be pure motion, give or take the import statements. Signed-off-by: John Snow Message-ID: <20250311034303.75779-37-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 420 +--------------------------------------- docs/sphinx/qapidoc_legacy.py | 439 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 441 insertions(+), 418 deletions(-) create mode 100644 docs/sphinx/qapidoc_legacy.py (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index dc72f3fd3f..f4abf42e7b 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -25,19 +25,16 @@ https://www.sphinx-doc.org/en/master/development/index.html """ import os -import re import sys -import textwrap from typing import List from docutils import nodes from docutils.parsers.rst import Directive, directives -from docutils.statemachine import ViewList -from qapi.error import QAPIError, QAPISemError +from qapi.error import QAPIError from qapi.gen import QAPISchemaVisitor -from qapi.parser import QAPIDoc from qapi.schema import QAPISchema +from qapidoc_legacy import QAPISchemaGenRSTVisitor from sphinx import addnodes from sphinx.directives.code import CodeBlock from sphinx.errors import ExtensionError @@ -48,419 +45,6 @@ from sphinx.util.nodes import nested_parse_with_titles __version__ = "1.0" -def dedent(text: str) -> str: - # Adjust indentation to make description text parse as paragraph. - - lines = text.splitlines(True) - if re.match(r"\s+", lines[0]): - # First line is indented; description started on the line after - # the name. dedent the whole block. - return textwrap.dedent(text) - - # Descr started on same line. Dedent line 2+. - return lines[0] + textwrap.dedent("".join(lines[1:])) - - -# Disable black auto-formatter until re-enabled: -# fmt: off - - -class QAPISchemaGenRSTVisitor(QAPISchemaVisitor): - """A QAPI schema visitor which generates docutils/Sphinx nodes - - This class builds up a tree of docutils/Sphinx nodes corresponding - to documentation for the various QAPI objects. To use it, first - create a QAPISchemaGenRSTVisitor object, and call its - visit_begin() method. Then you can call one of the two methods - 'freeform' (to add documentation for a freeform documentation - chunk) or 'symbol' (to add documentation for a QAPI symbol). These - will cause the visitor to build up the tree of document - nodes. Once you've added all the documentation via 'freeform' and - 'symbol' method calls, you can call 'get_document_nodes' to get - the final list of document nodes (in a form suitable for returning - from a Sphinx directive's 'run' method). - """ - def __init__(self, sphinx_directive): - self._cur_doc = None - self._sphinx_directive = sphinx_directive - self._top_node = nodes.section() - self._active_headings = [self._top_node] - - def _make_dlitem(self, term, defn): - """Return a dlitem node with the specified term and definition. - - term should be a list of Text and literal nodes. - defn should be one of: - - a string, which will be handed to _parse_text_into_node - - a list of Text and literal nodes, which will be put into - a paragraph node - """ - dlitem = nodes.definition_list_item() - dlterm = nodes.term('', '', *term) - dlitem += dlterm - if defn: - dldef = nodes.definition() - if isinstance(defn, list): - dldef += nodes.paragraph('', '', *defn) - else: - self._parse_text_into_node(defn, dldef) - dlitem += dldef - return dlitem - - def _make_section(self, title): - """Return a section node with optional title""" - section = nodes.section(ids=[self._sphinx_directive.new_serialno()]) - if title: - section += nodes.title(title, title) - return section - - def _nodes_for_ifcond(self, ifcond, with_if=True): - """Return list of Text, literal nodes for the ifcond - - Return a list which gives text like ' (If: condition)'. - If with_if is False, we don't return the "(If: " and ")". - """ - - doc = ifcond.docgen() - if not doc: - return [] - doc = nodes.literal('', doc) - if not with_if: - return [doc] - - nodelist = [nodes.Text(' ('), nodes.strong('', 'If: ')] - nodelist.append(doc) - nodelist.append(nodes.Text(')')) - return nodelist - - def _nodes_for_one_member(self, member): - """Return list of Text, literal nodes for this member - - Return a list of doctree nodes which give text like - 'name: type (optional) (If: ...)' suitable for use as the - 'term' part of a definition list item. - """ - term = [nodes.literal('', member.name)] - if member.type.doc_type(): - term.append(nodes.Text(': ')) - term.append(nodes.literal('', member.type.doc_type())) - if member.optional: - term.append(nodes.Text(' (optional)')) - if member.ifcond.is_present(): - term.extend(self._nodes_for_ifcond(member.ifcond)) - return term - - def _nodes_for_variant_when(self, branches, variant): - """Return list of Text, literal nodes for variant 'when' clause - - Return a list of doctree nodes which give text like - 'when tagname is variant (If: ...)' suitable for use in - the 'branches' part of a definition list. - """ - term = [nodes.Text(' when '), - nodes.literal('', branches.tag_member.name), - nodes.Text(' is '), - nodes.literal('', '"%s"' % variant.name)] - if variant.ifcond.is_present(): - term.extend(self._nodes_for_ifcond(variant.ifcond)) - return term - - def _nodes_for_members(self, doc, what, base=None, branches=None): - """Return list of doctree nodes for the table of members""" - dlnode = nodes.definition_list() - for section in doc.args.values(): - term = self._nodes_for_one_member(section.member) - # TODO drop fallbacks when undocumented members are outlawed - if section.text: - defn = dedent(section.text) - else: - defn = [nodes.Text('Not documented')] - - dlnode += self._make_dlitem(term, defn) - - if base: - dlnode += self._make_dlitem([nodes.Text('The members of '), - nodes.literal('', base.doc_type())], - None) - - if branches: - for v in branches.variants: - if v.type.name == 'q_empty': - continue - assert not v.type.is_implicit() - term = [nodes.Text('The members of '), - nodes.literal('', v.type.doc_type())] - term.extend(self._nodes_for_variant_when(branches, v)) - dlnode += self._make_dlitem(term, None) - - if not dlnode.children: - return [] - - section = self._make_section(what) - section += dlnode - return [section] - - def _nodes_for_enum_values(self, doc): - """Return list of doctree nodes for the table of enum values""" - seen_item = False - dlnode = nodes.definition_list() - for section in doc.args.values(): - termtext = [nodes.literal('', section.member.name)] - if section.member.ifcond.is_present(): - termtext.extend(self._nodes_for_ifcond(section.member.ifcond)) - # TODO drop fallbacks when undocumented members are outlawed - if section.text: - defn = dedent(section.text) - else: - defn = [nodes.Text('Not documented')] - - dlnode += self._make_dlitem(termtext, defn) - seen_item = True - - if not seen_item: - return [] - - section = self._make_section('Values') - section += dlnode - return [section] - - def _nodes_for_arguments(self, doc, arg_type): - """Return list of doctree nodes for the arguments section""" - if arg_type and not arg_type.is_implicit(): - assert not doc.args - section = self._make_section('Arguments') - dlnode = nodes.definition_list() - dlnode += self._make_dlitem( - [nodes.Text('The members of '), - nodes.literal('', arg_type.name)], - None) - section += dlnode - return [section] - - return self._nodes_for_members(doc, 'Arguments') - - def _nodes_for_features(self, doc): - """Return list of doctree nodes for the table of features""" - seen_item = False - dlnode = nodes.definition_list() - for section in doc.features.values(): - dlnode += self._make_dlitem( - [nodes.literal('', section.member.name)], dedent(section.text)) - seen_item = True - - if not seen_item: - return [] - - section = self._make_section('Features') - section += dlnode - return [section] - - def _nodes_for_sections(self, doc): - """Return list of doctree nodes for additional sections""" - nodelist = [] - for section in doc.sections: - if section.kind == QAPIDoc.Kind.TODO: - # Hide TODO: sections - continue - - if section.kind == QAPIDoc.Kind.PLAIN: - # Sphinx cannot handle sectionless titles; - # Instead, just append the results to the prior section. - container = nodes.container() - self._parse_text_into_node(section.text, container) - nodelist += container.children - continue - - snode = self._make_section(section.kind.name.title()) - self._parse_text_into_node(dedent(section.text), snode) - nodelist.append(snode) - return nodelist - - def _nodes_for_if_section(self, ifcond): - """Return list of doctree nodes for the "If" section""" - nodelist = [] - if ifcond.is_present(): - snode = self._make_section('If') - snode += nodes.paragraph( - '', '', *self._nodes_for_ifcond(ifcond, with_if=False) - ) - nodelist.append(snode) - return nodelist - - def _add_doc(self, typ, sections): - """Add documentation for a command/object/enum... - - We assume we're documenting the thing defined in self._cur_doc. - typ is the type of thing being added ("Command", "Object", etc) - - sections is a list of nodes for sections to add to the definition. - """ - - doc = self._cur_doc - snode = nodes.section(ids=[self._sphinx_directive.new_serialno()]) - snode += nodes.title('', '', *[nodes.literal(doc.symbol, doc.symbol), - nodes.Text(' (' + typ + ')')]) - self._parse_text_into_node(doc.body.text, snode) - for s in sections: - if s is not None: - snode += s - self._add_node_to_current_heading(snode) - - def visit_enum_type(self, name, info, ifcond, features, members, prefix): - doc = self._cur_doc - self._add_doc('Enum', - self._nodes_for_enum_values(doc) - + self._nodes_for_features(doc) - + self._nodes_for_sections(doc) - + self._nodes_for_if_section(ifcond)) - - def visit_object_type(self, name, info, ifcond, features, - base, members, branches): - doc = self._cur_doc - if base and base.is_implicit(): - base = None - self._add_doc('Object', - self._nodes_for_members(doc, 'Members', base, branches) - + self._nodes_for_features(doc) - + self._nodes_for_sections(doc) - + self._nodes_for_if_section(ifcond)) - - def visit_alternate_type(self, name, info, ifcond, features, - alternatives): - doc = self._cur_doc - self._add_doc('Alternate', - self._nodes_for_members(doc, 'Members') - + self._nodes_for_features(doc) - + self._nodes_for_sections(doc) - + self._nodes_for_if_section(ifcond)) - - def visit_command(self, name, info, ifcond, features, arg_type, - ret_type, gen, success_response, boxed, allow_oob, - allow_preconfig, coroutine): - doc = self._cur_doc - self._add_doc('Command', - self._nodes_for_arguments(doc, arg_type) - + self._nodes_for_features(doc) - + self._nodes_for_sections(doc) - + self._nodes_for_if_section(ifcond)) - - def visit_event(self, name, info, ifcond, features, arg_type, boxed): - doc = self._cur_doc - self._add_doc('Event', - self._nodes_for_arguments(doc, arg_type) - + self._nodes_for_features(doc) - + self._nodes_for_sections(doc) - + self._nodes_for_if_section(ifcond)) - - def symbol(self, doc, entity): - """Add documentation for one symbol to the document tree - - This is the main entry point which causes us to add documentation - nodes for a symbol (which could be a 'command', 'object', 'event', - etc). We do this by calling 'visit' on the schema entity, which - will then call back into one of our visit_* methods, depending - on what kind of thing this symbol is. - """ - self._cur_doc = doc - entity.visit(self) - self._cur_doc = None - - def _start_new_heading(self, heading, level): - """Start a new heading at the specified heading level - - Create a new section whose title is 'heading' and which is placed - in the docutils node tree as a child of the most recent level-1 - heading. Subsequent document sections (commands, freeform doc chunks, - etc) will be placed as children of this new heading section. - """ - if len(self._active_headings) < level: - raise QAPISemError(self._cur_doc.info, - 'Level %d subheading found outside a ' - 'level %d heading' - % (level, level - 1)) - snode = self._make_section(heading) - self._active_headings[level - 1] += snode - self._active_headings = self._active_headings[:level] - self._active_headings.append(snode) - return snode - - def _add_node_to_current_heading(self, node): - """Add the node to whatever the current active heading is""" - self._active_headings[-1] += node - - def freeform(self, doc): - """Add a piece of 'freeform' documentation to the document tree - - A 'freeform' document chunk doesn't relate to any particular - symbol (for instance, it could be an introduction). - - If the freeform document starts with a line of the form - '= Heading text', this is a section or subsection heading, with - the heading level indicated by the number of '=' signs. - """ - - # QAPIDoc documentation says free-form documentation blocks - # must have only a body section, nothing else. - assert not doc.sections - assert not doc.args - assert not doc.features - self._cur_doc = doc - - text = doc.body.text - if re.match(r'=+ ', text): - # Section/subsection heading (if present, will always be - # the first line of the block) - (heading, _, text) = text.partition('\n') - (leader, _, heading) = heading.partition(' ') - node = self._start_new_heading(heading, len(leader)) - if text == '': - return - else: - node = nodes.container() - - self._parse_text_into_node(text, node) - self._cur_doc = None - - def _parse_text_into_node(self, doctext, node): - """Parse a chunk of QAPI-doc-format text into the node - - The doc comment can contain most inline rST markup, including - bulleted and enumerated lists. - As an extra permitted piece of markup, @var will be turned - into ``var``. - """ - - # Handle the "@var means ``var`` case - doctext = re.sub(r'@([\w-]+)', r'``\1``', doctext) - - rstlist = ViewList() - for line in doctext.splitlines(): - # The reported line number will always be that of the start line - # of the doc comment, rather than the actual location of the error. - # Being more precise would require overhaul of the QAPIDoc class - # to track lines more exactly within all the sub-parts of the doc - # comment, as well as counting lines here. - rstlist.append(line, self._cur_doc.info.fname, - self._cur_doc.info.line) - # Append a blank line -- in some cases rST syntax errors get - # attributed to the line after one with actual text, and if there - # isn't anything in the ViewList corresponding to that then Sphinx - # 1.6's AutodocReporter will then misidentify the source/line location - # in the error message (usually attributing it to the top-level - # .rst file rather than the offending .json file). The extra blank - # line won't affect the rendered output. - rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line) - self._sphinx_directive.do_parse(rstlist, node) - - def get_document_node(self): - """Return the root docutils node which makes up the document""" - return self._top_node - - -# Turn the black formatter on for the rest of the file. -# fmt: on - - class QAPISchemaGenDepVisitor(QAPISchemaVisitor): """A QAPI schema visitor which adds Sphinx dependencies each module diff --git a/docs/sphinx/qapidoc_legacy.py b/docs/sphinx/qapidoc_legacy.py new file mode 100644 index 0000000000..679f38356b --- /dev/null +++ b/docs/sphinx/qapidoc_legacy.py @@ -0,0 +1,439 @@ +# coding=utf-8 +# +# QEMU qapidoc QAPI file parsing extension +# +# Copyright (c) 2020 Linaro +# +# This work is licensed under the terms of the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + +""" +qapidoc is a Sphinx extension that implements the qapi-doc directive + +The purpose of this extension is to read the documentation comments +in QAPI schema files, and insert them all into the current document. + +It implements one new rST directive, "qapi-doc::". +Each qapi-doc:: directive takes one argument, which is the +pathname of the schema file to process, relative to the source tree. + +The docs/conf.py file must set the qapidoc_srctree config value to +the root of the QEMU source tree. + +The Sphinx documentation on writing extensions is at: +https://www.sphinx-doc.org/en/master/development/index.html +""" + +import re +import textwrap + +from docutils import nodes +from docutils.statemachine import ViewList +from qapi.error import QAPISemError +from qapi.gen import QAPISchemaVisitor +from qapi.parser import QAPIDoc + + +def dedent(text: str) -> str: + # Adjust indentation to make description text parse as paragraph. + + lines = text.splitlines(True) + if re.match(r"\s+", lines[0]): + # First line is indented; description started on the line after + # the name. dedent the whole block. + return textwrap.dedent(text) + + # Descr started on same line. Dedent line 2+. + return lines[0] + textwrap.dedent("".join(lines[1:])) + + +class QAPISchemaGenRSTVisitor(QAPISchemaVisitor): + """A QAPI schema visitor which generates docutils/Sphinx nodes + + This class builds up a tree of docutils/Sphinx nodes corresponding + to documentation for the various QAPI objects. To use it, first + create a QAPISchemaGenRSTVisitor object, and call its + visit_begin() method. Then you can call one of the two methods + 'freeform' (to add documentation for a freeform documentation + chunk) or 'symbol' (to add documentation for a QAPI symbol). These + will cause the visitor to build up the tree of document + nodes. Once you've added all the documentation via 'freeform' and + 'symbol' method calls, you can call 'get_document_nodes' to get + the final list of document nodes (in a form suitable for returning + from a Sphinx directive's 'run' method). + """ + def __init__(self, sphinx_directive): + self._cur_doc = None + self._sphinx_directive = sphinx_directive + self._top_node = nodes.section() + self._active_headings = [self._top_node] + + def _make_dlitem(self, term, defn): + """Return a dlitem node with the specified term and definition. + + term should be a list of Text and literal nodes. + defn should be one of: + - a string, which will be handed to _parse_text_into_node + - a list of Text and literal nodes, which will be put into + a paragraph node + """ + dlitem = nodes.definition_list_item() + dlterm = nodes.term('', '', *term) + dlitem += dlterm + if defn: + dldef = nodes.definition() + if isinstance(defn, list): + dldef += nodes.paragraph('', '', *defn) + else: + self._parse_text_into_node(defn, dldef) + dlitem += dldef + return dlitem + + def _make_section(self, title): + """Return a section node with optional title""" + section = nodes.section(ids=[self._sphinx_directive.new_serialno()]) + if title: + section += nodes.title(title, title) + return section + + def _nodes_for_ifcond(self, ifcond, with_if=True): + """Return list of Text, literal nodes for the ifcond + + Return a list which gives text like ' (If: condition)'. + If with_if is False, we don't return the "(If: " and ")". + """ + + doc = ifcond.docgen() + if not doc: + return [] + doc = nodes.literal('', doc) + if not with_if: + return [doc] + + nodelist = [nodes.Text(' ('), nodes.strong('', 'If: ')] + nodelist.append(doc) + nodelist.append(nodes.Text(')')) + return nodelist + + def _nodes_for_one_member(self, member): + """Return list of Text, literal nodes for this member + + Return a list of doctree nodes which give text like + 'name: type (optional) (If: ...)' suitable for use as the + 'term' part of a definition list item. + """ + term = [nodes.literal('', member.name)] + if member.type.doc_type(): + term.append(nodes.Text(': ')) + term.append(nodes.literal('', member.type.doc_type())) + if member.optional: + term.append(nodes.Text(' (optional)')) + if member.ifcond.is_present(): + term.extend(self._nodes_for_ifcond(member.ifcond)) + return term + + def _nodes_for_variant_when(self, branches, variant): + """Return list of Text, literal nodes for variant 'when' clause + + Return a list of doctree nodes which give text like + 'when tagname is variant (If: ...)' suitable for use in + the 'branches' part of a definition list. + """ + term = [nodes.Text(' when '), + nodes.literal('', branches.tag_member.name), + nodes.Text(' is '), + nodes.literal('', '"%s"' % variant.name)] + if variant.ifcond.is_present(): + term.extend(self._nodes_for_ifcond(variant.ifcond)) + return term + + def _nodes_for_members(self, doc, what, base=None, branches=None): + """Return list of doctree nodes for the table of members""" + dlnode = nodes.definition_list() + for section in doc.args.values(): + term = self._nodes_for_one_member(section.member) + # TODO drop fallbacks when undocumented members are outlawed + if section.text: + defn = dedent(section.text) + else: + defn = [nodes.Text('Not documented')] + + dlnode += self._make_dlitem(term, defn) + + if base: + dlnode += self._make_dlitem([nodes.Text('The members of '), + nodes.literal('', base.doc_type())], + None) + + if branches: + for v in branches.variants: + if v.type.name == 'q_empty': + continue + assert not v.type.is_implicit() + term = [nodes.Text('The members of '), + nodes.literal('', v.type.doc_type())] + term.extend(self._nodes_for_variant_when(branches, v)) + dlnode += self._make_dlitem(term, None) + + if not dlnode.children: + return [] + + section = self._make_section(what) + section += dlnode + return [section] + + def _nodes_for_enum_values(self, doc): + """Return list of doctree nodes for the table of enum values""" + seen_item = False + dlnode = nodes.definition_list() + for section in doc.args.values(): + termtext = [nodes.literal('', section.member.name)] + if section.member.ifcond.is_present(): + termtext.extend(self._nodes_for_ifcond(section.member.ifcond)) + # TODO drop fallbacks when undocumented members are outlawed + if section.text: + defn = dedent(section.text) + else: + defn = [nodes.Text('Not documented')] + + dlnode += self._make_dlitem(termtext, defn) + seen_item = True + + if not seen_item: + return [] + + section = self._make_section('Values') + section += dlnode + return [section] + + def _nodes_for_arguments(self, doc, arg_type): + """Return list of doctree nodes for the arguments section""" + if arg_type and not arg_type.is_implicit(): + assert not doc.args + section = self._make_section('Arguments') + dlnode = nodes.definition_list() + dlnode += self._make_dlitem( + [nodes.Text('The members of '), + nodes.literal('', arg_type.name)], + None) + section += dlnode + return [section] + + return self._nodes_for_members(doc, 'Arguments') + + def _nodes_for_features(self, doc): + """Return list of doctree nodes for the table of features""" + seen_item = False + dlnode = nodes.definition_list() + for section in doc.features.values(): + dlnode += self._make_dlitem( + [nodes.literal('', section.member.name)], dedent(section.text)) + seen_item = True + + if not seen_item: + return [] + + section = self._make_section('Features') + section += dlnode + return [section] + + def _nodes_for_sections(self, doc): + """Return list of doctree nodes for additional sections""" + nodelist = [] + for section in doc.sections: + if section.kind == QAPIDoc.Kind.TODO: + # Hide TODO: sections + continue + + if section.kind == QAPIDoc.Kind.PLAIN: + # Sphinx cannot handle sectionless titles; + # Instead, just append the results to the prior section. + container = nodes.container() + self._parse_text_into_node(section.text, container) + nodelist += container.children + continue + + snode = self._make_section(section.kind.name.title()) + self._parse_text_into_node(dedent(section.text), snode) + nodelist.append(snode) + return nodelist + + def _nodes_for_if_section(self, ifcond): + """Return list of doctree nodes for the "If" section""" + nodelist = [] + if ifcond.is_present(): + snode = self._make_section('If') + snode += nodes.paragraph( + '', '', *self._nodes_for_ifcond(ifcond, with_if=False) + ) + nodelist.append(snode) + return nodelist + + def _add_doc(self, typ, sections): + """Add documentation for a command/object/enum... + + We assume we're documenting the thing defined in self._cur_doc. + typ is the type of thing being added ("Command", "Object", etc) + + sections is a list of nodes for sections to add to the definition. + """ + + doc = self._cur_doc + snode = nodes.section(ids=[self._sphinx_directive.new_serialno()]) + snode += nodes.title('', '', *[nodes.literal(doc.symbol, doc.symbol), + nodes.Text(' (' + typ + ')')]) + self._parse_text_into_node(doc.body.text, snode) + for s in sections: + if s is not None: + snode += s + self._add_node_to_current_heading(snode) + + def visit_enum_type(self, name, info, ifcond, features, members, prefix): + doc = self._cur_doc + self._add_doc('Enum', + self._nodes_for_enum_values(doc) + + self._nodes_for_features(doc) + + self._nodes_for_sections(doc) + + self._nodes_for_if_section(ifcond)) + + def visit_object_type(self, name, info, ifcond, features, + base, members, branches): + doc = self._cur_doc + if base and base.is_implicit(): + base = None + self._add_doc('Object', + self._nodes_for_members(doc, 'Members', base, branches) + + self._nodes_for_features(doc) + + self._nodes_for_sections(doc) + + self._nodes_for_if_section(ifcond)) + + def visit_alternate_type(self, name, info, ifcond, features, + alternatives): + doc = self._cur_doc + self._add_doc('Alternate', + self._nodes_for_members(doc, 'Members') + + self._nodes_for_features(doc) + + self._nodes_for_sections(doc) + + self._nodes_for_if_section(ifcond)) + + def visit_command(self, name, info, ifcond, features, arg_type, + ret_type, gen, success_response, boxed, allow_oob, + allow_preconfig, coroutine): + doc = self._cur_doc + self._add_doc('Command', + self._nodes_for_arguments(doc, arg_type) + + self._nodes_for_features(doc) + + self._nodes_for_sections(doc) + + self._nodes_for_if_section(ifcond)) + + def visit_event(self, name, info, ifcond, features, arg_type, boxed): + doc = self._cur_doc + self._add_doc('Event', + self._nodes_for_arguments(doc, arg_type) + + self._nodes_for_features(doc) + + self._nodes_for_sections(doc) + + self._nodes_for_if_section(ifcond)) + + def symbol(self, doc, entity): + """Add documentation for one symbol to the document tree + + This is the main entry point which causes us to add documentation + nodes for a symbol (which could be a 'command', 'object', 'event', + etc). We do this by calling 'visit' on the schema entity, which + will then call back into one of our visit_* methods, depending + on what kind of thing this symbol is. + """ + self._cur_doc = doc + entity.visit(self) + self._cur_doc = None + + def _start_new_heading(self, heading, level): + """Start a new heading at the specified heading level + + Create a new section whose title is 'heading' and which is placed + in the docutils node tree as a child of the most recent level-1 + heading. Subsequent document sections (commands, freeform doc chunks, + etc) will be placed as children of this new heading section. + """ + if len(self._active_headings) < level: + raise QAPISemError(self._cur_doc.info, + 'Level %d subheading found outside a ' + 'level %d heading' + % (level, level - 1)) + snode = self._make_section(heading) + self._active_headings[level - 1] += snode + self._active_headings = self._active_headings[:level] + self._active_headings.append(snode) + return snode + + def _add_node_to_current_heading(self, node): + """Add the node to whatever the current active heading is""" + self._active_headings[-1] += node + + def freeform(self, doc): + """Add a piece of 'freeform' documentation to the document tree + + A 'freeform' document chunk doesn't relate to any particular + symbol (for instance, it could be an introduction). + + If the freeform document starts with a line of the form + '= Heading text', this is a section or subsection heading, with + the heading level indicated by the number of '=' signs. + """ + + # QAPIDoc documentation says free-form documentation blocks + # must have only a body section, nothing else. + assert not doc.sections + assert not doc.args + assert not doc.features + self._cur_doc = doc + + text = doc.body.text + if re.match(r'=+ ', text): + # Section/subsection heading (if present, will always be + # the first line of the block) + (heading, _, text) = text.partition('\n') + (leader, _, heading) = heading.partition(' ') + node = self._start_new_heading(heading, len(leader)) + if text == '': + return + else: + node = nodes.container() + + self._parse_text_into_node(text, node) + self._cur_doc = None + + def _parse_text_into_node(self, doctext, node): + """Parse a chunk of QAPI-doc-format text into the node + + The doc comment can contain most inline rST markup, including + bulleted and enumerated lists. + As an extra permitted piece of markup, @var will be turned + into ``var``. + """ + + # Handle the "@var means ``var`` case + doctext = re.sub(r'@([\w-]+)', r'``\1``', doctext) + + rstlist = ViewList() + for line in doctext.splitlines(): + # The reported line number will always be that of the start line + # of the doc comment, rather than the actual location of the error. + # Being more precise would require overhaul of the QAPIDoc class + # to track lines more exactly within all the sub-parts of the doc + # comment, as well as counting lines here. + rstlist.append(line, self._cur_doc.info.fname, + self._cur_doc.info.line) + # Append a blank line -- in some cases rST syntax errors get + # attributed to the line after one with actual text, and if there + # isn't anything in the ViewList corresponding to that then Sphinx + # 1.6's AutodocReporter will then misidentify the source/line location + # in the error message (usually attributing it to the top-level + # .rst file rather than the offending .json file). The extra blank + # line won't affect the rendered output. + rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line) + self._sphinx_directive.do_parse(rstlist, node) + + def get_document_node(self): + """Return the root docutils node which makes up the document""" + return self._top_node -- cgit 1.4.1 From 45d483a851051bcea3a32b1ce7367180a8cf7bae Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:35 -0400 Subject: docs/qapidoc: Fix static typing on qapidoc.py Now that the legacy code is factored out, fix up the typing on the remaining code in qapidoc.py. Add a type ignore to qapi_legacy.py to prevent the errors there from bleeding out into qapidoc.py. Signed-off-by: John Snow Message-ID: <20250311034303.75779-38-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 40 +++++++++++++++++++++++++--------------- docs/sphinx/qapidoc_legacy.py | 1 + 2 files changed, 26 insertions(+), 15 deletions(-) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index f4abf42e7b..5246832b68 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -24,17 +24,18 @@ The Sphinx documentation on writing extensions is at: https://www.sphinx-doc.org/en/master/development/index.html """ +from __future__ import annotations + import os import sys -from typing import List +from typing import TYPE_CHECKING from docutils import nodes from docutils.parsers.rst import Directive, directives from qapi.error import QAPIError -from qapi.gen import QAPISchemaVisitor -from qapi.schema import QAPISchema +from qapi.schema import QAPISchema, QAPISchemaVisitor -from qapidoc_legacy import QAPISchemaGenRSTVisitor +from qapidoc_legacy import QAPISchemaGenRSTVisitor # type: ignore from sphinx import addnodes from sphinx.directives.code import CodeBlock from sphinx.errors import ExtensionError @@ -42,6 +43,15 @@ from sphinx.util.docutils import switch_source_input from sphinx.util.nodes import nested_parse_with_titles +if TYPE_CHECKING: + from typing import Any, List, Sequence + + from docutils.statemachine import StringList + + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata + + __version__ = "1.0" @@ -53,11 +63,11 @@ class QAPISchemaGenDepVisitor(QAPISchemaVisitor): schema file associated with each module in the QAPI input. """ - def __init__(self, env, qapidir): + def __init__(self, env: Any, qapidir: str) -> None: self._env = env self._qapidir = qapidir - def visit_module(self, name): + def visit_module(self, name: str) -> None: if name != "./builtin": qapifile = self._qapidir + "/" + name self._env.note_dependency(os.path.abspath(qapifile)) @@ -65,10 +75,10 @@ class QAPISchemaGenDepVisitor(QAPISchemaVisitor): class NestedDirective(Directive): - def run(self): + def run(self) -> Sequence[nodes.Node]: raise NotImplementedError - def do_parse(self, rstlist, node): + def do_parse(self, rstlist: StringList, node: nodes.Node) -> None: """ Parse rST source lines and add them to the specified node @@ -93,15 +103,15 @@ class QAPIDocDirective(NestedDirective): } has_content = False - def new_serialno(self): + def new_serialno(self) -> str: """Return a unique new ID string suitable for use as a node's ID""" env = self.state.document.settings.env return "qapidoc-%d" % env.new_serialno("qapidoc") - def transmogrify(self, schema) -> nodes.Element: + def transmogrify(self, schema: QAPISchema) -> nodes.Element: raise NotImplementedError - def legacy(self, schema) -> nodes.Element: + def legacy(self, schema: QAPISchema) -> nodes.Element: vis = QAPISchemaGenRSTVisitor(self) vis.visit_begin(schema) for doc in schema.docs: @@ -109,9 +119,9 @@ class QAPIDocDirective(NestedDirective): vis.symbol(doc, schema.lookup_entity(doc.symbol)) else: vis.freeform(doc) - return vis.get_document_node() + return vis.get_document_node() # type: ignore - def run(self): + def run(self) -> Sequence[nodes.Node]: env = self.state.document.settings.env qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] qapidir = os.path.dirname(qapifile) @@ -185,7 +195,7 @@ class QMPExample(CodeBlock, NestedDirective): ) return node - def admonition_wrap(self, *content) -> List[nodes.Node]: + def admonition_wrap(self, *content: nodes.Node) -> List[nodes.Node]: title = "Example:" if "title" in self.options: title = f"{title} {self.options['title']}" @@ -231,7 +241,7 @@ class QMPExample(CodeBlock, NestedDirective): return self.admonition_wrap(*content_nodes) -def setup(app): +def setup(app: Sphinx) -> ExtensionMetadata: """Register qapi-doc directive with Sphinx""" app.add_config_value("qapidoc_srctree", None, "env") app.add_directive("qapi-doc", QAPIDocDirective) diff --git a/docs/sphinx/qapidoc_legacy.py b/docs/sphinx/qapidoc_legacy.py index 679f38356b..13520f4c26 100644 --- a/docs/sphinx/qapidoc_legacy.py +++ b/docs/sphinx/qapidoc_legacy.py @@ -1,4 +1,5 @@ # coding=utf-8 +# type: ignore # # QEMU qapidoc QAPI file parsing extension # -- cgit 1.4.1 From bd7c1496aa695f7b7a560a2caf29f4b84f8752af Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:37 -0400 Subject: docs/qapidoc: add transmogrifier class stub Add the beginnings of the Transmogrifier class by adding the rST conversion helpers that will be used to build the virtual rST document. This version of the class does not actually "do anything" yet; each individual feature is added one-at-a-time in the forthcoming commits. Signed-off-by: John Snow Message-ID: <20250311034303.75779-40-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 3 deletions(-) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index 5246832b68..c243bb6faa 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -26,14 +26,17 @@ https://www.sphinx-doc.org/en/master/development/index.html from __future__ import annotations +from contextlib import contextmanager import os import sys from typing import TYPE_CHECKING from docutils import nodes from docutils.parsers.rst import Directive, directives +from docutils.statemachine import StringList from qapi.error import QAPIError from qapi.schema import QAPISchema, QAPISchemaVisitor +from qapi.source import QAPISourceInfo from qapidoc_legacy import QAPISchemaGenRSTVisitor # type: ignore from sphinx import addnodes @@ -44,9 +47,12 @@ from sphinx.util.nodes import nested_parse_with_titles if TYPE_CHECKING: - from typing import Any, List, Sequence - - from docutils.statemachine import StringList + from typing import ( + Any, + Generator, + List, + Sequence, + ) from sphinx.application import Sphinx from sphinx.util.typing import ExtensionMetadata @@ -55,6 +61,67 @@ if TYPE_CHECKING: __version__ = "1.0" +class Transmogrifier: + def __init__(self) -> None: + self._result = StringList() + self.indent = 0 + + # General-purpose rST generation functions + + def get_indent(self) -> str: + return " " * self.indent + + @contextmanager + def indented(self) -> Generator[None]: + self.indent += 1 + try: + yield + finally: + self.indent -= 1 + + def add_line_raw(self, line: str, source: str, *lineno: int) -> None: + """Append one line of generated reST to the output.""" + + # NB: Sphinx uses zero-indexed lines; subtract one. + lineno = tuple((n - 1 for n in lineno)) + + if line.strip(): + # not a blank line + self._result.append( + self.get_indent() + line.rstrip("\n"), source, *lineno + ) + else: + self._result.append("", source, *lineno) + + def add_line(self, content: str, info: QAPISourceInfo) -> None: + # NB: We *require* an info object; this works out OK because we + # don't document built-in objects that don't have + # one. Everything else should. + self.add_line_raw(content, info.fname, info.line) + + def add_lines( + self, + content: str, + info: QAPISourceInfo, + ) -> None: + lines = content.splitlines(True) + for i, line in enumerate(lines): + self.add_line_raw(line, info.fname, info.line + i) + + def ensure_blank_line(self) -> None: + # Empty document -- no blank line required. + if not self._result: + return + + # Last line isn't blank, add one. + if self._result[-1].strip(): # pylint: disable=no-member + fname, line = self._result.info(-1) + assert isinstance(line, int) + # New blank line is credited to one-after the current last line. + # +2: correct for zero/one index, then increment by one. + self.add_line_raw("", fname, line + 2) + + class QAPISchemaGenDepVisitor(QAPISchemaVisitor): """A QAPI schema visitor which adds Sphinx dependencies each module -- cgit 1.4.1 From 5edd7411c4f25a43400c5b8d6e5647603942f36b Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:38 -0400 Subject: docs/qapidoc: add visit_module() method This method annotates the start of a new module, crediting the source location to the first line of the module file. Signed-off-by: John Snow Message-ID: <20250311034303.75779-41-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index c243bb6faa..6de8c90054 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -28,6 +28,7 @@ from __future__ import annotations from contextlib import contextmanager import os +from pathlib import Path import sys from typing import TYPE_CHECKING @@ -121,6 +122,14 @@ class Transmogrifier: # +2: correct for zero/one index, then increment by one. self.add_line_raw("", fname, line + 2) + # Transmogrification core methods + + def visit_module(self, path: str) -> None: + name = Path(path).stem + # module directives are credited to the first line of a module file. + self.add_line_raw(f".. qapi:module:: {name}", path, 1) + self.ensure_blank_line() + class QAPISchemaGenDepVisitor(QAPISchemaVisitor): """A QAPI schema visitor which adds Sphinx dependencies each module -- cgit 1.4.1 From f0b2fe99f63023e25b827449d54ae3623339217e Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:40 -0400 Subject: docs/qapidoc: add visit_freeform() method Add the transmogrifier implementation for converting freeform doc blocks to rST. Signed-off-by: John Snow Message-ID: <20250311034303.75779-43-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index 6de8c90054..ddad604145 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -29,6 +29,7 @@ from __future__ import annotations from contextlib import contextmanager import os from pathlib import Path +import re import sys from typing import TYPE_CHECKING @@ -55,6 +56,8 @@ if TYPE_CHECKING: Sequence, ) + from qapi.parser import QAPIDoc + from sphinx.application import Sphinx from sphinx.util.typing import ExtensionMetadata @@ -130,6 +133,47 @@ class Transmogrifier: self.add_line_raw(f".. qapi:module:: {name}", path, 1) self.ensure_blank_line() + def visit_freeform(self, doc: QAPIDoc) -> None: + # TODO: Once the old qapidoc transformer is deprecated, freeform + # sections can be updated to pure rST, and this transformed removed. + # + # For now, translate our micro-format into rST. Code adapted + # from Peter Maydell's freeform(). + + assert len(doc.all_sections) == 1, doc.all_sections + body = doc.all_sections[0] + text = body.text + info = doc.info + + if re.match(r"=+ ", text): + # Section/subsection heading (if present, will always be the + # first line of the block) + (heading, _, text) = text.partition("\n") + (leader, _, heading) = heading.partition(" ") + # Implicit +1 for heading in the containing .rst doc + level = len(leader) + 1 + + # https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#sections + markers = ' #*=_^"' + overline = level <= 2 + marker = markers[level] + + self.ensure_blank_line() + # This credits all 2 or 3 lines to the single source line. + if overline: + self.add_line(marker * len(heading), info) + self.add_line(heading, info) + self.add_line(marker * len(heading), info) + self.ensure_blank_line() + + # Eat blank line(s) and advance info + trimmed = text.lstrip("\n") + text = trimmed + info = info.next_line(len(text) - len(trimmed) + 1) + + self.add_lines(text, info) + self.ensure_blank_line() + class QAPISchemaGenDepVisitor(QAPISchemaVisitor): """A QAPI schema visitor which adds Sphinx dependencies each module -- cgit 1.4.1 From 803df114fd68b5e7a3d6c60b162c0013cf6966e6 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:41 -0400 Subject: docs/qapidoc: add preamble() method This method adds the options/preamble to each definition block. Notably, :since: and :ifcond: are added, as are any "special features" such as :deprecated: and :unstable:. If conditionals, if attached to special features, are currently unhandled in this patch and will be addressed at a future date. We currently do not have any if conditionals attached to special features. Signed-off-by: John Snow Message-ID: <20250311034303.75779-44-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index ddad604145..f56aa6d1fd 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -37,7 +37,12 @@ from docutils import nodes from docutils.parsers.rst import Directive, directives from docutils.statemachine import StringList from qapi.error import QAPIError -from qapi.schema import QAPISchema, QAPISchemaVisitor +from qapi.parser import QAPIDoc +from qapi.schema import ( + QAPISchema, + QAPISchemaDefinition, + QAPISchemaVisitor, +) from qapi.source import QAPISourceInfo from qapidoc_legacy import QAPISchemaGenRSTVisitor # type: ignore @@ -56,8 +61,6 @@ if TYPE_CHECKING: Sequence, ) - from qapi.parser import QAPIDoc - from sphinx.application import Sphinx from sphinx.util.typing import ExtensionMetadata @@ -125,6 +128,38 @@ class Transmogrifier: # +2: correct for zero/one index, then increment by one. self.add_line_raw("", fname, line + 2) + # Transmogrification helpers + + def preamble(self, ent: QAPISchemaDefinition) -> None: + """ + Generate option lines for QAPI entity directives. + """ + if ent.doc and ent.doc.since: + assert ent.doc.since.kind == QAPIDoc.Kind.SINCE + # Generated from the entity's docblock; info location is exact. + self.add_line(f":since: {ent.doc.since.text}", ent.doc.since.info) + + if ent.ifcond.is_present(): + doc = ent.ifcond.docgen() + assert ent.info + # Generated from entity definition; info location is approximate. + self.add_line(f":ifcond: {doc}", ent.info) + + # Hoist special features such as :deprecated: and :unstable: + # into the options block for the entity. If, in the future, new + # special features are added, qapi-domain will chirp about + # unrecognized options and fail until they are handled in + # qapi-domain. + for feat in ent.features: + if feat.is_special(): + # FIXME: handle ifcond if present. How to display that + # information is TBD. + # Generated from entity def; info location is approximate. + assert feat.info + self.add_line(f":{feat.name}:", feat.info) + + self.ensure_blank_line() + # Transmogrification core methods def visit_module(self, path: str) -> None: -- cgit 1.4.1 From 56e1adf293037ec3324a7f20d54d50cd671dc281 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:42 -0400 Subject: docs/qapidoc: add visit_paragraph() method This transforms "formerly known as untagged sections" into our pure intermediate rST format. These sections are already pure rST, so this method doesn't do a whole lot except ensure appropriate newlines. Signed-off-by: John Snow Message-ID: <20250311034303.75779-45-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index f56aa6d1fd..a9f98d4657 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -130,6 +130,15 @@ class Transmogrifier: # Transmogrification helpers + def visit_paragraph(self, section: QAPIDoc.Section) -> None: + # Squelch empty paragraphs. + if not section.text: + return + + self.ensure_blank_line() + self.add_lines(section.text, section.info) + self.ensure_blank_line() + def preamble(self, ent: QAPISchemaDefinition) -> None: """ Generate option lines for QAPI entity directives. -- cgit 1.4.1 From e9fbf1a0c6c2df9c53bb577b4245cf48914687fe Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:43 -0400 Subject: docs/qapidoc: add visit_errors() method Notably, this method does not currently address the formatting issues present with the "errors" section in QAPIDoc and just vomits the text verbatim into the rST doc, with somewhat inconsistent results. To be addressed in a future patch. Signed-off-by: John Snow Message-ID: <20250311034303.75779-46-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index a9f98d4657..c17cb9f9b1 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -139,6 +139,12 @@ class Transmogrifier: self.add_lines(section.text, section.info) self.ensure_blank_line() + def visit_errors(self, section: QAPIDoc.Section) -> None: + # FIXME: the formatting for errors may be inconsistent and may + # or may not require different newline placement to ensure + # proper rendering as a nested list. + self.add_lines(f":error:\n{section.text}", section.info) + def preamble(self, ent: QAPISchemaDefinition) -> None: """ Generate option lines for QAPI entity directives. -- cgit 1.4.1 From 3a396a865be6a55322baa854c470c43c0a9f64e6 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:44 -0400 Subject: docs/qapidoc: add format_type() method This method is responsible for generating a type name for a given member with the correct annotations for the QAPI domain. Features and enums do not *have* types, so they return None. Everything else returns the type name with a "?" suffix if that type is optional, and ensconced in [brackets] if it's an array type. Signed-off-by: John Snow Message-ID: <20250311034303.75779-47-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index c17cb9f9b1..5144bb965a 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -40,7 +40,13 @@ from qapi.error import QAPIError from qapi.parser import QAPIDoc from qapi.schema import ( QAPISchema, + QAPISchemaArrayType, QAPISchemaDefinition, + QAPISchemaEnumMember, + QAPISchemaFeature, + QAPISchemaMember, + QAPISchemaObjectTypeMember, + QAPISchemaType, QAPISchemaVisitor, ) from qapi.source import QAPISourceInfo @@ -58,7 +64,9 @@ if TYPE_CHECKING: Any, Generator, List, + Optional, Sequence, + Union, ) from sphinx.application import Sphinx @@ -128,6 +136,30 @@ class Transmogrifier: # +2: correct for zero/one index, then increment by one. self.add_line_raw("", fname, line + 2) + def format_type( + self, ent: Union[QAPISchemaDefinition | QAPISchemaMember] + ) -> Optional[str]: + if isinstance(ent, (QAPISchemaEnumMember, QAPISchemaFeature)): + return None + + qapi_type = ent + optional = False + if isinstance(ent, QAPISchemaObjectTypeMember): + qapi_type = ent.type + optional = ent.optional + + if isinstance(qapi_type, QAPISchemaArrayType): + ret = f"[{qapi_type.element_type.doc_type()}]" + else: + assert isinstance(qapi_type, QAPISchemaType) + tmp = qapi_type.doc_type() + assert tmp + ret = tmp + if optional: + ret += "?" + + return ret + # Transmogrification helpers def visit_paragraph(self, section: QAPIDoc.Section) -> None: -- cgit 1.4.1 From 604df9bb001fc66f77d349ce63e870af84b42d96 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:45 -0400 Subject: docs/qapidoc: add add_field() and generate_field() helper methods These are simple rST generation methods that assist in getting the types and formatting correct for a field list entry. add_field() is a more raw, direct call while generate_field() is intended to be used for generating the correct field from a member object. Signed-off-by: John Snow Message-ID: <20250311034303.75779-48-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index 5144bb965a..2f85fe0bc3 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -136,6 +136,20 @@ class Transmogrifier: # +2: correct for zero/one index, then increment by one. self.add_line_raw("", fname, line + 2) + def add_field( + self, + kind: str, + name: str, + body: str, + info: QAPISourceInfo, + typ: Optional[str] = None, + ) -> None: + if typ: + text = f":{kind} {typ} {name}: {body}" + else: + text = f":{kind} {name}: {body}" + self.add_lines(text, info) + def format_type( self, ent: Union[QAPISchemaDefinition | QAPISchemaMember] ) -> Optional[str]: @@ -160,6 +174,16 @@ class Transmogrifier: return ret + def generate_field( + self, + kind: str, + member: QAPISchemaMember, + body: str, + info: QAPISourceInfo, + ) -> None: + typ = self.format_type(member) + self.add_field(kind, member.name, body, info, typ) + # Transmogrification helpers def visit_paragraph(self, section: QAPIDoc.Section) -> None: -- cgit 1.4.1 From 6c43b008c4b6338f49b6dffb82437285bf98b97a Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:46 -0400 Subject: docs/qapidoc: add visit_feature() method This adds a simple ":feat name: lorem ipsum ..." line to the generated rST document, so at the moment it's only for "top level" features. Features not attached directly to a QAPI definition are not currently handled! This is a small regression over the prior documentation generator that will be addressed in a future patch. Signed-off-by: John Snow Message-ID: <20250311034303.75779-49-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index 2f85fe0bc3..208d7ca144 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -195,6 +195,15 @@ class Transmogrifier: self.add_lines(section.text, section.info) self.ensure_blank_line() + def visit_feature(self, section: QAPIDoc.ArgSection) -> None: + # FIXME - ifcond for features is not handled at all yet! + # Proposal: decorate the right-hand column with some graphical + # element to indicate conditional availability? + assert section.text # Guaranteed by parser.py + assert section.member + + self.generate_field("feat", section.member, section.text, section.info) + def visit_errors(self, section: QAPIDoc.Section) -> None: # FIXME: the formatting for errors may be inconsistent and may # or may not require different newline placement to ensure -- cgit 1.4.1 From 38a349ff5b9ae583fe8a66e3e507ea9954b1aeb1 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:47 -0400 Subject: docs/qapidoc: prepare to record entity being transmogrified Prepare to keep a record of which entity we're working on documenting for the purposes of being able to change certain generative features conditionally and create stronger assertions. If you find yourself asking: "Wait, but where does the current entity actually get recorded?!", you're right! That part comes with the visit_entity() implementation, which gets added later. This patch is front-loaded for the sake of type checking in the forthcoming commits before visit_entity() is ready to be added. Signed-off-by: John Snow Message-ID: <20250311034303.75779-50-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index 208d7ca144..47c2eeef87 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -78,9 +78,15 @@ __version__ = "1.0" class Transmogrifier: def __init__(self) -> None: + self._curr_ent: Optional[QAPISchemaDefinition] = None self._result = StringList() self.indent = 0 + @property + def entity(self) -> QAPISchemaDefinition: + assert self._curr_ent is not None + return self._curr_ent + # General-purpose rST generation functions def get_indent(self) -> str: -- cgit 1.4.1 From 52c806cad08bfed53bf11c1a49bb203cc53deea0 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:48 -0400 Subject: docs/qapidoc: add visit_returns() method Generates :return: fields for explicit returns statements. Note that this does not presently handle undocumented returns, which is handled in a later commit. Signed-off-by: John Snow Message-ID: <20250311034303.75779-51-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index 47c2eeef87..eb8841099c 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -41,6 +41,7 @@ from qapi.parser import QAPIDoc from qapi.schema import ( QAPISchema, QAPISchemaArrayType, + QAPISchemaCommand, QAPISchemaDefinition, QAPISchemaEnumMember, QAPISchemaFeature, @@ -210,6 +211,20 @@ class Transmogrifier: self.generate_field("feat", section.member, section.text, section.info) + def visit_returns(self, section: QAPIDoc.Section) -> None: + assert isinstance(self.entity, QAPISchemaCommand) + rtype = self.entity.ret_type + # q_empty can produce None, but we won't be documenting anything + # without an explicit return statement in the doc block, and we + # should not have any such explicit statements when there is no + # return value. + assert rtype + + typ = self.format_type(rtype) + assert typ + assert section.text + self.add_field("return", typ, section.text, section.info) + def visit_errors(self, section: QAPIDoc.Section) -> None: # FIXME: the formatting for errors may be inconsistent and may # or may not require different newline placement to ensure -- cgit 1.4.1 From dbf51d15fdbb5410e21540d47c16f505413ce1eb Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:49 -0400 Subject: docs/qapidoc: add visit_member() method This method is used for generating the "members" of a wide variety of things, including structs, unions, enums, alternates, etc. The field name it uses to do so is dependent on the type of entity the "member" belongs to. Currently, IF conditionals for individual members are not handled or rendered, a small regression from the prior documentation generator. This will be fixed in a future patch. Signed-off-by: John Snow Message-ID: <20250311034303.75779-52-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index eb8841099c..a8e19487d0 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -78,6 +78,16 @@ __version__ = "1.0" class Transmogrifier: + # Field names used for different entity types: + field_types = { + "enum": "value", + "struct": "memb", + "union": "memb", + "event": "memb", + "command": "arg", + "alternate": "alt", + } + def __init__(self) -> None: self._curr_ent: Optional[QAPISchemaDefinition] = None self._result = StringList() @@ -88,6 +98,10 @@ class Transmogrifier: assert self._curr_ent is not None return self._curr_ent + @property + def member_field_type(self) -> str: + return self.field_types[self.entity.meta] + # General-purpose rST generation functions def get_indent(self) -> str: @@ -202,6 +216,19 @@ class Transmogrifier: self.add_lines(section.text, section.info) self.ensure_blank_line() + def visit_member(self, section: QAPIDoc.ArgSection) -> None: + # FIXME: ifcond for members + # TODO: features for members (documented at entity-level, + # but sometimes defined per-member. Should we add such + # information to member descriptions when we can?) + assert section.text and section.member + self.generate_field( + self.member_field_type, + section.member, + section.text, + section.info, + ) + def visit_feature(self, section: QAPIDoc.ArgSection) -> None: # FIXME - ifcond for features is not handled at all yet! # Proposal: decorate the right-hand column with some graphical -- cgit 1.4.1 From 8cb0a41490364400bc6b1c361b8e1abf9c21bd76 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:50 -0400 Subject: docs/qapidoc: add visit_sections() method Implement the actual main dispatch method that processes and handles the list of doc sections for a given QAPI entity. Process doc sections in strict source order. This is good; reordering doc text is undesirable. Improvement over the old doc generator, which can reorder doc comments that don't adhere to (largely unspoken) conventions. Signed-off-by: John Snow Message-ID: <20250311034303.75779-53-jsnow@redhat.com> Acked-by: Markus Armbruster [Commit message extended] Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index a8e19487d0..83022b15ca 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -288,6 +288,31 @@ class Transmogrifier: self.ensure_blank_line() + def visit_sections(self, ent: QAPISchemaDefinition) -> None: + sections = ent.doc.all_sections if ent.doc else [] + + # Add sections in source order: + for section in sections: + if section.kind == QAPIDoc.Kind.PLAIN: + self.visit_paragraph(section) + elif section.kind == QAPIDoc.Kind.MEMBER: + assert isinstance(section, QAPIDoc.ArgSection) + self.visit_member(section) + elif section.kind == QAPIDoc.Kind.FEATURE: + assert isinstance(section, QAPIDoc.ArgSection) + self.visit_feature(section) + elif section.kind in (QAPIDoc.Kind.SINCE, QAPIDoc.Kind.TODO): + # Since is handled in preamble, TODO is skipped intentionally. + pass + elif section.kind == QAPIDoc.Kind.RETURNS: + self.visit_returns(section) + elif section.kind == QAPIDoc.Kind.ERRORS: + self.visit_errors(section) + else: + assert False + + self.ensure_blank_line() + # Transmogrification core methods def visit_module(self, path: str) -> None: -- cgit 1.4.1 From c05de7235a24dd1719ee6132fc45802b15ce49df Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:51 -0400 Subject: docs/qapidoc: add visit_entity() Finally, the core entry method for a qapi entity. Signed-off-by: John Snow Message-ID: <20250311034303.75779-54-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index 83022b15ca..aaf5b6e22b 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -78,6 +78,8 @@ __version__ = "1.0" class Transmogrifier: + # pylint: disable=too-many-public-methods + # Field names used for different entity types: field_types = { "enum": "value", @@ -362,6 +364,25 @@ class Transmogrifier: self.add_lines(text, info) self.ensure_blank_line() + def visit_entity(self, ent: QAPISchemaDefinition) -> None: + assert ent.info + + try: + self._curr_ent = ent + + # Squish structs and unions together into an "object" directive. + meta = ent.meta + if meta in ("struct", "union"): + meta = "object" + + # This line gets credited to the start of the /definition/. + self.add_line(f".. qapi:{meta}:: {ent.name}", ent.info) + with self.indented(): + self.preamble(ent) + self.visit_sections(ent) + finally: + self._curr_ent = None + class QAPISchemaGenDepVisitor(QAPISchemaVisitor): """A QAPI schema visitor which adds Sphinx dependencies each module -- cgit 1.4.1 From 5c1636f7cc4bfbe5737e3acca009a97dfa188879 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:52 -0400 Subject: docs/qapidoc: implement transmogrify() method This is the true top-level processor for the new transmogrifier; responsible both for generating the intermediate rST and then running the nested parse on that generated document to produce the final docutils tree that is then - very finally - postprocessed by sphinx for final rendering to HTML &c. Signed-off-by: John Snow Message-ID: <20250311034303.75779-55-jsnow@redhat.com> Acked-by: Markus Armbruster [Use the opportunity to move the __version__ assignment to where PEP 8 wants it] Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index aaf5b6e22b..a0016f8853 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -2,6 +2,7 @@ # # QEMU qapidoc QAPI file parsing extension # +# Copyright (c) 2024-2025 Red Hat # Copyright (c) 2020 Linaro # # This work is licensed under the terms of the GNU GPLv2 or later. @@ -26,6 +27,8 @@ https://www.sphinx-doc.org/en/master/development/index.html from __future__ import annotations +__version__ = "2.0" + from contextlib import contextmanager import os from pathlib import Path @@ -56,6 +59,7 @@ from qapidoc_legacy import QAPISchemaGenRSTVisitor # type: ignore from sphinx import addnodes from sphinx.directives.code import CodeBlock from sphinx.errors import ExtensionError +from sphinx.util import logging from sphinx.util.docutils import switch_source_input from sphinx.util.nodes import nested_parse_with_titles @@ -74,7 +78,7 @@ if TYPE_CHECKING: from sphinx.util.typing import ExtensionMetadata -__version__ = "1.0" +logger = logging.getLogger(__name__) class Transmogrifier: @@ -95,6 +99,10 @@ class Transmogrifier: self._result = StringList() self.indent = 0 + @property + def result(self) -> StringList: + return self._result + @property def entity(self) -> QAPISchemaDefinition: assert self._curr_ent is not None @@ -438,7 +446,43 @@ class QAPIDocDirective(NestedDirective): return "qapidoc-%d" % env.new_serialno("qapidoc") def transmogrify(self, schema: QAPISchema) -> nodes.Element: - raise NotImplementedError + logger.info("Transmogrifying QAPI to rST ...") + vis = Transmogrifier() + modules = set() + + for doc in schema.docs: + module_source = doc.info.fname + if module_source not in modules: + vis.visit_module(module_source) + modules.add(module_source) + + if doc.symbol: + ent = schema.lookup_entity(doc.symbol) + assert isinstance(ent, QAPISchemaDefinition) + vis.visit_entity(ent) + else: + vis.visit_freeform(doc) + + logger.info("Transmogrification complete.") + + contentnode = nodes.section() + content = vis.result + titles_allowed = True + + logger.info("Transmogrifier running nested parse ...") + with switch_source_input(self.state, content): + if titles_allowed: + node: nodes.Element = nodes.section() + node.document = self.state.document + nested_parse_with_titles(self.state, content, contentnode) + else: + node = nodes.paragraph() + node.document = self.state.document + self.state.nested_parse(content, 0, contentnode) + logger.info("Transmogrifier's nested parse completed.") + sys.stdout.flush() + + return contentnode def legacy(self, schema: QAPISchema) -> nodes.Element: vis = QAPISchemaGenRSTVisitor(self) @@ -572,6 +616,7 @@ class QMPExample(CodeBlock, NestedDirective): def setup(app: Sphinx) -> ExtensionMetadata: """Register qapi-doc directive with Sphinx""" + app.setup_extension("qapi_domain") app.add_config_value("qapidoc_srctree", None, "env") app.add_directive("qapi-doc", QAPIDocDirective) app.add_directive("qmp-example", QMPExample) -- cgit 1.4.1 From c9b6f988037b3d6f042871b149c7fd307bed8475 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:53 -0400 Subject: docs/qapidoc: process @foo into ``foo`` Add support for the special QAPI doc syntax to process @references as ``preformatted text``. At the moment, there are no actual cross-references for individual members, so there is nothing to link against. For now, process it identically to how we did in the old qapidoc system. Signed-off-by: John Snow Message-ID: <20250311034303.75779-56-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index a0016f8853..ebdf709594 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -303,6 +303,9 @@ class Transmogrifier: # Add sections in source order: for section in sections: + # @var is translated to ``var``: + section.text = re.sub(r"@([\w-]+)", r"``\1``", section.text) + if section.kind == QAPIDoc.Kind.PLAIN: self.visit_paragraph(section) elif section.kind == QAPIDoc.Kind.MEMBER: -- cgit 1.4.1 From 7f6f24aaf55b567df2729d283cf8eac57c399493 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:54 -0400 Subject: docs/qapidoc: add intermediate output debugger Add debugging output for the qapidoc transmogrifier - setting DEBUG=1 will produce .ir files (one for each qapidoc directive) that write the generated rst file to disk to allow for easy debugging and verification of the generated document. Signed-off-by: John Snow Message-ID: <20250311034303.75779-57-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index ebdf709594..8ddebf73f2 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -37,7 +37,7 @@ import sys from typing import TYPE_CHECKING from docutils import nodes -from docutils.parsers.rst import Directive, directives +from docutils.parsers.rst import directives from docutils.statemachine import StringList from qapi.error import QAPIError from qapi.parser import QAPIDoc @@ -60,7 +60,7 @@ from sphinx import addnodes from sphinx.directives.code import CodeBlock from sphinx.errors import ExtensionError from sphinx.util import logging -from sphinx.util.docutils import switch_source_input +from sphinx.util.docutils import SphinxDirective, switch_source_input from sphinx.util.nodes import nested_parse_with_titles @@ -414,7 +414,7 @@ class QAPISchemaGenDepVisitor(QAPISchemaVisitor): super().visit_module(name) -class NestedDirective(Directive): +class NestedDirective(SphinxDirective): def run(self) -> Sequence[nodes.Node]: raise NotImplementedError @@ -483,10 +483,43 @@ class QAPIDocDirective(NestedDirective): node.document = self.state.document self.state.nested_parse(content, 0, contentnode) logger.info("Transmogrifier's nested parse completed.") - sys.stdout.flush() + if self.env.app.verbosity >= 2 or os.environ.get("DEBUG"): + argname = "_".join(Path(self.arguments[0]).parts) + name = Path(argname).stem + ".ir" + self.write_intermediate(content, name) + + sys.stdout.flush() return contentnode + def write_intermediate(self, content: StringList, filename: str) -> None: + logger.info( + "writing intermediate rST for '%s' to '%s'", + self.arguments[0], + filename, + ) + + srctree = Path(self.env.app.config.qapidoc_srctree).resolve() + outlines = [] + lcol_width = 0 + + for i, line in enumerate(content): + src, lineno = content.info(i) + srcpath = Path(src).resolve() + srcpath = srcpath.relative_to(srctree) + + lcol = f"{srcpath}:{lineno:04d}" + lcol_width = max(lcol_width, len(lcol)) + outlines.append((lcol, line)) + + with open(filename, "w", encoding="UTF-8") as outfile: + for lcol, rcol in outlines: + outfile.write(lcol.rjust(lcol_width)) + outfile.write(" |") + if rcol: + outfile.write(f" {rcol}") + outfile.write("\n") + def legacy(self, schema: QAPISchema) -> nodes.Element: vis = QAPISchemaGenRSTVisitor(self) vis.visit_begin(schema) -- cgit 1.4.1 From 1884492e64da659323a8da4de98b344bc689f62a Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:55 -0400 Subject: docs/qapidoc: Add "the members of" pointers Add "the members of ..." pointers to Members and Arguments lists where appropriate, with clickable cross-references - so it's a slight improvement over the old system :) This patch is meant to be a temporary solution until we can review and merge the inliner. The implementation of this patch is a little bit of a hack: Sphinx is not designed to allow you to mix fields of different "type"; i.e. mixing member descriptions and free-form text under the same heading. To accomplish this with a minimum of hackery, we technically document a "dummy field" and then just strip off the documentation for that dummy field in a post-processing step. We use the "q_dummy" variable for this purpose, then strip it back out before final processing. If this processing step should fail, you'll see warnings for a bad cross-reference. (So if you don't see any, it must be working!) Signed-off-by: John Snow Message-ID: <20250311034303.75779-58-jsnow@redhat.com> Acked-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapi_domain.py | 22 ++++++++++++++++-- docs/sphinx/qapidoc.py | 58 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapi_domain.py b/docs/sphinx/qapi_domain.py index ca3f3a7e2d..7ff618d8cd 100644 --- a/docs/sphinx/qapi_domain.py +++ b/docs/sphinx/qapi_domain.py @@ -433,6 +433,24 @@ class QAPIObject(QAPIDescription): self._validate_field(field) +class SpecialTypedField(CompatTypedField): + def make_field(self, *args: Any, **kwargs: Any) -> nodes.field: + ret = super().make_field(*args, **kwargs) + + # Look for the characteristic " -- " text node that Sphinx + # inserts for each TypedField entry ... + for node in ret.traverse(lambda n: str(n) == " -- "): + par = node.parent + if par.children[0].astext() != "q_dummy": + continue + + # If the first node's text is q_dummy, this is a dummy + # field we want to strip down to just its contents. + del par.children[:-1] + + return ret + + class QAPICommand(QAPIObject): """Description of a QAPI Command.""" @@ -440,7 +458,7 @@ class QAPICommand(QAPIObject): doc_field_types.extend( [ # :arg TypeName ArgName: descr - CompatTypedField( + SpecialTypedField( "argument", label=_("Arguments"), names=("arg",), @@ -508,7 +526,7 @@ class QAPIObjectWithMembers(QAPIObject): doc_field_types.extend( [ # :member type name: descr - CompatTypedField( + SpecialTypedField( "member", label=_("Members"), names=("memb",), diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index 8ddebf73f2..a2d6f648a2 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -47,8 +47,10 @@ from qapi.schema import ( QAPISchemaCommand, QAPISchemaDefinition, QAPISchemaEnumMember, + QAPISchemaEvent, QAPISchemaFeature, QAPISchemaMember, + QAPISchemaObjectType, QAPISchemaObjectTypeMember, QAPISchemaType, QAPISchemaVisitor, @@ -298,11 +300,61 @@ class Transmogrifier: self.ensure_blank_line() + def _insert_member_pointer(self, ent: QAPISchemaDefinition) -> None: + + def _get_target( + ent: QAPISchemaDefinition, + ) -> Optional[QAPISchemaDefinition]: + if isinstance(ent, (QAPISchemaCommand, QAPISchemaEvent)): + return ent.arg_type + if isinstance(ent, QAPISchemaObjectType): + return ent.base + return None + + target = _get_target(ent) + if target is not None and not target.is_implicit(): + assert ent.info + self.add_field( + self.member_field_type, + "q_dummy", + f"The members of :qapi:type:`{target.name}`.", + ent.info, + "q_dummy", + ) + + if isinstance(ent, QAPISchemaObjectType) and ent.branches is not None: + for variant in ent.branches.variants: + if variant.type.name == "q_empty": + continue + assert ent.info + self.add_field( + self.member_field_type, + "q_dummy", + f" When ``{ent.branches.tag_member.name}`` is " + f"``{variant.name}``: " + f"The members of :qapi:type:`{variant.type.name}`.", + ent.info, + "q_dummy", + ) + def visit_sections(self, ent: QAPISchemaDefinition) -> None: sections = ent.doc.all_sections if ent.doc else [] + # Determine the index location at which we should generate + # documentation for "The members of ..." pointers. This should + # go at the end of the members section(s) if any. Note that + # index 0 is assumed to be a plain intro section, even if it is + # empty; and that a members section if present will always + # immediately follow the opening PLAIN section. + gen_index = 1 + if len(sections) > 1: + while sections[gen_index].kind == QAPIDoc.Kind.MEMBER: + gen_index += 1 + if gen_index >= len(sections): + break + # Add sections in source order: - for section in sections: + for i, section in enumerate(sections): # @var is translated to ``var``: section.text = re.sub(r"@([\w-]+)", r"``\1``", section.text) @@ -324,6 +376,10 @@ class Transmogrifier: else: assert False + # Generate "The members of ..." entries if necessary: + if i == gen_index - 1: + self._insert_member_pointer(ent) + self.ensure_blank_line() # Transmogrification core methods -- cgit 1.4.1 From 565274da10fe46bf8c30a8175e39753e4fedb60e Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Mar 2025 23:42:56 -0400 Subject: docs/qapidoc: generate entries for undocumented members Presently, we never have any empty text entries for members. The next patch will explicitly generate such sections, so enable support for it in advance. The parser will generate placeholder sections to indicate undocumented members, but it's the qapidoc generator that's responsible for deciding what to do with that stub section. Signed-off-by: John Snow Message-ID: <20250311034303.75779-59-jsnow@redhat.com> Acked-by: Markus Armbruster [Tweak the stub section text] Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'docs/sphinx/qapidoc.py') diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index a2d6f648a2..432fef04b0 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -233,11 +233,12 @@ class Transmogrifier: # TODO: features for members (documented at entity-level, # but sometimes defined per-member. Should we add such # information to member descriptions when we can?) - assert section.text and section.member + assert section.member self.generate_field( self.member_field_type, section.member, - section.text, + # TODO drop fallbacks when undocumented members are outlawed + section.text if section.text else "Not documented", section.info, ) -- cgit 1.4.1