From 7137a96099644734cd6045313823840d4cecd5e8 Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 9 Oct 2020 12:15:27 -0400 Subject: qapi: Prefer explicit relative imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All of the QAPI include statements are changed to be package-aware, as explicit relative imports. A quirk of Python packages is that the name of the package exists only *outside* of the package. This means that to a module inside of the qapi folder, there is inherently no such thing as the "qapi" package. The reason these imports work is because the "qapi" package exists in the context of the caller -- the execution shim, where sys.path includes a directory that has a 'qapi' folder in it. When we write "from qapi import sibling", we are NOT referencing the folder 'qapi', but rather "any package named qapi in sys.path". If you should so happen to have a 'qapi' package in your path, it will use *that* package. When we write "from .sibling import foo", we always reference explicitly our sibling module; guaranteeing consistency in *where* we are importing these modules from. This can be useful when working with virtual environments and packages in development mode. In development mode, a package is installed as a series of symlinks that forwards to your same source files. The problem arises because code quality checkers will follow "import qapi.x" to the "installed" version instead of the sibling file and -- even though they are the same file -- they have different module paths, and this causes cyclic import problems, false positive type mismatch errors, and more. It can also be useful when dealing with hierarchical packages, e.g. if we allow qemu.core.qmp, qemu.qapi.parser, etc. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20201009161558.107041-6-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts/qapi/schema.py') diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index d1307ec661..676185d1a7 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -18,10 +18,10 @@ import os import re from collections import OrderedDict -from qapi.common import c_name, pointer_suffix -from qapi.error import QAPIError, QAPISemError -from qapi.expr import check_exprs -from qapi.parser import QAPISchemaParser +from .common import c_name, pointer_suffix +from .error import QAPIError, QAPISemError +from .expr import check_exprs +from .parser import QAPISchemaParser class QAPISchemaEntity: -- cgit 1.4.1 From 67fea575023a9b2871414857770aafd334a6e39d Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 9 Oct 2020 12:15:29 -0400 Subject: qapi: enforce import order/styling with isort While we're mucking around with imports, we might as well formalize the style we use. Let's use isort to do it for us. lines_after_imports=2: Use two lines after imports, to match PEP8's desire to have "two lines before and after" class definitions, which are likely to start immediately after imports. force_sort_within_sections: Intermingles "from x" and "import x" style statements, such that sorting is always performed strictly on the module name itself. force_grid_wrap=4: Four or more imports from a single module will force the one-per-line style that's more git-friendly. This will generally happen for 'typing' imports. multi_line_output=3: Uses the one-per-line indented style for long imports. include_trailing_comma: Adds a comma to the last import in a group, which makes git conflicts nicer to deal with, generally. line_length: 72 is chosen to match PEP8's "docstrings and comments" line length limit. If you have a single line import that exceeds 72 characters, your names are too long! Suggested-by: Cleber Rosa Signed-off-by: John Snow Reviewed-by: Cleber Rosa Tested-by: Cleber Rosa Reviewed-by: Markus Armbruster Message-Id: <20201009161558.107041-8-jsnow@redhat.com> Signed-off-by: Markus Armbruster --- scripts/qapi/.isort.cfg | 7 +++++++ scripts/qapi/expr.py | 3 ++- scripts/qapi/introspect.py | 7 +++++-- scripts/qapi/parser.py | 2 +- scripts/qapi/schema.py | 2 +- 5 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 scripts/qapi/.isort.cfg (limited to 'scripts/qapi/schema.py') diff --git a/scripts/qapi/.isort.cfg b/scripts/qapi/.isort.cfg new file mode 100644 index 0000000000..643caa1fbd --- /dev/null +++ b/scripts/qapi/.isort.cfg @@ -0,0 +1,7 @@ +[settings] +force_grid_wrap=4 +force_sort_within_sections=True +include_trailing_comma=True +line_length=72 +lines_after_imports=2 +multi_line_output=3 diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index bb4dc55f56..2fcaaa2497 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -14,8 +14,9 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -import re from collections import OrderedDict +import re + from .common import c_name from .error import QAPISemError diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py index 42016a7e66..fafec94e02 100644 --- a/scripts/qapi/introspect.py +++ b/scripts/qapi/introspect.py @@ -17,8 +17,11 @@ from .common import ( mcgen, ) from .gen import QAPISchemaMonolithicCVisitor -from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType, - QAPISchemaType) +from .schema import ( + QAPISchemaArrayType, + QAPISchemaBuiltinType, + QAPISchemaType, +) def _make_tree(obj, ifcond, features, extra=None): diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index 7298f5dbd1..e7b9d670ad 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -14,9 +14,9 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. +from collections import OrderedDict import os import re -from collections import OrderedDict from .error import QAPIParseError, QAPISemError from .source import QAPISourceInfo diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index 676185d1a7..71ebb1e396 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -14,9 +14,9 @@ # TODO catching name collisions in generated code would be nice +from collections import OrderedDict import os import re -from collections import OrderedDict from .common import c_name, pointer_suffix from .error import QAPIError, QAPISemError -- cgit 1.4.1 From 42c0dd122299cf2aa6ef8668afe95f4c332833df Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 9 Oct 2020 12:15:30 -0400 Subject: qapi: delint using flake8 Petty style guide fixes and line length enforcement. Not a big win, not a big loss, but flake8 passes 100% on the qapi module, which gives us an easy baseline to enforce hereafter. A note on the flake8 exception: flake8 will warn on *any* bare except, but pylint's is context-aware and will suppress the warning if you re-raise the exception. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20201009161558.107041-9-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/.flake8 | 2 ++ scripts/qapi/commands.py | 3 ++- scripts/qapi/schema.py | 8 +++++--- scripts/qapi/visit.py | 16 +++++++++++----- 4 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 scripts/qapi/.flake8 (limited to 'scripts/qapi/schema.py') diff --git a/scripts/qapi/.flake8 b/scripts/qapi/.flake8 new file mode 100644 index 0000000000..6b158c68b8 --- /dev/null +++ b/scripts/qapi/.flake8 @@ -0,0 +1,2 @@ +[flake8] +extend-ignore = E722 # Prefer pylint's bare-except checks to flake8's diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py index e06c10afcd..cde0c1e777 100644 --- a/scripts/qapi/commands.py +++ b/scripts/qapi/commands.py @@ -65,7 +65,8 @@ def gen_call(name, arg_type, boxed, ret_type): def gen_marshal_output(ret_type): return mcgen(''' -static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in, QObject **ret_out, Error **errp) +static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in, + QObject **ret_out, Error **errp) { Visitor *v; diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index 71ebb1e396..afd750989e 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -536,7 +536,7 @@ class QAPISchemaVariants: v.set_defined_in(name) def check(self, schema, seen): - if not self.tag_member: # flat union + if not self.tag_member: # flat union self.tag_member = seen.get(c_name(self._tag_name)) base = "'base'" # Pointing to the base type when not implicit would be @@ -824,7 +824,7 @@ class QAPISchema: self._entity_dict = {} self._module_dict = OrderedDict() self._schema_dir = os.path.dirname(fname) - self._make_module(None) # built-ins + self._make_module(None) # built-ins self._make_module(fname) self._predefining = True self._def_predefineds() @@ -968,7 +968,9 @@ class QAPISchema: # But it's not tight: the disjunction need not imply it. We # may end up compiling useless wrapper types. # TODO kill simple unions or implement the disjunction - assert (ifcond or []) == typ._ifcond # pylint: disable=protected-access + + # pylint: disable=protected-access + assert (ifcond or []) == typ._ifcond else: self._def_entity(QAPISchemaObjectType( name, info, None, ifcond, None, None, members, None)) diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py index ea277e7704..9fdbe5b9ef 100644 --- a/scripts/qapi/visit.py +++ b/scripts/qapi/visit.py @@ -31,7 +31,9 @@ def gen_visit_decl(name, scalar=False): if not scalar: c_type += '*' return mcgen(''' -bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_type)sobj, Error **errp); + +bool visit_type_%(c_name)s(Visitor *v, const char *name, + %(c_type)sobj, Error **errp); ''', c_name=c_name(name), c_type=c_type) @@ -125,7 +127,8 @@ bool visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp) def gen_visit_list(name, element_type): return mcgen(''' -bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp) +bool visit_type_%(c_name)s(Visitor *v, const char *name, + %(c_name)s **obj, Error **errp) { bool ok = false; %(c_name)s *tail; @@ -158,7 +161,8 @@ out_obj: def gen_visit_enum(name): return mcgen(''' -bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s *obj, Error **errp) +bool visit_type_%(c_name)s(Visitor *v, const char *name, + %(c_name)s *obj, Error **errp) { int value = *obj; bool ok = visit_type_enum(v, name, &value, &%(c_name)s_lookup, errp); @@ -172,7 +176,8 @@ bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s *obj, Error def gen_visit_alternate(name, variants): ret = mcgen(''' -bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp) +bool visit_type_%(c_name)s(Visitor *v, const char *name, + %(c_name)s **obj, Error **errp) { bool ok = false; @@ -247,7 +252,8 @@ out_obj: def gen_visit_object(name, base, members, variants): return mcgen(''' -bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp) +bool visit_type_%(c_name)s(Visitor *v, const char *name, + %(c_name)s **obj, Error **errp) { bool ok = false; -- cgit 1.4.1 From a7aa64a6aed6d1979881a07dc3d889fc7366cce2 Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 9 Oct 2020 12:15:34 -0400 Subject: qapi/common.py: delint with pylint At this point, that just means using a consistent strategy for constant names. constants get UPPER_CASE and names not used externally get a leading underscore. As a preference, while renaming constants to be UPPERCASE, move them to the head of the file. Generally, it's nice to be able to audit the code that runs on import in one central place. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20201009161558.107041-13-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/common.py | 18 ++++++++---------- scripts/qapi/schema.py | 14 +++++++------- 2 files changed, 15 insertions(+), 17 deletions(-) (limited to 'scripts/qapi/schema.py') diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py index b35318b72c..a417b6029c 100644 --- a/scripts/qapi/common.py +++ b/scripts/qapi/common.py @@ -14,6 +14,11 @@ import re +EATSPACE = '\033EATSPACE.' +POINTER_SUFFIX = ' *' + EATSPACE +_C_NAME_TRANS = str.maketrans('.-', '__') + + # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2 # ENUM24_Name -> ENUM24_NAME @@ -42,9 +47,6 @@ def c_enum_const(type_name, const_name, prefix=None): return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper() -c_name_trans = str.maketrans('.-', '__') - - # Map @name to a valid C identifier. # If @protect, avoid returning certain ticklish identifiers (like # C keywords) by prepending 'q_'. @@ -82,17 +84,13 @@ def c_name(name, protect=True): 'not_eq', 'or', 'or_eq', 'xor', 'xor_eq']) # namespace pollution: polluted_words = set(['unix', 'errno', 'mips', 'sparc', 'i386']) - name = name.translate(c_name_trans) + name = name.translate(_C_NAME_TRANS) if protect and (name in c89_words | c99_words | c11_words | gcc_words | cpp_words | polluted_words): return 'q_' + name return name -eatspace = '\033EATSPACE.' -pointer_suffix = ' *' + eatspace - - class Indentation: """ Indentation level management. @@ -132,12 +130,12 @@ indent = Indentation() # Generate @code with @kwds interpolated. -# Obey indent, and strip eatspace. +# Obey indent, and strip EATSPACE. def cgen(code, **kwds): raw = code % kwds if indent: raw = re.sub(r'^(?!(#|$))', str(indent), raw, flags=re.MULTILINE) - return re.sub(re.escape(eatspace) + r' *', '', raw) + return re.sub(re.escape(EATSPACE) + r' *', '', raw) def mcgen(code, **kwds): diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index afd750989e..7c01592956 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -18,7 +18,7 @@ from collections import OrderedDict import os import re -from .common import c_name, pointer_suffix +from .common import POINTER_SUFFIX, c_name from .error import QAPIError, QAPISemError from .expr import check_exprs from .parser import QAPISchemaParser @@ -309,7 +309,7 @@ class QAPISchemaArrayType(QAPISchemaType): return True def c_type(self): - return c_name(self.name) + pointer_suffix + return c_name(self.name) + POINTER_SUFFIX def json_type(self): return 'array' @@ -430,7 +430,7 @@ class QAPISchemaObjectType(QAPISchemaType): def c_type(self): assert not self.is_implicit() - return c_name(self.name) + pointer_suffix + return c_name(self.name) + POINTER_SUFFIX def c_unboxed_type(self): return c_name(self.name) @@ -504,7 +504,7 @@ class QAPISchemaAlternateType(QAPISchemaType): v.connect_doc(doc) def c_type(self): - return c_name(self.name) + pointer_suffix + return c_name(self.name) + POINTER_SUFFIX def json_type(self): return 'value' @@ -899,7 +899,7 @@ class QAPISchema: self._make_array_type(name, None) def _def_predefineds(self): - for t in [('str', 'string', 'char' + pointer_suffix), + for t in [('str', 'string', 'char' + POINTER_SUFFIX), ('number', 'number', 'double'), ('int', 'int', 'int64_t'), ('int8', 'int', 'int8_t'), @@ -912,8 +912,8 @@ class QAPISchema: ('uint64', 'int', 'uint64_t'), ('size', 'int', 'uint64_t'), ('bool', 'boolean', 'bool'), - ('any', 'value', 'QObject' + pointer_suffix), - ('null', 'null', 'QNull' + pointer_suffix)]: + ('any', 'value', 'QObject' + POINTER_SUFFIX), + ('null', 'null', 'QNull' + POINTER_SUFFIX)]: self._def_builtin_type(*t) self.the_empty_object_type = QAPISchemaObjectType( 'q_empty', None, None, None, None, None, [], None) -- cgit 1.4.1 From 7e09d7882dc1c9464c0fda260fe288011c612adc Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 9 Oct 2020 12:15:40 -0400 Subject: qapi: establish mypy type-checking baseline Fix a minor typing issue, and then establish a mypy type-checking baseline. Like pylint, this should be run from the folder above: > mypy --config-file=qapi/mypy.ini qapi/ This is designed and tested for mypy 0.770 or greater. Signed-off-by: John Snow Tested-by: Eduardo Habkost Reviewed-by: Cleber Rosa Tested-by: Cleber Rosa Message-Id: <20201009161558.107041-19-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/mypy.ini | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/qapi/schema.py | 3 ++- 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 scripts/qapi/mypy.ini (limited to 'scripts/qapi/schema.py') diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini new file mode 100644 index 0000000000..00fac15dc6 --- /dev/null +++ b/scripts/qapi/mypy.ini @@ -0,0 +1,60 @@ +[mypy] +strict = True +strict_optional = False +disallow_untyped_calls = False +python_version = 3.6 + +[mypy-qapi.commands] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.error] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.events] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.expr] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.gen] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.introspect] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.parser] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.schema] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.source] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.types] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False + +[mypy-qapi.visit] +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index 7c01592956..720449feee 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -17,6 +17,7 @@ from collections import OrderedDict import os import re +from typing import Optional from .common import POINTER_SUFFIX, c_name from .error import QAPIError, QAPISemError @@ -25,7 +26,7 @@ from .parser import QAPISchemaParser class QAPISchemaEntity: - meta = None + meta: Optional[str] = None def __init__(self, name, info, doc, ifcond=None, features=None): assert name is None or isinstance(name, str) -- cgit 1.4.1