diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/coverity-scan/COMPONENTS.md | 154 | ||||
| -rw-r--r-- | scripts/coverity-scan/model.c (renamed from scripts/coverity-model.c) | 0 | ||||
| -rw-r--r-- | scripts/decodetree.py | 172 | ||||
| -rw-r--r-- | scripts/qapi/error.py | 47 | ||||
| -rw-r--r-- | scripts/qapi/expr.py | 442 | ||||
| -rw-r--r-- | scripts/qapi/mypy.ini | 10 | ||||
| -rw-r--r-- | scripts/qapi/parser.py | 14 | ||||
| -rw-r--r-- | scripts/qapi/pylintrc | 4 | ||||
| -rw-r--r-- | scripts/qapi/schema.py | 4 | ||||
| -rwxr-xr-x | scripts/qemu-binfmt-conf.sh | 4 |
10 files changed, 666 insertions, 185 deletions
diff --git a/scripts/coverity-scan/COMPONENTS.md b/scripts/coverity-scan/COMPONENTS.md new file mode 100644 index 0000000000..02a3447dab --- /dev/null +++ b/scripts/coverity-scan/COMPONENTS.md @@ -0,0 +1,154 @@ +This is the list of currently configured Coverity components: + +alpha + ~ (/qemu)?((/include)?/hw/alpha/.*|/target/alpha/.*) + +arm + ~ (/qemu)?((/include)?/hw/arm/.*|(/include)?/hw/.*/(arm|allwinner-a10|bcm28|digic|exynos|imx|omap|stellaris|pxa2xx|versatile|zynq|cadence).*|/hw/net/xgmac.c|/hw/ssi/xilinx_spips.c|/target/arm/.*) + +avr + ~ (/qemu)?((/include)?/hw/avr/.*|/target/avr/.*) + +cris + ~ (/qemu)?((/include)?/hw/cris/.*|/target/cris/.*) + +hexagon + ~ (/qemu)?(/target/hexagon/.*) + +hppa + ~ (/qemu)?((/include)?/hw/hppa/.*|/target/hppa/.*) + +i386 + ~ (/qemu)?((/include)?/hw/i386/.*|/target/i386/.*|/hw/intc/[^/]*apic[^/]*\.c) + +lm32 + ~ (/qemu)?((/include)?/hw/lm32/.*|/target/lm32/.*|/hw/.*/(milkymist|lm32).*) + +m68k + ~ (/qemu)?((/include)?/hw/m68k/.*|/target/m68k/.*|(/include)?/hw(/.*)?/mcf.*) + +microblaze + ~ (/qemu)?((/include)?/hw/microblaze/.*|/target/microblaze/.*) + +mips + ~ (/qemu)?((/include)?/hw/mips/.*|/target/mips/.*) + +nios2 + ~ (/qemu)?((/include)?/hw/nios2/.*|/target/nios2/.*) + +ppc + ~ (/qemu)?((/include)?/hw/ppc/.*|/target/ppc/.*|/hw/pci-host/(uninorth.*|dec.*|prep.*|ppc.*)|/hw/misc/macio/.*|(/include)?/hw/.*/(xics|openpic|spapr).*) + +riscv + ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*) + +rx + ~ (/qemu)?((/include)?/hw/rx/.*|/target/rx/.*) + +s390 + ~ (/qemu)?((/include)?/hw/s390x/.*|/target/s390x/.*|/hw/.*/s390_.*) + +sh4 + ~ (/qemu)?((/include)?/hw/sh4/.*|/target/sh4/.*) + +sparc + ~ (/qemu)?((/include)?/hw/sparc(64)?.*|/target/sparc/.*|/hw/.*/grlib.*|/hw/display/cg3.c) + +tilegx + ~ (/qemu)?(/target/tilegx/.*) + +tricore + ~ (/qemu)?((/include)?/hw/tricore/.*|/target/tricore/.*) + +unicore32 + ~ (/qemu)?((/include)?/hw/unicore32/.*|/target/unicore32/.*) + +9pfs + ~ (/qemu)?(/hw/9pfs/.*|/fsdev/.*) + +audio + ~ (/qemu)?((/include)?/(audio|hw/audio)/.*) + +block + ~ (/qemu)?(/block.*|(/include?)(/hw)?/(block|storage-daemon)/.*|(/include)?/hw/ide/.*|/qemu-(img|io).*|/util/(aio|async|thread-pool).*) + +char + ~ (/qemu)?(/qemu-char\.c|/include/sysemu/char\.h|(/include)?/hw/char/.*) + +capstone + ~ (/qemu)?(/capstone/.*) + +crypto + ~ (/qemu)?((/include)?/crypto/.*|/hw/.*/crypto.*) + +disas + ~ (/qemu)?((/include)?/disas.*) + +fpu + ~ (/qemu)?((/include)?(/fpu|/libdecnumber)/.*) + +io + ~ (/qemu)?((/include)?/io/.*) + +ipmi + ~ (/qemu)?((/include)?/hw/ipmi/.*) + +libvixl + ~ (/qemu)?(/disas/libvixl/.*) + +migration + ~ (/qemu)?((/include)?/migration/.*) + +monitor + ~ (/qemu)?(/qapi.*|/qobject/.*|/monitor\..*|/[hq]mp\..*) + +nbd + ~ (/qemu)?(/nbd/.*|/include/block/nbd.*|/qemu-nbd\.c) + +net + ~ (/qemu)?((/include)?(/hw)?/(net|rdma)/.*) + +pci + ~ (/qemu)?(/hw/pci.*|/include/hw/pci.*) + +qemu-ga + ~ (/qemu)?(/qga/.*) + +scsi + ~ (/qemu)?(/scsi/.*|/hw/scsi/.*|/include/hw/scsi/.*) + +slirp + ~ (/qemu)?(/.*slirp.*) + +tcg + ~ (/qemu)?(/accel/tcg/.*|/replay/.*|/(.*/)?softmmu.*) + +trace + ~ (/qemu)?(/.*trace.*\.[ch]) + +ui + ~ (/qemu)?((/include)?(/ui|/hw/display|/hw/input)/.*) + +usb + ~ (/qemu)?(/hw/usb/.*|/include/hw/usb/.*) + +user + ~ (/qemu)?(/linux-user/.*|/bsd-user/.*|/user-exec\.c|/thunk\.c|/include/exec/user/.*) + +util + ~ (/qemu)?(/util/.*|/include/qemu/.*) + +xen + ~ (/qemu)?(.*/xen.*) + +virtiofsd + ~ (/qemu)?(/tools/virtiofsd/.*) + +(headers) + ~ (/qemu)?(/include/.*) + +testlibs + ~ (/qemu)?(/tests/qtest(/libqos/.*|/libqtest.*)) + +tests + ~ (/qemu)?(/tests/.*) diff --git a/scripts/coverity-model.c b/scripts/coverity-scan/model.c index 2c0346ff25..2c0346ff25 100644 --- a/scripts/coverity-model.c +++ b/scripts/coverity-scan/model.c diff --git a/scripts/decodetree.py b/scripts/decodetree.py index 4637b633e7..a03dc6b5e3 100644 --- a/scripts/decodetree.py +++ b/scripts/decodetree.py @@ -27,6 +27,7 @@ import sys import getopt insnwidth = 32 +bitop_width = 32 insnmask = 0xffffffff variablewidth = False fields = {} @@ -59,9 +60,9 @@ def error_with_file(file, lineno, *args): prefix = '' if file: - prefix += '{0}:'.format(file) + prefix += f'{file}:' if lineno: - prefix += '{0}:'.format(lineno) + prefix += f'{lineno}:' if prefix: prefix += ' ' print(prefix, end='error: ', file=sys.stderr) @@ -102,6 +103,23 @@ def str_fields(fields): return r[1:] +def whex(val): + """Return a hex string for val padded for insnwidth""" + global insnwidth + return f'0x{val:0{insnwidth // 4}x}' + + +def whexC(val): + """Return a hex string for val padded for insnwidth, + and with the proper suffix for a C constant.""" + suffix = '' + if val >= 0x100000000: + suffix = 'ull' + elif val >= 0x80000000: + suffix = 'u' + return whex(val) + suffix + + def str_match_bits(bits, mask): """Return a string pretty-printing BITS/MASK""" global insnwidth @@ -147,11 +165,15 @@ def is_contiguous(bits): return -1 -def eq_fields_for_args(flds_a, flds_b): - if len(flds_a) != len(flds_b): +def eq_fields_for_args(flds_a, arg): + if len(flds_a) != len(arg.fields): return False + # Only allow inference on default types + for t in arg.types: + if t != 'int': + return False for k, a in flds_a.items(): - if k not in flds_b: + if k not in arg.fields: return False return True @@ -184,11 +206,9 @@ class Field: return str(self.pos) + ':' + s + str(self.len) def str_extract(self): - if self.sign: - extr = 'sextract32' - else: - extr = 'extract32' - return '{0}(insn, {1}, {2})'.format(extr, self.pos, self.len) + global bitop_width + s = 's' if self.sign else '' + return f'{s}extract{bitop_width}(insn, {self.pos}, {self.len})' def __eq__(self, other): return self.sign == other.sign and self.mask == other.mask @@ -209,14 +229,15 @@ class MultiField: return str(self.subs) def str_extract(self): + global bitop_width ret = '0' pos = 0 for f in reversed(self.subs): + ext = f.str_extract() if pos == 0: - ret = f.str_extract() + ret = ext else: - ret = 'deposit32({0}, {1}, {2}, {3})' \ - .format(ret, pos, 32 - pos, f.str_extract()) + ret = f'deposit{bitop_width}({ret}, {pos}, {bitop_width - pos}, {ext})' pos += f.len return ret @@ -296,10 +317,11 @@ class ParameterField: class Arguments: """Class representing the extracted fields of a format""" - def __init__(self, nm, flds, extern): + def __init__(self, nm, flds, types, extern): self.name = nm self.extern = extern - self.fields = sorted(flds) + self.fields = flds + self.types = types def __str__(self): return self.name + ' ' + str(self.fields) @@ -310,8 +332,8 @@ class Arguments: def output_def(self): if not self.extern: output('typedef struct {\n') - for n in self.fields: - output(' int ', n, ';\n') + for (n, t) in zip(self.fields, self.types): + output(f' {t} {n};\n') output('} ', self.struct_name(), ';\n\n') # end Arguments @@ -477,11 +499,8 @@ class IncMultiPattern(MultiPattern): if outermask != p.fixedmask: innermask = p.fixedmask & ~outermask innerbits = p.fixedbits & ~outermask - output(ind, 'if ((insn & ', - '0x{0:08x}) == 0x{1:08x}'.format(innermask, innerbits), - ') {\n') - output(ind, ' /* ', - str_match_bits(p.fixedbits, p.fixedmask), ' */\n') + output(ind, f'if ((insn & {whexC(innermask)}) == {whexC(innerbits)}) {{\n') + output(ind, f' /* {str_match_bits(p.fixedbits, p.fixedmask)} */\n') p.output_code(i + 4, extracted, p.fixedbits, p.fixedmask) output(ind, '}\n') else: @@ -500,12 +519,12 @@ class Tree: def str1(self, i): ind = str_indent(i) - r = '{0}{1:08x}'.format(ind, self.fixedmask) + r = ind + whex(self.fixedmask) if self.format: r += ' ' + self.format.name r += ' [\n' for (b, s) in self.subs: - r += '{0} {1:08x}:\n'.format(ind, b) + r += ind + f' {whex(b)}:\n' r += s.str1(i + 4) + '\n' r += ind + ']' return r @@ -529,16 +548,16 @@ class Tree: if sh > 0: # Propagate SH down into the local functions. def str_switch(b, sh=sh): - return '(insn >> {0}) & 0x{1:x}'.format(sh, b >> sh) + return f'(insn >> {sh}) & {b >> sh:#x}' def str_case(b, sh=sh): - return '0x{0:x}'.format(b >> sh) + return hex(b >> sh) else: def str_switch(b): - return 'insn & 0x{0:08x}'.format(b) + return f'insn & {whexC(b)}' def str_case(b): - return '0x{0:08x}'.format(b) + return whexC(b) output(ind, 'switch (', str_switch(self.thismask), ') {\n') for b, s in sorted(self.subs): @@ -663,11 +682,11 @@ def parse_field(lineno, name, toks): subtoks = t.split(':') sign = False else: - error(lineno, 'invalid field token "{0}"'.format(t)) + error(lineno, f'invalid field token "{t}"') po = int(subtoks[0]) le = int(subtoks[1]) if po + le > insnwidth: - error(lineno, 'field {0} too large'.format(t)) + error(lineno, f'field {t} too large') f = Field(sign, po, le) subs.append(f) width += le @@ -705,21 +724,27 @@ def parse_arguments(lineno, name, toks): global anyextern flds = [] + types = [] extern = False - for t in toks: - if re.fullmatch('!extern', t): + for n in toks: + if re.fullmatch('!extern', n): extern = True anyextern = True continue - if not re.fullmatch(re_C_ident, t): - error(lineno, 'invalid argument set token "{0}"'.format(t)) - if t in flds: - error(lineno, 'duplicate argument "{0}"'.format(t)) - flds.append(t) + if re.fullmatch(re_C_ident + ':' + re_C_ident, n): + (n, t) = n.split(':') + elif re.fullmatch(re_C_ident, n): + t = 'int' + else: + error(lineno, f'invalid argument set token "{n}"') + if n in flds: + error(lineno, f'duplicate argument "{n}"') + flds.append(n) + types.append(t) if name in arguments: error(lineno, 'duplicate argument set', name) - arguments[name] = Arguments(name, flds, extern) + arguments[name] = Arguments(name, flds, types, extern) # end parse_arguments @@ -746,11 +771,11 @@ def infer_argument_set(flds): global decode_function for arg in arguments.values(): - if eq_fields_for_args(flds, arg.fields): + if eq_fields_for_args(flds, arg): return arg name = decode_function + str(len(arguments)) - arg = Arguments(name, flds.keys(), False) + arg = Arguments(name, flds.keys(), ['int'] * len(flds), False) arguments[name] = arg return arg @@ -883,14 +908,14 @@ def parse_generic(lineno, parent_pat, name, toks): flen = flen[1:] shift = int(flen, 10) if shift + width > insnwidth: - error(lineno, 'field {0} exceeds insnwidth'.format(fname)) + error(lineno, f'field {fname} exceeds insnwidth') f = Field(sign, insnwidth - width - shift, shift) flds = add_field(lineno, flds, fname, f) fixedbits <<= shift fixedmask <<= shift undefmask <<= shift else: - error(lineno, 'invalid token "{0}"'.format(t)) + error(lineno, f'invalid token "{t}"') width += shift if variablewidth and width < insnwidth and width % 8 == 0: @@ -902,7 +927,7 @@ def parse_generic(lineno, parent_pat, name, toks): # We should have filled in all of the bits of the instruction. elif not (is_format and width == 0) and width != insnwidth: - error(lineno, 'definition has {0} bits'.format(width)) + error(lineno, f'definition has {width} bits') # Do not check for fields overlapping fields; one valid usage # is to be able to duplicate fields via import. @@ -920,8 +945,7 @@ def parse_generic(lineno, parent_pat, name, toks): if arg: for f in flds.keys(): if f not in arg.fields: - error(lineno, 'field {0} not in argument set {1}' - .format(f, arg.name)) + error(lineno, f'field {f} not in argument set {arg.name}') else: arg = infer_argument_set(flds) if name in formats: @@ -948,13 +972,12 @@ def parse_generic(lineno, parent_pat, name, toks): arg = fmt.base for f in flds.keys(): if f not in arg.fields: - error(lineno, 'field {0} not in argument set {1}' - .format(f, arg.name)) + error(lineno, f'field {f} not in argument set {arg.name}') if f in fmt.fields.keys(): - error(lineno, 'field {0} set by format and pattern'.format(f)) + error(lineno, f'field {f} set by format and pattern') for f in arg.fields: if f not in flds.keys() and f not in fmt.fields.keys(): - error(lineno, 'field {0} not initialized'.format(f)) + error(lineno, f'field {f} not initialized') pat = Pattern(name, lineno, fmt, fixedbits, fixedmask, undefmask, fieldmask, flds, width) parent_pat.pats.append(pat) @@ -962,19 +985,19 @@ def parse_generic(lineno, parent_pat, name, toks): # Validate the masks that we have assembled. if fieldmask & fixedmask: - error(lineno, 'fieldmask overlaps fixedmask (0x{0:08x} & 0x{1:08x})' - .format(fieldmask, fixedmask)) + error(lineno, 'fieldmask overlaps fixedmask ', + f'({whex(fieldmask)} & {whex(fixedmask)})') if fieldmask & undefmask: - error(lineno, 'fieldmask overlaps undefmask (0x{0:08x} & 0x{1:08x})' - .format(fieldmask, undefmask)) + error(lineno, 'fieldmask overlaps undefmask ', + f'({whex(fieldmask)} & {whex(undefmask)})') if fixedmask & undefmask: - error(lineno, 'fixedmask overlaps undefmask (0x{0:08x} & 0x{1:08x})' - .format(fixedmask, undefmask)) + error(lineno, 'fixedmask overlaps undefmask ', + f'({whex(fixedmask)} & {whex(undefmask)})') if not is_format: allbits = fieldmask | fixedmask | undefmask if allbits != insnmask: - error(lineno, 'bits left unspecified (0x{0:08x})' - .format(allbits ^ insnmask)) + error(lineno, 'bits left unspecified ', + f'({whex(allbits ^ insnmask)})') # end parse_general @@ -1085,7 +1108,7 @@ def parse_file(f, parent_pat): elif re.fullmatch(re_pat_ident, name): parse_generic(start_lineno, parent_pat, name, toks) else: - error(lineno, 'invalid token "{0}"'.format(name)) + error(lineno, f'invalid token "{name}"') toks = [] if nesting != 0: @@ -1104,10 +1127,9 @@ class SizeTree: def str1(self, i): ind = str_indent(i) - r = '{0}{1:08x}'.format(ind, self.mask) - r += ' [\n' + r = ind + whex(self.mask) + ' [\n' for (b, s) in self.subs: - r += '{0} {1:08x}:\n'.format(ind, b) + r += ind + f' {whex(b)}:\n' r += s.str1(i + 4) + '\n' r += ind + ']' return r @@ -1120,9 +1142,8 @@ class SizeTree: # If we need to load more bytes to test, do so now. if extracted < self.width: - output(ind, 'insn = ', decode_function, - '_load_bytes(ctx, insn, {0}, {1});\n' - .format(extracted // 8, self.width // 8)); + output(ind, f'insn = {decode_function}_load_bytes', + f'(ctx, insn, {extracted // 8}, {self.width // 8});\n') extracted = self.width # Attempt to aid the compiler in producing compact switch statements. @@ -1131,16 +1152,16 @@ class SizeTree: if sh > 0: # Propagate SH down into the local functions. def str_switch(b, sh=sh): - return '(insn >> {0}) & 0x{1:x}'.format(sh, b >> sh) + return f'(insn >> {sh}) & {b >> sh:#x}' def str_case(b, sh=sh): - return '0x{0:x}'.format(b >> sh) + return hex(b >> sh) else: def str_switch(b): - return 'insn & 0x{0:08x}'.format(b) + return f'insn & {whexC(b)}' def str_case(b): - return '0x{0:08x}'.format(b) + return whexC(b) output(ind, 'switch (', str_switch(self.mask), ') {\n') for b, s in sorted(self.subs): @@ -1162,8 +1183,7 @@ class SizeLeaf: self.width = w def str1(self, i): - ind = str_indent(i) - return '{0}{1:08x}'.format(ind, self.mask) + return str_indent(i) + whex(self.mask) def __str__(self): return self.str1(0) @@ -1174,9 +1194,8 @@ class SizeLeaf: # If we need to load more bytes, do so now. if extracted < self.width: - output(ind, 'insn = ', decode_function, - '_load_bytes(ctx, insn, {0}, {1});\n' - .format(extracted // 8, self.width // 8)); + output(ind, f'insn = {decode_function}_load_bytes', + f'(ctx, insn, {extracted // 8}, {self.width // 8});\n') extracted = self.width output(ind, 'return insn;\n') # end SizeLeaf @@ -1210,7 +1229,7 @@ def build_size_tree(pats, width, outerbits, outermask): for p in pats: pnames.append(p.name + ':' + p.file + ':' + str(p.lineno)) error_with_file(pats[0].file, pats[0].lineno, - 'overlapping patterns size {0}:'.format(width), pnames) + f'overlapping patterns size {width}:', pnames) bins = {} for i in pats: @@ -1264,6 +1283,7 @@ def main(): global insntype global insnmask global decode_function + global bitop_width global variablewidth global anyextern @@ -1293,6 +1313,10 @@ def main(): if insnwidth == 16: insntype = 'uint16_t' insnmask = 0xffff + elif insnwidth == 64: + insntype = 'uint64_t' + insnmask = 0xffffffffffffffff + bitop_width = 64 elif insnwidth != 32: error(0, 'cannot handle insns of width', insnwidth) else: diff --git a/scripts/qapi/error.py b/scripts/qapi/error.py index ae60d9e2fe..e35e4ddb26 100644 --- a/scripts/qapi/error.py +++ b/scripts/qapi/error.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- # -# QAPI error classes -# # Copyright (c) 2017-2019 Red Hat Inc. # # Authors: @@ -11,15 +9,36 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. +""" +QAPI error classes + +Common error classes used throughout the package. Additional errors may +be defined in other modules. At present, `QAPIParseError` is defined in +parser.py. +""" + +from typing import Optional + +from .source import QAPISourceInfo + class QAPIError(Exception): - def __init__(self, info, col, msg): - Exception.__init__(self) + """Base class for all exceptions from the QAPI package.""" + + +class QAPISourceError(QAPIError): + """Error class for all exceptions identifying a source location.""" + def __init__(self, + info: Optional[QAPISourceInfo], + msg: str, + col: Optional[int] = None): + super().__init__() self.info = info - self.col = col self.msg = msg + self.col = col - def __str__(self): + def __str__(self) -> str: + assert self.info is not None loc = str(self.info) if self.col is not None: assert self.info.line is not None @@ -27,17 +46,5 @@ class QAPIError(Exception): return loc + ': ' + self.msg -class QAPIParseError(QAPIError): - def __init__(self, parser, msg): - col = 1 - for ch in parser.src[parser.line_pos:parser.pos]: - if ch == '\t': - col = (col + 7) % 8 + 1 - else: - col += 1 - super().__init__(parser.info, col, msg) - - -class QAPISemError(QAPIError): - def __init__(self, info, msg): - super().__init__(info, None, msg) +class QAPISemError(QAPISourceError): + """Error class for semantic QAPI errors.""" diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 540b3982b1..496f7e0333 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -1,40 +1,97 @@ # -*- coding: utf-8 -*- # -# Check (context-free) QAPI schema expression structure -# # Copyright IBM, Corp. 2011 -# Copyright (c) 2013-2019 Red Hat Inc. +# Copyright (c) 2013-2021 Red Hat Inc. # # Authors: # Anthony Liguori <aliguori@us.ibm.com> # Markus Armbruster <armbru@redhat.com> # Eric Blake <eblake@redhat.com> # Marc-André Lureau <marcandre.lureau@redhat.com> +# John Snow <jsnow@redhat.com> # # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -from collections import OrderedDict +""" +Normalize and validate (context-free) QAPI schema expression structures. + +`QAPISchemaParser` parses a QAPI schema into abstract syntax trees +consisting of dict, list, str, bool, and int nodes. This module ensures +that these nested structures have the correct type(s) and key(s) where +appropriate for the QAPI context-free grammar. + +The QAPI schema expression language allows for certain syntactic sugar; +this module also handles the normalization process of these nested +structures. + +See `check_exprs` for the main entry point. + +See `schema.QAPISchema` for processing into native Python data +structures and contextual semantic validation. +""" + import re +from typing import ( + Collection, + Dict, + Iterable, + List, + Optional, + Union, + cast, +) from .common import c_name from .error import QAPISemError +from .parser import QAPIDoc +from .source import QAPISourceInfo -# Names consist of letters, digits, -, and _, starting with a letter. -# An experimental name is prefixed with x-. A name of a downstream -# extension is prefixed with __RFQDN_. The latter prefix goes first. +# Deserialized JSON objects as returned by the parser. +# The values of this mapping are not necessary to exhaustively type +# here (and also not practical as long as mypy lacks recursive +# types), because the purpose of this module is to interrogate that +# type. +_JSONObject = Dict[str, object] + + +# See check_name_str(), below. valid_name = re.compile(r'(__[a-z0-9.-]+_)?' r'(x-)?' r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) -def check_name_is_str(name, info, source): +def check_name_is_str(name: object, + info: QAPISourceInfo, + source: str) -> None: + """ + Ensure that ``name`` is a ``str``. + + :raise QAPISemError: When ``name`` fails validation. + """ if not isinstance(name, str): raise QAPISemError(info, "%s requires a string name" % source) -def check_name_str(name, info, source): +def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: + """ + Ensure that ``name`` is a valid QAPI name. + + A valid name consists of ASCII letters, digits, ``-``, and ``_``, + starting with a letter. It may be prefixed by a downstream prefix + of the form __RFQDN_, or the experimental prefix ``x-``. If both + prefixes are present, the __RFDQN_ prefix goes first. + + A valid name cannot start with ``q_``, which is reserved. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + + :raise QAPISemError: When ``name`` fails validation. + :return: The stem of the valid name, with no prefixes. + """ # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' # and 'q_obj_*' implicit type names. match = valid_name.match(name) @@ -43,16 +100,44 @@ def check_name_str(name, info, source): return match.group(3) -def check_name_upper(name, info, source): +def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: + """ + Ensure that ``name`` is a valid event name. + + This means it must be a valid QAPI name as checked by + `check_name_str()`, but where the stem prohibits lowercase + characters and ``-``. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + + :raise QAPISemError: When ``name`` fails validation. + """ stem = check_name_str(name, info, source) if re.search(r'[a-z-]', stem): raise QAPISemError( info, "name of %s must not use lowercase or '-'" % source) -def check_name_lower(name, info, source, - permit_upper=False, - permit_underscore=False): +def check_name_lower(name: str, info: QAPISourceInfo, source: str, + permit_upper: bool = False, + permit_underscore: bool = False) -> None: + """ + Ensure that ``name`` is a valid command or member name. + + This means it must be a valid QAPI name as checked by + `check_name_str()`, but where the stem prohibits uppercase + characters and ``_``. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + :param permit_upper: Additionally permit uppercase. + :param permit_underscore: Additionally permit ``_``. + + :raise QAPISemError: When ``name`` fails validation. + """ stem = check_name_str(name, info, source) if ((not permit_upper and re.search(r'[A-Z]', stem)) or (not permit_underscore and '_' in stem)): @@ -60,13 +145,40 @@ def check_name_lower(name, info, source, info, "name of %s must not use uppercase or '_'" % source) -def check_name_camel(name, info, source): +def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: + """ + Ensure that ``name`` is a valid user-defined type name. + + This means it must be a valid QAPI name as checked by + `check_name_str()`, but where the stem must be in CamelCase. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + + :raise QAPISemError: When ``name`` fails validation. + """ stem = check_name_str(name, info, source) if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): raise QAPISemError(info, "name of %s must use CamelCase" % source) -def check_defn_name_str(name, info, meta): +def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: + """ + Ensure that ``name`` is a valid definition name. + + Based on the value of ``meta``, this means that: + - 'event' names adhere to `check_name_upper()`. + - 'command' names adhere to `check_name_lower()`. + - Else, meta is a type, and must pass `check_name_camel()`. + These names must not end with ``Kind`` nor ``List``. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param meta: Meta-type name of the QAPI expression. + + :raise QAPISemError: When ``name`` fails validation. + """ if meta == 'event': check_name_upper(name, info, meta) elif meta == 'command': @@ -75,14 +187,29 @@ def check_defn_name_str(name, info, meta): permit_underscore=name in info.pragma.command_name_exceptions) else: check_name_camel(name, info, meta) - if name.endswith('Kind') or name.endswith('List'): - raise QAPISemError( - info, "%s name should not end in '%s'" % (meta, name[-4:])) + if name.endswith('Kind') or name.endswith('List'): + raise QAPISemError( + info, "%s name should not end in '%s'" % (meta, name[-4:])) + +def check_keys(value: _JSONObject, + info: QAPISourceInfo, + source: str, + required: Collection[str], + optional: Collection[str]) -> None: + """ + Ensure that a dict has a specific set of keys. -def check_keys(value, info, source, required, optional): + :param value: The dict to check. + :param info: QAPI schema source file information. + :param source: Error string describing this ``value``. + :param required: Keys that *must* be present. + :param optional: Keys that *may* be present. - def pprint(elems): + :raise QAPISemError: When unknown keys are present. + """ + + def pprint(elems: Iterable[str]) -> str: return ', '.join("'" + e + "'" for e in sorted(elems)) missing = set(required) - set(value) @@ -92,7 +219,7 @@ def check_keys(value, info, source, required, optional): "%s misses key%s %s" % (source, 's' if len(missing) > 1 else '', pprint(missing))) - allowed = set(required + optional) + allowed = set(required) | set(optional) unknown = set(value) - allowed if unknown: raise QAPISemError( @@ -102,12 +229,22 @@ def check_keys(value, info, source, required, optional): pprint(unknown), pprint(allowed))) -def check_flags(expr, info): - for key in ['gen', 'success-response']: +def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Ensure flag members (if present) have valid values. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: + When certain flags have an invalid value, or when + incompatible flags are present. + """ + for key in ('gen', 'success-response'): if key in expr and expr[key] is not False: raise QAPISemError( info, "flag '%s' may only use false value" % key) - for key in ['boxed', 'allow-oob', 'allow-preconfig', 'coroutine']: + for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): if key in expr and expr[key] is not True: raise QAPISemError( info, "flag '%s' may only use true value" % key) @@ -120,47 +257,106 @@ def check_flags(expr, info): "are incompatible") -def check_if(expr, info, source): +def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: + """ + Normalize and validate the ``if`` member of an object. - def check_if_str(ifcond, info): - if not isinstance(ifcond, str): - raise QAPISemError( - info, - "'if' condition of %s must be a string or a list of strings" - % source) - if ifcond.strip() == '': - raise QAPISemError( - info, - "'if' condition '%s' of %s makes no sense" - % (ifcond, source)) + The ``if`` member may be either a ``str`` or a ``List[str]``. + A ``str`` value will be normalized to ``List[str]``. + :forms: + :sugared: ``Union[str, List[str]]`` + :canonical: ``List[str]`` + + :param expr: The expression containing the ``if`` member to validate. + :param info: QAPI schema source file information. + :param source: Error string describing ``expr``. + + :raise QAPISemError: + When the "if" member fails validation, or when there are no + non-empty conditions. + :return: None, ``expr`` is normalized in-place as needed. + """ ifcond = expr.get('if') if ifcond is None: return + if isinstance(ifcond, list): - if ifcond == []: + if not ifcond: raise QAPISemError( info, "'if' condition [] of %s is useless" % source) - for elt in ifcond: - check_if_str(elt, info) else: - check_if_str(ifcond, info) - expr['if'] = [ifcond] + # Normalize to a list + ifcond = expr['if'] = [ifcond] + + for elt in ifcond: + if not isinstance(elt, str): + raise QAPISemError( + info, + "'if' condition of %s must be a string or a list of strings" + % source) + if not elt.strip(): + raise QAPISemError( + info, + "'if' condition '%s' of %s makes no sense" + % (elt, source)) -def normalize_members(members): - if isinstance(members, OrderedDict): +def normalize_members(members: object) -> None: + """ + Normalize a "members" value. + + If ``members`` is a dict, for every value in that dict, if that + value is not itself already a dict, normalize it to + ``{'type': value}``. + + :forms: + :sugared: ``Dict[str, Union[str, TypeRef]]`` + :canonical: ``Dict[str, TypeRef]`` + + :param members: The members value to normalize. + + :return: None, ``members`` is normalized in-place as needed. + """ + if isinstance(members, dict): for key, arg in members.items(): if isinstance(arg, dict): continue members[key] = {'type': arg} -def check_type(value, info, source, - allow_array=False, allow_dict=False): +def check_type(value: Optional[object], + info: QAPISourceInfo, + source: str, + allow_array: bool = False, + allow_dict: Union[bool, str] = False) -> None: + """ + Normalize and validate the QAPI type of ``value``. + + Python types of ``str`` or ``None`` are always allowed. + + :param value: The value to check. + :param info: QAPI schema source file information. + :param source: Error string describing this ``value``. + :param allow_array: + Allow a ``List[str]`` of length 1, which indicates an array of + the type named by the list element. + :param allow_dict: + Allow a dict. Its members can be struct type members or union + branches. When the value of ``allow_dict`` is in pragma + ``member-name-exceptions``, the dict's keys may violate the + member naming rules. The dict members are normalized in place. + + :raise QAPISemError: When ``value`` fails validation. + :return: None, ``value`` is normalized in-place as needed. + """ if value is None: return + # Type name + if isinstance(value, str): + return + # Array type if isinstance(value, list): if not allow_array: @@ -171,20 +367,18 @@ def check_type(value, info, source, source) return - # Type name - if isinstance(value, str): - return - # Anonymous type if not allow_dict: raise QAPISemError(info, "%s should be a type name" % source) - if not isinstance(value, OrderedDict): + if not isinstance(value, dict): raise QAPISemError(info, "%s should be an object or type name" % source) - permissive = allow_dict in info.pragma.member_name_exceptions + permissive = False + if isinstance(allow_dict, str): + permissive = allow_dict in info.pragma.member_name_exceptions # value is a dictionary, check that each member is okay for (key, arg) in value.items(): @@ -202,24 +396,50 @@ def check_type(value, info, source, check_type(arg['type'], info, key_source, allow_array=True) -def check_features(features, info): +def check_features(features: Optional[object], + info: QAPISourceInfo) -> None: + """ + Normalize and validate the ``features`` member. + + ``features`` may be a ``list`` of either ``str`` or ``dict``. + Any ``str`` element will be normalized to ``{'name': element}``. + + :forms: + :sugared: ``List[Union[str, Feature]]`` + :canonical: ``List[Feature]`` + + :param features: The features member value to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``features`` fails validation. + :return: None, ``features`` is normalized in-place as needed. + """ if features is None: return if not isinstance(features, list): raise QAPISemError(info, "'features' must be an array") features[:] = [f if isinstance(f, dict) else {'name': f} for f in features] - for f in features: + for feat in features: source = "'features' member" - assert isinstance(f, dict) - check_keys(f, info, source, ['name'], ['if']) - check_name_is_str(f['name'], info, source) - source = "%s '%s'" % (source, f['name']) - check_name_lower(f['name'], info, source) - check_if(f, info, source) + assert isinstance(feat, dict) + check_keys(feat, info, source, ['name'], ['if']) + check_name_is_str(feat['name'], info, source) + source = "%s '%s'" % (source, feat['name']) + check_name_str(feat['name'], info, source) + check_if(feat, info, source) + + +def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as an ``enum`` definition. + :param expr: The expression to validate. + :param info: QAPI schema source file information. -def check_enum(expr, info): + :raise QAPISemError: When ``expr`` is not a valid ``enum``. + :return: None, ``expr`` is normalized in-place as needed. + """ name = expr['enum'] members = expr['data'] prefix = expr.get('prefix') @@ -241,23 +461,41 @@ def check_enum(expr, info): source = "%s '%s'" % (source, member_name) # Enum members may start with a digit if member_name[0].isdigit(): - member_name = 'd' + member_name # Hack: hide the digit + member_name = 'd' + member_name # Hack: hide the digit check_name_lower(member_name, info, source, permit_upper=permissive, permit_underscore=permissive) check_if(member, info, source) -def check_struct(expr, info): - name = expr['struct'] +def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as a ``struct`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``struct``. + :return: None, ``expr`` is normalized in-place as needed. + """ + name = cast(str, expr['struct']) # Checked in check_exprs members = expr['data'] check_type(members, info, "'data'", allow_dict=name) check_type(expr.get('base'), info, "'base'") -def check_union(expr, info): - name = expr['union'] +def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as a ``union`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: when ``expr`` is not a valid ``union``. + :return: None, ``expr`` is normalized in-place as needed. + """ + name = cast(str, expr['union']) # Checked in check_exprs base = expr.get('base') discriminator = expr.get('discriminator') members = expr['data'] @@ -271,6 +509,9 @@ def check_union(expr, info): raise QAPISemError(info, "'discriminator' requires 'base'") check_name_is_str(discriminator, info, "'discriminator'") + if not isinstance(members, dict): + raise QAPISemError(info, "'data' must be an object") + for (key, value) in members.items(): source = "'data' member '%s'" % key if discriminator is None: @@ -281,11 +522,24 @@ def check_union(expr, info): check_type(value['type'], info, source, allow_array=not base) -def check_alternate(expr, info): +def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as an ``alternate`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``alternate``. + :return: None, ``expr`` is normalized in-place as needed. + """ members = expr['data'] if not members: raise QAPISemError(info, "'data' must not be empty") + + if not isinstance(members, dict): + raise QAPISemError(info, "'data' must be an object") + for (key, value) in members.items(): source = "'data' member '%s'" % key check_name_lower(key, info, source) @@ -294,7 +548,16 @@ def check_alternate(expr, info): check_type(value['type'], info, source) -def check_command(expr, info): +def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as a ``command`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``command``. + :return: None, ``expr`` is normalized in-place as needed. + """ args = expr.get('data') rets = expr.get('returns') boxed = expr.get('boxed', False) @@ -305,7 +568,16 @@ def check_command(expr, info): check_type(rets, info, "'returns'", allow_array=True) -def check_event(expr, info): +def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as an ``event`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``event``. + :return: None, ``expr`` is normalized in-place as needed. + """ args = expr.get('data') boxed = expr.get('boxed', False) @@ -314,11 +586,33 @@ def check_event(expr, info): check_type(args, info, "'data'", allow_dict=not boxed) -def check_exprs(exprs): +def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: + """ + Validate and normalize a list of parsed QAPI schema expressions. + + This function accepts a list of expressions and metadata as returned + by the parser. It destructively normalizes the expressions in-place. + + :param exprs: The list of expressions to normalize and validate. + + :raise QAPISemError: When any expression fails validation. + :return: The same list of expressions (now modified). + """ for expr_elem in exprs: - expr = expr_elem['expr'] - info = expr_elem['info'] - doc = expr_elem.get('doc') + # Expression + assert isinstance(expr_elem['expr'], dict) + for key in expr_elem['expr'].keys(): + assert isinstance(key, str) + expr: _JSONObject = expr_elem['expr'] + + # QAPISourceInfo + assert isinstance(expr_elem['info'], QAPISourceInfo) + info: QAPISourceInfo = expr_elem['info'] + + # Optional[QAPIDoc] + tmp = expr_elem.get('doc') + assert tmp is None or isinstance(tmp, QAPIDoc) + doc: Optional[QAPIDoc] = tmp if 'include' in expr: continue @@ -338,8 +632,8 @@ def check_exprs(exprs): else: raise QAPISemError(info, "expression is missing metatype") - name = expr[meta] - check_name_is_str(name, info, "'%s'" % meta) + check_name_is_str(expr[meta], info, "'%s'" % meta) + name = cast(str, expr[meta]) info.set_defn(meta, name) check_defn_name_str(name, info, meta) diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini index 0a000d58b3..54ca4483d6 100644 --- a/scripts/qapi/mypy.ini +++ b/scripts/qapi/mypy.ini @@ -3,16 +3,6 @@ strict = True disallow_untyped_calls = False python_version = 3.6 -[mypy-qapi.error] -disallow_untyped_defs = False -disallow_incomplete_defs = False -check_untyped_defs = False - -[mypy-qapi.expr] -disallow_untyped_defs = False -disallow_incomplete_defs = False -check_untyped_defs = False - [mypy-qapi.parser] disallow_untyped_defs = False disallow_incomplete_defs = False diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index 58267c3db9..ca5e8e18e0 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -18,10 +18,22 @@ from collections import OrderedDict import os import re -from .error import QAPIParseError, QAPISemError +from .error import QAPISemError, QAPISourceError from .source import QAPISourceInfo +class QAPIParseError(QAPISourceError): + """Error class for all QAPI schema parsing errors.""" + def __init__(self, parser, msg): + col = 1 + for ch in parser.src[parser.line_pos:parser.pos]: + if ch == '\t': + col = (col + 7) % 8 + 1 + else: + col += 1 + super().__init__(parser.info, msg, col) + + class QAPISchemaParser: def __init__(self, fname, previously_included=None, incl_info=None): diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc index b9e077a164..88efbf71cb 100644 --- a/scripts/qapi/pylintrc +++ b/scripts/qapi/pylintrc @@ -2,9 +2,7 @@ # Add files or directories matching the regex patterns to the ignore list. # The regex matches against base names, not paths. -ignore-patterns=error.py, - expr.py, - parser.py, +ignore-patterns=parser.py, schema.py, diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index 703b446fd2..3a4172fb74 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -20,7 +20,7 @@ import re from typing import Optional from .common import POINTER_SUFFIX, c_name -from .error import QAPIError, QAPISemError +from .error import QAPISemError, QAPISourceError from .expr import check_exprs from .parser import QAPISchemaParser @@ -875,7 +875,7 @@ class QAPISchema: other_ent = self._entity_dict.get(ent.name) if other_ent: if other_ent.info: - where = QAPIError(other_ent.info, None, "previous definition") + where = QAPISourceError(other_ent.info, "previous definition") raise QAPISemError( ent.info, "'%s' is already defined\n%s" % (ent.name, where)) diff --git a/scripts/qemu-binfmt-conf.sh b/scripts/qemu-binfmt-conf.sh index 573b5dc6ac..7de996d536 100755 --- a/scripts/qemu-binfmt-conf.sh +++ b/scripts/qemu-binfmt-conf.sh @@ -294,7 +294,9 @@ package qemu-$cpu interpreter $qemu magic $magic mask $mask -credential $CREDENTIAL +credentials $CREDENTIAL +preserve $PRESERVE_ARG0 +fix_binary $PERSISTENT EOF } |