From 39a181581650f4d50f4445bc6276d9716cece050 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Wed, 16 Sep 2015 13:06:28 +0200 Subject: qapi: New QMP command query-qmp-schema for QMP introspection qapi/introspect.json defines the introspection schema. It's designed for QMP introspection, but should do for similar uses, such as QGA. The introspection schema does not reflect all the rules and restrictions that apply to QAPI schemata. A valid QAPI schema has an introspection value conforming to the introspection schema, but the converse is not true. Introspection lowers away a number of schema details, and makes implicit things explicit: * The built-in types are declared with their JSON type. All integer types are mapped to 'int', because how many bits we use internally is an implementation detail. It could be pressed into external interface service as very approximate range information, but that's a bad idea. If we need range information, we better do it properly. * Implicit type definitions are made explicit, and given auto-generated names: - Array types, named by appending "List" to the name of their element type, like in generated C. - The enumeration types implicitly defined by simple union types, named by appending "Kind" to the name of their simple union type, like in generated C. - Types that don't occur in generated C. Their names start with ':' so they don't clash with the user's names. * All type references are by name. * The struct and union types are generalized into an object type. * Base types are flattened. * Commands take a single argument and return a single result. Dictionary argument or list result is an implicit type definition. The empty object type is used when a command takes no arguments or produces no results. The argument is always of object type, but the introspection schema doesn't reflect that. The 'gen': false directive is omitted as implementation detail. The 'success-response' directive is omitted as well for now, even though it's not an implementation detail, because it's not used by QMP. * Events carry a single data value. Implicit type definition and empty object type use, just like for commands. The value is of object type, but the introspection schema doesn't reflect that. * Types not used by commands or events are omitted. Indirect use counts as use. * Optional members have a default, which can only be null right now Instead of a mandatory "optional" flag, we have an optional default. No default means mandatory, default null means optional without default value. Non-null is available for optional with default (possible future extension). * Clients should *not* look up types by name, because type names are not ABI. Look up the command or event you're interested in, then follow the references. TODO Should we hide the type names to eliminate the temptation? New generator scripts/qapi-introspect.py computes an introspection value for its input, and generates a C variable holding it. It can generate awfully long lines. Marked TODO. A new test-qmp-input-visitor test case feeds its result for both tests/qapi-schema/qapi-schema-test.json and qapi-schema.json to a QmpInputVisitor to verify it actually conforms to the schema. New QMP command query-qmp-schema takes its return value from that variable. Its reply is some 85KiBytes for me right now. If this turns out to be too much, we have a couple of options: * We can use shorter names in the JSON. Not the QMP style. * Optionally return the sub-schema for commands and events given as arguments. Right now qmp_query_schema() sends the string literal computed by qmp-introspect.py. To compute sub-schema at run time, we'd have to duplicate parts of qapi-introspect.py in C. Unattractive. * Let clients cache the output of query-qmp-schema. It changes only on QEMU upgrades, i.e. rarely. Provide a command query-qmp-schema-hash. Clients can have a cache indexed by hash, and re-query the schema only when they don't have it cached. Even simpler: put the hash in the QMP greeting. Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake --- scripts/qapi-introspect.py | 184 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 scripts/qapi-introspect.py (limited to 'scripts/qapi-introspect.py') diff --git a/scripts/qapi-introspect.py b/scripts/qapi-introspect.py new file mode 100644 index 0000000000..831d6d5944 --- /dev/null +++ b/scripts/qapi-introspect.py @@ -0,0 +1,184 @@ +# +# QAPI introspection generator +# +# Copyright (C) 2015 Red Hat, Inc. +# +# Authors: +# Markus Armbruster +# +# This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. + +from qapi import * + + +# Caveman's json.dumps() replacement (we're stuck at Python 2.4) +# TODO try to use json.dumps() once we get unstuck +def to_json(obj, level=0): + if obj is None: + ret = 'null' + elif isinstance(obj, str): + ret = '"' + obj.replace('"', r'\"') + '"' + elif isinstance(obj, list): + elts = [to_json(elt, level + 1) + for elt in obj] + ret = '[' + ', '.join(elts) + ']' + elif isinstance(obj, dict): + elts = ['"%s": %s' % (key.replace('"', r'\"'), + to_json(obj[key], level + 1)) + for key in sorted(obj.keys())] + ret = '{' + ', '.join(elts) + '}' + else: + assert False # not implemented + if level == 1: + ret = '\n' + ret + return ret + + +def to_c_string(string): + return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"' + + +class QAPISchemaGenIntrospectVisitor(QAPISchemaVisitor): + def __init__(self): + self.defn = None + self.decl = None + self._schema = None + self._jsons = None + self._used_types = None + + def visit_begin(self, schema): + self._schema = schema + self._jsons = [] + self._used_types = [] + return QAPISchemaType # don't visit types for now + + def visit_end(self): + # visit the types that are actually used + for typ in self._used_types: + typ.visit(self) + self._jsons.sort() + # generate C + # TODO can generate awfully long lines + name = prefix + 'qmp_schema_json' + self.decl = mcgen(''' +extern const char %(c_name)s[]; +''', + c_name=c_name(name)) + lines = to_json(self._jsons).split('\n') + c_string = '\n '.join([to_c_string(line) for line in lines]) + self.defn = mcgen(''' +const char %(c_name)s[] = %(c_string)s; +''', + c_name=c_name(name), + c_string=c_string) + self._schema = None + self._jsons = None + self._used_types = None + + def _use_type(self, typ): + # Map the various integer types to plain int + if typ.json_type() == 'int': + typ = self._schema.lookup_type('int') + elif (isinstance(typ, QAPISchemaArrayType) and + typ.element_type.json_type() == 'int'): + typ = self._schema.lookup_type('intList') + # Add type to work queue if new + if typ not in self._used_types: + self._used_types.append(typ) + return typ.name + + def _gen_json(self, name, mtype, obj): + obj['name'] = name + obj['meta-type'] = mtype + self._jsons.append(obj) + + def _gen_member(self, member): + ret = {'name': member.name, 'type': self._use_type(member.type)} + if member.optional: + ret['default'] = None + return ret + + def _gen_variants(self, tag_name, variants): + return {'tag': tag_name, + 'variants': [self._gen_variant(v) for v in variants]} + + def _gen_variant(self, variant): + return {'case': variant.name, 'type': self._use_type(variant.type)} + + def visit_builtin_type(self, name, info, json_type): + self._gen_json(name, 'builtin', {'json-type': json_type}) + + def visit_enum_type(self, name, info, values, prefix): + self._gen_json(name, 'enum', {'values': values}) + + def visit_array_type(self, name, info, element_type): + self._gen_json(name, 'array', + {'element-type': self._use_type(element_type)}) + + def visit_object_type_flat(self, name, info, members, variants): + obj = {'members': [self._gen_member(m) for m in members]} + if variants: + obj.update(self._gen_variants(variants.tag_member.name, + variants.variants)) + self._gen_json(name, 'object', obj) + + def visit_alternate_type(self, name, info, variants): + self._gen_json(name, 'alternate', + {'members': [{'type': self._use_type(m.type)} + for m in variants.variants]}) + + def visit_command(self, name, info, arg_type, ret_type, + gen, success_response): + arg_type = arg_type or self._schema.the_empty_object_type + ret_type = ret_type or self._schema.the_empty_object_type + self._gen_json(name, 'command', + {'arg-type': self._use_type(arg_type), + 'ret-type': self._use_type(ret_type)}) + + def visit_event(self, name, info, arg_type): + arg_type = arg_type or self._schema.the_empty_object_type + self._gen_json(name, 'event', {'arg-type': self._use_type(arg_type)}) + +(input_file, output_dir, do_c, do_h, prefix, dummy) = parse_command_line() + +c_comment = ''' +/* + * QAPI/QMP schema introspection + * + * Copyright (C) 2015 Red Hat, Inc. + * + * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. + * See the COPYING.LIB file in the top-level directory. + * + */ +''' +h_comment = ''' +/* + * QAPI/QMP schema introspection + * + * Copyright (C) 2015 Red Hat, Inc. + * + * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. + * See the COPYING.LIB file in the top-level directory. + * + */ +''' + +(fdef, fdecl) = open_output(output_dir, do_c, do_h, prefix, + 'qmp-introspect.c', 'qmp-introspect.h', + c_comment, h_comment) + +fdef.write(mcgen(''' +#include "%(prefix)sqmp-introspect.h" + +''', + prefix=prefix)) + +schema = QAPISchema(input_file) +gen = QAPISchemaGenIntrospectVisitor() +schema.visit(gen) +fdef.write(gen.defn) +fdecl.write(gen.decl) + +close_output(fdef, fdecl) -- cgit 1.4.1 From 1a9a507b2e3e90aa719c96b4c092e7fad7215f21 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Wed, 16 Sep 2015 13:06:29 +0200 Subject: qapi-introspect: Hide type names To eliminate the temptation for clients to look up types by name (which are not ABI), replace all type names by meaningless strings. Reduces output of query-schema by 13 out of 85KiB. As a debugging aid, provide option -u to suppress the hiding. Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <1442401589-24189-27-git-send-email-armbru@redhat.com> --- docs/qapi-code-gen.txt | 36 +++++++++++++++++------------------- qapi/introspect.json | 12 ++++-------- scripts/qapi-introspect.py | 41 +++++++++++++++++++++++++++++++++++------ 3 files changed, 56 insertions(+), 33 deletions(-) (limited to 'scripts/qapi-introspect.py') diff --git a/docs/qapi-code-gen.txt b/docs/qapi-code-gen.txt index f321d4b949..b1c8361d22 100644 --- a/docs/qapi-code-gen.txt +++ b/docs/qapi-code-gen.txt @@ -530,13 +530,16 @@ additional variant members depending on the value of meta-type. Each SchemaInfo object describes a wire ABI entity of a certain meta-type: a command, event or one of several kinds of type. -SchemaInfo for entities defined in the QAPI schema have the same name -as in the schema. This is the case for all commands and events, and -most types. +SchemaInfo for commands and events have the same name as in the QAPI +schema. Command and event names are part of the wire ABI, but type names are -not. Therefore, looking up a type by its name in the QAPI schema is -wrong. Look up the command or event, then follow references by name. +not. Therefore, the SchemaInfo for types have auto-generated +meaningless names. For readability, the examples in this section use +meaningful type names instead. + +To examine a type, start with a command or event using it, then follow +references by name. QAPI schema definitions not reachable that way are omitted. @@ -567,8 +570,7 @@ object type without members. The event may not have a data member on the wire then. Each command or event defined with dictionary-valued 'data' in the -QAPI schema implicitly defines an object type called ":obj-NAME-arg", -where NAME is the command or event's name. +QAPI schema implicitly defines an object type. Example: the SchemaInfo for EVENT_C from section Events @@ -623,12 +625,9 @@ Note that base types are "flattened": its members are included in the A simple union implicitly defines an enumeration type for its implicit discriminator (called "type" on the wire, see section Union types). -Such a type's name is made by appending "Kind" to the simple union's -name. A simple union implicitly defines an object type for each of its -variants. The type's name is ":obj-NAME-wrapper", where NAME is the -name of the name of the variant's type. +variants. Example: the SchemaInfo for simple union BlockdevOptions from section Union types @@ -659,8 +658,7 @@ Example: the SchemaInfo for BlockRef from section Alternate types The SchemaInfo for an array type has meta-type "array", and variant member "element-type", which names the array's element type. Array -types are implicitly defined. An array type's name is made by -appending "List" to its element type's name. +types are implicitly defined. Example: the SchemaInfo for ['str'] @@ -1067,13 +1065,13 @@ Example: [Uninteresting stuff omitted...] const char example_qmp_schema_json[] = "[" - "{\"arg-type\": \":empty\", \"meta-type\": \"event\", \"name\": \"MY_EVENT\"}, " + "{\"arg-type\": \"0\", \"meta-type\": \"event\", \"name\": \"MY_EVENT\"}, " + "{\"arg-type\": \"1\", \"meta-type\": \"command\", \"name\": \"my-command\", \"ret-type\": \"2\"}, " + "{\"members\": [], \"meta-type\": \"object\", \"name\": \"0\"}, " + "{\"members\": [{\"name\": \"arg1\", \"type\": \"2\"}], \"meta-type\": \"object\", \"name\": \"1\"}, " + "{\"members\": [{\"name\": \"integer\", \"type\": \"int\"}, {\"name\": \"string\", \"type\": \"str\"}], \"meta-type\": \"object\", \"name\": \"2\"}, " "{\"json-type\": \"int\", \"meta-type\": \"builtin\", \"name\": \"int\"}, " - "{\"json-type\": \"string\", \"meta-type\": \"builtin\", \"name\": \"str\"}, " - "{\"members\": [], \"meta-type\": \"object\", \"name\": \":empty\"}, " - "{\"members\": [{\"name\": \"arg1\", \"type\": \"UserDefOne\"}], \"meta-type\": \"object\", \"name\": \":obj-my-command-arg\"}, " - "{\"members\": [{\"name\": \"integer\", \"type\": \"int\"}, {\"name\": \"string\", \"type\": \"str\"}], \"meta-type\": \"object\", \"name\": \"UserDefOne\"}, " - "{\"arg-type\": \":obj-my-command-arg\", \"meta-type\": \"command\", \"name\": \"my-command\", \"ret-type\": \"UserDefOne\"}]"; + "{\"json-type\": \"string\", \"meta-type\": \"builtin\", \"name\": \"str\"}]"; $ cat qapi-generated/example-qmp-introspect.h [Uninteresting stuff omitted...] diff --git a/qapi/introspect.json b/qapi/introspect.json index 9c8ad53d7a..cc50dc6bcb 100644 --- a/qapi/introspect.json +++ b/qapi/introspect.json @@ -75,17 +75,13 @@ # @SchemaInfo # # @name: the entity's name, inherited from @base. -# Entities defined in the QAPI schema have the name defined in -# the schema. Implicitly defined entities have generated -# names. See docs/qapi-code-gen.txt section "Client JSON -# Protocol introspection" for details. +# Commands and events have the name defined in the QAPI schema. +# Unlike command and event names, type names are not part of +# the wire ABI. Consequently, type names are meaningless +# strings here. # # All references to other SchemaInfo are by name. # -# Command and event names are part of the wire ABI, but type names are -# not. Therefore, looking up a type by "well-known" name is wrong. -# Look up the command or event, then follow the references. -# # @meta-type: the entity's meta type, inherited from @base. # # Additional members depend on the value of @meta-type. diff --git a/scripts/qapi-introspect.py b/scripts/qapi-introspect.py index 831d6d5944..7d39320174 100644 --- a/scripts/qapi-introspect.py +++ b/scripts/qapi-introspect.py @@ -40,32 +40,37 @@ def to_c_string(string): class QAPISchemaGenIntrospectVisitor(QAPISchemaVisitor): - def __init__(self): + def __init__(self, unmask): + self._unmask = unmask self.defn = None self.decl = None self._schema = None self._jsons = None self._used_types = None + self._name_map = None def visit_begin(self, schema): self._schema = schema self._jsons = [] self._used_types = [] + self._name_map = {} return QAPISchemaType # don't visit types for now def visit_end(self): # visit the types that are actually used + jsons = self._jsons + self._jsons = [] for typ in self._used_types: typ.visit(self) - self._jsons.sort() # generate C # TODO can generate awfully long lines + jsons.extend(self._jsons) name = prefix + 'qmp_schema_json' self.decl = mcgen(''' extern const char %(c_name)s[]; ''', c_name=c_name(name)) - lines = to_json(self._jsons).split('\n') + lines = to_json(jsons).split('\n') c_string = '\n '.join([to_c_string(line) for line in lines]) self.defn = mcgen(''' const char %(c_name)s[] = %(c_string)s; @@ -75,6 +80,14 @@ const char %(c_name)s[] = %(c_string)s; self._schema = None self._jsons = None self._used_types = None + self._name_map = None + + def _name(self, name): + if self._unmask: + return name + if name not in self._name_map: + self._name_map[name] = '%d' % len(self._name_map) + return self._name_map[name] def _use_type(self, typ): # Map the various integer types to plain int @@ -86,9 +99,16 @@ const char %(c_name)s[] = %(c_string)s; # Add type to work queue if new if typ not in self._used_types: self._used_types.append(typ) - return typ.name + # Clients should examine commands and events, not types. Hide + # type names to reduce the temptation. Also saves a few + # characters. + if isinstance(typ, QAPISchemaBuiltinType): + return typ.name + return self._name(typ.name) def _gen_json(self, name, mtype, obj): + if mtype != 'command' and mtype != 'event' and mtype != 'builtin': + name = self._name(name) obj['name'] = name obj['meta-type'] = mtype self._jsons.append(obj) @@ -140,7 +160,16 @@ const char %(c_name)s[] = %(c_string)s; arg_type = arg_type or self._schema.the_empty_object_type self._gen_json(name, 'event', {'arg-type': self._use_type(arg_type)}) -(input_file, output_dir, do_c, do_h, prefix, dummy) = parse_command_line() +# Debugging aid: unmask QAPI schema's type names +# We normally mask them, because they're not QMP wire ABI +opt_unmask = False + +(input_file, output_dir, do_c, do_h, prefix, opts) = \ + parse_command_line("u", ["unmask-non-abi-names"]) + +for o, a in opts: + if o in ("-u", "--unmask-non-abi-names"): + opt_unmask = True c_comment = ''' /* @@ -176,7 +205,7 @@ fdef.write(mcgen(''' prefix=prefix)) schema = QAPISchema(input_file) -gen = QAPISchemaGenIntrospectVisitor() +gen = QAPISchemaGenIntrospectVisitor(opt_unmask) schema.visit(gen) fdef.write(gen.defn) fdecl.write(gen.decl) -- cgit 1.4.1