From 62815d85aed71eff7b6c3a524705180fb04f5d30 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:01 +0200 Subject: json: Redesign the callback to consume JSON values The classical way to structure parser and lexer is to have the client call the parser to get an abstract syntax tree, the parser call the lexer to get the next token, and the lexer call some function to get input characters. Another way to structure them would be to have the client feed characters to the lexer, the lexer feed tokens to the parser, and the parser feed abstract syntax trees to some callback provided by the client. This way is more easily integrated into an event loop that dispatches input characters as they arrive. Our JSON parser is kind of between the two. The lexer feeds tokens to a "streamer" instead of a real parser. The streamer accumulates tokens until it got the sequence of tokens that comprise a single JSON value (it counts curly braces and square brackets to decide). It feeds those token sequences to a callback provided by the client. The callback passes each token sequence to the parser, and gets back an abstract syntax tree. I figure it was done that way to make a straightforward recursive descent parser possible. "Get next token" becomes "pop the first token off the token sequence". Drawback: we need to store a complete token sequence. Each token eats 13 + input characters + malloc overhead bytes. Observations: 1. This is not the only way to use recursive descent. If we replaced "get next token" by a coroutine yield, we could do without a streamer. 2. The lexer reports errors by passing a JSON_ERROR token to the streamer. This communicates the offending input characters and their location, but no more. 3. The streamer reports errors by passing a null token sequence to the callback. The (already poor) lexical error information is thrown away. 4. Having the callback receive a token sequence duplicates the code to convert token sequence to abstract syntax tree in every callback. 5. Known bug: the streamer silently drops incomplete token sequences. This commit rectifies 4. by lifting the call of the parser from the callbacks into the streamer. Later commits will address 3. and 5. The lifting removes a bug from qjson.c's parse_json(): it passed a pointer to a non-null Error * in certain cases, as demonstrated by check-qjson.c. json_parser_parse() is now unused. It's a stupid wrapper around json_parser_parse_err(). Drop it, and rename json_parser_parse_err() to json_parser_parse(). Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <20180823164025.12553-35-armbru@redhat.com> --- qobject/qjson.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'qobject/qjson.c') diff --git a/qobject/qjson.c b/qobject/qjson.c index ab4040f235..7395556069 100644 --- a/qobject/qjson.c +++ b/qobject/qjson.c @@ -13,8 +13,6 @@ #include "qemu/osdep.h" #include "qapi/error.h" -#include "qapi/qmp/json-lexer.h" -#include "qapi/qmp/json-parser.h" #include "qapi/qmp/json-streamer.h" #include "qapi/qmp/qjson.h" #include "qapi/qmp/qbool.h" @@ -27,16 +25,16 @@ typedef struct JSONParsingState { JSONMessageParser parser; - va_list *ap; QObject *result; Error *err; } JSONParsingState; -static void parse_json(JSONMessageParser *parser, GQueue *tokens) +static void consume_json(void *opaque, QObject *json, Error *err) { - JSONParsingState *s = container_of(parser, JSONParsingState, parser); + JSONParsingState *s = opaque; - s->result = json_parser_parse_err(tokens, s->ap, &s->err); + s->result = json; + error_propagate(&s->err, err); } /* @@ -54,9 +52,7 @@ static QObject *qobject_from_jsonv(const char *string, va_list *ap, { JSONParsingState state = {}; - state.ap = ap; - - json_message_parser_init(&state.parser, parse_json); + json_message_parser_init(&state.parser, consume_json, &state, ap); json_message_parser_feed(&state.parser, string, strlen(string)); json_message_parser_flush(&state.parser); json_message_parser_destroy(&state.parser); -- cgit 1.4.1 From 2a4794ba146d6560bd77ca840ff6908f81d585f4 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:10 +0200 Subject: qjson: Fix qobject_from_json() & friends for multiple values qobject_from_json() & friends use the consume_json() callback to receive either a value or an error from the parser. When they are fed a string that contains more than either one JSON value or one JSON syntax error, consume_json() gets called multiple times. When the last call receives a value, qobject_from_json() returns that value. Any other values are leaked. When any call receives an error, qobject_from_json() sets the first error received. Any other errors are thrown away. When values follow errors, qobject_from_json() returns both a value and sets an error. That's bad. Impact: * block.c's parse_json_protocol() ignores and leaks the value. It's used to to parse pseudo-filenames starting with "json:". The pseudo-filenames can come from the user or from image meta-data such as a QCOW2 image's backing file name. * vl.c's parse_display_qapi() ignores and leaks the error. It's used to parse the argument of command line option -display. * vl.c's main() case QEMU_OPTION_blockdev ignores the error and leaves it in @err. main() will then pass a pointer to a non-null Error * to net_init_clients(), which is forbidden. It can lead to assertion failure or other misbehavior. * check-qjson.c's multiple_values() demonstrates the badness. * The other callers are not affected since they only pass strings with exactly one JSON value or, in the case of negative tests, one error. The impact on the _nofail() functions is relatively harmless. They abort when any call receives an error. Else they return the last value, and leak the others, if any. Fix consume_json() as follows. On the first call, save value and error as before. On subsequent calls, if any, don't save them. If the first call saved a value, the next call, if any, replaces the value by an "Expecting at most one JSON value" error. Take care not to leak values or errors that aren't saved. Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <20180823164025.12553-44-armbru@redhat.com> --- qobject/qjson.c | 15 ++++++++++++++- tests/check-qjson.c | 10 +++------- 2 files changed, 17 insertions(+), 8 deletions(-) (limited to 'qobject/qjson.c') diff --git a/qobject/qjson.c b/qobject/qjson.c index 7395556069..7f69036487 100644 --- a/qobject/qjson.c +++ b/qobject/qjson.c @@ -33,8 +33,21 @@ static void consume_json(void *opaque, QObject *json, Error *err) { JSONParsingState *s = opaque; + assert(!json != !err); + assert(!s->result || !s->err); + + if (s->result) { + qobject_unref(s->result); + s->result = NULL; + error_setg(&s->err, "Expecting at most one JSON value"); + } + if (s->err) { + qobject_unref(json); + error_free(err); + return; + } s->result = json; - error_propagate(&s->err, err); + s->err = err; } /* diff --git a/tests/check-qjson.c b/tests/check-qjson.c index f344ad921c..f9438370d9 100644 --- a/tests/check-qjson.c +++ b/tests/check-qjson.c @@ -1443,17 +1443,13 @@ static void multiple_values(void) Error *err = NULL; QObject *obj; - /* BUG this leaks the syntax tree for "false" */ obj = qobject_from_json("false true", &err); - g_assert(qbool_get_bool(qobject_to(QBool, obj))); - g_assert(!err); - qobject_unref(obj); + error_free_or_abort(&err); + g_assert(obj == NULL); - /* BUG simultaneously succeeds and fails */ obj = qobject_from_json("} true", &err); - g_assert(qbool_get_bool(qobject_to(QBool, obj))); error_free_or_abort(&err); - qobject_unref(obj); + g_assert(obj == NULL); } int main(int argc, char **argv) -- cgit 1.4.1 From dd98e8481992741a6b5ec0bdfcee05c1c8f602d6 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:14 +0200 Subject: qjson: Have qobject_from_json() & friends reject empty and blank The last case where qobject_from_json() & friends return null without setting an error is empty or blank input. Callers: * block.c's parse_json_protocol() reports "Could not parse the JSON options". It's marked as a work-around, because it also covered actual bugs, but they got fixed in the previous few commits. * qobject_input_visitor_new_str() reports "JSON parse error". Also marked as work-around. The recent fixes have made this unreachable, because it currently gets called only for input starting with '{'. * check-qjson.c's empty_input() and blank_input() demonstrate the behavior. * The other callers are not affected since they only pass input with exactly one JSON value or, in the case of negative tests, one error. Fail with "Expecting a JSON value" instead of returning null, and simplify callers. Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <20180823164025.12553-48-armbru@redhat.com> --- block.c | 5 ----- qapi/qobject-input-visitor.c | 5 ----- qobject/qjson.c | 4 ++++ tests/check-qjson.c | 12 ++++++++++-- 4 files changed, 14 insertions(+), 12 deletions(-) (limited to 'qobject/qjson.c') diff --git a/block.c b/block.c index 6161dbe3eb..0dbb1fcc7b 100644 --- a/block.c +++ b/block.c @@ -1478,11 +1478,6 @@ static QDict *parse_json_filename(const char *filename, Error **errp) options_obj = qobject_from_json(filename, errp); if (!options_obj) { - /* Work around qobject_from_json() lossage TODO fix that */ - if (errp && !*errp) { - error_setg(errp, "Could not parse the JSON options"); - return NULL; - } error_prepend(errp, "Could not parse the JSON options: "); return NULL; } diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c index da57f4cc24..3e88b27f9e 100644 --- a/qapi/qobject-input-visitor.c +++ b/qapi/qobject-input-visitor.c @@ -725,11 +725,6 @@ Visitor *qobject_input_visitor_new_str(const char *str, if (is_json) { obj = qobject_from_json(str, errp); if (!obj) { - /* Work around qobject_from_json() lossage TODO fix that */ - if (errp && !*errp) { - error_setg(errp, "JSON parse error"); - return NULL; - } return NULL; } args = qobject_to(QDict, obj); diff --git a/qobject/qjson.c b/qobject/qjson.c index 7f69036487..b9ccae2c2a 100644 --- a/qobject/qjson.c +++ b/qobject/qjson.c @@ -70,6 +70,10 @@ static QObject *qobject_from_jsonv(const char *string, va_list *ap, json_message_parser_flush(&state.parser); json_message_parser_destroy(&state.parser); + if (!state.result && !state.err) { + error_setg(&state.err, "Expecting a JSON value"); + } + error_propagate(errp, state.err); return state.result; } diff --git a/tests/check-qjson.c b/tests/check-qjson.c index 0ca4b3c823..936258ddd4 100644 --- a/tests/check-qjson.c +++ b/tests/check-qjson.c @@ -1291,13 +1291,21 @@ static void simple_interpolation(void) static void empty_input(void) { - QObject *obj = qobject_from_json("", &error_abort); + Error *err = NULL; + QObject *obj; + + obj = qobject_from_json("", &err); + error_free_or_abort(&err); g_assert(obj == NULL); } static void blank_input(void) { - QObject *obj = qobject_from_json("\n ", &error_abort); + Error *err = NULL; + QObject *obj; + + obj = qobject_from_json("\n ", &err); + error_free_or_abort(&err); g_assert(obj == NULL); } -- cgit 1.4.1 From 86cdf9ec8dec2763702cc52fa412d108a5dc9608 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:20 +0200 Subject: json: Clean up headers The JSON parser has three public headers, json-lexer.h, json-parser.h, json-streamer.h. They all contain stuff that is of no interest outside qobject/json-*.c. Collect the public interface in include/qapi/qmp/json-parser.h, and everything else in qobject/json-parser-int.h. Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <20180823164025.12553-54-armbru@redhat.com> --- include/qapi/qmp/json-lexer.h | 50 ------------------------------------- include/qapi/qmp/json-parser.h | 36 +++++++++++++++++++++------ include/qapi/qmp/json-streamer.h | 46 ---------------------------------- monitor.c | 2 +- qga/main.c | 2 +- qobject/json-lexer.c | 3 +-- qobject/json-parser-int.h | 54 ++++++++++++++++++++++++++++++++++++++++ qobject/json-parser.c | 4 +-- qobject/json-streamer.c | 4 +-- qobject/qjson.c | 2 +- tests/libqtest.c | 2 +- 11 files changed, 90 insertions(+), 115 deletions(-) delete mode 100644 include/qapi/qmp/json-lexer.h delete mode 100644 include/qapi/qmp/json-streamer.h create mode 100644 qobject/json-parser-int.h (limited to 'qobject/qjson.c') diff --git a/include/qapi/qmp/json-lexer.h b/include/qapi/qmp/json-lexer.h deleted file mode 100644 index 508fc7bdaf..0000000000 --- a/include/qapi/qmp/json-lexer.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * JSON lexer - * - * Copyright IBM, Corp. 2009 - * - * Authors: - * Anthony Liguori - * - * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. - * See the COPYING.LIB file in the top-level directory. - * - */ - -#ifndef QEMU_JSON_LEXER_H -#define QEMU_JSON_LEXER_H - - -typedef enum json_token_type { - JSON_MIN = 100, - JSON_LCURLY = JSON_MIN, - JSON_RCURLY, - JSON_LSQUARE, - JSON_RSQUARE, - JSON_COLON, - JSON_COMMA, - JSON_INTEGER, - JSON_FLOAT, - JSON_KEYWORD, - JSON_STRING, - JSON_INTERP, - JSON_SKIP, - JSON_ERROR, - JSON_END_OF_INPUT, -} JSONTokenType; - -typedef struct JSONLexer { - int start_state, state; - GString *token; - int x, y; -} JSONLexer; - -void json_lexer_init(JSONLexer *lexer, bool enable_interpolation); - -void json_lexer_feed(JSONLexer *lexer, const char *buffer, size_t size); - -void json_lexer_flush(JSONLexer *lexer); - -void json_lexer_destroy(JSONLexer *lexer); - -#endif diff --git a/include/qapi/qmp/json-parser.h b/include/qapi/qmp/json-parser.h index 55f75954c3..7345a9bd5c 100644 --- a/include/qapi/qmp/json-parser.h +++ b/include/qapi/qmp/json-parser.h @@ -1,5 +1,5 @@ /* - * JSON Parser + * JSON Parser * * Copyright IBM, Corp. 2009 * @@ -11,14 +11,36 @@ * */ -#ifndef QEMU_JSON_PARSER_H -#define QEMU_JSON_PARSER_H +#ifndef QAPI_QMP_JSON_PARSER_H +#define QAPI_QMP_JSON_PARSER_H -#include "qapi/qmp/json-lexer.h" +typedef struct JSONLexer { + int start_state, state; + GString *token; + int x, y; +} JSONLexer; -typedef struct JSONToken JSONToken; +typedef struct JSONMessageParser { + void (*emit)(void *opaque, QObject *json, Error *err); + void *opaque; + va_list *ap; + JSONLexer lexer; + int brace_count; + int bracket_count; + GQueue tokens; + uint64_t token_size; +} JSONMessageParser; -JSONToken *json_token(JSONTokenType type, int x, int y, GString *tokstr); -QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp); +void json_message_parser_init(JSONMessageParser *parser, + void (*emit)(void *opaque, QObject *json, + Error *err), + void *opaque, va_list *ap); + +void json_message_parser_feed(JSONMessageParser *parser, + const char *buffer, size_t size); + +void json_message_parser_flush(JSONMessageParser *parser); + +void json_message_parser_destroy(JSONMessageParser *parser); #endif diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h deleted file mode 100644 index 29950ac37c..0000000000 --- a/include/qapi/qmp/json-streamer.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * JSON streaming support - * - * Copyright IBM, Corp. 2009 - * - * Authors: - * Anthony Liguori - * - * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. - * See the COPYING.LIB file in the top-level directory. - * - */ - -#ifndef QEMU_JSON_STREAMER_H -#define QEMU_JSON_STREAMER_H - -#include "qapi/qmp/json-lexer.h" - -typedef struct JSONMessageParser -{ - void (*emit)(void *opaque, QObject *json, Error *err); - void *opaque; - va_list *ap; - JSONLexer lexer; - int brace_count; - int bracket_count; - GQueue tokens; - uint64_t token_size; -} JSONMessageParser; - -void json_message_process_token(JSONLexer *lexer, GString *input, - JSONTokenType type, int x, int y); - -void json_message_parser_init(JSONMessageParser *parser, - void (*emit)(void *opaque, QObject *json, - Error *err), - void *opaque, va_list *ap); - -void json_message_parser_feed(JSONMessageParser *parser, - const char *buffer, size_t size); - -void json_message_parser_flush(JSONMessageParser *parser); - -void json_message_parser_destroy(JSONMessageParser *parser); - -#endif diff --git a/monitor.c b/monitor.c index 3dbdcb5190..021c11b1bf 100644 --- a/monitor.c +++ b/monitor.c @@ -58,7 +58,7 @@ #include "qapi/qmp/qnum.h" #include "qapi/qmp/qstring.h" #include "qapi/qmp/qjson.h" -#include "qapi/qmp/json-streamer.h" +#include "qapi/qmp/json-parser.h" #include "qapi/qmp/qlist.h" #include "qom/object_interfaces.h" #include "trace-root.h" diff --git a/qga/main.c b/qga/main.c index b74e1241ef..6d70242d05 100644 --- a/qga/main.c +++ b/qga/main.c @@ -18,7 +18,7 @@ #include #include #endif -#include "qapi/qmp/json-streamer.h" +#include "qapi/qmp/json-parser.h" #include "qapi/qmp/qdict.h" #include "qapi/qmp/qjson.h" #include "qapi/qmp/qstring.h" diff --git a/qobject/json-lexer.c b/qobject/json-lexer.c index 06ec67dc45..e1745a3d95 100644 --- a/qobject/json-lexer.c +++ b/qobject/json-lexer.c @@ -12,8 +12,7 @@ */ #include "qemu/osdep.h" -#include "qapi/qmp/json-lexer.h" -#include "qapi/qmp/json-streamer.h" +#include "json-parser-int.h" #define MAX_TOKEN_SIZE (64ULL << 20) diff --git a/qobject/json-parser-int.h b/qobject/json-parser-int.h new file mode 100644 index 0000000000..ceaa890ec6 --- /dev/null +++ b/qobject/json-parser-int.h @@ -0,0 +1,54 @@ +/* + * JSON Parser + * + * Copyright IBM, Corp. 2009 + * + * Authors: + * Anthony Liguori + * + * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. + * See the COPYING.LIB file in the top-level directory. + * + */ + +#ifndef JSON_PARSER_INT_H +#define JSON_PARSER_INT_H + +#include "qapi/qmp/json-parser.h" + + +typedef enum json_token_type { + JSON_MIN = 100, + JSON_LCURLY = JSON_MIN, + JSON_RCURLY, + JSON_LSQUARE, + JSON_RSQUARE, + JSON_COLON, + JSON_COMMA, + JSON_INTEGER, + JSON_FLOAT, + JSON_KEYWORD, + JSON_STRING, + JSON_INTERP, + JSON_SKIP, + JSON_ERROR, + JSON_END_OF_INPUT, +} JSONTokenType; + +typedef struct JSONToken JSONToken; + +/* json-lexer.c */ +void json_lexer_init(JSONLexer *lexer, bool enable_interpolation); +void json_lexer_feed(JSONLexer *lexer, const char *buffer, size_t size); +void json_lexer_flush(JSONLexer *lexer); +void json_lexer_destroy(JSONLexer *lexer); + +/* json-streamer.c */ +void json_message_process_token(JSONLexer *lexer, GString *input, + JSONTokenType type, int x, int y); + +/* json-parser.c */ +JSONToken *json_token(JSONTokenType type, int x, int y, GString *tokstr); +QObject *json_parser_parse(GQueue *tokens, va_list *ap, Error **errp); + +#endif diff --git a/qobject/json-parser.c b/qobject/json-parser.c index a247875f14..7449684f1c 100644 --- a/qobject/json-parser.c +++ b/qobject/json-parser.c @@ -22,9 +22,7 @@ #include "qapi/qmp/qnull.h" #include "qapi/qmp/qnum.h" #include "qapi/qmp/qstring.h" -#include "qapi/qmp/json-parser.h" -#include "qapi/qmp/json-lexer.h" -#include "qapi/qmp/json-streamer.h" +#include "json-parser-int.h" struct JSONToken { JSONTokenType type; diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c index da53e770e9..47dd7ea576 100644 --- a/qobject/json-streamer.c +++ b/qobject/json-streamer.c @@ -13,9 +13,7 @@ #include "qemu/osdep.h" #include "qapi/error.h" -#include "qapi/qmp/json-lexer.h" -#include "qapi/qmp/json-parser.h" -#include "qapi/qmp/json-streamer.h" +#include "json-parser-int.h" #define MAX_TOKEN_SIZE (64ULL << 20) #define MAX_TOKEN_COUNT (2ULL << 20) diff --git a/qobject/qjson.c b/qobject/qjson.c index b9ccae2c2a..db36101f3b 100644 --- a/qobject/qjson.c +++ b/qobject/qjson.c @@ -13,7 +13,7 @@ #include "qemu/osdep.h" #include "qapi/error.h" -#include "qapi/qmp/json-streamer.h" +#include "qapi/qmp/json-parser.h" #include "qapi/qmp/qjson.h" #include "qapi/qmp/qbool.h" #include "qapi/qmp/qdict.h" diff --git a/tests/libqtest.c b/tests/libqtest.c index 5973a67652..d635c5bea0 100644 --- a/tests/libqtest.c +++ b/tests/libqtest.c @@ -24,7 +24,7 @@ #include "qemu-common.h" #include "qemu/cutils.h" #include "qapi/error.h" -#include "qapi/qmp/json-streamer.h" +#include "qapi/qmp/json-parser.h" #include "qapi/qmp/qdict.h" #include "qapi/qmp/qjson.h" #include "qapi/qmp/qlist.h" -- cgit 1.4.1