From da949d495ddd4d36f2c9750eb4d70f4135018199 Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrangé" Date: Tue, 16 Sep 2025 09:16:35 +0100 Subject: tracetool: add test suite for tracetool with reference output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When reviewing tracetool patches it is often very unclear what the expected output will be for the generated backends. Compounding this is that a default build will only enable the 'log' trace backend, so developers won't see generated code for other backends without making a special effort. Some backends are also platform specific, so can't be enabled in QEMU builds, even though tracetool could generate the code. To address this, introduce a test suite for tracetool which is conceptually similar to the qapi-schema test. It is a simple python program that runs tracetool and compares the actual output to historical reference output kept in git. The test directly emits TAP format logs for ease of integration with meson. This can be run with make check-tracetool to make it easier for developers changing generated output, the sample expected content can be auto-recreated QEMU_TEST_REGENERATE=1 make check-tracetool and the changes reviewed and added to the commit. This will also assist reviewers interpreting the change. Developers are reminded of this in the test output on failure: $ make check-tracetool 1/6 qemu:tracetool / dtrace OK 0.14s 2/6 qemu:tracetool / ftrace FAIL 0.06s exit status 1 ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――― 1..2 ok 1 - ftrace.c # not ok 1 - ftrace.h (set QEMU_TEST_REGENERATE=1 to recreate reference output if tracetool generator was intentionally changed) ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――― 3/6 qemu:tracetool / log OK 0.06s 4/6 qemu:tracetool / simple OK 0.06s 5/6 qemu:tracetool / syslog OK 0.06s 6/6 qemu:tracetool / ust OK 0.11s Summary of Failures: 2/6 qemu:tracetool / ftrace FAIL 0.06s exit status 1 Signed-off-by: Daniel P. Berrangé Message-id: 20250916081638.764020-6-berrange@redhat.com Signed-off-by: Stefan Hajnoczi --- docs/devel/testing/main.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'docs/devel/testing') diff --git a/docs/devel/testing/main.rst b/docs/devel/testing/main.rst index 2b5cb0c148..11f05c0006 100644 --- a/docs/devel/testing/main.rst +++ b/docs/devel/testing/main.rst @@ -178,6 +178,34 @@ parser (either fixing a bug or extending/modifying the syntax). To do this: ``qapi-schema += foo.json`` +.. _tracetool-tests: + +Tracetool tests +~~~~~~~~~~~~~~~ + +The tracetool tests validate the generated source files used for defining +probes for various tracing backends and source formats. The test operates +by running the tracetool program against a sample trace-events file, and +comparing the generated output against known good reference output. The +tests can be run with: + +.. code:: + + make check-tracetool + +The reference output is stored in files under tests/tracetool, and when +the tracetool backend/format output is intentionally changed, the reference +files need to be updated. This can be automated by setting the +QEMU_TEST_REGENERATE=1 environment variable: + +.. code:: + + QEMU_TEST_REGENERATE=1 make check-tracetool + +The resulting changes must be reviewed by the author to ensure they match +the intended results, before adding the updated reference output to the +same commit that alters the generator code. + check-block ~~~~~~~~~~~ -- cgit 1.4.1 From 4b1e71591254a7e924b6b295868edc6550ed0420 Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrangé" Date: Tue, 16 Sep 2025 09:16:37 +0100 Subject: qapi: switch to use QEMU_TEST_REGENERATE env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The QAPI_TEST_UPDATE env var can be set when running the QAPI schema tests to regenerate the reference output. For consistent naming with the tracetool test, change the env var name to QEMU_TEST_REGENERATE. The test is modified to provide a hint about use of the new env var and it is also added to the developer documentation.document its usage. Reviewed-by: Stefan Hajnoczi Signed-off-by: Daniel P. Berrangé Reviewed-by: Markus Armbruster Message-id: 20250916081638.764020-8-berrange@redhat.com Signed-off-by: Stefan Hajnoczi --- docs/devel/testing/main.rst | 12 ++++++++++++ tests/qapi-schema/test-qapi.py | 7 +++++-- 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'docs/devel/testing') diff --git a/docs/devel/testing/main.rst b/docs/devel/testing/main.rst index 11f05c0006..0662766b5c 100644 --- a/docs/devel/testing/main.rst +++ b/docs/devel/testing/main.rst @@ -178,6 +178,18 @@ parser (either fixing a bug or extending/modifying the syntax). To do this: ``qapi-schema += foo.json`` +The reference output can be automatically updated to match the latest QAPI +code generator by running the tests with the QEMU_TEST_REGENERATE environment +variable set. + +.. code:: + + QEMU_TEST_REGENERATE=1 make check-qapi-schema + +The resulting changes must be reviewed by the author to ensure they match +the intended results before adding the updated reference output to the +same commit that alters the generator code. + .. _tracetool-tests: Tracetool tests diff --git a/tests/qapi-schema/test-qapi.py b/tests/qapi-schema/test-qapi.py index 4be930228c..cf7fb8a6df 100755 --- a/tests/qapi-schema/test-qapi.py +++ b/tests/qapi-schema/test-qapi.py @@ -165,7 +165,7 @@ def test_and_diff(test_name, dir_name, update): if actual_out == expected_out and actual_err == expected_err: return 0 - print("%s %s" % (test_name, 'UPDATE' if update else 'FAIL'), + print("%s: %s" % (test_name, 'UPDATE' if update else 'FAIL'), file=sys.stderr) out_diff = difflib.unified_diff(expected_out, actual_out, outfp.name) err_diff = difflib.unified_diff(expected_err, actual_err, errfp.name) @@ -173,6 +173,9 @@ def test_and_diff(test_name, dir_name, update): sys.stdout.writelines(err_diff) if not update: + print(("\n%s: set QEMU_TEST_REGENERATE=1 to recreate reference output" + + "if the QAPI schema generator was intentionally changed") % test_name, + file=sys.stderr) return 1 try: @@ -197,7 +200,7 @@ def main(argv): parser.add_argument('-d', '--dir', action='store', default='', help="directory containing tests") parser.add_argument('-u', '--update', action='store_true', - default='QAPI_TEST_UPDATE' in os.environ, + default='QEMU_TEST_REGENERATE' in os.environ, help="update expected test results") parser.add_argument('tests', nargs='*', metavar='TEST', action='store') args = parser.parse_args() -- cgit 1.4.1