From 555b27a75072d2298a5c37f81df41036b780e181 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 1 Sep 2020 09:58:48 -0400 Subject: mtest2make: split environment from test command Pass the environment and test command in separate macro arguments, so that we will be able to insert a test driver in the next patch. Signed-off-by: Paolo Bonzini --- scripts/mtest2make.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'scripts/mtest2make.py') diff --git a/scripts/mtest2make.py b/scripts/mtest2make.py index d7a51bf97e..f4ee4d3994 100644 --- a/scripts/mtest2make.py +++ b/scripts/mtest2make.py @@ -19,16 +19,16 @@ class Suite(object): print(''' SPEED = quick -# $1 = test command, $2 = test name -.test-human-tap = $1 < /dev/null | ./scripts/tap-driver.pl --test-name="$2" $(if $(V),,--show-failures-only) -.test-human-exitcode = $1 < /dev/null -.test-tap-tap = $1 < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $2/" || true -.test-tap-exitcode = printf "%s\\n" 1..1 "`$1 < /dev/null > /dev/null || echo "not "`ok 1 $2" -.test.print = echo $(if $(V),'$1','Running test $2') >&3 +# $1 = environment, $2 = test command, $3 = test name +.test-human-tap = $1 $2 < /dev/null | ./scripts/tap-driver.pl --test-name="$3" $(if $(V),,--show-failures-only) +.test-human-exitcode = $1 $2 < /dev/null +.test-tap-tap = $1 $2 < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $3/" || true +.test-tap-exitcode = printf "%s\\n" 1..1 "`$1 $2 < /dev/null > /dev/null || echo "not "`ok 1 $3" +.test.print = echo $(if $(V),'$1 $2','Running test $3') >&3 .test.env = MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$(( $${RANDOM:-0} % 255 + 1))} # $1 = test name, $2 = test target (human or tap) -.test.run = $(call .test.print,$(.test.cmd.$1),$(.test.name.$1)) && $(call .test-$2-$(.test.driver.$1),$(.test.cmd.$1),$(.test.name.$1)) +.test.run = $(call .test.print,$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1)) && $(call .test-$2-$(.test.driver.$1),$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1)) define .test.human_k @exec 3>&1; rc=0; $(foreach TEST, $1, $(call .test.run,$(TEST),human) || rc=$$?;) \\ @@ -65,7 +65,7 @@ for test in json.load(sys.stdin): test['cmd'][0] = executable else: test['cmd'][0] = executable - cmd = '$(.test.env) %s %s' % (env, ' '.join((shlex.quote(x) for x in test['cmd']))) + cmd = ' '.join((shlex.quote(x) for x in test['cmd'])) if test['workdir'] is not None: cmd = '(cd %s && %s)' % (shlex.quote(test['workdir']), cmd) driver = test['protocol'] if 'protocol' in test else 'exitcode' @@ -73,6 +73,7 @@ for test in json.load(sys.stdin): i += 1 print('.test.name.%d := %s' % (i, test['name'])) print('.test.driver.%d := %s' % (i, driver)) + print('.test.env.%d := $(.test.env) %s' % (i, env)) print('.test.cmd.%d := %s' % (i, cmd)) test_suites = test['suite'] or ['default'] -- cgit 1.4.1 From 42d729e12c9a03e1140d681a5fdeceb10c4fa548 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 1 Sep 2020 09:58:48 -0400 Subject: mtest2make: split working directory from test command Pass the working directory and test command in separate macro arguments, so that we will be able to insert a test driver in the next patch. Signed-off-by: Paolo Bonzini --- scripts/mtest2make.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'scripts/mtest2make.py') diff --git a/scripts/mtest2make.py b/scripts/mtest2make.py index f4ee4d3994..c709b37f28 100644 --- a/scripts/mtest2make.py +++ b/scripts/mtest2make.py @@ -19,16 +19,16 @@ class Suite(object): print(''' SPEED = quick -# $1 = environment, $2 = test command, $3 = test name -.test-human-tap = $1 $2 < /dev/null | ./scripts/tap-driver.pl --test-name="$3" $(if $(V),,--show-failures-only) -.test-human-exitcode = $1 $2 < /dev/null -.test-tap-tap = $1 $2 < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $3/" || true -.test-tap-exitcode = printf "%s\\n" 1..1 "`$1 $2 < /dev/null > /dev/null || echo "not "`ok 1 $3" +# $1 = environment, $2 = test command, $3 = test name, $4 = dir +.test-human-tap = $1 $(if $4,(cd $4 && $2),$2) < /dev/null | ./scripts/tap-driver.pl --test-name="$3" $(if $(V),,--show-failures-only) +.test-human-exitcode = $1 $(if $4,(cd $4 && $2),$2) < /dev/null +.test-tap-tap = $1 $(if $4,(cd $4 && $2),$2) < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $3/" || true +.test-tap-exitcode = printf "%s\\n" 1..1 "`$1 $(if $4,(cd $4 && $2),$2) < /dev/null > /dev/null || echo "not "`ok 1 $3" .test.print = echo $(if $(V),'$1 $2','Running test $3') >&3 .test.env = MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$(( $${RANDOM:-0} % 255 + 1))} # $1 = test name, $2 = test target (human or tap) -.test.run = $(call .test.print,$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1)) && $(call .test-$2-$(.test.driver.$1),$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1)) +.test.run = $(call .test.print,$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1)) && $(call .test-$2-$(.test.driver.$1),$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1),$(.test.dir.$1)) define .test.human_k @exec 3>&1; rc=0; $(foreach TEST, $1, $(call .test.run,$(TEST),human) || rc=$$?;) \\ @@ -66,11 +66,11 @@ for test in json.load(sys.stdin): else: test['cmd'][0] = executable cmd = ' '.join((shlex.quote(x) for x in test['cmd'])) - if test['workdir'] is not None: - cmd = '(cd %s && %s)' % (shlex.quote(test['workdir']), cmd) driver = test['protocol'] if 'protocol' in test else 'exitcode' i += 1 + if test['workdir'] is not None: + print('.test.dir.%d := %s' % (i, shlex.quote(test['workdir']))) print('.test.name.%d := %s' % (i, test['name'])) print('.test.driver.%d := %s' % (i, driver)) print('.test.env.%d := $(.test.env) %s' % (i, env)) -- cgit 1.4.1 From d322e84eef850d1e16bca81ac9b6811f13a8d7a7 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 1 Sep 2020 09:14:41 -0400 Subject: mtest2make: hide output of successful tests The softfloat tests are quite noisy; before the Meson conversion they buffered the output in a file and emitted the output only if the test failed. Tweak mtest2make.py so that the courtesy is extended to all non-TAP tests. Signed-off-by: Paolo Bonzini --- scripts/mtest2make.py | 2 +- scripts/test-driver.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 scripts/test-driver.py (limited to 'scripts/mtest2make.py') diff --git a/scripts/mtest2make.py b/scripts/mtest2make.py index c709b37f28..27425080cf 100644 --- a/scripts/mtest2make.py +++ b/scripts/mtest2make.py @@ -21,7 +21,7 @@ SPEED = quick # $1 = environment, $2 = test command, $3 = test name, $4 = dir .test-human-tap = $1 $(if $4,(cd $4 && $2),$2) < /dev/null | ./scripts/tap-driver.pl --test-name="$3" $(if $(V),,--show-failures-only) -.test-human-exitcode = $1 $(if $4,(cd $4 && $2),$2) < /dev/null +.test-human-exitcode = $1 $(PYTHON) scripts/test-driver.py $(if $4,-C$4) $(if $(V),--verbose) -- $2 < /dev/null .test-tap-tap = $1 $(if $4,(cd $4 && $2),$2) < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $3/" || true .test-tap-exitcode = printf "%s\\n" 1..1 "`$1 $(if $4,(cd $4 && $2),$2) < /dev/null > /dev/null || echo "not "`ok 1 $3" .test.print = echo $(if $(V),'$1 $2','Running test $3') >&3 diff --git a/scripts/test-driver.py b/scripts/test-driver.py new file mode 100644 index 0000000000..eef74b29a8 --- /dev/null +++ b/scripts/test-driver.py @@ -0,0 +1,35 @@ +#! /usr/bin/env python3 + +# Wrapper for tests that hides the output if they succeed. +# Used by "make check" +# +# Copyright (C) 2020 Red Hat, Inc. +# +# Author: Paolo Bonzini + +import subprocess +import sys +import os +import argparse + +parser = argparse.ArgumentParser(description='Test driver for QEMU') +parser.add_argument('-C', metavar='DIR', dest='dir', default='.', + help='change to DIR before doing anything else') +parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', + help='be more verbose') +parser.add_argument('test_args', nargs=argparse.REMAINDER) + +args = parser.parse_args() +os.chdir(args.dir) + +test_args = args.test_args +if test_args[0] == '--': + test_args = test_args[1:] + +if args.verbose: + result = subprocess.run(test_args, stdout=None, stderr=None) +else: + result = subprocess.run(test_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode: + sys.stdout.buffer.write(result.stdout) +sys.exit(result.returncode) -- cgit 1.4.1 From 40d9b74eafc9db5d19b92e32932ba915e6e2e664 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 2 Sep 2020 07:19:43 -0400 Subject: mtest2make: unify tests that appear in multiple suites Whenever a test appears in multiple suites, the rules generated by mtest2make are currently running it twice. Instead, after this patch we generate a phony target for each test and we have a generic "run-tests" target depend on all the tests that were chosen on the command line. Tests that appear in multiple suites will be added to the prerequisites just once. This has other advantages: it removes the handling of -k and it increases parallelism. Signed-off-by: Paolo Bonzini --- scripts/mtest2make.py | 74 ++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 36 deletions(-) (limited to 'scripts/mtest2make.py') diff --git a/scripts/mtest2make.py b/scripts/mtest2make.py index 27425080cf..af6dd0d3b1 100644 --- a/scripts/mtest2make.py +++ b/scripts/mtest2make.py @@ -24,33 +24,20 @@ SPEED = quick .test-human-exitcode = $1 $(PYTHON) scripts/test-driver.py $(if $4,-C$4) $(if $(V),--verbose) -- $2 < /dev/null .test-tap-tap = $1 $(if $4,(cd $4 && $2),$2) < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $3/" || true .test-tap-exitcode = printf "%s\\n" 1..1 "`$1 $(if $4,(cd $4 && $2),$2) < /dev/null > /dev/null || echo "not "`ok 1 $3" -.test.print = echo $(if $(V),'$1 $2','Running test $3') >&3 +.test.human-print = echo $(if $(V),'$1 $2','Running test $3') && .test.env = MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$(( $${RANDOM:-0} % 255 + 1))} # $1 = test name, $2 = test target (human or tap) -.test.run = $(call .test.print,$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1)) && $(call .test-$2-$(.test.driver.$1),$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1),$(.test.dir.$1)) +.test.run = $(call .test.$2-print,$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1)) $(call .test-$2-$(.test.driver.$1),$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1),$(.test.dir.$1)) -define .test.human_k - @exec 3>&1; rc=0; $(foreach TEST, $1, $(call .test.run,$(TEST),human) || rc=$$?;) \\ - exit $$rc -endef -define .test.human_no_k - $(foreach TEST, $1, @exec 3>&1; $(call .test.run,$(TEST),human) -) -endef -.test.human = \\ - $(if $(findstring k, $(MAKEFLAGS)), $(.test.human_k), $(.test.human_no_k)) - -define .test.tap - @exec 3>&1; { $(foreach TEST, $1, $(call .test.run,$(TEST),tap); ) } \\ - | ./scripts/tap-merge.pl | tee "$@" \\ - | ./scripts/tap-driver.pl $(if $(V),, --show-failures-only) -endef +.test.output-format = human ''') -suites = defaultdict(Suite) +introspect = json.load(sys.stdin) i = 0 -for test in json.load(sys.stdin): + +def process_tests(test, suites): + global i env = ' '.join(('%s=%s' % (shlex.quote(k), shlex.quote(v)) for k, v in test['env'].items())) executable = test['cmd'][0] @@ -75,6 +62,9 @@ for test in json.load(sys.stdin): print('.test.driver.%d := %s' % (i, driver)) print('.test.env.%d := $(.test.env) %s' % (i, env)) print('.test.cmd.%d := %s' % (i, cmd)) + print('.PHONY: run-test-%d' % (i,)) + print('run-test-%d: all' % (i,)) + print('\t@$(call .test.run,%d,$(.test.output-format))' % (i,)) test_suites = test['suite'] or ['default'] is_slow = any(s.endswith('-slow') for s in test_suites) @@ -89,22 +79,34 @@ for test in json.load(sys.stdin): suites[s].tests.append(i) suites[s].executables.add(executable) -print('.PHONY: check check-report.tap') -print('check:') -print('check-report.tap:') -print('\t@cat $^ | scripts/tap-merge.pl >$@') -for name, suite in suites.items(): +def emit_prolog(suites, prefix): + all_tap = ' '.join(('%s-report-%s.tap' % (prefix, k) for k in suites.keys())) + print('.PHONY: %s %s-report.tap %s' % (prefix, prefix, all_tap)) + print('%s: run-tests' % (prefix,)) + print('%s-report.tap %s: %s-report%%.tap: all' % (prefix, all_tap, prefix)) + print('''\t$(MAKE) .test.output-format=tap --quiet -Otarget V=1 %s$* | ./scripts/tap-merge.pl | tee "$@" \\ + | ./scripts/tap-driver.pl $(if $(V),, --show-failures-only)''' % (prefix, )) + +def emit_suite(name, suite, prefix): executables = ' '.join(suite.executables) slow_test_numbers = ' '.join((str(x) for x in suite.slow_tests)) test_numbers = ' '.join((str(x) for x in suite.tests)) - print('.test.suite-quick.%s := %s' % (name, test_numbers)) - print('.test.suite-slow.%s := $(.test.suite-quick.%s) %s' % (name, name, slow_test_numbers)) - print('check-build: %s' % executables) - print('.PHONY: check-%s' % name) - print('.PHONY: check-report-%s.tap' % name) - print('check: check-%s' % name) - print('check-%s: all %s' % (name, executables)) - print('\t$(call .test.human, $(.test.suite-$(SPEED).%s))' % (name, )) - print('check-report.tap: check-report-%s.tap' % name) - print('check-report-%s.tap: %s' % (name, executables)) - print('\t$(call .test.tap, $(.test.suite-$(SPEED).%s))' % (name, )) + target = '%s-%s' % (prefix, name) + print('.test.quick.%s := %s' % (target, test_numbers)) + print('.test.slow.%s := $(.test.quick.%s) %s' % (target, target, slow_test_numbers)) + print('%s-build: %s' % (prefix, executables)) + print('.PHONY: %s' % (target, )) + print('.PHONY: %s-report-%s.tap' % (prefix, name)) + print('%s: run-tests' % (target, )) + print('ifneq ($(filter %s %s, $(MAKECMDGOALS)),)' % (target, prefix)) + print('.tests += $(.test.$(SPEED).%s)' % (target, )) + print('endif') + +testsuites = defaultdict(Suite) +for test in introspect: + process_tests(test, testsuites) +emit_prolog(testsuites, 'check') +for name, suite in testsuites.items(): + emit_suite(name, suite, 'check') + +print('run-tests: $(patsubst %, run-test-%, $(.tests))') -- cgit 1.4.1 From 9ed7247a5969cf101ac3d22995ae06730b7da3ea Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 2 Sep 2020 07:25:19 -0400 Subject: meson: convert the speed tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use meson benchmark() for them, adjust mtest2make.py for that. A new target "make bench" can be used to run all benchmarks. Signed-off-by: Marc-André Lureau Message-Id: <20200828110734.1638685-14-marcandre.lureau@redhat.com> [Rewrite mtest2make part. - Paolo] Signed-off-by: Paolo Bonzini --- Makefile | 3 ++- scripts/mtest2make.py | 9 ++++++++- tests/Makefile.include | 15 ++++----------- tests/meson.build | 15 +++++++++++++++ 4 files changed, 29 insertions(+), 13 deletions(-) (limited to 'scripts/mtest2make.py') diff --git a/Makefile b/Makefile index ed354c43b0..d4d6a67d96 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ ${ninja-targets-c_COMPILER} ${ninja-targets-cpp_COMPILER}: .var.command += -MP # reread (and MESON won't be empty anymore). ifneq ($(MESON),) Makefile.mtest: build.ninja scripts/mtest2make.py - $(MESON) introspect --tests | $(PYTHON) scripts/mtest2make.py > $@ + $(MESON) introspect --tests --benchmarks | $(PYTHON) scripts/mtest2make.py > $@ -include Makefile.mtest endif @@ -283,6 +283,7 @@ help: @echo '' @echo 'Test targets:' $(call print-help,check,Run all tests (check-help for details)) + $(call print-help,bench,Run all benchmarks) $(call print-help,docker,Help about targets running tests inside containers) $(call print-help,vm-help,Help about targets running tests inside VM) @echo '' diff --git a/scripts/mtest2make.py b/scripts/mtest2make.py index af6dd0d3b1..9cbb2e374d 100644 --- a/scripts/mtest2make.py +++ b/scripts/mtest2make.py @@ -103,10 +103,17 @@ def emit_suite(name, suite, prefix): print('endif') testsuites = defaultdict(Suite) -for test in introspect: +for test in introspect['tests']: process_tests(test, testsuites) emit_prolog(testsuites, 'check') for name, suite in testsuites.items(): emit_suite(name, suite, 'check') +benchsuites = defaultdict(Suite) +for test in introspect['benchmarks']: + process_tests(test, benchsuites) +emit_prolog(benchsuites, 'bench') +for name, suite in benchsuites.items(): + emit_suite(name, suite, 'bench') + print('run-tests: $(patsubst %, run-test-%, $(.tests))') diff --git a/tests/Makefile.include b/tests/Makefile.include index 0388a0e4fd..fe22ccfcc6 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -38,16 +38,8 @@ export SRC_PATH SYSEMU_TARGET_LIST := $(subst -softmmu.mak,,$(notdir \ $(wildcard $(SRC_PATH)/default-configs/*-softmmu.mak))) -check-speed-$(CONFIG_BLOCK) += tests/benchmark-crypto-hash$(EXESUF) -check-speed-$(CONFIG_BLOCK) += tests/benchmark-crypto-hmac$(EXESUF) -check-speed-$(CONFIG_BLOCK) += tests/benchmark-crypto-cipher$(EXESUF) - QEMU_CFLAGS += -I$(SRC_PATH)/tests -I$(SRC_PATH)/tests/qtest -tests/benchmark-crypto-hash$(EXESUF): tests/benchmark-crypto-hash.o $(test-crypto-obj-y) -tests/benchmark-crypto-hmac$(EXESUF): tests/benchmark-crypto-hmac.o $(test-crypto-obj-y) -tests/benchmark-crypto-cipher$(EXESUF): tests/benchmark-crypto-cipher.o $(test-crypto-obj-y) - tests/migration/stress$(EXESUF): tests/migration/stress.o $(call quiet-command, $(LINKPROG) -static -O3 $(PTHREAD_LIB) -o $@ $< ,"LINK","$(TARGET_DIR)$@") @@ -97,9 +89,6 @@ define do_test_tap "TAP","$@") endef -check-speed: $(check-speed-y) - $(call do_test_human, $^) - # Per guest TCG tests BUILD_TCG_TARGET_RULES=$(patsubst %,build-tcg-tests-%, $(TARGET_DIRS)) @@ -212,6 +201,10 @@ check-clean: clean: check-clean +# For backwards compatibility + +check-speed: bench-speed + # Build the help program automatically -include $(wildcard tests/*.d) diff --git a/tests/meson.build b/tests/meson.build index 3a3066b292..94e34f72a8 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -111,6 +111,8 @@ test_deps = { 'test-qht-par': qht_bench, } +benchs = {} + if have_block tests += { 'test-coroutine': [testblock], @@ -167,6 +169,11 @@ if have_block if 'CONFIG_NETTLE' in config_host or 'CONFIG_GCRYPT' in config_host tests += {'test-crypto-pbkdf': [io]} endif + benchs += { + 'benchmark-crypto-hash': [crypto], + 'benchmark-crypto-hmac': [crypto], + 'benchmark-crypto-cipher': [crypto], + } endif if have_system @@ -226,6 +233,14 @@ foreach test_name, extra: tests suite: ['unit']) endforeach +foreach bench_name, deps: benchs + exe = executable(bench_name, bench_name + '.c', + dependencies: [qemuutil] + deps) + benchmark(bench_name, exe, + args: ['--tap', '-k'], + protocol: 'tap', + suite: ['speed']) +endforeach if have_tools and 'CONFIG_VHOST_USER' in config_host executable('vhost-user-bridge', -- cgit 1.4.1