diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/cleanup-trace-events.pl | 23 | ||||
| -rwxr-xr-x | scripts/grepy.sh | 3 | ||||
| -rw-r--r-- | scripts/mtest2make.py | 100 | ||||
| -rwxr-xr-x | scripts/ninjatool.py | 8 | ||||
| -rwxr-xr-x | scripts/oss-fuzz/build.sh | 2 | ||||
| -rw-r--r-- | scripts/qapi/parser.py | 24 | ||||
| -rw-r--r-- | scripts/test-driver.py | 35 | ||||
| -rw-r--r-- | scripts/tracetool/format/d.py | 15 | ||||
| -rw-r--r-- | scripts/tracetool/vcpu.py | 2 | ||||
| -rw-r--r-- | scripts/undefsym.py | 49 | ||||
| -rwxr-xr-x | scripts/undefsym.sh | 20 |
11 files changed, 191 insertions, 90 deletions
diff --git a/scripts/cleanup-trace-events.pl b/scripts/cleanup-trace-events.pl index d4f0e4cab5..c40d2fcc50 100755 --- a/scripts/cleanup-trace-events.pl +++ b/scripts/cleanup-trace-events.pl @@ -15,12 +15,15 @@ use warnings; use strict; use File::Basename; -my $buf = ''; +my @files = (); +my $events = ''; my %seen = (); sub out { - print $buf; - $buf = ''; + print sort @files; + print $events; + @files = (); + $events = ''; %seen = (); } @@ -31,16 +34,18 @@ open(IN, $in) or die "open $in: $!"; chdir($dir) or die "chdir $dir: $!"; while (<IN>) { - if (/^(disable |(tcg) |vcpu )*([a-z_0-9]+)\(/i) { - my $pat = "trace_$3"; - $pat .= '_tcg' if (defined $2); - open GREP, '-|', 'git', 'grep', '-lw', '--max-depth', '1', $pat + if (/^(disable |(tcg) |(vcpu) )*([a-z_0-9]+)\(/i) { + my $pat = "trace_$4"; + $pat .= '_tcg' if defined $2; + open GREP, '-|', 'git', 'grep', '-lw', + defined $3 ? () : ('--max-depth', '1'), + $pat or die "run git grep: $!"; while (my $fname = <GREP>) { chomp $fname; next if $seen{$fname} || $fname eq 'trace-events'; $seen{$fname} = 1; - $buf = "# $fname\n" . $buf; + push @files, "# $fname\n"; } unless (close GREP) { die "close git grep: $!" @@ -53,7 +58,7 @@ while (<IN>) { } elsif (!/^#|^$/) { warn "unintelligible line"; } - $buf .= $_; + $events .= $_; } out; diff --git a/scripts/grepy.sh b/scripts/grepy.sh deleted file mode 100755 index aee46ddc8d..0000000000 --- a/scripts/grepy.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -grep -h '=y$' "$@" | sort -u diff --git a/scripts/mtest2make.py b/scripts/mtest2make.py index d7a51bf97e..9cbb2e374d 100644 --- a/scripts/mtest2make.py +++ b/scripts/mtest2make.py @@ -19,38 +19,25 @@ class Suite(object): print(''' SPEED = quick -# $1 = test command, $2 = test name -.test-human-tap = $1 < /dev/null | ./scripts/tap-driver.pl --test-name="$2" $(if $(V),,--show-failures-only) -.test-human-exitcode = $1 < /dev/null -.test-tap-tap = $1 < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $2/" || true -.test-tap-exitcode = printf "%s\\n" 1..1 "`$1 < /dev/null > /dev/null || echo "not "`ok 1 $2" -.test.print = echo $(if $(V),'$1','Running test $2') >&3 +# $1 = environment, $2 = test command, $3 = test name, $4 = dir +.test-human-tap = $1 $(if $4,(cd $4 && $2),$2) < /dev/null | ./scripts/tap-driver.pl --test-name="$3" $(if $(V),,--show-failures-only) +.test-human-exitcode = $1 $(PYTHON) scripts/test-driver.py $(if $4,-C$4) $(if $(V),--verbose) -- $2 < /dev/null +.test-tap-tap = $1 $(if $4,(cd $4 && $2),$2) < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $3/" || true +.test-tap-exitcode = printf "%s\\n" 1..1 "`$1 $(if $4,(cd $4 && $2),$2) < /dev/null > /dev/null || echo "not "`ok 1 $3" +.test.human-print = echo $(if $(V),'$1 $2','Running test $3') && .test.env = MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$(( $${RANDOM:-0} % 255 + 1))} # $1 = test name, $2 = test target (human or tap) -.test.run = $(call .test.print,$(.test.cmd.$1),$(.test.name.$1)) && $(call .test-$2-$(.test.driver.$1),$(.test.cmd.$1),$(.test.name.$1)) - -define .test.human_k - @exec 3>&1; rc=0; $(foreach TEST, $1, $(call .test.run,$(TEST),human) || rc=$$?;) \\ - exit $$rc -endef -define .test.human_no_k - $(foreach TEST, $1, @exec 3>&1; $(call .test.run,$(TEST),human) -) -endef -.test.human = \\ - $(if $(findstring k, $(MAKEFLAGS)), $(.test.human_k), $(.test.human_no_k)) - -define .test.tap - @exec 3>&1; { $(foreach TEST, $1, $(call .test.run,$(TEST),tap); ) } \\ - | ./scripts/tap-merge.pl | tee "$@" \\ - | ./scripts/tap-driver.pl $(if $(V),, --show-failures-only) -endef +.test.run = $(call .test.$2-print,$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1)) $(call .test-$2-$(.test.driver.$1),$(.test.env.$1),$(.test.cmd.$1),$(.test.name.$1),$(.test.dir.$1)) + +.test.output-format = human ''') -suites = defaultdict(Suite) +introspect = json.load(sys.stdin) i = 0 -for test in json.load(sys.stdin): + +def process_tests(test, suites): + global i env = ' '.join(('%s=%s' % (shlex.quote(k), shlex.quote(v)) for k, v in test['env'].items())) executable = test['cmd'][0] @@ -65,15 +52,19 @@ for test in json.load(sys.stdin): test['cmd'][0] = executable else: test['cmd'][0] = executable - cmd = '$(.test.env) %s %s' % (env, ' '.join((shlex.quote(x) for x in test['cmd']))) - if test['workdir'] is not None: - cmd = '(cd %s && %s)' % (shlex.quote(test['workdir']), cmd) + cmd = ' '.join((shlex.quote(x) for x in test['cmd'])) driver = test['protocol'] if 'protocol' in test else 'exitcode' i += 1 + if test['workdir'] is not None: + print('.test.dir.%d := %s' % (i, shlex.quote(test['workdir']))) print('.test.name.%d := %s' % (i, test['name'])) print('.test.driver.%d := %s' % (i, driver)) + print('.test.env.%d := $(.test.env) %s' % (i, env)) print('.test.cmd.%d := %s' % (i, cmd)) + print('.PHONY: run-test-%d' % (i,)) + print('run-test-%d: all' % (i,)) + print('\t@$(call .test.run,%d,$(.test.output-format))' % (i,)) test_suites = test['suite'] or ['default'] is_slow = any(s.endswith('-slow') for s in test_suites) @@ -88,22 +79,41 @@ for test in json.load(sys.stdin): suites[s].tests.append(i) suites[s].executables.add(executable) -print('.PHONY: check check-report.tap') -print('check:') -print('check-report.tap:') -print('\t@cat $^ | scripts/tap-merge.pl >$@') -for name, suite in suites.items(): +def emit_prolog(suites, prefix): + all_tap = ' '.join(('%s-report-%s.tap' % (prefix, k) for k in suites.keys())) + print('.PHONY: %s %s-report.tap %s' % (prefix, prefix, all_tap)) + print('%s: run-tests' % (prefix,)) + print('%s-report.tap %s: %s-report%%.tap: all' % (prefix, all_tap, prefix)) + print('''\t$(MAKE) .test.output-format=tap --quiet -Otarget V=1 %s$* | ./scripts/tap-merge.pl | tee "$@" \\ + | ./scripts/tap-driver.pl $(if $(V),, --show-failures-only)''' % (prefix, )) + +def emit_suite(name, suite, prefix): executables = ' '.join(suite.executables) slow_test_numbers = ' '.join((str(x) for x in suite.slow_tests)) test_numbers = ' '.join((str(x) for x in suite.tests)) - print('.test.suite-quick.%s := %s' % (name, test_numbers)) - print('.test.suite-slow.%s := $(.test.suite-quick.%s) %s' % (name, name, slow_test_numbers)) - print('check-build: %s' % executables) - print('.PHONY: check-%s' % name) - print('.PHONY: check-report-%s.tap' % name) - print('check: check-%s' % name) - print('check-%s: all %s' % (name, executables)) - print('\t$(call .test.human, $(.test.suite-$(SPEED).%s))' % (name, )) - print('check-report.tap: check-report-%s.tap' % name) - print('check-report-%s.tap: %s' % (name, executables)) - print('\t$(call .test.tap, $(.test.suite-$(SPEED).%s))' % (name, )) + target = '%s-%s' % (prefix, name) + print('.test.quick.%s := %s' % (target, test_numbers)) + print('.test.slow.%s := $(.test.quick.%s) %s' % (target, target, slow_test_numbers)) + print('%s-build: %s' % (prefix, executables)) + print('.PHONY: %s' % (target, )) + print('.PHONY: %s-report-%s.tap' % (prefix, name)) + print('%s: run-tests' % (target, )) + print('ifneq ($(filter %s %s, $(MAKECMDGOALS)),)' % (target, prefix)) + print('.tests += $(.test.$(SPEED).%s)' % (target, )) + print('endif') + +testsuites = defaultdict(Suite) +for test in introspect['tests']: + process_tests(test, testsuites) +emit_prolog(testsuites, 'check') +for name, suite in testsuites.items(): + emit_suite(name, suite, 'check') + +benchsuites = defaultdict(Suite) +for test in introspect['benchmarks']: + process_tests(test, benchsuites) +emit_prolog(benchsuites, 'bench') +for name, suite in benchsuites.items(): + emit_suite(name, suite, 'bench') + +print('run-tests: $(patsubst %, run-test-%, $(.tests))') diff --git a/scripts/ninjatool.py b/scripts/ninjatool.py index ba6bd9a2a6..627a1cab45 100755 --- a/scripts/ninjatool.py +++ b/scripts/ninjatool.py @@ -34,6 +34,7 @@ import os import re import json import argparse +import hashlib import shutil @@ -51,6 +52,9 @@ else: normpath = os.path.normpath +def sha1_text(text): + return hashlib.sha1(text.encode()).hexdigest() + # ---- lexer and parser ---- PATH_RE = r"[^$\s:|]+|\$[$ :]|\$[a-zA-Z0-9_-]+|\$\{[a-zA-Z0-9_.-]+\}" @@ -767,7 +771,6 @@ class Ninja2Make(NinjaParserEventsWithVars): self.build_vars = defaultdict(lambda: dict()) self.rule_targets = defaultdict(lambda: list()) self.stamp_targets = defaultdict(lambda: list()) - self.num_stamp = defaultdict(lambda: 0) self.all_outs = set() self.all_ins = set() self.all_phony = set() @@ -903,8 +906,7 @@ class Ninja2Make(NinjaParserEventsWithVars): if len(out) == 1: stamp = out[0] + '.stamp' else: - stamp = '%s%d.stamp' %(rule, self.num_stamp[rule]) - self.num_stamp[rule] += 1 + stamp = '%s@%s.stamp' % (rule, sha1_text(targets)[0:11]) self.print('%s: %s; @:' % (targets, stamp)) self.print('%s: %s | %s; ${ninja-command-restat}' % (stamp, inputs, orderonly)) self.rule_targets[rule].append(stamp) diff --git a/scripts/oss-fuzz/build.sh b/scripts/oss-fuzz/build.sh index f0b7442c96..d16207eb67 100755 --- a/scripts/oss-fuzz/build.sh +++ b/scripts/oss-fuzz/build.sh @@ -81,7 +81,7 @@ rm qemu-fuzz-i386 # Build a second time to build the final binary with correct rpath ../configure --disable-werror --cc="$CC" --cxx="$CXX" --enable-fuzzing \ --prefix="$DEST_DIR" --bindir="$DEST_DIR" --datadir="$DEST_DIR/data/" \ - --extra-cflags="$EXTRA_CFLAGS" --extra-ldflags="-Wl,-rpath,'\$\$ORIGIN/lib'" \ + --extra-cflags="$EXTRA_CFLAGS" --extra-ldflags="-Wl,-rpath,\$ORIGIN/lib" \ --target-list="i386-softmmu" make "-j$(nproc)" qemu-fuzz-i386 V=1 diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index abadacbb0e..165925ca72 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -52,8 +52,8 @@ class QAPISchemaParser: info = self.info if self.tok == '#': self.reject_expr_doc(cur_doc) - cur_doc = self.get_doc(info) - self.docs.append(cur_doc) + for cur_doc in self.get_doc(info): + self.docs.append(cur_doc) continue expr = self.get_expr(False) @@ -270,7 +270,8 @@ class QAPISchemaParser: raise QAPIParseError( self, "junk after '##' at start of documentation comment") - doc = QAPIDoc(self, info) + docs = [] + cur_doc = QAPIDoc(self, info) self.accept(False) while self.tok == '#': if self.val.startswith('##'): @@ -279,10 +280,20 @@ class QAPISchemaParser: raise QAPIParseError( self, "junk after '##' at end of documentation comment") - doc.end_comment() + cur_doc.end_comment() + docs.append(cur_doc) self.accept() - return doc - doc.append(self.val) + return docs + if self.val.startswith('# ='): + if cur_doc.symbol: + raise QAPIParseError( + self, + "unexpected '=' markup in definition documentation") + if cur_doc.body.text: + cur_doc.end_comment() + docs.append(cur_doc) + cur_doc = QAPIDoc(self, info) + cur_doc.append(self.val) self.accept(False) raise QAPIParseError(self, "documentation comment must end with '##'") @@ -311,7 +322,6 @@ class QAPIDoc: def __init__(self, name=None): # optional section name (argument/member or section name) self.name = name - # the list of lines for this section self.text = '' def append(self, line): diff --git a/scripts/test-driver.py b/scripts/test-driver.py new file mode 100644 index 0000000000..eef74b29a8 --- /dev/null +++ b/scripts/test-driver.py @@ -0,0 +1,35 @@ +#! /usr/bin/env python3 + +# Wrapper for tests that hides the output if they succeed. +# Used by "make check" +# +# Copyright (C) 2020 Red Hat, Inc. +# +# Author: Paolo Bonzini <pbonzini@redhat.com> + +import subprocess +import sys +import os +import argparse + +parser = argparse.ArgumentParser(description='Test driver for QEMU') +parser.add_argument('-C', metavar='DIR', dest='dir', default='.', + help='change to DIR before doing anything else') +parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', + help='be more verbose') +parser.add_argument('test_args', nargs=argparse.REMAINDER) + +args = parser.parse_args() +os.chdir(args.dir) + +test_args = args.test_args +if test_args[0] == '--': + test_args = test_args[1:] + +if args.verbose: + result = subprocess.run(test_args, stdout=None, stderr=None) +else: + result = subprocess.run(test_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode: + sys.stdout.buffer.write(result.stdout) +sys.exit(result.returncode) diff --git a/scripts/tracetool/format/d.py b/scripts/tracetool/format/d.py index 0afb5f3f47..353722f89c 100644 --- a/scripts/tracetool/format/d.py +++ b/scripts/tracetool/format/d.py @@ -13,6 +13,7 @@ __email__ = "stefanha@redhat.com" from tracetool import out +from sys import platform # Reserved keywords from @@ -34,7 +35,8 @@ def generate(events, backend, group): # SystemTap's dtrace(1) warns about empty "provider qemu {}" but is happy # with an empty file. Avoid the warning. - if not events: + # But dtrace on macOS can't deal with empty files. + if not events and platform != "darwin": return out('/* This file is autogenerated by tracetool, do not edit. */' @@ -44,6 +46,17 @@ def generate(events, backend, group): for e in events: args = [] for type_, name in e.args: + if platform == "darwin": + # macOS dtrace accepts only C99 _Bool + if type_ == 'bool': + type_ = '_Bool' + if type_ == 'bool *': + type_ = '_Bool *' + # It converts int8_t * in probe points to char * in header + # files and introduces [-Wpointer-sign] warning. + # Avoid it by changing probe type to signed char * beforehand. + if type_ == 'int8_t *': + type_ = 'signed char *' if name in RESERVED_WORDS: name += '_' args.append(type_ + ' ' + name) diff --git a/scripts/tracetool/vcpu.py b/scripts/tracetool/vcpu.py index b54e4f4e7a..868b4cb04c 100644 --- a/scripts/tracetool/vcpu.py +++ b/scripts/tracetool/vcpu.py @@ -24,7 +24,7 @@ def transform_event(event): assert "tcg-trans" not in event.properties assert "tcg-exec" not in event.properties - event.args = Arguments([("CPUState *", "__cpu"), event.args]) + event.args = Arguments([("void *", "__cpu"), event.args]) if "tcg" in event.properties: fmt = "\"cpu=%p \"" event.fmt = [fmt + event.fmt[0], diff --git a/scripts/undefsym.py b/scripts/undefsym.py new file mode 100644 index 0000000000..69a895cd26 --- /dev/null +++ b/scripts/undefsym.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# Before a shared module's DSO is produced, a static library is built for it +# and passed to this script. The script generates -Wl,-u options to force +# the inclusion of symbol from libqemuutil.a if the shared modules need them, +# This is necessary because the modules may use functions not needed by the +# executable itself, which would cause the function to not be linked in. +# Then the DSO loading would fail because of the missing symbol. + + +import sys +import subprocess + +def filter_lines_set(stdout, from_staticlib): + linesSet = set() + for line in stdout.splitlines(): + tokens = line.split(b' ') + if len(tokens) >= 1: + if len(tokens) > 1: + if from_staticlib and tokens[1] == b'U': + continue + if not from_staticlib and tokens[1] != b'U': + continue + new_line = b'-Wl,-u,' + tokens[0] + if not new_line in linesSet: + linesSet.add(new_line) + return linesSet + +def main(args): + if len(args) <= 3: + sys.exit(0) + + nm = args[1] + staticlib = args[2] + pc = subprocess.run([nm, "-P", "-g", staticlib], stdout=subprocess.PIPE) + if pc.returncode != 0: + sys.exit(1) + staticlib_syms = filter_lines_set(pc.stdout, True) + + shared_modules = args[3:] + pc = subprocess.run([nm, "-P", "-g"] + shared_modules, stdout=subprocess.PIPE) + if pc.returncode != 0: + sys.exit(1) + modules_undef_syms = filter_lines_set(pc.stdout, False) + lines = sorted(staticlib_syms.intersection(modules_undef_syms)) + sys.stdout.buffer.write(b'\n'.join(lines)) + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/undefsym.sh b/scripts/undefsym.sh deleted file mode 100755 index b9ec332e95..0000000000 --- a/scripts/undefsym.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /usr/bin/env bash - -# Before a shared module's DSO is produced, a static library is built for it -# and passed to this script. The script generates -Wl,-u options to force -# the inclusion of symbol from libqemuutil.a if the shared modules need them, -# This is necessary because the modules may use functions not needed by the -# executable itself, which would cause the function to not be linked in. -# Then the DSO loading would fail because of the missing symbol. - -if test $# -le 2; then - exit 0 -fi - -NM=$1 -staticlib=$2 -shift 2 -# Find symbols defined in static libraries and undefined in shared modules -comm -12 \ - <( $NM -P -g $staticlib | awk '$2!="U"{print "-Wl,-u," $1}' | sort -u) \ - <( $NM -P -g "$@" | awk '$2=="U"{print "-Wl,-u," $1}' | sort -u) |