diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/qapi/common.py | 8 | ||||
| -rw-r--r-- | scripts/qapi/main.py | 6 | ||||
| -rw-r--r-- | scripts/qapi/parser.py | 232 | ||||
| -rw-r--r-- | scripts/qapi/pylintrc | 1 | ||||
| -rw-r--r-- | scripts/qapi/schema.py | 11 | ||||
| -rw-r--r-- | scripts/qapi/source.py | 13 | ||||
| -rwxr-xr-x | scripts/simplebench/bench-backup.py | 95 | ||||
| -rwxr-xr-x | scripts/simplebench/bench_block_job.py | 42 | ||||
| -rw-r--r-- | scripts/simplebench/simplebench.py | 28 |
9 files changed, 338 insertions, 98 deletions
diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py index cbd3fd81d3..6ad1eeb61d 100644 --- a/scripts/qapi/common.py +++ b/scripts/qapi/common.py @@ -12,7 +12,7 @@ # See the COPYING file in the top-level directory. import re -from typing import Optional, Sequence +from typing import Match, Optional, Sequence #: Magic string that gets removed along with all space to its right. @@ -210,3 +210,9 @@ def gen_endif(ifcond: Sequence[str]) -> str: #endif /* %(cond)s */ ''', cond=ifc) return ret + + +def must_match(pattern: str, string: str) -> Match[str]: + match = re.match(pattern, string) + assert match is not None + return match diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py index 703e7ed1ed..f2ea6e0ce4 100644 --- a/scripts/qapi/main.py +++ b/scripts/qapi/main.py @@ -8,11 +8,11 @@ This is the main entry point for generating C code from the QAPI schema. """ import argparse -import re import sys from typing import Optional from .commands import gen_commands +from .common import must_match from .error import QAPIError from .events import gen_events from .introspect import gen_introspect @@ -22,9 +22,7 @@ from .visit import gen_visit def invalid_prefix_char(prefix: str) -> Optional[str]: - match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix) - # match cannot be None, but mypy cannot infer that. - assert match is not None + match = must_match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix) if match.end() != len(prefix): return prefix[match.end()] return None diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index ca5e8e18e0..f03ba2cfec 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -17,14 +17,26 @@ from collections import OrderedDict import os import re - +from typing import ( + Dict, + List, + Optional, + Set, + Union, +) + +from .common import must_match from .error import QAPISemError, QAPISourceError from .source import QAPISourceInfo +# Return value alias for get_expr(). +_ExprValue = Union[List[object], Dict[str, object], str, bool] + + class QAPIParseError(QAPISourceError): """Error class for all QAPI schema parsing errors.""" - def __init__(self, parser, msg): + def __init__(self, parser: 'QAPISchemaParser', msg: str): col = 1 for ch in parser.src[parser.line_pos:parser.pos]: if ch == '\t': @@ -35,31 +47,69 @@ class QAPIParseError(QAPISourceError): class QAPISchemaParser: + """ + Parse QAPI schema source. - def __init__(self, fname, previously_included=None, incl_info=None): - previously_included = previously_included or set() - previously_included.add(os.path.abspath(fname)) + Parse a JSON-esque schema file and process directives. See + qapi-code-gen.txt section "Schema Syntax" for the exact syntax. + Grammatical validation is handled later by `expr.check_exprs()`. - try: - fp = open(fname, 'r', encoding='utf-8') - self.src = fp.read() - except IOError as e: - raise QAPISemError(incl_info or QAPISourceInfo(None, None, None), - "can't read %s file '%s': %s" - % ("include" if incl_info else "schema", - fname, - e.strerror)) + :param fname: Source file name. + :param previously_included: + The absolute names of previously included source files, + if being invoked from another parser. + :param incl_info: + `QAPISourceInfo` belonging to the parent module. + ``None`` implies this is the root module. - if self.src == '' or self.src[-1] != '\n': - self.src += '\n' + :ivar exprs: Resulting parsed expressions. + :ivar docs: Resulting parsed documentation blocks. + + :raise OSError: For problems reading the root schema document. + :raise QAPIError: For errors in the schema source. + """ + def __init__(self, + fname: str, + previously_included: Optional[Set[str]] = None, + incl_info: Optional[QAPISourceInfo] = None): + self._fname = fname + self._included = previously_included or set() + self._included.add(os.path.abspath(self._fname)) + self.src = '' + + # Lexer state (see `accept` for details): + self.info = QAPISourceInfo(self._fname, incl_info) + self.tok: Union[None, str] = None + self.pos = 0 self.cursor = 0 - self.info = QAPISourceInfo(fname, 1, incl_info) + self.val: Optional[Union[bool, str]] = None self.line_pos = 0 - self.exprs = [] - self.docs = [] - self.accept() + + # Parser output: + self.exprs: List[Dict[str, object]] = [] + self.docs: List[QAPIDoc] = [] + + # Showtime! + self._parse() + + def _parse(self) -> None: + """ + Parse the QAPI schema document. + + :return: None. Results are stored in ``.exprs`` and ``.docs``. + """ cur_doc = None + # May raise OSError; allow the caller to handle it. + with open(self._fname, 'r', encoding='utf-8') as fp: + self.src = fp.read() + if self.src == '' or self.src[-1] != '\n': + self.src += '\n' + + # Prime the lexer: + self.accept() + + # Parse until done: while self.tok is not None: info = self.info if self.tok == '#': @@ -68,7 +118,11 @@ class QAPISchemaParser: self.docs.append(cur_doc) continue - expr = self.get_expr(False) + expr = self.get_expr() + if not isinstance(expr, dict): + raise QAPISemError( + info, "top-level expression must be an object") + if 'include' in expr: self.reject_expr_doc(cur_doc) if len(expr) != 1: @@ -77,12 +131,12 @@ class QAPISchemaParser: if not isinstance(include, str): raise QAPISemError(info, "value of 'include' must be a string") - incl_fname = os.path.join(os.path.dirname(fname), + incl_fname = os.path.join(os.path.dirname(self._fname), include) self.exprs.append({'expr': {'include': incl_fname}, 'info': info}) exprs_include = self._include(include, info, incl_fname, - previously_included) + self._included) if exprs_include: self.exprs.extend(exprs_include.exprs) self.docs.extend(exprs_include.docs) @@ -109,17 +163,22 @@ class QAPISchemaParser: self.reject_expr_doc(cur_doc) @staticmethod - def reject_expr_doc(doc): + def reject_expr_doc(doc: Optional['QAPIDoc']) -> None: if doc and doc.symbol: raise QAPISemError( doc.info, "documentation for '%s' is not followed by the definition" % doc.symbol) - def _include(self, include, info, incl_fname, previously_included): + @staticmethod + def _include(include: str, + info: QAPISourceInfo, + incl_fname: str, + previously_included: Set[str] + ) -> Optional['QAPISchemaParser']: incl_abs_fname = os.path.abspath(incl_fname) # catch inclusion cycle - inf = info + inf: Optional[QAPISourceInfo] = info while inf: if incl_abs_fname == os.path.abspath(inf.fname): raise QAPISemError(info, "inclusion loop for %s" % include) @@ -129,34 +188,86 @@ class QAPISchemaParser: if incl_abs_fname in previously_included: return None - return QAPISchemaParser(incl_fname, previously_included, info) - - def _check_pragma_list_of_str(self, name, value, info): - if (not isinstance(value, list) - or any([not isinstance(elt, str) for elt in value])): + try: + return QAPISchemaParser(incl_fname, previously_included, info) + except OSError as err: raise QAPISemError( info, - "pragma %s must be a list of strings" % name) + f"can't read include file '{incl_fname}': {err.strerror}" + ) from err + + @staticmethod + def _pragma(name: str, value: object, info: QAPISourceInfo) -> None: + + def check_list_str(name: str, value: object) -> List[str]: + if (not isinstance(value, list) or + any(not isinstance(elt, str) for elt in value)): + raise QAPISemError( + info, + "pragma %s must be a list of strings" % name) + return value + + pragma = info.pragma - def _pragma(self, name, value, info): if name == 'doc-required': if not isinstance(value, bool): raise QAPISemError(info, "pragma 'doc-required' must be boolean") - info.pragma.doc_required = value + pragma.doc_required = value elif name == 'command-name-exceptions': - self._check_pragma_list_of_str(name, value, info) - info.pragma.command_name_exceptions = value + pragma.command_name_exceptions = check_list_str(name, value) elif name == 'command-returns-exceptions': - self._check_pragma_list_of_str(name, value, info) - info.pragma.command_returns_exceptions = value + pragma.command_returns_exceptions = check_list_str(name, value) elif name == 'member-name-exceptions': - self._check_pragma_list_of_str(name, value, info) - info.pragma.member_name_exceptions = value + pragma.member_name_exceptions = check_list_str(name, value) else: raise QAPISemError(info, "unknown pragma '%s'" % name) - def accept(self, skip_comment=True): + def accept(self, skip_comment: bool = True) -> None: + """ + Read and store the next token. + + :param skip_comment: + When false, return COMMENT tokens ("#"). + This is used when reading documentation blocks. + + :return: + None. Several instance attributes are updated instead: + + - ``.tok`` represents the token type. See below for values. + - ``.info`` describes the token's source location. + - ``.val`` is the token's value, if any. See below. + - ``.pos`` is the buffer index of the first character of + the token. + + * Single-character tokens: + + These are "{", "}", ":", ",", "[", and "]". + ``.tok`` holds the single character and ``.val`` is None. + + * Multi-character tokens: + + * COMMENT: + + This token is not normally returned by the lexer, but it can + be when ``skip_comment`` is False. ``.tok`` is "#", and + ``.val`` is a string including all chars until end-of-line, + including the "#" itself. + + * STRING: + + ``.tok`` is "'", the single quote. ``.val`` contains the + string, excluding the surrounding quotes. + + * TRUE and FALSE: + + ``.tok`` is either "t" or "f", ``.val`` will be the + corresponding bool value. + + * EOF: + + ``.tok`` and ``.val`` will both be None at EOF. + """ while True: self.tok = self.src[self.cursor] self.pos = self.cursor @@ -216,12 +327,12 @@ class QAPISchemaParser: elif not self.tok.isspace(): # Show up to next structural, whitespace or quote # character - match = re.match('[^[\\]{}:,\\s\'"]+', - self.src[self.cursor-1:]) + match = must_match('[^[\\]{}:,\\s\'"]+', + self.src[self.cursor-1:]) raise QAPIParseError(self, "stray '%s'" % match.group(0)) - def get_members(self): - expr = OrderedDict() + def get_members(self) -> Dict[str, object]: + expr: Dict[str, object] = OrderedDict() if self.tok == '}': self.accept() return expr @@ -229,13 +340,15 @@ class QAPISchemaParser: raise QAPIParseError(self, "expected string or '}'") while True: key = self.val + assert isinstance(key, str) # Guaranteed by tok == "'" + self.accept() if self.tok != ':': raise QAPIParseError(self, "expected ':'") self.accept() if key in expr: raise QAPIParseError(self, "duplicate key '%s'" % key) - expr[key] = self.get_expr(True) + expr[key] = self.get_expr() if self.tok == '}': self.accept() return expr @@ -245,16 +358,16 @@ class QAPISchemaParser: if self.tok != "'": raise QAPIParseError(self, "expected string") - def get_values(self): - expr = [] + def get_values(self) -> List[object]: + expr: List[object] = [] if self.tok == ']': self.accept() return expr - if self.tok not in "{['tf": + if self.tok not in tuple("{['tf"): raise QAPIParseError( self, "expected '{', '[', ']', string, or boolean") while True: - expr.append(self.get_expr(True)) + expr.append(self.get_expr()) if self.tok == ']': self.accept() return expr @@ -262,16 +375,16 @@ class QAPISchemaParser: raise QAPIParseError(self, "expected ',' or ']'") self.accept() - def get_expr(self, nested): - if self.tok != '{' and not nested: - raise QAPIParseError(self, "expected '{'") + def get_expr(self) -> _ExprValue: + expr: _ExprValue if self.tok == '{': self.accept() expr = self.get_members() elif self.tok == '[': self.accept() expr = self.get_values() - elif self.tok in "'tf": + elif self.tok in tuple("'tf"): + assert isinstance(self.val, (str, bool)) expr = self.val self.accept() else: @@ -279,7 +392,7 @@ class QAPISchemaParser: self, "expected '{', '[', string, or boolean") return expr - def get_doc(self, info): + def get_doc(self, info: QAPISourceInfo) -> List['QAPIDoc']: if self.val != '##': raise QAPIParseError( self, "junk after '##' at start of documentation comment") @@ -288,6 +401,7 @@ class QAPISchemaParser: cur_doc = QAPIDoc(self, info) self.accept(False) while self.tok == '#': + assert isinstance(self.val, str) if self.val.startswith('##'): # End of doc comment if self.val != '##': @@ -346,7 +460,7 @@ class QAPIDoc: # Strip leading spaces corresponding to the expected indent level # Blank lines are always OK. if line: - indent = re.match(r'\s*', line).end() + indent = must_match(r'\s*', line).end() if indent < self._indent: raise QAPIParseError( self._parser, @@ -482,7 +596,7 @@ class QAPIDoc: # from line and replace it with spaces so that 'f' has the # same index as it did in the original line and can be # handled the same way we will handle following lines. - indent = re.match(r'@\S*:\s*', line).end() + indent = must_match(r'@\S*:\s*', line).end() line = line[indent:] if not line: # Line was just the "@arg:" header; following lines @@ -517,7 +631,7 @@ class QAPIDoc: # from line and replace it with spaces so that 'f' has the # same index as it did in the original line and can be # handled the same way we will handle following lines. - indent = re.match(r'@\S*:\s*', line).end() + indent = must_match(r'@\S*:\s*', line).end() line = line[indent:] if not line: # Line was just the "@arg:" header; following lines @@ -563,7 +677,7 @@ class QAPIDoc: # from line and replace it with spaces so that 'f' has the # same index as it did in the original line and can be # handled the same way we will handle following lines. - indent = re.match(r'\S*:\s*', line).end() + indent = must_match(r'\S*:\s*', line).end() line = line[indent:] if not line: # Line was just the "Section:" header; following lines diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc index 88efbf71cb..c5275d5f59 100644 --- a/scripts/qapi/pylintrc +++ b/scripts/qapi/pylintrc @@ -43,6 +43,7 @@ good-names=i, _, fp, # fp = open(...) fd, # fd = os.open(...) + ch, [VARIABLES] diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index 3a4172fb74..d1d27ff7ee 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -20,7 +20,7 @@ import re from typing import Optional from .common import POINTER_SUFFIX, c_name -from .error import QAPISemError, QAPISourceError +from .error import QAPIError, QAPISemError, QAPISourceError from .expr import check_exprs from .parser import QAPISchemaParser @@ -849,7 +849,14 @@ class QAPISchemaEvent(QAPISchemaEntity): class QAPISchema: def __init__(self, fname): self.fname = fname - parser = QAPISchemaParser(fname) + + try: + parser = QAPISchemaParser(fname) + except OSError as err: + raise QAPIError( + f"can't read schema file '{fname}': {err.strerror}" + ) from err + exprs = check_exprs(parser.exprs) self.docs = parser.docs self._entity_list = [] diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py index 03b6ede082..04193cc964 100644 --- a/scripts/qapi/source.py +++ b/scripts/qapi/source.py @@ -10,7 +10,6 @@ # See the COPYING file in the top-level directory. import copy -import sys from typing import List, Optional, TypeVar @@ -32,10 +31,9 @@ class QAPISchemaPragma: class QAPISourceInfo: T = TypeVar('T', bound='QAPISourceInfo') - def __init__(self, fname: str, line: int, - parent: Optional['QAPISourceInfo']): + def __init__(self, fname: str, parent: Optional['QAPISourceInfo']): self.fname = fname - self.line = line + self.line = 1 self.parent = parent self.pragma: QAPISchemaPragma = ( parent.pragma if parent else QAPISchemaPragma() @@ -53,12 +51,7 @@ class QAPISourceInfo: return info def loc(self) -> str: - if self.fname is None: - return sys.argv[0] - ret = self.fname - if self.line is not None: - ret += ':%d' % self.line - return ret + return f"{self.fname}:{self.line}" def in_defn(self) -> str: if self.defn_name: diff --git a/scripts/simplebench/bench-backup.py b/scripts/simplebench/bench-backup.py index 33a1ecfefa..5a0675c593 100755 --- a/scripts/simplebench/bench-backup.py +++ b/scripts/simplebench/bench-backup.py @@ -23,7 +23,7 @@ import json import simplebench from results_to_text import results_to_text -from bench_block_job import bench_block_copy, drv_file, drv_nbd +from bench_block_job import bench_block_copy, drv_file, drv_nbd, drv_qcow2 def bench_func(env, case): @@ -37,29 +37,56 @@ def bench_func(env, case): def bench(args): test_cases = [] - sources = {} - targets = {} - for d in args.dir: - label, path = d.split(':') # paths with colon not supported - sources[label] = drv_file(path + '/test-source') - targets[label] = drv_file(path + '/test-target') + # paths with colon not supported, so we just split by ':' + dirs = dict(d.split(':') for d in args.dir) + nbd_drv = None if args.nbd: nbd = args.nbd.split(':') host = nbd[0] port = '10809' if len(nbd) == 1 else nbd[1] - drv = drv_nbd(host, port) - sources['nbd'] = drv - targets['nbd'] = drv + nbd_drv = drv_nbd(host, port) for t in args.test: src, dst = t.split(':') - test_cases.append({ - 'id': t, - 'source': sources[src], - 'target': targets[dst] - }) + if src == 'nbd' and dst == 'nbd': + raise ValueError("Can't use 'nbd' label for both src and dst") + + if (src == 'nbd' or dst == 'nbd') and not nbd_drv: + raise ValueError("'nbd' label used but --nbd is not given") + + if src == 'nbd': + source = nbd_drv + elif args.qcow2_sources: + source = drv_qcow2(drv_file(dirs[src] + '/test-source.qcow2')) + else: + source = drv_file(dirs[src] + '/test-source') + + if dst == 'nbd': + test_cases.append({'id': t, 'source': source, 'target': nbd_drv}) + continue + + if args.target_cache == 'both': + target_caches = ['direct', 'cached'] + else: + target_caches = [args.target_cache] + + for c in target_caches: + o_direct = c == 'direct' + fname = dirs[dst] + '/test-target' + if args.compressed: + fname += '.qcow2' + target = drv_file(fname, o_direct=o_direct) + if args.compressed: + target = drv_qcow2(target) + + test_id = t + if args.target_cache == 'both': + test_id += f'({c})' + + test_cases.append({'id': test_id, 'source': source, + 'target': target}) binaries = [] # list of (<label>, <path>, [<options>]) for i, q in enumerate(args.env): @@ -106,6 +133,13 @@ def bench(args): elif opt.startswith('max-workers='): x_perf['max-workers'] = int(opt.split('=')[1]) + backup_options = {} + if x_perf: + backup_options['x-perf'] = x_perf + + if args.compressed: + backup_options['compress'] = True + if is_mirror: assert not x_perf test_envs.append({ @@ -117,11 +151,13 @@ def bench(args): test_envs.append({ 'id': f'backup({label})\n' + '\n'.join(opts), 'cmd': 'blockdev-backup', - 'cmd-options': {'x-perf': x_perf} if x_perf else {}, + 'cmd-options': backup_options, 'qemu-binary': path }) - result = simplebench.bench(bench_func, test_envs, test_cases, count=3) + result = simplebench.bench(bench_func, test_envs, test_cases, + count=args.count, initial_run=args.initial_run, + drop_caches=args.drop_caches) with open('results.json', 'w') as f: json.dump(result, f, indent=4) print(results_to_text(result)) @@ -163,5 +199,30 @@ default port 10809). Use it in tests, label is "nbd" p.add_argument('--test', nargs='+', help='''\ Tests, in form source-dir-label:target-dir-label''', action=ExtendAction) + p.add_argument('--compressed', help='''\ +Use compressed backup. It automatically means +automatically creating qcow2 target with +lazy_refcounts for each test run''', action='store_true') + p.add_argument('--qcow2-sources', help='''\ +Use test-source.qcow2 images as sources instead of +test-source raw images''', action='store_true') + p.add_argument('--target-cache', help='''\ +Setup cache for target nodes. Options: + direct: default, use O_DIRECT and aio=native + cached: use system cache (Qemu default) and aio=threads (Qemu default) + both: generate two test cases for each src:dst pair''', + default='direct', choices=('direct', 'cached', 'both')) + + p.add_argument('--count', type=int, default=3, help='''\ +Number of test runs per table cell''') + + # BooleanOptionalAction helps to support --no-initial-run option + p.add_argument('--initial-run', action=argparse.BooleanOptionalAction, + help='''\ +Do additional initial run per cell which doesn't count in result, +default true''') + + p.add_argument('--drop-caches', action='store_true', help='''\ +Do "sync; echo 3 > /proc/sys/vm/drop_caches" before each test run''') bench(p.parse_args()) diff --git a/scripts/simplebench/bench_block_job.py b/scripts/simplebench/bench_block_job.py index 7332845c1c..4f03c12169 100755 --- a/scripts/simplebench/bench_block_job.py +++ b/scripts/simplebench/bench_block_job.py @@ -21,6 +21,7 @@ import sys import os +import subprocess import socket import json @@ -69,6 +70,10 @@ def bench_block_job(cmd, cmd_args, qemu_args): vm.shutdown() return {'error': 'block-job failed: ' + str(e), 'vm-log': vm.get_log()} + if 'error' in e['data']: + vm.shutdown() + return {'error': 'block-job failed: ' + e['data']['error'], + 'vm-log': vm.get_log()} end_ms = e['timestamp']['seconds'] * 1000000 + \ e['timestamp']['microseconds'] finally: @@ -77,11 +82,34 @@ def bench_block_job(cmd, cmd_args, qemu_args): return {'seconds': (end_ms - start_ms) / 1000000.0} +def get_image_size(path): + out = subprocess.run(['qemu-img', 'info', '--out=json', path], + stdout=subprocess.PIPE, check=True).stdout + return json.loads(out)['virtual-size'] + + +def get_blockdev_size(obj): + img = obj['filename'] if 'filename' in obj else obj['file']['filename'] + return get_image_size(img) + + # Bench backup or mirror def bench_block_copy(qemu_binary, cmd, cmd_options, source, target): """Helper to run bench_block_job() for mirror or backup""" assert cmd in ('blockdev-backup', 'blockdev-mirror') + if target['driver'] == 'qcow2': + try: + os.remove(target['file']['filename']) + except OSError: + pass + + subprocess.run(['qemu-img', 'create', '-f', 'qcow2', + target['file']['filename'], + str(get_blockdev_size(source))], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, check=True) + source['node-name'] = 'source' target['node-name'] = 'target' @@ -96,9 +124,13 @@ def bench_block_copy(qemu_binary, cmd, cmd_options, source, target): '-blockdev', json.dumps(target)]) -def drv_file(filename): - return {'driver': 'file', 'filename': filename, - 'cache': {'direct': True}, 'aio': 'native'} +def drv_file(filename, o_direct=True): + node = {'driver': 'file', 'filename': filename} + if o_direct: + node['cache'] = {'direct': True} + node['aio'] = 'native' + + return node def drv_nbd(host, port): @@ -106,6 +138,10 @@ def drv_nbd(host, port): 'server': {'type': 'inet', 'host': host, 'port': port}} +def drv_qcow2(file): + return {'driver': 'qcow2', 'file': file} + + if __name__ == '__main__': import sys diff --git a/scripts/simplebench/simplebench.py b/scripts/simplebench/simplebench.py index f61513af90..8efca2af98 100644 --- a/scripts/simplebench/simplebench.py +++ b/scripts/simplebench/simplebench.py @@ -19,9 +19,17 @@ # import statistics +import subprocess +import time -def bench_one(test_func, test_env, test_case, count=5, initial_run=True): +def do_drop_caches(): + subprocess.run('sync; echo 3 > /proc/sys/vm/drop_caches', shell=True, + check=True) + + +def bench_one(test_func, test_env, test_case, count=5, initial_run=True, + slow_limit=100, drop_caches=False): """Benchmark one test-case test_func -- benchmarking function with prototype @@ -36,6 +44,9 @@ def bench_one(test_func, test_env, test_case, count=5, initial_run=True): test_case -- test case - opaque second argument for test_func count -- how many times to call test_func, to calculate average initial_run -- do initial run of test_func, which don't get into result + slow_limit -- stop at slow run (that exceedes the slow_limit by seconds). + (initial run is not measured) + drop_caches -- drop caches before each run Returns dict with the following fields: 'runs': list of test_func results @@ -49,15 +60,25 @@ def bench_one(test_func, test_env, test_case, count=5, initial_run=True): """ if initial_run: print(' #initial run:') + do_drop_caches() print(' ', test_func(test_env, test_case)) runs = [] for i in range(count): + t = time.time() + print(' #run {}'.format(i+1)) + do_drop_caches() res = test_func(test_env, test_case) print(' ', res) runs.append(res) + if time.time() - t > slow_limit: + print(' - run is too slow, stop here') + break + + count = len(runs) + result = {'runs': runs} succeeded = [r for r in runs if ('seconds' in r or 'iops' in r)] @@ -71,7 +92,10 @@ def bench_one(test_func, test_env, test_case, count=5, initial_run=True): dim = 'seconds' result['dimension'] = dim result['average'] = statistics.mean(r[dim] for r in succeeded) - result['stdev'] = statistics.stdev(r[dim] for r in succeeded) + if len(succeeded) == 1: + result['stdev'] = 0 + else: + result['stdev'] = statistics.stdev(r[dim] for r in succeeded) if len(succeeded) < count: result['n-failed'] = count - len(succeeded) |