diff options
| -rw-r--r-- | src/focaccia/benchmark.py | 31 | ||||
| -rw-r--r-- | src/focaccia/qemu/_qemu_tool.py | 100 | ||||
| -rwxr-xr-x | src/focaccia/tools/capture_transforms.py | 11 | ||||
| -rwxr-xr-x | src/focaccia/tools/validate_qemu.py | 10 |
4 files changed, 130 insertions, 22 deletions
diff --git a/src/focaccia/benchmark.py b/src/focaccia/benchmark.py new file mode 100644 index 0000000..c7476a2 --- /dev/null +++ b/src/focaccia/benchmark.py @@ -0,0 +1,31 @@ +from time import perf_counter + +class Timer: + def __init__(self, name: str = "Timer", paused: bool = False, iterations: int = 1, enabled: bool = True): + self.name = name + self.iterations = iterations + self.total_time = 0 + self.paused = paused + self.enabled = enabled + if self.enabled: + print(f'{self.name}: start timer, do {self.iterations} iterations') + self.start_time = perf_counter() + else: + self.iterations = 1 + + def pause(self): + if self.enabled and not self.paused: + self.paused = True + self.total_time += (perf_counter() - self.start_time) + + def unpause(self): + if self.enabled and self.paused: + self.paused = False + self.start_time = perf_counter() + + def log_time(self): + if self.enabled: + if not self.paused: + self.total_time += (perf_counter() - self.start_time) + time = self.total_time / self.iterations + print(f'{self.name}: took {time:.5f} seconds') diff --git a/src/focaccia/qemu/_qemu_tool.py b/src/focaccia/qemu/_qemu_tool.py index 7ca556b..88ce02a 100644 --- a/src/focaccia/qemu/_qemu_tool.py +++ b/src/focaccia/qemu/_qemu_tool.py @@ -10,8 +10,11 @@ import re import gdb import logging import traceback +import subprocess +import time from typing import Iterable, Optional +from focaccia.benchmark import Timer import focaccia.parser as parser from focaccia.arch import supported_architectures, Arch from focaccia.compare import compare_symbolic, Error, ErrorTypes @@ -538,10 +541,54 @@ def main(): raise NotImplementedError(f'Deterministic log {args.deterministic_log} specified but ' 'Focaccia built without deterministic log support') - try: - gdb_server = GDBServerStateIterator(args.remote, detlog) - except Exception as e: - raise Exception(f'Unable to perform basic GDB setup: {e}') + # Benchmark native QEMU execution + if args.benchmark_execution_continue: + try: + timer = Timer("Emulator execution", paused=True, iterations=10) + for i in range(timer.iterations): + qemu_process = subprocess.Popen( + [f"qemu-{args.guest_arch}", "-singlestep", "-g", args.remote.split(':')[1], args.executable], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + time.sleep(0.5) + timer.unpause() + gdb_server = GDBServerStateIterator(args.remote, detlog) + gdb.execute("continue") + qemu_process.wait() + timer.pause() + timer.log_time() + exit(0) + except Exception as e: + raise Exception(f'Unable to benchmark QEMU: {e}') + if args.benchmark_execution_stepping: + try: + timer = Timer("Emulator execution", paused=True, iterations=10) + for i in range(timer.iterations): + try: + qemu_process = subprocess.Popen( + [f"qemu-{args.guest_arch}", "-g", args.remote.split(':')[1], args.executable], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + time.sleep(0.5) + timer.unpause() + gdb_server = GDBServerStateIterator(args.remote, detlog) + state_iter = iter(gdb_server) + while True: + cur_state = next(state_iter) + except StopIteration: + timer.pause() + timer.log_time() + exit(0) + except Exception as e: + raise Exception(f'Unable to benchmark QEMU: {e}') + + if not args.benchmark_trace_test: + try: + gdb_server = GDBServerStateIterator(args.remote, detlog) + except Exception as e: + raise Exception(f'Unable to perform basic GDB setup: {e}') try: executable: str | None = None @@ -565,26 +612,43 @@ def main(): # Use symbolic trace to collect concrete trace from QEMU try: - conc_states, matched_transforms = collect_conc_trace( - gdb_server, - symb_transforms.states, - symb_transforms.env.start_address, - symb_transforms.env.stop_address) + timer = Timer("Emulator tracing", iterations=10, paused=True, enabled=args.benchmark_trace_test) + for i in range(timer.iterations): + if timer.enabled: + qemu_process = subprocess.Popen( + [f"qemu-{args.guest_arch}", "-g", args.remote.split(':')[1], executable], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + time.sleep(0.5) + timer.unpause() + gdb_server = GDBServerStateIterator(args.remote, detlog) + + conc_states, matched_transforms = collect_conc_trace( + gdb_server, + symb_transforms.states, + symb_transforms.env.start_address, + symb_transforms.env.stop_address) + timer.pause() + timer.log_time() except Exception as e: raise Exception(f'Failed to collect concolic trace from QEMU: {e}') # Verify and print result if not args.quiet: try: - res = compare_symbolic(conc_states, matched_transforms) - if qemu_crash["crashed"]: - res.append({ - 'pc': qemu_crash["pc"], - 'txl': None, - 'ref': qemu_crash["ref"], - 'errors': qemu_crash["errors"], - 'snap': qemu_crash["snap"], - }) + timer = Timer("Emulator testing", iterations=10, enabled=args.benchmark_trace_test) + for i in range(timer.iterations): + res = compare_symbolic(conc_states, matched_transforms) + if qemu_crash["crashed"]: + res.append({ + 'pc': qemu_crash["pc"], + 'txl': None, + 'ref': qemu_crash["ref"], + 'errors': qemu_crash["errors"], + 'snap': qemu_crash["snap"], + }) + timer.log_time() print_result(res, verbosity[args.error_level]) except Exception as e: raise Exception('Error occured when comparing with symbolic equations: {e}') diff --git a/src/focaccia/tools/capture_transforms.py b/src/focaccia/tools/capture_transforms.py index 268af36..6a8a38a 100755 --- a/src/focaccia/tools/capture_transforms.py +++ b/src/focaccia/tools/capture_transforms.py @@ -4,7 +4,7 @@ import sys import argparse import logging -from focaccia import parser, utils +from focaccia import parser, utils, benchmark from focaccia.trace import TraceEnvironment from focaccia.native.tracer import SymbolicTracer from focaccia.deterministic import DeterministicLog @@ -51,6 +51,10 @@ def main(): type=utils.to_num, help='Set a time limit for executing an instruction symbolically, skip' 'instruction when limit is exceeded') + prog.add_argument('--benchmark', + default=False, + action='store_true', + help='Benchmark the trace function') args = prog.parse_args() if args.debug: @@ -75,7 +79,10 @@ def main(): tracer = SymbolicTracer(env, remote=args.remote, cross_validate=args.debug, force=args.force) - trace = tracer.trace(time_limit=args.insn_time_limit) + timer = benchmark.Timer("Native tracing", iterations=10, enabled=args.benchmark) + for i in range(timer.iterations): + trace = tracer.trace(time_limit=args.insn_time_limit) + timer.log_time() with open(args.output, 'w') as file: parser.serialize_transformations(trace, file) diff --git a/src/focaccia/tools/validate_qemu.py b/src/focaccia/tools/validate_qemu.py index 4b5160f..f1823b9 100755 --- a/src/focaccia/tools/validate_qemu.py +++ b/src/focaccia/tools/validate_qemu.py @@ -76,7 +76,7 @@ memory, and stepping forward by single instructions. type=str, choices=supported_architectures.keys(), help='Architecture of the emulated guest' - '(Only required when using --use-socket)') + '(Only required when using --use-socket or --benchmark-executioe)') prog.add_argument('--remote', type=str, help='The hostname:port pair at which to find a QEMU GDB server.') @@ -86,6 +86,12 @@ memory, and stepping forward by single instructions. help='GDB binary to invoke.') prog.add_argument('--deterministic-log', default=None, help='The directory containing rr traces') + prog.add_argument('--benchmark-execution-continue', default=False, action='store_true', + help="Benchmark QEMU's execution of binary without tracing (continue mode) (overrides other flags)") + prog.add_argument('--benchmark-execution-stepping', default=False, action='store_true', + help="Benchmark QEMU's execution of binary without tracing (stepping mode) (overrides other flags)") + prog.add_argument('--benchmark-trace-test', default=False, action='store_true', + help="Benchmark Focaccia's tracing and testing of QEMU") return prog def quoted(s: str) -> str: @@ -105,7 +111,7 @@ def main(): env = os.environ.copy() # Differentiate between the QEMU GDB server and QEMU plugin interfaces - if args.use_socket: + if args.use_socket and not args.benchmark_execution: if not args.guest_arch: argparser.error('--guest-arch is required when --use-socket is specified') |