Add benchmarking

5 files changed, 130 insertions, 20 deletions

diff --git a/qemu b/qemu
-Subproject 3b2a0fb80eb9b6b5f216fa69069e66210466f5e
+Subproject f2da205cb4142259d9bc6b9d4596ebbe2426fe4
diff --git a/src/focaccia/benchmark.py b/src/focaccia/benchmark.py
new file mode 100644
index 0000000..c7476a2
--- /dev/null
+++ b/src/focaccia/benchmark.py
@@ -0,0 +1,31 @@
+from time import perf_counter
+
+class Timer:
+    def __init__(self, name: str = "Timer", paused: bool = False, iterations: int = 1, enabled: bool = True):
+        self.name = name
+        self.iterations = iterations
+        self.total_time = 0
+        self.paused = paused
+        self.enabled = enabled
+        if self.enabled:
+            print(f'{self.name}: start timer, do {self.iterations} iterations')
+            self.start_time = perf_counter()
+        else:
+            self.iterations = 1
+
+    def pause(self):
+        if self.enabled and not self.paused:
+            self.paused = True
+            self.total_time += (perf_counter() - self.start_time)
+
+    def unpause(self):
+        if self.enabled and self.paused:
+            self.paused = False
+            self.start_time = perf_counter()
+
+    def log_time(self):
+        if self.enabled:
+            if not self.paused:
+                self.total_time += (perf_counter() - self.start_time)
+            time = self.total_time / self.iterations
+            print(f'{self.name}: took {time:.5f} seconds')
diff --git a/src/focaccia/qemu/_qemu_tool.py b/src/focaccia/qemu/_qemu_tool.py
index c9a471f..ca95553 100644
--- a/src/focaccia/qemu/_qemu_tool.py
+++ b/src/focaccia/qemu/_qemu_tool.py
@@ -9,8 +9,11 @@ work to do.
 import logging
 import traceback
 import pyroaring
+import subprocess
+import time
 from typing import Iterable, Optional
 
+from focaccia.benchmark import Timer
 import focaccia.parser as parser
 from focaccia.compare import compare_symbolic, Error, ErrorTypes
 from focaccia.snapshot import (
@@ -246,10 +249,54 @@ def main():
         raise NotImplementedError(f'Deterministic log {args.deterministic_log} specified but '
                                    'Focaccia built without deterministic log support')
 
-    try:
-        gdb_server = GDBServerStateIterator(args.remote, detlog, args.schedule)
-    except Exception as e:
-        raise Exception(f'Unable to perform basic GDB setup: {e}')
+    # Benchmark native QEMU execution
+    if args.benchmark_execution_continue:
+        try:
+            timer = Timer("Emulator execution", paused=True, iterations=10)
+            for i in range(timer.iterations):
+                qemu_process = subprocess.Popen(
+                    [f"qemu-{args.guest_arch}", "-singlestep", "-g", args.remote.split(':')[1], args.executable],
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL
+                )
+                time.sleep(0.5)
+                timer.unpause()
+                gdb_server = GDBServerStateIterator(args.remote, detlog)
+                gdb.execute("continue")
+                qemu_process.wait()
+                timer.pause()
+            timer.log_time()
+            exit(0)
+        except Exception as e:
+            raise Exception(f'Unable to benchmark QEMU: {e}')
+    if args.benchmark_execution_stepping:
+        try:
+            timer = Timer("Emulator execution", paused=True, iterations=10)
+            for i in range(timer.iterations):
+                try:
+                    qemu_process = subprocess.Popen(
+                            [f"qemu-{args.guest_arch}", "-g", args.remote.split(':')[1], args.executable],
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL
+                    )
+                    time.sleep(0.5)
+                    timer.unpause()
+                    gdb_server = GDBServerStateIterator(args.remote, detlog)
+                    state_iter = iter(gdb_server)
+                    while True:
+                        cur_state = next(state_iter)
+                except StopIteration:
+                    timer.pause()
+            timer.log_time()
+            exit(0)
+        except Exception as e:
+            raise Exception(f'Unable to benchmark QEMU: {e}')
+
+    if not args.benchmark_trace_test:
+        try:
+            gdb_server = GDBServerStateIterator(args.remote, detlog)
+        except Exception as e:
+            raise Exception(f'Unable to perform basic GDB setup: {e}')
 
     try:
         executable: str | None = None
@@ -277,24 +324,43 @@ def main():
 
     # Use symbolic trace to collect concrete trace from QEMU
     try:
-        conc_states, matched_transforms = collect_conc_trace(
-            gdb_server,
-            symb_transforms)
+        timer = Timer("Emulator tracing", iterations=10, paused=True, enabled=args.benchmark_trace_test)
+        for i in range(timer.iterations):
+            if timer.enabled:
+                qemu_process = subprocess.Popen(
+                        [f"qemu-{args.guest_arch}", "-g", args.remote.split(':')[1], executable],
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL
+                )
+                time.sleep(0.5)
+                timer.unpause()
+                gdb_server = GDBServerStateIterator(args.remote, detlog)
+
+            conc_states, matched_transforms = collect_conc_trace(
+                gdb_server,
+                symb_transforms.states,
+                symb_transforms.env.start_address,
+                symb_transforms.env.stop_address)
+            timer.pause()
+        timer.log_time()
     except Exception as e:
         raise Exception(f'Failed to collect concolic trace from QEMU: {e}')
 
     # Verify and print result
     if not args.quiet:
         try:
-            res = compare_symbolic(conc_states, matched_transforms)
-            if qemu_crash["crashed"]:
-                res.append({
-                    'pc': qemu_crash["pc"],
-                    'txl': None,
-                    'ref': qemu_crash["ref"],
-                    'errors': qemu_crash["errors"],
-                    'snap': qemu_crash["snap"],
-                })
+            timer = Timer("Emulator testing", iterations=10, enabled=args.benchmark_trace_test)
+            for i in range(timer.iterations):
+                res = compare_symbolic(conc_states, matched_transforms)
+                if qemu_crash["crashed"]:
+                    res.append({
+                        'pc': qemu_crash["pc"],
+                        'txl': None,
+                        'ref': qemu_crash["ref"],
+                        'errors': qemu_crash["errors"],
+                        'snap': qemu_crash["snap"],
+                    })
+            timer.log_time()
             print_result(res, verbosity[args.error_level])
         except Exception as e:
             raise Exception('Error occured when comparing with symbolic equations: {e}')
diff --git a/src/focaccia/tools/capture_transforms.py b/src/focaccia/tools/capture_transforms.py
index d69c786..4cc6139 100755
--- a/src/focaccia/tools/capture_transforms.py
+++ b/src/focaccia/tools/capture_transforms.py
@@ -4,7 +4,7 @@ import sys
 import argparse
 import logging
 
-from focaccia import parser, utils
+from focaccia import parser, utils, benchmark
 from focaccia.trace import TraceEnvironment
 from focaccia.native.tracer import SymbolicTracer
 from focaccia.deterministic import DeterministicLog
@@ -55,6 +55,10 @@ def main():
                       default='json',
                       choices=['json', 'msgpack'],
                       help='Symbolic trace output format')
+    prog.add_argument('--benchmark',
+                      default=False,
+                      action='store_true',
+                      help='Benchmark the trace function')
     args = prog.parse_args()
 
     if args.debug:
@@ -79,7 +83,10 @@ def main():
     tracer = SymbolicTracer(env, remote=args.remote, cross_validate=args.debug,
                             force=args.force)
 
-    trace = tracer.trace(time_limit=args.insn_time_limit)
+    timer = benchmark.Timer("Native tracing", iterations=10, enabled=args.benchmark)
+    for i in range(timer.iterations):
+        trace = tracer.trace(time_limit=args.insn_time_limit)
+    timer.log_time()
 
     parser.serialize_transformations(trace, args.output, args.out_type)
 
diff --git a/src/focaccia/tools/validate_qemu.py b/src/focaccia/tools/validate_qemu.py
index cdb4263..48f9f6d 100755
--- a/src/focaccia/tools/validate_qemu.py
+++ b/src/focaccia/tools/validate_qemu.py
@@ -76,7 +76,7 @@ memory, and stepping forward by single instructions.
                       type=str,
                       choices=supported_architectures.keys(),
                       help='Architecture of the emulated guest'
-                           '(Only required when using --use-socket)')
+                           '(Only required when using --use-socket or --benchmark-executioe)')
     prog.add_argument('--remote',
                       type=str,
                       help='The hostname:port pair at which to find a QEMU GDB server.')
@@ -94,6 +94,12 @@ memory, and stepping forward by single instructions.
                       default=False,
                       action='store_true',
                       help='Enables scheduling (experimental)')
+    prog.add_argument('--benchmark-execution-continue', default=False, action='store_true',
+                      help="Benchmark QEMU's execution of binary without tracing (continue mode) (overrides other flags)")
+    prog.add_argument('--benchmark-execution-stepping', default=False, action='store_true',
+                      help="Benchmark QEMU's execution of binary without tracing (stepping mode) (overrides other flags)")
+    prog.add_argument('--benchmark-trace-test', default=False, action='store_true',
+                      help="Benchmark Focaccia's tracing and testing of QEMU")
     return prog
 
 def quoted(s: str) -> str:
@@ -113,7 +119,7 @@ def main():
     env = os.environ.copy()
 
     # Differentiate between the QEMU GDB server and QEMU plugin interfaces
-    if args.use_socket:
+    if args.use_socket and not args.benchmark_execution:
         if not args.guest_arch:
             argparser.error('--guest-arch is required when --use-socket is specified')