Revert benchmark additions in scripts

3 files changed, 22 insertions, 97 deletions

diff --git a/src/focaccia/qemu/_qemu_tool.py b/src/focaccia/qemu/_qemu_tool.py
index ca95553..0e33d28 100644
--- a/src/focaccia/qemu/_qemu_tool.py
+++ b/src/focaccia/qemu/_qemu_tool.py
@@ -13,7 +13,6 @@ import subprocess
 import time
 from typing import Iterable, Optional
 
-from focaccia.benchmark import Timer
 import focaccia.parser as parser
 from focaccia.compare import compare_symbolic, Error, ErrorTypes
 from focaccia.snapshot import (
@@ -249,54 +248,10 @@ def main():
         raise NotImplementedError(f'Deterministic log {args.deterministic_log} specified but '
                                    'Focaccia built without deterministic log support')
 
-    # Benchmark native QEMU execution
-    if args.benchmark_execution_continue:
-        try:
-            timer = Timer("Emulator execution", paused=True, iterations=10)
-            for i in range(timer.iterations):
-                qemu_process = subprocess.Popen(
-                    [f"qemu-{args.guest_arch}", "-singlestep", "-g", args.remote.split(':')[1], args.executable],
-                    stdout=subprocess.DEVNULL,
-                    stderr=subprocess.DEVNULL
-                )
-                time.sleep(0.5)
-                timer.unpause()
-                gdb_server = GDBServerStateIterator(args.remote, detlog)
-                gdb.execute("continue")
-                qemu_process.wait()
-                timer.pause()
-            timer.log_time()
-            exit(0)
-        except Exception as e:
-            raise Exception(f'Unable to benchmark QEMU: {e}')
-    if args.benchmark_execution_stepping:
-        try:
-            timer = Timer("Emulator execution", paused=True, iterations=10)
-            for i in range(timer.iterations):
-                try:
-                    qemu_process = subprocess.Popen(
-                            [f"qemu-{args.guest_arch}", "-g", args.remote.split(':')[1], args.executable],
-                        stdout=subprocess.DEVNULL,
-                        stderr=subprocess.DEVNULL
-                    )
-                    time.sleep(0.5)
-                    timer.unpause()
-                    gdb_server = GDBServerStateIterator(args.remote, detlog)
-                    state_iter = iter(gdb_server)
-                    while True:
-                        cur_state = next(state_iter)
-                except StopIteration:
-                    timer.pause()
-            timer.log_time()
-            exit(0)
-        except Exception as e:
-            raise Exception(f'Unable to benchmark QEMU: {e}')
-
-    if not args.benchmark_trace_test:
-        try:
-            gdb_server = GDBServerStateIterator(args.remote, detlog)
-        except Exception as e:
-            raise Exception(f'Unable to perform basic GDB setup: {e}')
+    try:
+        gdb_server = GDBServerStateIterator(args.remote, detlog)
+    except Exception as e:
+        raise Exception(f'Unable to perform basic GDB setup: {e}')
 
     try:
         executable: str | None = None
@@ -324,43 +279,26 @@ def main():
 
     # Use symbolic trace to collect concrete trace from QEMU
     try:
-        timer = Timer("Emulator tracing", iterations=10, paused=True, enabled=args.benchmark_trace_test)
-        for i in range(timer.iterations):
-            if timer.enabled:
-                qemu_process = subprocess.Popen(
-                        [f"qemu-{args.guest_arch}", "-g", args.remote.split(':')[1], executable],
-                    stdout=subprocess.DEVNULL,
-                    stderr=subprocess.DEVNULL
-                )
-                time.sleep(0.5)
-                timer.unpause()
-                gdb_server = GDBServerStateIterator(args.remote, detlog)
-
-            conc_states, matched_transforms = collect_conc_trace(
-                gdb_server,
-                symb_transforms.states,
-                symb_transforms.env.start_address,
-                symb_transforms.env.stop_address)
-            timer.pause()
-        timer.log_time()
+        conc_states, matched_transforms = collect_conc_trace(
+            gdb_server,
+            symb_transforms.states,
+            symb_transforms.env.start_address,
+            symb_transforms.env.stop_address)
     except Exception as e:
         raise Exception(f'Failed to collect concolic trace from QEMU: {e}')
 
     # Verify and print result
     if not args.quiet:
         try:
-            timer = Timer("Emulator testing", iterations=10, enabled=args.benchmark_trace_test)
-            for i in range(timer.iterations):
-                res = compare_symbolic(conc_states, matched_transforms)
-                if qemu_crash["crashed"]:
-                    res.append({
-                        'pc': qemu_crash["pc"],
-                        'txl': None,
-                        'ref': qemu_crash["ref"],
-                        'errors': qemu_crash["errors"],
-                        'snap': qemu_crash["snap"],
-                    })
-            timer.log_time()
+            res = compare_symbolic(conc_states, matched_transforms)
+            if qemu_crash["crashed"]:
+                res.append({
+                    'pc': qemu_crash["pc"],
+                    'txl': None,
+                    'ref': qemu_crash["ref"],
+                    'errors': qemu_crash["errors"],
+                    'snap': qemu_crash["snap"],
+                })
             print_result(res, verbosity[args.error_level])
         except Exception as e:
             raise Exception('Error occured when comparing with symbolic equations: {e}')
diff --git a/src/focaccia/tools/capture_transforms.py b/src/focaccia/tools/capture_transforms.py
index 4cc6139..d69c786 100755
--- a/src/focaccia/tools/capture_transforms.py
+++ b/src/focaccia/tools/capture_transforms.py
@@ -4,7 +4,7 @@ import sys
 import argparse
 import logging
 
-from focaccia import parser, utils, benchmark
+from focaccia import parser, utils
 from focaccia.trace import TraceEnvironment
 from focaccia.native.tracer import SymbolicTracer
 from focaccia.deterministic import DeterministicLog
@@ -55,10 +55,6 @@ def main():
                       default='json',
                       choices=['json', 'msgpack'],
                       help='Symbolic trace output format')
-    prog.add_argument('--benchmark',
-                      default=False,
-                      action='store_true',
-                      help='Benchmark the trace function')
     args = prog.parse_args()
 
     if args.debug:
@@ -83,10 +79,7 @@ def main():
     tracer = SymbolicTracer(env, remote=args.remote, cross_validate=args.debug,
                             force=args.force)
 
-    timer = benchmark.Timer("Native tracing", iterations=10, enabled=args.benchmark)
-    for i in range(timer.iterations):
-        trace = tracer.trace(time_limit=args.insn_time_limit)
-    timer.log_time()
+    trace = tracer.trace(time_limit=args.insn_time_limit)
 
     parser.serialize_transformations(trace, args.output, args.out_type)
 
diff --git a/src/focaccia/tools/validate_qemu.py b/src/focaccia/tools/validate_qemu.py
index 48f9f6d..cdb4263 100755
--- a/src/focaccia/tools/validate_qemu.py
+++ b/src/focaccia/tools/validate_qemu.py
@@ -76,7 +76,7 @@ memory, and stepping forward by single instructions.
                       type=str,
                       choices=supported_architectures.keys(),
                       help='Architecture of the emulated guest'
-                           '(Only required when using --use-socket or --benchmark-executioe)')
+                           '(Only required when using --use-socket)')
     prog.add_argument('--remote',
                       type=str,
                       help='The hostname:port pair at which to find a QEMU GDB server.')
@@ -94,12 +94,6 @@ memory, and stepping forward by single instructions.
                       default=False,
                       action='store_true',
                       help='Enables scheduling (experimental)')
-    prog.add_argument('--benchmark-execution-continue', default=False, action='store_true',
-                      help="Benchmark QEMU's execution of binary without tracing (continue mode) (overrides other flags)")
-    prog.add_argument('--benchmark-execution-stepping', default=False, action='store_true',
-                      help="Benchmark QEMU's execution of binary without tracing (stepping mode) (overrides other flags)")
-    prog.add_argument('--benchmark-trace-test', default=False, action='store_true',
-                      help="Benchmark Focaccia's tracing and testing of QEMU")
     return prog
 
 def quoted(s: str) -> str:
@@ -119,7 +113,7 @@ def main():
     env = os.environ.copy()
 
     # Differentiate between the QEMU GDB server and QEMU plugin interfaces
-    if args.use_socket and not args.benchmark_execution:
+    if args.use_socket:
         if not args.guest_arch:
             argparser.error('--guest-arch is required when --use-socket is specified')