Add benchmark script

6 files changed, 245 insertions, 2 deletions

diff --git a/flake.nix b/flake.nix
index e94f156..8a012d3 100644
--- a/flake.nix
+++ b/flake.nix
@@ -356,6 +356,18 @@
 				};
 			};
 
+			benchmark-focaccia = {
+				type = "app";
+				program = let
+					wrapper = pkgs.writeShellScriptBin "benchmark-focaccia" ''
+						exec ${packages.focaccia}/bin/benchmark-focaccia --gdb "${gdbInternal}/bin/gdb" "$@"
+					'';
+				in "${wrapper}/bin/benchmark-focaccia";
+				meta = {
+					description = "Benchmark focaccia";
+				};
+			};
+
 			# Useful for synchronize the uv lockfile
 			uv-sync = {
 				type = "app";
diff --git a/pyproject.toml b/pyproject.toml
index 98365c1..8a53ecb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,6 +37,7 @@ focaccia = "focaccia.cli:main"
 convert = "focaccia.tools.convert:main"
 validate-qemu = "focaccia.tools.validate_qemu:main"
 capture-transforms = "focaccia.tools.capture_transforms:main"
+benchmark-focaccia = "focaccia.tools.benchmark_focaccia:main"
 
 [build-system]
 requires = ["hatchling"]
diff --git a/src/focaccia/benchmark/__init__.py b/src/focaccia/benchmark/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/focaccia/benchmark/__init__.py
diff --git a/src/focaccia/benchmark/_benchmark.py b/src/focaccia/benchmark/_benchmark.py
new file mode 100644
index 0000000..0562035
--- /dev/null
+++ b/src/focaccia/benchmark/_benchmark.py
@@ -0,0 +1,102 @@
+from focaccia.tools.benchmark_focaccia import make_argparser
+from focaccia.benchmark.timer import Timer
+from focaccia.qemu import _qemu_tool
+from focaccia.deterministic import DeterministicLog
+from focaccia.compare import compare_symbolic
+import focaccia.parser as parser
+
+import gdb
+import subprocess
+import time
+
+def main():
+    print("Benchmarking focaccia")
+    args = make_argparser().parse_args()
+
+    detlog = DeterministicLog(args.deterministic_log)
+    if args.deterministic_log and detlog.base_directory is None:
+        raise NotImplementedError(f'Deterministic log {args.deterministic_log} specified but '
+                                   'Focaccia built without deterministic log support')
+
+    # Emu exec continue
+    try:
+        timer = Timer("Emulator execution (continue)", paused=True, iterations=args.iterations)
+        for i in range(timer.iterations):
+            qemu_process = subprocess.Popen(
+                [f"qemu-{args.guest_arch}", "-singlestep", "-g", args.port, args.binary],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL
+            )
+            time.sleep(0.5)
+            timer.unpause()
+            gdb_server = _qemu_tool.GDBServerStateIterator(f"localhost:{args.port}", detlog)
+            gdb.execute("continue")
+            qemu_process.wait()
+            timer.pause()
+        timer.log_time()
+    except Exception as e:
+        raise Exception(f'Unable to benchmark QEMU: {e}')
+
+    # Emu exec stepping
+    try:
+        timer = Timer("Emulator execution (stepping)", paused=True, iterations=args.iterations)
+        for i in range(timer.iterations):
+            try:
+                qemu_process = subprocess.Popen(
+                        [f"qemu-{args.guest_arch}", "-g", args.port, args.binary],
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL
+                )
+                time.sleep(0.5)
+                timer.unpause()
+                gdb_server = _qemu_tool.GDBServerStateIterator(f"localhost:{args.port}", detlog)
+                state_iter = iter(gdb_server)
+                while True:
+                    cur_state = next(state_iter)
+            except StopIteration:
+                timer.pause()
+        timer.log_time()
+    except Exception as e:
+        raise Exception(f'Unable to benchmark QEMU: {e}')
+
+    try:
+        with open(f"/tmp/benchmark-{args.binary.split('/')[-1]}-symbolic.trace", 'r') as strace:
+            symb_transforms = parser.parse_transformations(strace)
+    except Exception as e:
+        raise Exception(f'Failed to parse state transformations from native trace: {e}')
+
+    # emu exec tracing
+    try:
+        timer = Timer("Emulator tracing", iterations=args.iterations, paused=True)
+        for i in range(timer.iterations):
+            if timer.enabled:
+                qemu_process = subprocess.Popen(
+                        [f"qemu-{args.guest_arch}", "-g", args.port, args.binary],
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL
+                )
+                time.sleep(0.5)
+                timer.unpause()
+                gdb_server = _qemu_tool.GDBServerStateIterator(f"localhost:{args.port}", detlog)
+
+            conc_states, matched_transforms = _qemu_tool.collect_conc_trace(
+                gdb_server,
+                symb_transforms.states,
+                symb_transforms.env.start_address,
+                symb_transforms.env.stop_address)
+            timer.pause()
+        timer.log_time()
+    except Exception as e:
+        raise Exception(f'Failed to collect concolic trace from QEMU: {e}')
+
+    # emu exec testing
+    try:
+        timer = Timer("Emulator testing", iterations=args.iterations)
+        for i in range(timer.iterations):
+            res = compare_symbolic(conc_states, matched_transforms)
+        timer.log_time()
+    except Exception as e:
+        raise Exception('Error occured when comparing with symbolic equations: {e}')
+
+if __name__ == "__main__":
+    main()
diff --git a/src/focaccia/benchmark.py b/src/focaccia/benchmark/timer.py
index c7476a2..65b37b8 100644
--- a/src/focaccia/benchmark.py
+++ b/src/focaccia/benchmark/timer.py
@@ -1,12 +1,13 @@
 from time import perf_counter
 
 class Timer:
-    def __init__(self, name: str = "Timer", paused: bool = False, iterations: int = 1, enabled: bool = True):
+    def __init__(self, name: str = "Timer", paused: bool = False, iterations: int = 1, file_path: str = "benchmark.txt", enabled: bool = True):
         self.name = name
-        self.iterations = iterations
+        self.iterations = int(iterations)
         self.total_time = 0
         self.paused = paused
         self.enabled = enabled
+        self.file = open(file_path, "a")
         if self.enabled:
             print(f'{self.name}: start timer, do {self.iterations} iterations')
             self.start_time = perf_counter()
@@ -29,3 +30,8 @@ class Timer:
                 self.total_time += (perf_counter() - self.start_time)
             time = self.total_time / self.iterations
             print(f'{self.name}: took {time:.5f} seconds')
+            self.file.write(f"{self.name}: {time:.5f} seconds\n")
+            self.file.close()
+
+    def write_binary(self, binary: str):
+        self.file.write(f"\n{binary}:\n")
diff --git a/src/focaccia/tools/benchmark_focaccia.py b/src/focaccia/tools/benchmark_focaccia.py
new file mode 100755
index 0000000..3b3019e
--- /dev/null
+++ b/src/focaccia/tools/benchmark_focaccia.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import logging
+import argparse
+import sysconfig
+import subprocess
+
+import focaccia.benchmark
+from focaccia.arch import supported_architectures
+from focaccia import utils, parser
+from focaccia.trace import TraceEnvironment
+from focaccia.native.tracer import SymbolicTracer
+from focaccia.deterministic import DeterministicLog
+from focaccia.benchmark.timer import Timer
+
+def make_argparser():
+    prog = argparse.ArgumentParser()
+    prog.description = 'Focaccia benchmark'
+    prog.add_argument('binary', help='The program to analyse.')
+    prog.add_argument('args', action='store', nargs=argparse.REMAINDER,
+                      help='Arguments to the program.')
+    prog.add_argument('--guest-arch',
+                      type=str,
+                      choices=supported_architectures.keys(),
+                      help='Architecture of the emulated guest')
+    prog.add_argument('-p', '--port',
+                      default='12345',
+                      help='Port to use for connection with QEMU')
+    prog.add_argument('-r', '--remote',
+                      default=False,
+                      help='Remote target to trace (e.g. 127.0.0.1:12345)')
+    prog.add_argument('-n', '--iterations',
+                      default='10',
+                      help='Number of iterations per benchmark')
+    prog.add_argument('-l', '--deterministic-log',
+                      help='Path of the directory storing the deterministic log produced by RR')
+    prog.add_argument('--gdb',
+                      type=str,
+                      default='gdb',
+                      help='GDB binary to invoke.')
+    prog.add_argument('-o', '--output',
+                      default='./benchmark.txt',
+                      help='Output file to save results')
+    return prog
+
+def quoted(s: str) -> str:
+    return f'"{s}"'
+
+def try_remove(l: list, v):
+    try:
+        l.remove(v)
+    except ValueError:
+        pass
+
+def main():
+    argparser = make_argparser()
+    args = argparser.parse_args()
+
+    # Test native tracing
+    detlog = DeterministicLog(args.deterministic_log)
+    if args.deterministic_log and detlog.base_directory is None:
+        raise NotImplementedError(f'Deterministic log {args.deterministic_log} specified but '
+                                   'Focaccia built without deterministic log support')
+
+    timer = Timer("Native tracing", iterations=args.iterations, file_path=args.output)
+    timer.write_binary(args.binary)
+    for i in range(timer.iterations):
+        env = TraceEnvironment(args.binary, args.args, utils.get_envp(),
+                               nondeterminism_log=detlog,
+                               start_address=None,
+                               stop_address=None)
+        tracer = SymbolicTracer(env, remote=args.remote, cross_validate=False,
+                            force=True)
+        trace = tracer.trace(time_limit=None)
+    timer.log_time()
+
+    with open(f"/tmp/benchmark-{args.binary.split('/')[-1]}-symbolic.trace", 'w') as file:
+        parser.serialize_transformations(trace, file)
+
+    # Get environment
+    env = os.environ.copy()
+    # QEMU GDB interface
+    script_dirname = os.path.dirname(focaccia.benchmark.__file__)
+    benchmark_path = os.path.join(script_dirname, '_benchmark.py')
+
+    # We have to remove all arguments we don't want to pass to the qemu tool
+    # manually here. Not nice, but what can you do..
+    argv = sys.argv
+    try_remove(argv, '--gdb')
+    try_remove(argv, args.gdb)
+
+    # Assemble the argv array passed to the qemu tool. GDB does not have a
+    # mechanism to pass arguments to a script that it executes, so we
+    # overwrite `sys.argv` manually before invoking the script.
+    argv_str = f'[{", ".join(quoted(a) for a in argv)}]'
+    path_str = f'[{", ".join(quoted(s) for s in sys.path)}]'
+
+    paths = sysconfig.get_paths()
+    candidates = [paths["purelib"], paths["platlib"]]
+    entries = [p for p in candidates if p and os.path.isdir(p)]
+    venv_site = entries[0]
+    env["PYTHONPATH"] = ','.join([script_dirname, venv_site])
+
+    print(f"GDB started with Python Path: {env['PYTHONPATH']}")
+    gdb_cmd = [
+        args.gdb,
+        '-nx',  # Don't parse any .gdbinits
+        '--batch',
+        '-ex',  'py import sys',
+        '-ex', f'py sys.argv = {argv_str}',
+        '-ex', f'py sys.path = {path_str}',
+        "-ex", f'py import site; site.addsitedir({venv_site!r})',
+        "-ex", f'py import site; site.addsitedir({script_dirname!r})',
+        '-x', benchmark_path
+    ]
+    proc = subprocess.Popen(gdb_cmd, env=env)
+
+    ret = proc.wait()
+    exit(ret)
+