1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
#!/usr/bin/env bash
import argparse
import asyncio
from pathlib import Path
import os
import sys
from context import Context, RemoteContext, LocalContext
from log import error, info
import experiments
async def run(
ctx: Context,
local_result_path: Path,
experiment_base: Path,
experiment: experiments.Experiments,
) -> None:
info(str(ctx))
"""
async with ctx as client:
info(f"Open new context ({client.context()})")
info("Copy data to context")
await client.sync_to(experiment_root=experiment_base)
info("Build and run benchmarks on remote machine")
await client.run([f"cd experiments/{experiment}", "make"])
info("Copy results back to local machine")
await client.sync_back(
remote_path=Path(f"experiments/{experiment}/results/"),
local_path=local_result_path,
)
info("Done running experiments")
"""
info("Start plotting measurements")
experiments.plot(local_result_path, experiment, format="pdf")
info("--------- Done ---------")
def run_local(args):
experiment = args.experiment
experiment_root = args.base
result_root = args.result
cleanup = args.cleanup
ctx = LocalContext(
delete=cleanup,
)
asyncio.run(
run(
ctx=ctx,
local_result_path=result_root,
experiment_base=experiment_root,
experiment=experiment,
)
)
pass
def run_remote(args):
remote_user = args.remote_user or os.getenv("MTE_REMOTE_USER")
if not remote_user:
error(f"--remote-user not provided and MTE_REMOTE_USER not set.")
sys.exit(1)
remote_host = args.remote_user or os.getenv("MTE_REMOTE_HOST")
if not remote_host:
error(f"--remote-host not provided and MTE_REMOTE_HOST not set.")
sys.exit(1)
remote_jump_user = args.remote_jump_user or os.getenv("MTE_REMOTE_JUMP_USER")
remote_jump_host = args.remote_jump_host or os.getenv("MTE_REMOTE_JUMP_HOST")
remote_jump_port = args.remote_jump_port or os.getenv("MTE_REMOTE_JUMP_PORT")
if (
any([remote_jump_user, remote_jump_host, remote_jump_port])
and all([remote_jump_user, remote_jump_host]) == False
):
error(
(
"must provide both "
"--remote-jump-user (or set MTE_REMOTE_JUMP_USER) and "
"--remote-jump-host (or set MTE_REMOTE_JUMP_HOST) "
"or set neither"
)
)
sys.exit(1)
experiment = args.experiment
experiment_root = args.base
result_root = args.result
cleanup = args.cleanup
ctx = RemoteContext(
remote_user=remote_user,
remote_host=remote_host,
jump_user=remote_jump_user,
jump_host=remote_jump_host,
jump_port=remote_jump_port,
delete=cleanup,
)
asyncio.run(
run(
ctx=ctx,
local_result_path=result_root,
experiment_base=experiment_root,
experiment=experiment,
)
)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog="benchmark-tool",
description="A tool for performing benchmarks on remote machines.",
)
experiment_root = Path(__file__).parent.parent / Path("experiments/")
_ = parser.add_argument(
"--base",
default=str(experiment_root),
type=str,
help="The base directory in which the experiments are implemented.",
)
result_root = Path(__file__).parent.parent / Path("results/")
_ = parser.add_argument(
"--result",
default=str(result_root),
type=str,
help="The base directory in which all the results are placed.",
)
_ = parser.add_argument(
"--experiment",
type=str,
choices=experiments.experiment_choices(),
required=True,
help="The experiment to run",
)
_ = parser.add_argument(
"--cleanup",
action="store_true",
help="Cleanup all the resources of the context",
)
subparsers = parser.add_subparsers(dest="command", required=True)
local = subparsers.add_parser("local", help="Run the experiment locally.")
local.set_defaults(func=run_local)
remote = subparsers.add_parser("remote", help="Run the experiment remotely")
remote.set_defaults(func=run_remote)
remote.add_argument(
"--remote-user", help="Remote username (or set MTE_REMOTE_USER)"
)
remote.add_argument("--remote-host", help="Remote host (or set MTE_REMOTE_HOST)")
remote.add_argument(
"--remote-jump-user", help="Remote jump user (or set MTE_REMOTE_JUMP_USER)"
)
remote.add_argument(
"--remote-jump-host", help="Remote jump host (or set MTE_REMOTE_JUMP_HOST)"
)
remote.add_argument(
"--remote-jump-port", help="Remote jump port (or set MTE_REMOTE_JUMP_PORT)"
)
args = parser.parse_args()
args.func(args)
|