|
| 1 | +#!/usr/bin/env python3 |
| 2 | +import os |
| 3 | +import shutil |
| 4 | +import subprocess |
| 5 | +import sys |
| 6 | +import time |
| 7 | +import csv |
| 8 | +import matplotlib.pyplot as plt |
| 9 | + |
| 10 | +# Constants |
| 11 | +COMMANDS = [ |
| 12 | + "schaeppert -vv -f dot nfa-{n}.dot iterate tmp/", |
| 13 | + # "schaeppert -f dot nfa-{n}.dot iterate tmp/", |
| 14 | + #"shepherd -vv -f dot nfa-{n}.dot", |
| 15 | + "shepherd -f dot nfa-{n}.dot", |
| 16 | +] |
| 17 | +TIMEOUT = 60 # seconds |
| 18 | +TMP_DIR = "tmp" |
| 19 | +LOGFILE_TEMPLATE = "log_{cmd_idx}_n{n}.txt" |
| 20 | + |
| 21 | +def ensure_empty_tmp(): |
| 22 | + if os.path.exists(TMP_DIR): |
| 23 | + shutil.rmtree(TMP_DIR) |
| 24 | + os.makedirs(TMP_DIR) |
| 25 | + |
| 26 | +def run_command(cmd, logfile, timeout): |
| 27 | + start = time.time() |
| 28 | + try: |
| 29 | + proc = subprocess.run( |
| 30 | + cmd, |
| 31 | + shell=True, |
| 32 | + stdout=subprocess.PIPE, |
| 33 | + stderr=subprocess.PIPE, |
| 34 | + timeout=timeout |
| 35 | + ) |
| 36 | + elapsed = time.time() - start |
| 37 | + with open(logfile, 'wb') as f: |
| 38 | + f.write(b'=== STDOUT ===\n') |
| 39 | + f.write(proc.stdout) |
| 40 | + f.write(b'\n=== STDERR ===\n') |
| 41 | + f.write(proc.stderr) |
| 42 | + return elapsed |
| 43 | + except subprocess.TimeoutExpired as e: |
| 44 | + with open(logfile, 'wb') as f: |
| 45 | + f.write(b'=== STDOUT ===\n') |
| 46 | + f.write((e.stdout or b'')) |
| 47 | + f.write(b'\n=== STDERR ===\n') |
| 48 | + f.write((e.stderr or b'')) |
| 49 | + f.write(b'\n=== TIMEOUT ===\n') |
| 50 | + return None |
| 51 | + |
| 52 | +def main(): |
| 53 | + if len(sys.argv) != 2: |
| 54 | + print(f"Usage: {sys.argv[0]} N", file=sys.stderr) |
| 55 | + sys.exit(1) |
| 56 | + try: |
| 57 | + N = int(sys.argv[1]) |
| 58 | + except ValueError: |
| 59 | + print("N must be an integer", file=sys.stderr) |
| 60 | + sys.exit(1) |
| 61 | + |
| 62 | + ensure_empty_tmp() |
| 63 | + results = [] |
| 64 | + for n in range(1, N + 1): |
| 65 | + row = {'n': n} |
| 66 | + for cmd_idx, cmd_template in enumerate(COMMANDS): |
| 67 | + cmd = cmd_template.format(n=n) |
| 68 | + logfile = LOGFILE_TEMPLATE.format(cmd_idx=cmd_idx, n=n) |
| 69 | + print(f"Running command {cmd_idx+1} for n={n}: {cmd_template}") |
| 70 | + t = run_command(cmd, logfile, TIMEOUT) |
| 71 | + if t is None: |
| 72 | + break |
| 73 | + row[f'cmd{cmd_idx+1}_time'] = t |
| 74 | + else: |
| 75 | + results.append(row) |
| 76 | + continue |
| 77 | + break |
| 78 | + |
| 79 | + # Write CSV to stdout |
| 80 | + fieldnames = ['n'] + [f'cmd{i+1}_time' for i in range(len(COMMANDS))] |
| 81 | + writer = csv.DictWriter(sys.stdout, fieldnames=fieldnames) |
| 82 | + writer.writeheader() |
| 83 | + for row in results: |
| 84 | + writer.writerow(row) |
| 85 | + |
| 86 | + # Plotting |
| 87 | + ns = [row['n'] for row in results] |
| 88 | + for cmd_idx, cmd_template in enumerate(COMMANDS): |
| 89 | + times = [row.get(f'cmd{cmd_idx+1}_time') for row in results] |
| 90 | + plt.plot(ns, times, label=cmd_template) |
| 91 | + plt.xlabel('n') |
| 92 | + plt.ylabel('Wallclock time (s)') |
| 93 | + plt.legend() |
| 94 | + plt.title('Command Running Times') |
| 95 | + plt.grid(True) |
| 96 | + plt.tight_layout() |
| 97 | + plt.savefig("benchmark_results.png") |
| 98 | + plt.show() |
| 99 | + |
| 100 | +if __name__ == '__main__': |
| 101 | + main() |
0 commit comments