mtllm-evaluation/eval.py at main · jaseci-labs/mtllm-evaluation · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
import os
from pyinstrument import Profiler
from pyinstrument.renderers import JSONRenderer
from cProfile import Profile
import importlib
from jaclang import jac_import
from loguru import logger
import json
import sys
import argparse
import time


def run_dspy_program(program_name, program_path, profiler, output_dir):
    os.makedirs(f"{output_dir}/{program_name}/dspy", exist_ok=True)
    program_path = program_path.replace(".py", "").replace("/", ".")

    results_file = open(f"{output_dir}/{program_name}/dspy/results.txt", "w")
    sys.stdout = results_file

    if profiler == "cProfile":
        pr = Profile()
        pr.enable()
    else:
        profiler = Profiler()
        profiler.start()

    try:
        module = importlib.import_module(program_path)
    except Exception as e:
        logger.error(f"Error while running {program_path}: {e}")

    if profiler == "cProfile":
        pr.disable()
        pr.dump_stats(f"{output_dir}/{program_name}/dspy/profile.prof")
    else:
        profiler.stop()
        with open(f"{output_dir}/{program_name}/dspy/profile.html", "w") as f:
            f.write(profiler.output_html())
        with open(f"{output_dir}/{program_name}/dspy/profile.json", "w") as f:
            json.dump(json.loads(profiler.output(JSONRenderer())), f, indent=4)

    sys.stdout = sys.__stdout__
    results_file.close()


def run_jac_program(program_name, program_path, profiler, output_dir):
    os.makedirs(f"{output_dir}/{program_name}/jac", exist_ok=True)
    program_path = os.path.abspath(program_path)
    program_dir, program_file = os.path.split(program_path)

    results_file = open(f"{output_dir}/{program_name}/jac/results.txt", "w")
    sys.stdout = results_file

    if profiler == "cProfile":
        pr = Profile()
        pr.enable()
    else:
        profiler = Profiler()
        profiler.start()

    try:
        module = jac_import(program_file.replace(".jac", ""), base_path=program_dir)
    except Exception as e:
        logger.error(f"Error while running {program_path}: {e}")

    if profiler == "cProfile":
        pr.disable()
        pr.dump_stats(f"{output_dir}/{program_name}/jac/profile.prof")
    else:
        profiler.stop()
        with open(f"{output_dir}/{program_name}/jac/profile.html", "w") as f:
            f.write(profiler.output_html())
        with open(f"{output_dir}/{program_name}/jac/profile.json", "w") as f:
            json.dump(json.loads(profiler.output(JSONRenderer())), f, indent=4)

    sys.stdout = sys.__stdout__
    results_file.close()


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--profiler",
        help="Which profiler to use",
        default="cProfile",
        type=str,
        choices=["cProfile", "pyinstrument"],
    )
    parser.add_argument(
        "--config",
        help="Location to the Eval config",
        default="eval.config.json",
        type=str,
    )
    parser.add_argument(
        "--output_dir",
        help="Output Directory Location",
        default="output",
        type=str,
    )
    parser.add_argument(
        "--impl",
        help="Implementation to use",
        default="both",
        type=str,
        choices=["both", "dspy", "jac"],
    )
    parser.add_argument(
        "-y",
        help="Skip confirmation",
        action="store_true",
        default=False,
    )
    args = parser.parse_args()
    logger.info(f"Using {args.profiler} as the profiler.")
    with open(args.config) as f:
        EVAL_PROBLEMS = json.load(f)
    if os.path.exists(args.output_dir) and not args.y:
        logger.info(f"Output directory exists. Do you want to overwrite it? (y/n)")
        if input().lower() != "y":
            logger.info("Exiting...")
            exit(0)
    for difficulty, PROBLEM_SET in EVAL_PROBLEMS.items():
        for problem_name, paths in PROBLEM_SET.items():
            if os.path.exists(f"{args.output_dir}/{problem_name}") and not args.y:
                logger.info(f"Output directory for {problem_name} exists.")
                logger.info(f"Do you want to overwrite it? (y/n)")
                if input().lower() != "y":
                    logger.info("Skipping...")
                    continue

            logger.info(f"Running {problem_name} problem from {difficulty} difficulty.")

            if args.impl == "jac" or args.impl == "both":
                logger.info(f"Running JAC program: {paths['jac']}")
                run_jac_program(
                    problem_name, paths["jac"], args.profiler, args.output_dir
                )
                time.sleep(60)

            if args.impl == "dspy" or args.impl == "both":
                logger.info(f"Running Dspy program: {paths['dspy']}")
                run_dspy_program(
                    problem_name, paths["dspy"], args.profiler, args.output_dir
                )
                time.sleep(60)