DeepNSM/eval_config.json at main · OSU-STARLAB/DeepNSM · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
{
    "dataset": {
        "name": "baartmar/nsm_dataset",
        "split": "test"
    },
    "output_path": "results",
    "models": [
        {
            "model": "baartmar/DeepNSM-8B",
            "use_system_prompt": true,
            "num_examples": 3,
            "max_batch_size": 8,
            "orig": "meta-llama/Llama-3.1-8B"
        },
        {
            "model": "baartmar/DeepNSM-1B",
            "use_system_prompt": true,
            "num_examples": 3,
            "max_batch_size": 8,
            "orig": "meta-llama/Llama-3.2-1B"
        },
        {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "use_system_prompt": true,
            "num_examples": 3,
            "max_batch_size": 8,
            "orig": ""
        },
        {
            "model": "meta-llama/Llama-3.2-1B-Instruct",
            "use_system_prompt": true,
            "num_examples": 3,
            "max_batch_size": 8,
            "orig": ""
        },
        {
            "model": "gpt-4o",
            "use_system_prompt": true,
            "num_examples": 3,
            "max_batch_size": 1,
            "orig": ""
        },
        {
            "model": "gpt-4o",
            "use_system_prompt": true,
            "num_examples": 3,
            "max_batch_size": 1,
            "orig": ""
        },
        {
            "model": "gemini-2.0-flash",
            "use_system_prompt": true,
            "num_examples": 3,
            "max_batch_size": 1,
            "orig": ""
        }
    ],
    "grader_models": [
        {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "max_batch_size": 8
        },
        {
            "model": "google/gemma-3-12b-it",
            "max_batch_size": 8
        },
        {
            "model": "mistralai/Mistral-7B-Instruct-v0.3",
            "max_batch_size": 8
        }
    ],
    "languages": [
        "alz",
        "rw",
        "dn",
        "din",
        "ab"
    ]
}