oracle_routing_analysis.py

#!/usr/bin/env python3
"""
Oracle Routing Analysis

Analyzes the maximum possible accuracy of a router that always chooses 
one of the correct models (if one exists) for each prompt.

For each unique original_id:
- If at least one model has performance=1.0, the oracle router is "correct"
- If no model has performance=1.0, the oracle router is "wrong"
"""

import pandas as pd
import sys

def analyze_oracle_routing(csv_path: str) -> None:
    print(f"Loading data from: {csv_path}")
    
    # Read the CSV file
    df = pd.read_csv(csv_path)
    
    print(f"Total rows: {len(df)}")
    print(f"Columns: {list(df.columns)}")
    
    # Get unique original_ids
    unique_ids = df['original_id'].unique()
    print(f"Unique prompts (original_id): {len(unique_ids)}")
    
    # Count unique models (llm column)
    unique_models = df['llm'].unique()
    print(f"Unique models: {len(unique_models)}")
    print(f"Models: {list(unique_models)}")
    
    # Group by original_id and check if any has performance=1.0
    oracle_correct = 0
    oracle_wrong = 0
    
    for orig_id in unique_ids:
        group = df[df['original_id'] == orig_id]
        max_perf = group['performance'].max()
        
        if max_perf == 1.0:
            oracle_correct += 1
        else:
            oracle_wrong += 1
    
    total_prompts = oracle_correct + oracle_wrong
    oracle_accuracy = (oracle_correct / total_prompts) * 100
    
    print("\n" + "="*50)
    print("ORACLE ROUTING ANALYSIS RESULTS")
    print("="*50)
    print(f"Total unique prompts: {total_prompts}")
    print(f"Prompts with at least one correct model: {oracle_correct}")
    print(f"Prompts with no correct model: {oracle_wrong}")
    print(f"\nOracle Routing Accuracy: {oracle_accuracy:.2f}%")
    print("="*50)
    
    # Additional analysis: distribution of how many models are correct per prompt
    print("\nAdditional Analysis:")
    correct_counts = []
    for orig_id in unique_ids:
        group = df[df['original_id'] == orig_id]
        correct_count = (group['performance'] == 1.0).sum()
        correct_counts.append(correct_count)
    
    correct_counts_series = pd.Series(correct_counts)
    print(f"\nDistribution of correct models per prompt:")
    for count, freq in sorted(correct_counts_series.value_counts().items()):
        print(f"  {count} correct model(s): {freq} prompts ({freq/total_prompts*100:.1f}%)")


if __name__ == "__main__":
    csv_path = sys.argv[1] if len(sys.argv) > 1 else "new_data/routerarena_irtrouter_format.csv"
    analyze_oracle_routing(csv_path)