evaluate.py

import json
from typing import Any
import numpy as np
import time
from collections import defaultdict
from tabulate import tabulate
import os
import dotenv
import logging
import argparse
from pympler import asizeof
import warnings
from utils.logger import setup_logging
warnings.filterwarnings("ignore")
from card_estimator import card_estimator
from utils.metrics import q_error, q_error_percentiles
from utils.plots import (
    plot_execution_times,
    q_error_plots,
    boxplots,
    regression_plot,
    hierarchical_regression_plot,
    regression_plot_vertical,
)
from utils.subplan_generation import annotate_subplans_with_bespoke_cards
from utils.end_to_end_evaluation import execute_annotated_queries


logger = logging.getLogger(__name__)
setup_logging(logging.INFO)

dotenv.load_dotenv()

percentiles = [50, 90, 95, 99]
regression_threshold = (
    1.0  
)
feedback_file = "outputs/feedback.json"
with open(feedback_file, "w") as f:
    json.dump({}, f)


def add_feedback(key: str, value: Any):
    with open(feedback_file, "r") as f:
        fb = json.load(f)
    fb[key] = value
    with open(feedback_file, "w") as f:
        json.dump(fb, f)
        f.write("\n")


def count_filters(filters):
    count = 0
    for f in filters:
        if "children" in f:
            count += count_filters(f["children"])
        else:
            count += 1
    return count


def filter_subplan_duplicates(subplans_dict: dict):
    subplans = []
    for _, values in subplans_dict.items():
        for _, subplan in values.items():
            subplans.append(subplan)
    seen = set()
    filtered = []
    for sp in subplans:
        key = (
            sp["q_error_pg"],
            # sp["q_error_bespoke"],
            sp["true_card"],
            len(sp["query_raw_info"][0])
            + len(sp["query_raw_info"][1])
            + len(sp["query_raw_info"][2]),
            "".join(sorted([t["name"] for t in sp["query_raw_info"][0]])),
        )  ## filter by these three values to identify duplicates as others can be permuted
        if key not in seen:
            seen.add(key)
            filtered.append(sp)
    logger.debug(
        f"Filtered {len(subplans) - len(filtered)} duplicate subplans, {len(filtered)} remaining"
    )
    return filtered


def annotate_subplans_with_q_errors():
    with open("outputs/job_subplans.json", "r") as f:
        subplans_dict = json.load(f)
    for sql, subplans in subplans_dict.items():
        for subplan, items in subplans.items():
            items["q_error_pg"] = q_error(items["true_card"], items["pg_card"])[0]
            items["q_error_bespoke"] = q_error(
                items["true_card"], items["bespoke_card"]
            )[0]
            items["bespoke_over"] = items["bespoke_card"] > items["true_card"]
            items["pg_over"] = items["pg_card"] > items["true_card"]
    with open("outputs/job_subplans.json", "w") as f:
        json.dump(subplans_dict, f, indent=2)


def root_analysis(subplans_dict: dict):
    logger.info("")
    logger.info(f"{'#'*50} Root Analysis {'#'*50}")
    root_plans = [list(subplans.values())[-1] for subplans in subplans_dict.values()]
    q_errors_pg = [items["q_error_pg"] for items in root_plans]
    q_errors_bespoke = [items["q_error_bespoke"] for items in root_plans]
    q_error_percentiles_bespoke = q_error_percentiles(q_errors_bespoke, percentiles)
    q_error_percentiles_pg = q_error_percentiles(q_errors_pg, percentiles)

    headers = ["Percentile"] + [f"{p}th" for p in percentiles]
    bespoke_row = ["Q-Error Bespoke"] + [
        f"{q_error_percentiles_bespoke[f'{p}th']:.0f}" for p in percentiles
    ]
    pg_row = ["Q-Error Postgres"] + [
        f"{q_error_percentiles_pg[f'{p}th']:.0f}" for p in percentiles
    ]
    rows = [bespoke_row, pg_row]
    logger.info(f"\n{tabulate(rows, headers=headers)}")

    total_regression_rate = sum(
        1
        for b, pg in zip(q_errors_bespoke, q_errors_pg)
        if b / pg > regression_threshold
    ) / len(q_errors_bespoke)
    logger.info(f"Root plan regression rate: {total_regression_rate:.2%}")

    q_error_plots(q_errors_bespoke, q_errors_pg, all=False)


def global_analysis(subplans_dict: dict):
    logger.info("")
    logger.info(f"{'#'*50} Global Analysis {'#'*50}")
    subplans = filter_subplan_duplicates(subplans_dict)
    q_errors_pg = [items["q_error_pg"] for items in subplans]
    q_errors_bespoke = [items["q_error_bespoke"] for items in subplans]
    q_error_percentiles_bespoke = q_error_percentiles(q_errors_bespoke, percentiles)
    q_error_percentiles_pg = q_error_percentiles(q_errors_pg, percentiles)

    headers = ["Percentile"] + [f"{p}th" for p in percentiles]
    bespoke_row = ["Q-Error Bespoke"] + [
        f"{q_error_percentiles_bespoke[f'{p}th']:.0f}" for p in percentiles
    ]
    pg_row = ["Q-Error Postgres"] + [
        f"{q_error_percentiles_pg[f'{p}th']:.0f}" for p in percentiles
    ]
    rows = [bespoke_row, pg_row]
    logger.info(f"\n{tabulate(rows, headers=headers)}")

    q_error_plots(q_errors_bespoke, q_errors_pg, all=True)
    total_regression_rate = sum(
        1
        for b, pg in zip(q_errors_bespoke, q_errors_pg)
        if b / pg > regression_threshold
    ) / len(q_errors_bespoke)
    logger.info(f"Global regression rate: {total_regression_rate:.2%}")

    add_feedback(
        "q_error_percentiles",
        {
            "bespoke": {p: round(q, 2) for p, q in q_error_percentiles_bespoke.items()},
            "pg": {p: round(q, 2) for p, q in q_error_percentiles_pg.items()},
        },
    )
    add_feedback("total_regression_rate", round(total_regression_rate, 4))


def num_table_analysis(subplans_dict: dict):
    logger.info("")
    logger.info(f"{'#'*50} Number of Tables Analysis {'#'*50}")
    subplans = filter_subplan_duplicates(subplans_dict)
    max_table_count = max([len(items["query_raw_info"][0]) for items in subplans])
    per_num_tables = {
        i: {
            "abs_q_err": {"bespoke": [], "postgres": []},
            "rel_q_err": {"bespoke": [], "postgres": []},
            "counts": 0,
            "regressions": 0,
        }
        for i in range(1, max_table_count + 1)
    }
    for subplan in subplans:
        num_tables = len(subplan["query_raw_info"][0])
        per_num_tables[num_tables]["abs_q_err"]["bespoke"].append(
            subplan["q_error_bespoke"]
        )
        per_num_tables[num_tables]["abs_q_err"]["postgres"].append(
            subplan["q_error_pg"]
        )
        per_num_tables[num_tables]["rel_q_err"]["bespoke"].append(
            subplan["q_error_bespoke"]
            if subplan["bespoke_over"]
            else 1.0 / subplan["q_error_bespoke"]
        )
        per_num_tables[num_tables]["rel_q_err"]["postgres"].append(
            subplan["q_error_pg"] if subplan["pg_over"] else 1.0 / subplan["q_error_pg"]
        )
        per_num_tables[num_tables]["counts"] += 1
        if subplan["q_error_bespoke"] / subplan["q_error_pg"] > regression_threshold:
            per_num_tables[num_tables]["regressions"] += 1
    headers = ["# Tables"] + [f"{i}" for i in range(1, max_table_count + 1)]
    bespoke_row = ["Bespoke Median Q-Error"] + [
        f"{np.median(per_num_tables[i]['abs_q_err']['bespoke']):.0f}"
        for i in range(1, max_table_count + 1)
    ]
    pg_row = ["Postgres Median Q-Error"] + [
        f"{np.median(per_num_tables[i]['abs_q_err']['postgres']):.0f}"
        for i in range(1, max_table_count + 1)
    ]
    rows = [bespoke_row, pg_row]
    logger.info(f"\n{tabulate(rows, headers=headers)}")

    boxplots(
        {i: per_num_tables[i]["abs_q_err"] for i in range(1, max_table_count + 1)},
        "Tables",
    )

    boxplots(
        {i: per_num_tables[i]["rel_q_err"] for i in range(1, max_table_count + 1)},
        "Tables",
        over_under=True,
    )

    regression_plot(
        {i: per_num_tables[i]["regressions"] for i in range(1, max_table_count + 1)},
        {i: per_num_tables[i]["counts"] for i in range(1, max_table_count + 1)},
        x_label="Number of Tables",
        x_values=list(range(1, max_table_count + 1)),
        tag="num_tables",
    )

    median_q_error_num_tables_pg = {}
    median_q_error_num_tables_bespoke = {}
    plus5_pg = []
    plus5_bespoke = []
    for i in range(1, max_table_count + 1):
        if i < 5:
            median_q_error_num_tables_pg[str(i)] = round(
                np.median(per_num_tables[i]["abs_q_err"]["postgres"])
            )
            median_q_error_num_tables_bespoke[str(i)] = round(
                np.median(per_num_tables[i]["abs_q_err"]["bespoke"])
            )
        else:
            plus5_pg.extend(per_num_tables[i]["abs_q_err"]["postgres"])
            plus5_bespoke.extend(per_num_tables[i]["abs_q_err"]["bespoke"])
    median_q_error_num_tables_pg[">=5"] = round(np.median(plus5_pg))
    median_q_error_num_tables_bespoke[">=5"] = round(np.median(plus5_bespoke))
    add_feedback(
        "median_q_error_num_tables",
        {
            "bespoke": median_q_error_num_tables_bespoke,
            "pg": median_q_error_num_tables_pg,
        },
    )


def num_filter_analysis(subplans_dict: dict):
    logger.info("")
    logger.info(f"{'#'*50} Number of Filters Analysis {'#'*50}")
    subplans = filter_subplan_duplicates(subplans_dict)
    max_filter_count = max(
        [count_filters(subplan["query_raw_info"][1]) for subplan in subplans]
    )
    # similar to num_table_analysis but for filters instead of tables
    per_num_filters = {
        i: {
            "abs_q_err": {"bespoke": [], "postgres": []},
            "rel_q_err": {"bespoke": [], "postgres": []},
            "counts": 0,
            "regressions": 0,
        }
        for i in range(0, max_filter_count + 1)
    }
    for subplan in subplans:
        num_filters = count_filters(subplan["query_raw_info"][1])
        per_num_filters[num_filters]["abs_q_err"]["bespoke"].append(
            subplan["q_error_bespoke"]
        )
        per_num_filters[num_filters]["abs_q_err"]["postgres"].append(
            subplan["q_error_pg"]
        )
        per_num_filters[num_filters]["rel_q_err"]["bespoke"].append(
            subplan["q_error_bespoke"]
            if subplan["bespoke_over"]
            else 1.0 / subplan["q_error_bespoke"]
        )
        per_num_filters[num_filters]["rel_q_err"]["postgres"].append(
            subplan["q_error_pg"] if subplan["pg_over"] else 1.0 / subplan["q_error_pg"]
        )
        per_num_filters[num_filters]["counts"] += 1
        if subplan["q_error_bespoke"] / subplan["q_error_pg"] > regression_threshold:
            per_num_filters[num_filters]["regressions"] += 1
    headers = ["# Filters"] + [f"{i}" for i in range(0, max_filter_count + 1)]
    bespoke_row = ["Bespoke Median Q-Error"] + [
        f"{np.median(per_num_filters[i]['abs_q_err']['bespoke']):.0f}"
        for i in range(0, max_filter_count + 1)
    ]
    pg_row = ["Postgres Median Q-Error"] + [
        f"{np.median(per_num_filters[i]['abs_q_err']['postgres']):.0f}"
        for i in range(0, max_filter_count + 1)
    ]
    rows = [bespoke_row, pg_row]
    logger.info(f"\n{tabulate(rows, headers=headers)}")

    boxplots(
        {i: per_num_filters[i]["abs_q_err"] for i in range(0, max_filter_count + 1)},
        "Filters",
    )

    boxplots(
        {i: per_num_filters[i]["rel_q_err"] for i in range(0, max_filter_count + 1)},
        "Filters",
        over_under=True,
    )

    regression_plot(
        {i: per_num_filters[i]["regressions"] for i in range(0, max_filter_count + 1)},
        {i: per_num_filters[i]["counts"] for i in range(0, max_filter_count + 1)},
        x_label="Number of Filters",
        x_values=list(range(0, max_filter_count + 1)),
        tag="num_filters",
    )


def table_and_col_analysis(subplans_dict: dict):
    logger.info("")
    logger.info(f"{'#'*50} Table and Column Analysis {'#'*50}")
    subplans = filter_subplan_duplicates(subplans_dict)
    with open("data/schema.json", "r") as f:
        schema = json.load(f)

    tables = {
        table: {
            "filter_cols": {
                col: {"counts": 0, "regressions": 0, "dtype": schema[table][col]}
                for col in cols
            },
            "counts": 0,
            "regressions": 0,
        }
        for table, cols in schema.items()
    }

    def traverse_filters(filters, alias):
        cols = []
        for f in filters:
            if "column" and "alias" in f:
                if alias == f["alias"]:
                    col_name = f["column"]
                    cols.append(col_name)
            if "children" in f:
                cols.extend(traverse_filters(f["children"], alias))
        return cols

    for subplan in subplans:
        for table in subplan["query_raw_info"][0]:
            t_name = table["name"]
            t_alias = table["alias"]

            tables[t_name]["counts"] += 1
            filter_cols = set(traverse_filters(subplan["query_raw_info"][1], t_alias))
            for col in filter_cols:
                tables[t_name]["filter_cols"][col]["counts"] += 1
            if (
                subplan["q_error_bespoke"] / subplan["q_error_pg"]
                > regression_threshold
            ):
                tables[t_name]["regressions"] += 1
                for col in filter_cols:
                    tables[t_name]["filter_cols"][col]["regressions"] += 1

    headers = ["Table"] + [f"{table}" for table in tables.keys()]
    appearances_row = ["# Plans"] + [f"{val["counts"]}" for val in tables.values()]
    regression_row = ["Regression Rate"] + [
        (f"{val["regressions"]/val["counts"]:.2%}" if val["counts"] > 0 else "N/A")
        for val in tables.values()
    ]
    logger.info(f"\n{tabulate([appearances_row, regression_row], headers=headers)}")

    regression_plot(
        {name: tables[name]["regressions"] for name in tables.keys()},
        {name: tables[name]["counts"] for name in tables.keys()},
        x_label="Table",
        x_values=tables.keys(),
        tag="table",
    )

    # add feedback on regression rates per table, if greater than 20%
    add_feedback(
       "regression_rate_per_table",
       {
           name: round(tables[name]["regressions"] / tables[name]["counts"], 4)
           for name in tables.keys()
           if round(tables[name]["regressions"] / tables[name]["counts"], 4) > 0.33
       },
    )

    ## clear all columns and tables that are never used
    filter_tables = {}
    for table_name in tables:
        # check if any filter columns have counts > 0, if not, skip the table entirely for filter analysis
        if not any(
            col_stats["counts"] > 0
            for col_stats in tables[table_name]["filter_cols"].values()
        ):
            continue
        filter_tables[table_name] = {"cols": {}}
        filter_tables[table_name]["cols"] = {
            col: stats
            for col, stats in tables[table_name]["filter_cols"].items()
            if stats["counts"] > 0
        }
    hierarchical_regression_plot(filter_tables, "filter_cols")

    # now plot regression rates per column datatype of filter columns
    x = sorted(
        set(
            filter_tables[t]["cols"][c]["dtype"]
            for t in filter_tables
            for c in filter_tables[t]["cols"]
        )
    )
    total_counts = {dtype: 0 for dtype in x}
    regression_count = {dtype: 0 for dtype in x}
    for t in filter_tables:
        for c in filter_tables[t]["cols"]:
            dtype = filter_tables[t]["cols"][c]["dtype"]
            total_counts[dtype] += filter_tables[t]["cols"][c]["counts"]
            regression_count[dtype] += filter_tables[t]["cols"][c]["regressions"]
    regression_plot(
        regression_count,
        total_counts,
        x_label="Column Data Type",
        x_values=x,
        tag="filter_col_dtype",
    )


def join_pair_analysis(subplans_dict: dict):
    logger.info("")
    logger.info(f"{'#'*50} Join Pair Analysis {'#'*50}")
    subplans = filter_subplan_duplicates(subplans_dict)
    with open("data/schema.json", "r") as f:
        schema = json.load(f)
    # initialize join_pairs dict
    join_pairs = {}
    for table1, cols1 in schema.items():
        for col1 in cols1:
            for table2, cols2 in schema.items():
                for col2 in cols2:
                    if (
                        table1 != table2
                        and schema[table1][col1] == schema[table2][col2]
                    ):
                        pair = tuple(sorted([f"{table1}.{col1}", f"{table2}.{col2}"]))
                        join_pairs[pair] = {
                            "counts": 0,
                            "regressions": 0,
                        }
    # traverse plans and count join pair usage and regressions
    for subplan in subplans:
        for join in subplan["query_raw_info"][2]:
            # resolve alias to table name for both sides of the join
            a1 = join["alias1"]
            a2 = join["alias2"]
            t1 = next(
                t["name"] for t in subplan["query_raw_info"][0] if t["alias"] == a1
            )
            t2 = next(
                t["name"] for t in subplan["query_raw_info"][0] if t["alias"] == a2
            )
            join_pair = tuple(
                sorted(
                    [
                        f"{t1}.{join['column1']}",
                        f"{t2}.{join['column2']}",
                    ]
                )
            )

            join_pairs[join_pair]["counts"] += 1
            if (
                subplan["q_error_bespoke"] / subplan["q_error_pg"]
                > regression_threshold
            ):
                join_pairs[join_pair]["regressions"] += 1
    # filter join pairs that are never used
    join_pairs = {
        pair: stats for pair, stats in join_pairs.items() if stats["counts"] > 0
    }
    # plot regression rates for join pairs
    # labesl are t1.col1 = t2.col2, introduce aliases by splitting table names on '_' and taking initials, e.g. customer_info.customer_id -> ci.customer_id, orders.customer_id -> o.customer_id, label would be ci.customer_id = o.customer_id
    x = []
    for pair in join_pairs.keys():
        t1, c1 = pair[0].split(".")
        t2, c2 = pair[1].split(".")
        alias1 = "".join([w[0] for w in t1.split("_")]) + "." + c1
        alias2 = "".join([w[0] for w in t2.split("_")]) + "." + c2
        x.append(f"{alias1} = {alias2}")

    regression_plot_vertical(
        {pair: stats["regressions"] for pair, stats in join_pairs.items()},
        {pair: stats["counts"] for pair, stats in join_pairs.items()},
        x_label="Join Pair",
        x_values=x,
        tag="join_pair",
    )


def filter_type_analysis(subplans_dict: dict):
    logger.info("")
    logger.info(f"{'#'*50} Filter Type Analysis {'#'*50}")
    subplans = filter_subplan_duplicates(subplans_dict)

    def visit_filters(filters):
        # Helper function to visit nested filters
        operators = []
        for f in filters:
            if "children" in f:
                operators.extend(visit_filters(f["children"]))
            else:
                operators.append(f["operator"])
        return operators

    per_filter_type = defaultdict(lambda: {"counts": 0, "regressions": 0})
    for subplan in subplans:
        operators = set(visit_filters(subplan["query_raw_info"][1]))
        for op in operators:
            per_filter_type[op]["counts"] += 1
        if subplan["q_error_bespoke"] / subplan["q_error_pg"] > regression_threshold:
            for op in operators:
                per_filter_type[op]["regressions"] += 1
    headers = ["Filter Type"] + [f"{i}" for i in per_filter_type.keys()]
    appearances_row = ["# Plans"] + [
        f"{per_filter_type[ftype]["counts"]}" for ftype in per_filter_type.keys()
    ]
    regression_row = ["Regression Rate"] + [
        (
            f"{per_filter_type[ftype]["regressions"]/per_filter_type[ftype]["counts"]:.2%}"
            if per_filter_type[ftype]["counts"] > 0
            else "N/A"
        )
        for ftype in per_filter_type.keys()
    ]
    logger.info(f"\n{tabulate([appearances_row, regression_row], headers=headers)}")

    x_values = sorted(list(per_filter_type.keys()))
    regression_plot(
        {x: per_filter_type[x]["regressions"] for x in x_values},
        {x: per_filter_type[x]["counts"] for x in x_values},
        x_label="Filter Type",
        x_values=x_values,
        tag="filter_type",
    )
    add_feedback(
        "regression_rate_per_filter_type",
        {
            x: round(
                per_filter_type[x]["regressions"] / per_filter_type[x]["counts"], 2
            )
            for x in x_values
        },
    )


def get_execution_times():
    with open("outputs/end_to_end_results.json", "r") as f:
        results = json.load(f)
    pg_time = sum(r["pg_time"] for r in results.values())
    bespoke_time = sum(r["bespoke_time"] for r in results.values())
    true_time = sum(r["true_time"] for r in results.values())
    return {"postgres": pg_time, "bespoke": bespoke_time, "true": true_time}


def setup_and_annotate(no_tqdm=False):
    estimator = card_estimator()
    logger.info(
        f"Estimator size before setup: {asizeof.asizeof(estimator) / (1024**2):.2f} MB"
    )
    logger.info("Setting up estimator...")
    start_time = time.perf_counter()
    estimator.setup()
    logger.info(
        f"Estimator setup completed in {time.perf_counter() - start_time} seconds"
    )
    estimator_size = asizeof.asizeof(estimator) / (1024**2)
    logger.info(f"Estimator size after setup: {estimator_size:.2f} MB")
    annotate_subplans_with_bespoke_cards(estimator, no_tqdm=no_tqdm)
    annotate_subplans_with_q_errors()
    add_feedback("estimator_size", f"{estimator_size:.2f} MB")


def outlier_analysis(subplans_dict: dict):
    logger.info("")
    logger.info(f"{'#'*50} Outlier Analysis {'#'*50}")
    subplans = filter_subplan_duplicates(subplans_dict)
    # identify subplans where bespoke q error is > 2x pg q error
    outliers = []
    for subplan in subplans:
        if (
            subplan["q_error_bespoke"] / subplan["q_error_pg"] > 2
            and len(subplan["query_raw_info"][0]) <= 4
        ):  # only consider outliers that have 4 or fewer tables to make it easier to analyze and fix them
            outliers.append(subplan)

    count_over = sum(1 for o in outliers if o["bespoke_over"])
    count_under = sum(1 for o in outliers if not o["bespoke_over"])
    outlier_plans = [
        {
            "plan": o["query_raw_info"],
            "true_card": o["true_card"],
            "bespoke_card": o["bespoke_card"],
            "q_error_bespoke": round(o["q_error_bespoke"], 2),
        }
        for o in outliers
    ]
    outlier_plans = sorted(
        outlier_plans, key=lambda x: x["q_error_bespoke"], reverse=True
    )
    add_feedback(
        "outliers",
        {
            "count_over_estimates": count_over,
            "count_under_estimates": count_under,
            "10_worst": outlier_plans[:10],
        },
    )
    with open("outputs/outliers.json", "w") as f:
        json.dump(outlier_plans, f, indent=2)


parser = argparse.ArgumentParser()
parser.add_argument("--end2end", action="store_true", help="Run end-to-end evaluation")
parser.add_argument(
    "--no_joins",
    action="store_true",
    help="Run analysis only on single table queries",
)
parser.add_argument(
    "--skip_setup",
    action="store_true",
    help="Skip setup and annotation, use existing outputs/job_subplans.json",
)
parser.add_argument(
    "--no_filters",
    action="store_true",
    help="Run analysis only on queries without filters",
)
parser.add_argument(
    "--no_tqdm", action="store_true", help="Show tqdm progress bars during analysis"
)
args = parser.parse_args()


if not (os.getenv("skip_setup") == "true" or args.skip_setup):
    setup_and_annotate(args.no_tqdm or os.getenv("no_tqdm") == "true")
if args.no_joins or os.getenv("no_joins") == "true":
    with open("outputs/job_subplans.json", "r") as f:
        subplans_dict = json.load(f)

    subplans_dict = {
        sql: dict(
            filter(
                lambda items: len(items[1]["query_raw_info"][0]) == 1, subplans.items()
            )
        )
        for sql, subplans in subplans_dict.items()
    }

elif args.no_filters or os.getenv("no_filters") == "true":
    with open("outputs/job_subplans.json", "r") as f:
        subplans_dict = json.load(f)
    subplans_dict = {
        sql: dict(
            filter(
                lambda items: count_filters(items[1]["query_raw_info"][1]) == 0,
                subplans.items(),
            )
        )
        for sql, subplans in subplans_dict.items()
    }

else:
    with open("outputs/job_subplans.json", "r") as f:
        subplans_dict = json.load(f)


global_analysis(subplans_dict)
if not (args.no_joins or os.getenv("no_joins") == "true") and not (
    args.no_filters or os.getenv("no_filters") == "true"
):
    root_analysis(subplans_dict)
if not args.no_joins and not os.getenv("no_joins") == "true":
    num_table_analysis(subplans_dict)
    join_pair_analysis(subplans_dict)
if not args.no_filters and not os.getenv("no_filters") == "true":
    num_filter_analysis(subplans_dict)
    table_and_col_analysis(subplans_dict)
    filter_type_analysis(subplans_dict)
outlier_analysis(subplans_dict)

if args.end2end or os.getenv("end2end") == "true":
    logger.info("Running end-to-end evaluation...")
    execute_annotated_queries(
        approaches_to_execute=["bespoke", "pg", "true"], warmup=True, repetitions=1
    )
    plot_execution_times()
    add_feedback("end2end_times", get_execution_times())