diff --git a/aeo_updates/natural_gas_price_regression/.gitignore b/aeo_updates/natural_gas_price_regression/.gitignore new file mode 100644 index 0000000..9b648df --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/.gitignore @@ -0,0 +1,34 @@ +# Python cache +__pycache__/ +*.pyc + +# Visualization outputs (PNGs, PDFs) +*.png +*.pdf + +# Validation and visualization results (all generated output) +results validation/ + +# Raw API data dumps (reproducible from EIA API) +outputs of alpha regression/raw_aeo_data/ +outputs of beta regression/raw_aeo_data/ + +# Beta raw data visualization subfolder +outputs of beta regression/bata raw data visualization/ + +# Beta regression intermediate outputs (keep cd_beta0.csv, national_beta.csv, beta_regression_summary.csv) +outputs of beta regression/regression_points.csv +outputs of beta regression/alpha_from_beta_by_scenario.csv +outputs of beta regression/alpha_from_beta_regression.csv +outputs of beta regression/alpha from beta regression/ + +# Notebook checkpoints +.ipynb_checkpoints/ + +# Stale nested duplicate directory +AEO_Updates/ + +# Old scripts (replaced by visualization.py) +results_visualization.py +results_validation.py +beta_raw_data_visualization.py diff --git a/aeo_updates/natural_gas_price_regression/README.md b/aeo_updates/natural_gas_price_regression/README.md new file mode 100644 index 0000000..5522342 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/README.md @@ -0,0 +1,91 @@ +# Natural Gas Price Regression + +This folder has the natural gas update workflow for ReEDS. + +## Model Note + +`alpha` differs by step: + +- Beta step (`alpha1`) is `alpha1(region, year)` and is shared across scenarios. +- Alpha step (`alpha2`) is `alpha2(region, year, scenario)`, so each output + scenario has its own alpha path. + +## Prerequisites + +### EIA API Key + +The pipeline fetches data from the EIA API and requires an API key. Set it as an environment variable before running: + +**Windows (persistent — open a new terminal after running):** +```cmd +setx EIA_API_KEY your_api_key_here +``` + +**Windows (current session only):** +```cmd +set EIA_API_KEY=your_api_key_here +``` + +**macOS/Linux:** +```bash +export EIA_API_KEY=your_api_key_here +``` + +> The key is read from the `EIA_API_KEY` environment variable. Do **not** hardcode it in `aeo_pipeline_config.json`. + +## Run order + +1. Edit `aeo_pipeline_config.json` with your settings. + +2. Run **beta regression first**. + +```bash +python aeo_beta_regression.py --config aeo_pipeline_config.json +``` + +3. Sync beta outputs into alpha inputs. + +```bash +python sync_beta_to_alpha_inputs.py --config aeo_pipeline_config.json +``` + +4. Run **alpha regression**. + +```bash +python aeo_alpha_regression.py --config aeo_pipeline_config.json +``` + +5. Generate all diagnostic plots and validation. + +```bash +python visualization.py --config aeo_pipeline_config.json +``` + +Skip individual parts with flags: +```bash +python visualization.py --skip-raw-scatter --skip-validation +``` + +Default output folder: `results validation` (all plots and validation CSVs). + +## Scenario Configuration + +Set explicit scenarios in `aeo_pipeline_config.json` under `scenarios`: + +- `scenarios.beta_regression.include`: scenarios used to estimate beta. +- `scenarios.alpha_regression.fetch`: scenarios fetched for alpha preprocessing. +- `scenarios.alpha_regression.outputs`: mapping from output suffix (`reference`, `HOG`, `LOG`, etc.) to scenario aliases. + +Use canonical IDs (for example `ref{aeo_year}`, `highogs`, `lowogs`) for a clear setup. + +## One-command batch (Windows) + +```bat +run_ng_pipeline.bat +``` + +Optional custom config: + +```bat +run_ng_pipeline.bat my_config.json +``` diff --git a/aeo_updates/natural_gas_price_regression/aeo_alpha_regression.py b/aeo_updates/natural_gas_price_regression/aeo_alpha_regression.py new file mode 100644 index 0000000..a21a6b0 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/aeo_alpha_regression.py @@ -0,0 +1,1269 @@ +""" +AEO Natural Gas Price Preprocessing Pipeline for ReEDS Inputs +============================================================= + +This script automates the preprocessing of AEO (Annual Energy Outlook) +natural gas data from the EIA API into input files for the ReEDS capacity +expansion model. + +NG Supply Curve Model +--------------------- +ReEDS models regional natural gas prices using a linear supply curve +framework. The regional delivered price of natural gas in each census +division is decomposed into three components: + + Price(r,t,s) = Alpha(r,t,s) + Beta_regional(r) × Demand_regional(r,t,s) + + Beta_national × Demand_national(t,s) + +Where: + - Price(r,t,s) : AEO NG price for region r, year t, scenario s (2024$/MMBtu) + - Alpha(r,t,s) : Intercept / base price component (2004$/MMBtu) + - Beta_regional(r) : Region-specific demand sensitivity (2004$/MMBtu per Quad) + - Demand_regional(r,t,s) : Electric sector NG demand in region r (Quads) + - Beta_national : National demand sensitivity (2004$/MMBtu per Quad) + - Demand_national(t,s) : Total national electric sector NG demand (Quads) + +Alpha is solved as the residual after removing demand-driven price effects: + + Alpha(r,t,s) = Price(r,t,s) × deflator - Beta_regional(r) × Demand_regional(r,t,s) + - Beta_national × Demand_national(t,s) + +Alpha is solved at scenario level and indexed by (region, year, scenario), +so each scenario keeps its own residual alpha path. + +The deflator converts from the AEO's native dollar year (e.g., 2024$) to +2004$, the base dollar year used internally by ReEDS for NG pricing. + +Data Sources +------------ +- Prices and demand projections: EIA AEO API (3 scenarios: Reference, High/Low O&G) +- Historical backfill: Local CSV files for pre-projection years +- Betas: Pre-computed regional and national sensitivity coefficients + +Output Files (per scenario) +--------------------------- +- alpha_AEO_{year}_{scenario}.csv : Regional alpha values (2004$/MMBtu) +- ng_AEO_{year}_{scenario}.csv : Regional NG prices (AEO dollar year) +- ng_demand_AEO_{year}_{scenario}.csv : Electric sector demand (Quads) +- ng_tot_demand_AEO_{year}_{scenario}.csv : Total sector demand (Quads) +- cd_beta0.csv : Electric sector regional betas + +Usage +----- + python aeo_alpha_regression.py --config aeo_pipeline_config.json + +Example config: see aeo_pipeline_config.json +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import re +import shutil +import sys +from pathlib import Path +from typing import Any + +import pandas as pd +import requests +from requests.adapters import HTTPAdapter +from urllib3 import disable_warnings +from urllib3.exceptions import InsecureRequestWarning +from urllib3.util.retry import Retry + +LOGGER = logging.getLogger("aeo_pipeline") + +# ============================================================================ +# Constants +# ============================================================================ + +# Mapping from normalized region names to canonical internal keys +CENDIV_CANONICAL = { + "newengland": "NewEngland", + "middleatlantic": "MiddleAtlantic", + "eastnorthcentral": "EastNorthCentral", + "westnorthcentral": "WestNorthCentral", + "southatlantic": "SouthAtlantic", + "eastsouthcentral": "EastSouthCentral", + "westsouthcentral": "WestSouthCentral", + "mountain": "Mountain", + "pacific": "Pacific", + "unitedstates": "UnitedStates", +} + +# Mapping from internal keys to ReEDS output format (underscore-separated) +CENDIV_OUTPUT = { + "NewEngland": "New_England", + "MiddleAtlantic": "Mid_Atlantic", + "EastNorthCentral": "East_North_Central", + "WestNorthCentral": "West_North_Central", + "SouthAtlantic": "South_Atlantic", + "EastSouthCentral": "East_South_Central", + "WestSouthCentral": "West_South_Central", + "Mountain": "Mountain", + "Pacific": "Pacific", +} + +# EIA AEO series names for NG data +NG_SERIES_NAMES = { + "price": "Energy Prices : Electric Power : Natural Gas", + "demand_elec": "Energy Use : Electric Power : Natural Gas", + "demand_total": "Energy Use : Total : Natural Gas", +} + +# Required output scenarios and their EIA API aliases +NG_OUTPUT_SCENARIOS = { + "reference": ["ref{aeo_year}", "Reference case", "Reference Case"], + "HOG": ["highogs", "High Oil and Gas Supply"], + "LOG": ["lowogs", "Low Oil and Gas Supply"], +} + +HISTORY_SUFFIX = "historical" + + +# ============================================================================ +# Utility Functions +# ============================================================================ + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="AEO NG update pipeline") + p.add_argument("--config", default="aeo_pipeline_config.json") + p.add_argument("--log-level", default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR"]) + p.add_argument("--aeo-year", type=int, default=None) + return p.parse_args() + + +def setup_logging(level: str) -> None: + logging.basicConfig( + level=getattr(logging, level.upper()), + format="%(asctime)s | %(levelname)s | %(message)s", + ) + + +def require(condition: bool, message: str) -> None: + """Assert a condition with a descriptive error message.""" + if not condition: + raise ValueError(message) + + +def normalize_token(value: Any) -> str: + """Normalize a string for case-insensitive, whitespace-insensitive matching.""" + if value is None: + return "" + return re.sub(r"[^a-z0-9]+", "", str(value).replace("\xa0", " ").strip().lower()) + + +def canonical_cendiv(value: Any) -> str: + """Convert a region name to its canonical internal key.""" + key = normalize_token(value) + return CENDIV_CANONICAL.get(key, re.sub(r"[^A-Za-z0-9]", "", str(value))) + + +def cendiv_output_label(cendiv: str) -> str: + """Convert an internal key to ReEDS output format (e.g., 'New_England').""" + require(cendiv in CENDIV_OUTPUT, + f"Unsupported census division for NG output: {cendiv}") + return CENDIV_OUTPUT[cendiv] + + +def output_label_to_cendiv(label: str) -> str: + """Convert a ReEDS output label back to the internal key.""" + norm = normalize_token(label.replace("_", " ")) + for cendiv, output in CENDIV_OUTPUT.items(): + if normalize_token(output.replace("_", " ")) == norm: + return cendiv + return canonical_cendiv(label) + + +def resolve_case_insensitive(path: Path) -> Path: + """Resolve a path with case-insensitive matching on each component.""" + if path.exists(): + return path + path = path.resolve() + current = Path(path.anchor) + for part in path.parts[1:]: + if not current.exists(): + return path + try: + matches = [p for p in current.iterdir() + if p.name.lower() == part.lower()] + except PermissionError: + return path + if not matches: + return path + current = matches[0] + return current + + +def resolve_path(base_dir: Path, configured_path: str) -> Path: + """Resolve a configured path relative to the base directory.""" + p = Path(configured_path) + if not p.is_absolute(): + p = base_dir / p + return resolve_case_insensitive(p) + + +def load_config(config_path: Path) -> dict[str, Any]: + """Load and validate the JSON configuration file.""" + require(config_path.exists(), f"Config not found: {config_path}") + with config_path.open("r", encoding="utf-8") as f: + cfg = json.load(f) + require(isinstance(cfg, dict), f"Config root must be an object: {config_path}") + return cfg + + +def resolve_api_key(config: dict[str, Any]) -> str: + """Resolve the EIA API key from environment, config, or legacy fallback.""" + api_cfg = config["api"] + env_var = api_cfg.get("key_env_var", "EIA_API_KEY") + env_key = os.getenv(env_var, "").strip() + if env_key: + return env_key + fallback = api_cfg.get("key_fallback", "").strip() + if fallback: + LOGGER.warning("Using key_fallback from config.") + return fallback + try: + from _eia_api_functions import api_key as legacy_api_key # type: ignore + if legacy_api_key: + LOGGER.warning("Using API key from _eia_api_functions.py fallback.") + return str(legacy_api_key) + except Exception: + pass + raise ValueError(f"Missing EIA API key. Set {env_var} or api.key_fallback.") + + +# ============================================================================ +# EIA API Client +# ============================================================================ + +class EiaClient: + """Client for fetching data from the EIA AEO API with retry logic.""" + + def __init__(self, api_cfg: dict[str, Any], api_key: str): + self.base_url = api_cfg["base_url"].rstrip("/") + self.verify_ssl = bool(api_cfg.get("verify_ssl", True)) + self.timeout = int(api_cfg.get("timeout_seconds", 60)) + self.api_key = api_key + if not self.verify_ssl: + disable_warnings(InsecureRequestWarning) + retries = Retry( + total=int(api_cfg.get("max_retries", 4)), + connect=int(api_cfg.get("max_retries", 4)), + read=int(api_cfg.get("max_retries", 4)), + status=int(api_cfg.get("max_retries", 4)), + backoff_factor=float(api_cfg.get("backoff_seconds", 1.0)), + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["GET"], + raise_on_status=False, + ) + self.session = requests.Session() + adapter = HTTPAdapter(max_retries=retries) + self.session.mount("https://", adapter) + self.session.mount("http://", adapter) + + def get_json(self, path: str, + params: list[tuple[str, str]] | None = None) -> dict[str, Any]: + full_path = path if path.startswith("/") else f"/{path}" + query = [("api_key", self.api_key)] + if params: + query.extend(params) + resp = self.session.get( + f"{self.base_url}{full_path}", + params=query, + timeout=self.timeout, + verify=self.verify_ssl, + ) + resp.raise_for_status() + payload = resp.json() + require("response" in payload, + f"Unexpected payload from {path}: {payload}") + return payload + + def get_facets(self, aeo_year: int, facet: str) -> list[dict[str, Any]]: + return self.get_json( + f"/aeo/{aeo_year}/facet/{facet}")["response"]["facets"] + + def get_data(self, path: str, + params: list[tuple[str, str]]) -> pd.DataFrame: + payload = self.get_json(path, params) + warnings = payload["response"].get("warnings", []) + for w in warnings: + LOGGER.warning("EIA warning: %s | %s", + w.get("warning"), w.get("description")) + data = payload["response"].get("data", []) + require(data, f"No data from endpoint {path}") + return pd.DataFrame(data) + + +# ============================================================================ +# EIA API Resolution Helpers +# ============================================================================ + +def resolve_region_ids(client: EiaClient, aeo_year: int, + regions: list[str]) -> dict[str, str]: + """Map region display names to EIA region IDs.""" + facets = client.get_facets(aeo_year, "regionId") + region_map = {normalize_token(item["name"]): str(item["id"]) + for item in facets} + out: dict[str, str] = {} + for name in regions: + key = normalize_token(name) + require(key in region_map, f"Region not found in EIA facets: {name}") + out[name] = region_map[key] + return out + + +def resolve_series_ids(client: EiaClient, aeo_year: int, + series_name: str) -> list[str]: + """Find EIA series IDs matching a given series name.""" + facets = client.get_facets(aeo_year, "seriesId") + ids = [str(item["id"]) for item in facets + if normalize_token(item.get("name")) == normalize_token(series_name)] + require(ids, f"Series not found: {series_name}") + return ids + + +def resolve_ng_scenarios( + client: EiaClient, + aeo_year: int, + include_scenarios: list[str] | None = None, +) -> pd.DataFrame: + """ + Discover available AEO scenarios for NG data. + + If include_scenarios is provided, only those scenarios are kept + (matching by scenario id or scenario name, case-insensitive). + """ + facets = client.get_facets(aeo_year, "scenario") + rows: list[dict[str, str]] = [] + for item in facets: + scenario_id = str(item.get("id", "")).strip() + scenario_name = str(item.get("name", "")).strip() + if not scenario_id: + continue + # Skip legacy composite IDs (e.g., "aeo2023ref") + if normalize_token(scenario_id).startswith("aeo"): + continue + rows.append({"scenario_id": scenario_id, + "scenario_name": scenario_name}) + require(rows, "No NG scenarios left after filtering legacy IDs.") + + if include_scenarios: + picked: list[dict[str, str]] = [] + missing: list[str] = [] + for raw in include_scenarios: + token = normalize_token(str(raw).replace("{aeo_year}", str(aeo_year))) + found = None + for row in rows: + sid = str(row["scenario_id"]) + sname = str(row["scenario_name"]) + if normalize_token(sid) == token or normalize_token(sname) == token: + found = row + break + if found is None: + missing.append(str(raw)) + continue + if all(str(r["scenario_id"]) != str(found["scenario_id"]) for r in picked): + picked.append(found) + require( + not missing, + f"Configured alpha_regression.fetch scenarios not found: {missing}", + ) + out = pd.DataFrame(picked) + LOGGER.info( + "Selected NG scenarios from config (%d): %s", + len(out), + out["scenario_id"].tolist(), + ) + return out.reset_index(drop=True) + + out = ( + pd.DataFrame(rows) + .drop_duplicates(subset=["scenario_id"]) + .sort_values("scenario_id") + .reset_index(drop=True) + ) + LOGGER.info("Selected NG scenarios (%d): %s", len(out), out["scenario_id"].tolist()) + return out + + +def resolve_output_scenario_aliases( + aeo_year: int, + configured_outputs: Any, +) -> dict[str, list[str]]: + """Build output scenario alias map from config, with defaults.""" + if configured_outputs is None: + raw_map: dict[str, list[str]] = NG_OUTPUT_SCENARIOS + elif isinstance(configured_outputs, dict): + raw_map = {} + for suffix, aliases in configured_outputs.items(): + require( + isinstance(aliases, list), + f"alpha_regression.outputs['{suffix}'] must be a list.", + ) + raw_map[str(suffix)] = [str(a) for a in aliases] + elif isinstance(configured_outputs, list): + raw_map = {} + for row in configured_outputs: + require( + isinstance(row, dict), + "alpha_regression.outputs list entries must be objects.", + ) + suffix = str(row.get("file_suffix", "")).strip() + require(suffix, "alpha_regression.outputs entry missing file_suffix.") + aliases = row.get("aliases", []) + require( + isinstance(aliases, list), + f"alpha_regression.outputs entry '{suffix}' aliases must be a list.", + ) + vals = [str(a).strip() for a in aliases if str(a).strip()] + scenario_id = str(row.get("scenario_id", "")).strip() + if scenario_id: + vals.insert(0, scenario_id) + require(vals, f"alpha_regression.outputs entry '{suffix}' has no aliases.") + raw_map[suffix] = vals + else: + raise ValueError( + "alpha_regression.outputs must be either an object or a list of objects." + ) + + out: dict[str, list[str]] = {} + for suffix, aliases in raw_map.items(): + suffix_txt = str(suffix).strip() + require(suffix_txt, "alpha_regression.outputs contains an empty file_suffix.") + clean_aliases = [str(a).strip() for a in aliases if str(a).strip()] + require(clean_aliases, f"alpha_regression.outputs['{suffix_txt}'] has no aliases.") + # Keep original strings for logs and output; matching happens via normalize_token. + out[suffix_txt] = clean_aliases + require(out, "alpha_regression.outputs resolved to an empty mapping.") + LOGGER.info( + "Configured alpha output scenario map for AEO %d: %s", + aeo_year, + {k: v for k, v in out.items()}, + ) + return out + + +def resolve_output_scenarios( + available_scenarios: pd.DataFrame, + aeo_year: int, + output_aliases: dict[str, list[str]], +) -> pd.DataFrame: + """Match available scenarios to configured output scenario aliases.""" + require(not available_scenarios.empty, + "No available scenarios to resolve NG output scenarios.") + rows: list[dict[str, str]] = [] + for suffix, aliases in output_aliases.items(): + alias_tokens = {normalize_token(a.replace("{aeo_year}", str(aeo_year))) + for a in aliases} + found_row = None + for row in available_scenarios.itertuples(index=False): + sid = str(row.scenario_id) + sname = str(row.scenario_name) + if (normalize_token(sid) in alias_tokens + or normalize_token(sname) in alias_tokens): + found_row = row + break + require(found_row is not None, + f"Could not resolve required NG output scenario: '{suffix}'") + rows.append({ + "scenario_id": str(found_row.scenario_id), + "scenario_name": str(found_row.scenario_name), + "file_suffix": str(suffix), + }) + out = pd.DataFrame(rows).reset_index(drop=True) + require( + out["scenario_id"].nunique() == len(out), + "alpha_regression.outputs resolved to duplicate scenario_id values. " + "Use distinct output scenarios.", + ) + LOGGER.info("Resolved NG output scenarios: %s", + out[["scenario_id", "file_suffix"]].to_dict(orient="records")) + return out + + +def resolve_ng_region_order(config: dict[str, Any]) -> list[str]: + """Get the ordered list of census divisions from the config.""" + configured = config["ng"]["regions"] + order = [output_label_to_cendiv(x) for x in configured] + require(len(order) == len(set(order)), "NG regions contains duplicates.") + return order + + +# ============================================================================ +# Data Fetching +# ============================================================================ + +def fetch_aeo_series_by_scenario( + client: EiaClient, + aeo_year: int, + series_name: str, + scenario_ids: list[str], + region_ids: list[str], + value_col: str, + start_year: int, + end_year: int, + raw_output_path: Path | None = None, +) -> pd.DataFrame: + """ + Fetch a single AEO series (price/demand) for all scenarios and regions. + + Returns a DataFrame with columns: + [scenario_id, cendiv, year, ] + """ + series_ids = resolve_series_ids(client, aeo_year, series_name) + params: list[tuple[str, str]] = [ + ("data[]", "value"), + ("start", str(start_year)), + ("end", str(end_year)), + ] + params.extend(("facets[scenario][]", sid) for sid in scenario_ids) + params.extend(("facets[regionId][]", rid) for rid in region_ids) + params.extend(("facets[seriesId][]", sid) for sid in series_ids) + + df = client.get_data(f"/aeo/{aeo_year}/data", params=params) + if raw_output_path is not None: + raw_output_path.parent.mkdir(parents=True, exist_ok=True) + raw_export = df.copy() + raw_export.insert(0, "series_name", series_name) + raw_export.to_csv(raw_output_path, index=False, float_format="%.6f") + LOGGER.info( + "Wrote raw AEO rows for '%s' to %s (%d rows)", + series_name, raw_output_path, len(raw_export) + ) + need = {"scenario", "regionName", "period", "value"} + require( + not (need - set(df.columns)), + f"Missing columns for series '{series_name}': " + f"{sorted(need - set(df.columns))}", + ) + + df["scenario_id"] = df["scenario"].astype(str) + df["cendiv"] = df["regionName"].map(canonical_cendiv) + df["year"] = pd.to_numeric(df["period"], errors="coerce") + df[value_col] = pd.to_numeric(df["value"], errors="coerce") + df = df.dropna(subset=["scenario_id", "cendiv", "year", value_col]).copy() + df["year"] = df["year"].astype(int) + df = df[df["scenario_id"].isin(scenario_ids)].copy() + + # Verify no conflicting duplicate values + uniq = df.groupby(["scenario_id", "cendiv", "year"])[value_col].nunique() + require( + (uniq <= 1).all(), + f"Conflicting duplicate values for series '{series_name}'. " + f"Samples: {uniq[uniq > 1].head().to_dict()}", + ) + out = df.groupby( + ["scenario_id", "cendiv", "year"], as_index=False + )[value_col].mean() + LOGGER.info("Fetched %s rows for series '%s': %d", + value_col, series_name, len(out)) + return out + + +# ============================================================================ +# Data Validation and Filtering +# ============================================================================ + +def filter_complete_ng_scenarios( + scenario_table: pd.DataFrame, + series_frames: dict[str, pd.DataFrame], + region_order: list[str], + start_year: int, + end_year: int, +) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]]: + """Drop scenarios that have incomplete region × year coverage.""" + expected_count = len(region_order) * (end_year - start_year + 1) + keep = set(scenario_table["scenario_id"].tolist()) + + for name, frame in series_frames.items(): + counts = frame.groupby("scenario_id").size().to_dict() + complete = {sid for sid in keep + if counts.get(sid, 0) == expected_count} + dropped = sorted(keep - complete) + if dropped: + LOGGER.warning( + "Series '%s' has incomplete coverage for %s-%s. " + "Dropping scenarios: %s", + name, start_year, end_year, dropped, + ) + keep = complete + + require(keep, "No NG scenarios remain after coverage filtering.") + out_scen = (scenario_table[scenario_table["scenario_id"].isin(keep)] + .copy().reset_index(drop=True)) + out_frames = { + name: frame[frame["scenario_id"].isin(keep)].copy() + for name, frame in series_frames.items() + } + return out_scen, out_frames + + +def validate_ng_coverage( + frame: pd.DataFrame, + scenarios: list[str], + region_order: list[str], + start_year: int, + end_year: int, + label: str, +) -> None: + """Verify that a DataFrame has complete scenario × region × year coverage.""" + expected = { + (sid, reg, yr) + for sid in scenarios + for reg in region_order + for yr in range(start_year, end_year + 1) + } + actual = { + (r.scenario_id, r.cendiv, int(r.year)) + for r in frame[["scenario_id", "cendiv", "year"]].itertuples(index=False) + } + missing = expected - actual + require( + not missing, + f"{label} missing scenario/region/year combos. " + f"Sample: {list(sorted(missing))[:10]}", + ) + + +# ============================================================================ +# Historical Data Handling +# ============================================================================ + +def resolve_history_file(input_dir: Path, stem: str, + history_suffix: str) -> Path: + """Locate a historical data CSV file.""" + file_path = input_dir / f"{stem}_{history_suffix}.csv" + require(file_path.exists(), + f"History source file not found: {file_path}") + return file_path + + +def load_history_wide_file( + file_path: Path, + value_col: str, + region_order: list[str], +) -> pd.DataFrame: + """ + Load a wide-format historical CSV and melt to long format. + + Expected CSV format: year/t column + one column per region (output labels). + Returns: DataFrame with [cendiv, year, ]. + """ + require(file_path.exists(), + f"History source file not found: {file_path}") + df = pd.read_csv(file_path) + year_col = ("year" if "year" in df.columns + else ("t" if "t" in df.columns else None)) + require(year_col is not None, + f"History file missing 'year' or 't' column: {file_path}") + df = df.rename(columns={year_col: "year"}) + for col in df.columns: + if col != "year": + df[col] = pd.to_numeric(df[col], errors="coerce") + melted = (df.melt(id_vars=["year"], var_name="region_out", + value_name=value_col) + .dropna(subset=[value_col]).copy()) + melted["cendiv"] = melted["region_out"].map(output_label_to_cendiv) + melted["year"] = pd.to_numeric(melted["year"], errors="coerce").astype(int) + melted = melted[melted["cendiv"].isin(region_order)].copy() + return melted[["cendiv", "year", value_col]] + + +def apply_reference_history_to_all_scenarios( + frame: pd.DataFrame, + value_col: str, + history_frame: pd.DataFrame, + scenario_ids: list[str], + projection_start_year: int, +) -> pd.DataFrame: + """ + Backfill pre-projection years with historical data. + + Historical data is replicated identically across all scenarios, + since AEO scenarios share the same historical period. + """ + hist = history_frame[history_frame["year"] < projection_start_year].copy() + if hist.empty: + return frame + replicated = pd.concat( + [hist.assign(scenario_id=sid) for sid in scenario_ids], + ignore_index=True, + ) + future = frame[frame["year"] >= projection_start_year].copy() + out = pd.concat([future, replicated], ignore_index=True) + out = out.groupby( + ["scenario_id", "cendiv", "year"], as_index=False + )[value_col].mean() + return out + + +def _append_year_to_history_csv( + csv_path: Path, + year: int, + data_frame: pd.DataFrame, + scenario_id: str, + value_col: str, +) -> None: + """Append a single year from the reference scenario to a history CSV if missing.""" + if not csv_path.exists(): + return + existing = pd.read_csv(csv_path) + year_col = "year" if "year" in existing.columns else "t" + if year_col not in existing.columns: + return + if year in existing[year_col].values: + return # Already present + + row_data = data_frame[ + (data_frame["scenario_id"] == scenario_id) + & (data_frame["year"] == year) + ].copy() + if row_data.empty: + LOGGER.warning("No data for year %d to append to %s.", year, csv_path.name) + return + + row_data["region_out"] = row_data["cendiv"].map(cendiv_output_label) + wide = row_data.pivot_table( + index="year", columns="region_out", values=value_col, aggfunc="mean", + ).reset_index().rename(columns={"year": year_col}) + + for col in existing.columns: + if col not in wide.columns: + wide[col] = float("nan") + wide = wide[existing.columns] + + updated = pd.concat([existing, wide], ignore_index=True) + updated = updated.sort_values(year_col).reset_index(drop=True) + updated.to_csv(csv_path, index=False, float_format="%.5f") + LOGGER.info("Appended year %d to %s.", year, csv_path.name) + + +# ============================================================================ +# Beta Loading +# ============================================================================ + +def load_regional_betas(beta_path: Path, + region_order: list[str]) -> dict[str, float]: + """ + Load regional beta coefficients from a CSV file. + + The beta file (cd_beta0.csv) contains electric-sector-only betas, + representing the price sensitivity to regional electric sector demand: + Beta_regional(r) in units of 2004$/MMBtu per Quad + """ + require(beta_path.exists(), + f"Regional beta file not found: {beta_path}") + df = pd.read_csv(beta_path) + cendiv_col = next( + (c for c in df.columns if normalize_token(c).endswith("cendiv")), + None, + ) + value_col = next( + (c for c in df.columns if normalize_token(c) == "value"), + None, + ) + require(cendiv_col is not None and value_col is not None, + "Regional beta file missing cendiv/value columns.") + df["cendiv"] = df[cendiv_col].map(output_label_to_cendiv) + df["value"] = pd.to_numeric(df[value_col], errors="coerce") + df = df.dropna(subset=["cendiv", "value"]).copy() + betas = {row.cendiv: float(row.value) + for row in df[["cendiv", "value"]].itertuples(index=False)} + missing = [c for c in region_order if c not in betas] + require(not missing, f"Regional beta file missing regions: {missing}") + return betas + + +# ============================================================================ +# Alpha Computation (Core Calculation) +# ============================================================================ + +def compute_ng_alpha( + price_2004: pd.DataFrame, + demand_elec: pd.DataFrame, + beta_regional: dict[str, float], + beta_national: float, + first_model_year: int, +) -> pd.DataFrame: + """ + Compute NG alpha with a scenario-specific index alpha(region, year, scenario). + + For each scenario we compute an implied residual: + + alpha_2004 = price_2004 - beta_reg * q_reg - beta_nat * q_nat + + ReEDS sets both NG beta terms to zero in the first model year, so the + first-year alpha must carry the full converted price level. + + Parameters + ---------- + price_2004 : DataFrame + Columns: [scenario_id, cendiv, year, price_2004] + demand_elec : DataFrame + Columns: [scenario_id, cendiv, year, demand_elec_quads] + beta_regional : dict + {cendiv: beta_value} for each census division + beta_national : float + National beta coefficient (2004$/MMBtu per Quad) + first_model_year : int + First modeled year in ReEDS. NG elasticity is disabled in this year. + + Returns + ------- + DataFrame with columns: [scenario_id, cendiv, year, alpha_2004]. + """ + # Merge price and regional demand. + merged = price_2004.merge( + demand_elec, + on=["scenario_id", "cendiv", "year"], + how="inner", + ) + + # Compute national demand: sum of regional electric demands per scenario-year. + q_nat = ( + demand_elec + .groupby(["scenario_id", "year"], as_index=False)["demand_elec_quads"] + .sum() + .rename(columns={"demand_elec_quads": "q_nat"}) + ) + merged = merged.merge(q_nat, on=["scenario_id", "year"], how="left") + + # Map regional betas. + merged["beta_reg"] = merged["cendiv"].map(beta_regional) + require( + not merged["beta_reg"].isna().any(), + "Missing regional beta values while computing NG alpha.", + ) + require( + not merged["q_nat"].isna().any(), + "Missing national demand values while computing NG alpha.", + ) + + # Scenario-level residual alpha. + merged["alpha_2004"] = ( + merged["price_2004"] + - merged["beta_reg"] * merged["demand_elec_quads"] + - beta_national * merged["q_nat"] + ) + first_year_mask = merged["year"] == first_model_year + merged.loc[first_year_mask, "alpha_2004"] = merged.loc[first_year_mask, "price_2004"] + out = merged[["scenario_id", "cendiv", "year", "alpha_2004"]].copy() + out = out.sort_values(["scenario_id", "cendiv", "year"]).reset_index(drop=True) + return out + + +# ============================================================================ +# Output Writing +# ============================================================================ +def pivot_ng_series( + frame: pd.DataFrame, + scenario_id: str, + value_col: str, + region_order: list[str], +) -> pd.DataFrame: + """Pivot a long-format series into wide format (year × region).""" + subset = frame[frame["scenario_id"] == scenario_id].copy() + subset["region_out"] = subset["cendiv"].map(cendiv_output_label) + pivot = subset.pivot_table( + index="year", columns="region_out", + values=value_col, aggfunc="mean", + ) + ordered_cols = [cendiv_output_label(c) for c in region_order] + pivot = pivot.reindex(columns=ordered_cols) + pivot = pivot.sort_index().reset_index() + return pivot + + +def write_ng_outputs( + scenario_table: pd.DataFrame, + price_raw: pd.DataFrame, + demand_elec: pd.DataFrame, + demand_total: pd.DataFrame, + alpha_2004: pd.DataFrame, + beta_regional: dict[str, float], + region_order: list[str], + config: dict[str, Any], + base_dir: Path, +) -> None: + """ + Write all NG output files to the configured output directory. + + Output files per scenario: + - ng_AEO_{year}_{suffix}.csv : NG prices (AEO native dollar year) + - ng_demand_AEO_{year}_{suffix}.csv : Electric sector demand (Quads) + - ng_tot_demand_AEO_{year}_{suffix}.csv : Total sector demand (Quads) + - alpha_AEO_{year}_{suffix}.csv : Alpha values (2004$/MMBtu) + + Shared output files: + - cd_beta0.csv : Electric sector regional betas + """ + ng_cfg = config["ng"] + aeo_year = int(config["aeo_year"]) + out_dir = resolve_path(base_dir, config["paths"]["output_dir"]) + out_dir.mkdir(parents=True, exist_ok=True) + + written_files: list[Path] = [] + + # --- Per-scenario output files --- + for row in scenario_table.itertuples(index=False): + scenario_id = str(row.scenario_id) + file_suffix = getattr(row, "file_suffix", None) + require(bool(file_suffix), + f"Missing output suffix for NG scenario '{scenario_id}'") + suffix = str(file_suffix) + + price_wide = pivot_ng_series( + price_raw, scenario_id, "ng_price", region_order) + elec_wide = pivot_ng_series( + demand_elec, scenario_id, "demand_elec_quads", region_order) + total_wide = pivot_ng_series( + demand_total, scenario_id, "demand_total_quads", region_order) + alpha_wide = pivot_ng_series( + alpha_2004, scenario_id, "alpha_2004", region_order + ).rename(columns={"year": "t"}) + + fn_price = out_dir / f"ng_AEO_{aeo_year}_{suffix}.csv" + fn_elec = out_dir / f"ng_demand_AEO_{aeo_year}_{suffix}.csv" + fn_total = out_dir / f"ng_tot_demand_AEO_{aeo_year}_{suffix}.csv" + fn_alpha = out_dir / f"alpha_AEO_{aeo_year}_{suffix}.csv" + + price_wide.to_csv(fn_price, index=False, float_format="%.6f") + elec_wide.to_csv(fn_elec, index=False, float_format="%.6f") + total_wide.to_csv(fn_total, index=False, float_format="%.6f") + alpha_wide.to_csv(fn_alpha, index=False, float_format="%.6f") + written_files.extend([fn_price, fn_elec, fn_total, fn_alpha]) + + # --- Beta output files (copy from input) --- + input_dir = resolve_path( + base_dir, + config["paths"].get("input_dir", config["paths"]["output_dir"]), + ) + beta_files = ["cd_beta0.csv", "national_beta.csv"] + for beta_name in beta_files: + src = resolve_case_insensitive(input_dir / beta_name) + dst = out_dir / beta_name + if src.exists(): + shutil.copy2(src, dst) + written_files.append(dst) + else: + LOGGER.warning("%s not found at %s; skipping copy.", beta_name, src) + + LOGGER.info("Wrote NG outputs to %s (%d files)", out_dir, len(written_files)) + + +# ============================================================================ +# Main Pipeline +# ============================================================================ + +def run_ng_pipeline(config: dict[str, Any], base_dir: Path) -> None: + """ + Execute the full NG preprocessing pipeline: + + 1. Connect to EIA API and discover available scenarios + 2. Fetch price & demand series for all scenarios and regions + 3. Backfill historical data for pre-projection years + 4. Validate completeness of all data + 5. Convert prices from AEO dollar year to 2004$ using the deflator + 6. Compute alpha values using the supply curve inversion: + Alpha = Price_2004 - Beta_reg × Q_reg - Beta_nat × Q_nat + 7. Write output CSV files for ReEDS consumption + """ + api_key = resolve_api_key(config) + client = EiaClient(config["api"], api_key) + + ng_cfg = config["ng"] + aeo_year = int(config["aeo_year"]) + start_year = int(config["start_year"]) + end_year = int(config["end_year"]) + region_order = resolve_ng_region_order(config) + out_dir = resolve_path(base_dir, config["paths"]["output_dir"]) + out_dir.mkdir(parents=True, exist_ok=True) + raw_dir = out_dir / "raw_aeo_data" + raw_dir.mkdir(parents=True, exist_ok=True) + + scenarios_cfg = config.get("scenarios", {}) + if scenarios_cfg is None: + scenarios_cfg = {} + require(isinstance(scenarios_cfg, dict), "Config key 'scenarios' must be an object.") + alpha_scen_cfg = scenarios_cfg.get("alpha_regression", {}) + if alpha_scen_cfg is None: + alpha_scen_cfg = {} + require(isinstance(alpha_scen_cfg, dict), "Config key 'scenarios.alpha_regression' must be an object.") + + fetch_cfg = alpha_scen_cfg.get("fetch") + require( + fetch_cfg is None or isinstance(fetch_cfg, list), + "Config key 'scenarios.alpha_regression.fetch' must be a list when provided.", + ) + fetch_scenarios = [str(x).strip() for x in (fetch_cfg or []) if str(x).strip()] + output_aliases = resolve_output_scenario_aliases( + aeo_year, + alpha_scen_cfg.get("outputs"), + ) + + # ---- Step 1: Discover and resolve scenarios ---- + LOGGER.info("Step 1: Resolving AEO %d scenarios...", aeo_year) + all_scenarios = resolve_ng_scenarios( + client, + aeo_year, + include_scenarios=fetch_scenarios or None, + ) + output_scenarios = resolve_output_scenarios( + available_scenarios=all_scenarios, + aeo_year=aeo_year, + output_aliases=output_aliases, + ) + pd.DataFrame(client.get_facets(aeo_year, "scenario")).to_csv( + raw_dir / "scenario_facets.csv", index=False + ) + all_scenarios.to_csv(raw_dir / "selected_scenarios_all.csv", index=False) + output_scenarios.to_csv(raw_dir / "selected_scenarios_outputs.csv", index=False) + scenario_ids = all_scenarios["scenario_id"].tolist() + region_ids = list( + resolve_region_ids(client, aeo_year, ng_cfg["regions"]).values() + ) + pd.DataFrame(client.get_facets(aeo_year, "regionId")).to_csv( + raw_dir / "region_facets.csv", index=False + ) + + # ---- Step 2: Fetch price and demand data from EIA API ---- + LOGGER.info("Step 2: Fetching AEO data from EIA API...") + price_raw = fetch_aeo_series_by_scenario( + client, aeo_year, + NG_SERIES_NAMES["price"], + scenario_ids, region_ids, + "ng_price", start_year, end_year, + raw_output_path=raw_dir / "aeo_raw_ng_price.csv", + ) + demand_elec = fetch_aeo_series_by_scenario( + client, aeo_year, + NG_SERIES_NAMES["demand_elec"], + scenario_ids, region_ids, + "demand_elec_quads", start_year, end_year, + raw_output_path=raw_dir / "aeo_raw_ng_demand_electric_power.csv", + ) + demand_total = fetch_aeo_series_by_scenario( + client, aeo_year, + NG_SERIES_NAMES["demand_total"], + scenario_ids, region_ids, + "demand_total_quads", start_year, end_year, + raw_output_path=raw_dir / "aeo_raw_ng_demand_total.csv", + ) + + # ---- Step 3: Backfill historical data ---- + LOGGER.info("Step 3: Backfilling historical data...") + projection_start_year = int(min( + price_raw["year"].min(), + demand_elec["year"].min(), + demand_total["year"].min(), + )) + validation_start_year = projection_start_year + + if start_year < projection_start_year: + input_dir = resolve_path( + base_dir, + config["paths"].get("input_dir", config["paths"]["output_dir"]), + ) + try: + history_price = load_history_wide_file( + resolve_history_file(input_dir, "ng_AEO", HISTORY_SUFFIX), + "ng_price", region_order, + ) + history_elec = load_history_wide_file( + resolve_history_file( + input_dir, "ng_demand_AEO", HISTORY_SUFFIX), + "demand_elec_quads", region_order, + ) + history_total = load_history_wide_file( + resolve_history_file( + input_dir, "ng_tot_demand_AEO", HISTORY_SUFFIX), + "demand_total_quads", region_order, + ) + price_raw = apply_reference_history_to_all_scenarios( + price_raw, "ng_price", history_price, + scenario_ids, projection_start_year, + ) + demand_elec = apply_reference_history_to_all_scenarios( + demand_elec, "demand_elec_quads", history_elec, + scenario_ids, projection_start_year, + ) + demand_total = apply_reference_history_to_all_scenarios( + demand_total, "demand_total_quads", history_total, + scenario_ids, projection_start_year, + ) + validation_start_year = start_year + LOGGER.info( + "Backfilled %d-%d from historical files in %s.", + start_year, projection_start_year - 1, input_dir, + ) + except Exception as exc: + LOGGER.warning( + "Could not backfill NG history (%s). " + "Using available years %d-%d only.", + exc, projection_start_year, end_year, + ) + validation_start_year = projection_start_year + + # ---- Step 4: Filter and validate data completeness ---- + LOGGER.info("Step 4: Validating data completeness...") + all_scenarios, series_frames = filter_complete_ng_scenarios( + scenario_table=all_scenarios, + series_frames={ + "price": price_raw, + "demand_elec": demand_elec, + "demand_total": demand_total, + }, + region_order=region_order, + start_year=validation_start_year, + end_year=end_year, + ) + price_raw = series_frames["price"] + demand_elec = series_frames["demand_elec"] + demand_total = series_frames["demand_total"] + scenario_ids = all_scenarios["scenario_id"].tolist() + output_scenarios = ( + output_scenarios[output_scenarios["scenario_id"].isin(scenario_ids)] + .copy().reset_index(drop=True) + ) + require( + len(output_scenarios) == len(output_aliases), + "One or more required output scenarios were dropped by coverage filtering.", + ) + validate_ng_coverage( + price_raw, scenario_ids, region_order, + validation_start_year, end_year, "NG price", + ) + validate_ng_coverage( + demand_elec, scenario_ids, region_order, + validation_start_year, end_year, "NG electric demand", + ) + validate_ng_coverage( + demand_total, scenario_ids, region_order, + validation_start_year, end_year, "NG total demand", + ) + + # ---- Step 5: Convert prices to 2004$ ---- + LOGGER.info("Step 5: Converting prices to 2004$ (deflator=%.6f)...", + float(ng_cfg["price_deflator_to_2004"])) + deflator = float(ng_cfg["price_deflator_to_2004"]) + price_2004 = price_raw.copy() + price_2004["price_2004"] = price_2004["ng_price"] * deflator + + # ---- Step 6: Compute alpha values ---- + LOGGER.info("Step 6: Computing NG alpha values...") + beta_path = resolve_path(base_dir, ng_cfg["regional_beta_path"]) + beta_regional = load_regional_betas(beta_path, region_order) + national_beta_path = resolve_case_insensitive(beta_path.parent / "national_beta.csv") + require( + national_beta_path.exists(), + f"National beta file not found: {national_beta_path}", + ) + national_beta_df = pd.read_csv(national_beta_path) + require( + "beta" in national_beta_df.columns, + f"National beta file missing 'beta' column: {national_beta_path}", + ) + beta_vals = pd.to_numeric(national_beta_df["beta"], errors="coerce").dropna() + require( + not beta_vals.empty, + f"National beta file has no numeric beta value: {national_beta_path}", + ) + beta_national = float(beta_vals.iloc[0]) + LOGGER.info( + " Beta_national = %.10f (2004$/MMBtu per Quad, source=%s)", + beta_national, + national_beta_path, + ) + LOGGER.info( + " Beta_regional: %s", + {cendiv_output_label(k): f"{v:.6f}" for k, v in beta_regional.items()}, + ) + + alpha_2004 = compute_ng_alpha( + price_2004, + demand_elec, + beta_regional, + beta_national, + first_model_year=start_year, + ) + validate_ng_coverage( + alpha_2004, scenario_ids, region_order, + validation_start_year, end_year, "NG alpha", + ) + + # ---- Step 7: Write output files ---- + LOGGER.info("Step 7: Writing output files...") + write_ng_outputs( + scenario_table=output_scenarios, + price_raw=price_raw, + demand_elec=demand_elec, + demand_total=demand_total, + alpha_2004=alpha_2004, + beta_regional=beta_regional, + region_order=region_order, + config=config, + base_dir=base_dir, + ) + + # ---- Step 8: Append projection_start_year to history CSVs ---- + ref_rows = output_scenarios[ + output_scenarios["file_suffix"] == "reference" + ] + if not ref_rows.empty: + ref_sid = str(ref_rows.iloc[0]["scenario_id"]) + hist_dir = resolve_path( + base_dir, + config["paths"].get("input_dir", config["paths"]["output_dir"]), + ) + for stem, vcol, df in [ + ("ng_AEO", "ng_price", price_raw), + ("ng_demand_AEO", "demand_elec_quads", demand_elec), + ("ng_tot_demand_AEO", "demand_total_quads", demand_total), + ]: + _append_year_to_history_csv( + hist_dir / f"{stem}_{HISTORY_SUFFIX}.csv", + projection_start_year, df, ref_sid, vcol, + ) + + +# ============================================================================ +# Entry Point +# ============================================================================ + +def main() -> int: + args = parse_args() + setup_logging(args.log_level) + + script_dir = Path(__file__).resolve().parent + cfg_path = Path(args.config) + if not cfg_path.is_absolute(): + cwd_candidate = resolve_case_insensitive( + (Path.cwd() / cfg_path).resolve()) + script_candidate = resolve_case_insensitive( + (script_dir / cfg_path).resolve()) + cfg_path = (cwd_candidate if cwd_candidate.exists() + else script_candidate) + + cfg = load_config(cfg_path) + if args.aeo_year is not None: + cfg["aeo_year"] = int(args.aeo_year) + + run_ng_pipeline(cfg, script_dir) + LOGGER.info("NG pipeline complete.") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as exc: + LOGGER.exception("Pipeline failed: %s", exc) + sys.exit(1) + + diff --git a/aeo_updates/natural_gas_price_regression/aeo_beta_regression.py b/aeo_updates/natural_gas_price_regression/aeo_beta_regression.py new file mode 100644 index 0000000..5413931 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/aeo_beta_regression.py @@ -0,0 +1,735 @@ +""" +NG Beta Regression: Estimate Regional and National Betas from AEO Scenarios +============================================================================ + +This script is a preprocessing step that runs BEFORE the main aeo_pipeline.py. +It estimates the beta coefficients used in the ReEDS NG supply curve model. + +Method +------ +The NG supply curve decomposes regional prices as: + + Price(r,t) = Alpha(r,t) + Beta_reg(r) * Q_reg(r,t) + Beta_nat * Q_nat(t) + +Alpha varies by region and year (absorbing all time-varying effects). +Beta_reg and Beta_nat are structural parameters estimated via regression. + +Estimation uses a joint fixed-effects regression across all regions: + + Price_2004(r,t,s) = Alpha(r,t) + Beta_reg(r) * Q_reg(r,t,s) + Beta_nat * Q_nat(t,s) + +Alpha(r,t) is absorbed by demeaning each variable across scenarios within +each (region, year) group. The joint OLS then estimates Beta_nat and all +regional Beta_reg values simultaneously. + +Data: By default uses ALL available years for selected AEO scenarios +EXCEPT High/Low O&G Supply (held out for alpha computation), but +scenario and year selection can be overridden in config under +scenarios.beta_regression. + +Usage +----- + python aeo_beta_regression.py --config aeo_pipeline_config.json + +Outputs +------- + outputs of beta regression/cd_beta0.csv : Regional betas + outputs of beta regression/national_beta.csv : National beta + outputs of beta regression/beta_regression_summary.csv : Full summary + outputs of beta regression/alpha_from_beta_regression.csv : Shared alpha(region, year) + outputs of beta regression/alpha_from_beta_by_scenario.csv : Scenario residual alpha candidates + outputs of beta regression/alpha from beta regression/alpha_from_beta_shared.csv : Wide shared alpha + outputs of beta regression/*.png : Generated by results_visualization.py +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import re +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import requests +from requests.adapters import HTTPAdapter + +from urllib3 import disable_warnings +from urllib3.exceptions import InsecureRequestWarning +from urllib3.util.retry import Retry + +LOGGER = logging.getLogger("beta_regression") + +# ============================================================================ +# Constants +# ============================================================================ + +REGION_TO_LABEL = { + "New England": "New_England", + "Middle Atlantic": "Mid_Atlantic", + "East North Central": "East_North_Central", + "West North Central": "West_North_Central", + "South Atlantic": "South_Atlantic", + "East South Central": "East_South_Central", + "West South Central": "West_South_Central", + "Mountain": "Mountain", + "Pacific": "Pacific", +} + +NG_SERIES_NAMES = { + "price": "Energy Prices : Electric Power : Natural Gas", + "demand_elec": "Energy Use : Electric Power : Natural Gas", +} + +# Scenarios to EXCLUDE from regression (held out for alpha computation) +EXCLUDE_SCENARIOS = {"highogs", "lowogs", "high oil and gas supply", + "low oil and gas supply"} + + +# ============================================================================ +# Helpers +# ============================================================================ + +def _norm(value: Any) -> str: + if value is None: + return "" + return re.sub(r"[^a-z0-9]+", "", + str(value).replace("\xa0", " ").strip().lower()) + + +def _region_label(region_name: str) -> str: + if region_name in REGION_TO_LABEL: + return REGION_TO_LABEL[region_name] + norm = _norm(region_name) + for name, label in REGION_TO_LABEL.items(): + if _norm(name) == norm or _norm(label) == norm: + return label + raise ValueError(f"Unknown region: {region_name}") + + +def _calc_r2(y: np.ndarray, y_hat: np.ndarray) -> float: + """R2 for demeaned data (where mean ~= 0, so SS_tot = sum(y^2)).""" + ss_res = float(np.sum((y - y_hat) ** 2)) + ss_tot = float(np.sum(y ** 2)) + return 1 - ss_res / ss_tot if ss_tot > 0 else 0.0 + + +def _parse_optional_int(value: Any, config_key: str) -> int | None: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"Config key '{config_key}' must be an integer.") from exc + + +# ============================================================================ +# EIA API Client +# ============================================================================ + +class EiaClient: + def __init__(self, api_cfg: dict[str, Any], api_key: str): + self.base_url = api_cfg["base_url"].rstrip("/") + self.verify_ssl = bool(api_cfg.get("verify_ssl", True)) + self.timeout = int(api_cfg.get("timeout_seconds", 60)) + self.api_key = api_key + if not self.verify_ssl: + disable_warnings(InsecureRequestWarning) + retries = Retry( + total=int(api_cfg.get("max_retries", 4)), + backoff_factor=float(api_cfg.get("backoff_seconds", 1.0)), + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["GET"], + raise_on_status=False, + ) + self.session = requests.Session() + self.session.mount("https://", HTTPAdapter(max_retries=retries)) + self.session.mount("http://", HTTPAdapter(max_retries=retries)) + + def _get(self, path: str, + params: list[tuple[str, str]] | None = None) -> dict: + url = f"{self.base_url}/{path.lstrip('/')}" + query = [("api_key", self.api_key)] + (params or []) + resp = self.session.get(url, params=query, + timeout=self.timeout, verify=self.verify_ssl) + resp.raise_for_status() + payload = resp.json() + if "response" not in payload: + raise ValueError(f"Unexpected payload from {path}") + return payload + + def get_facets(self, aeo_year: int, facet: str) -> list[dict]: + return self._get( + f"/aeo/{aeo_year}/facet/{facet}")["response"]["facets"] + + def get_data(self, aeo_year: int, + params: list[tuple[str, str]]) -> pd.DataFrame: + payload = self._get(f"/aeo/{aeo_year}/data", params) + for w in payload["response"].get("warnings", []): + LOGGER.warning("EIA: %s | %s", + w.get("warning"), w.get("description")) + data = payload["response"].get("data", []) + if not data: + raise ValueError(f"No data returned for AEO {aeo_year}") + return pd.DataFrame(data) + + +# ============================================================================ +# Scenario Discovery +# ============================================================================ + +def discover_regression_scenarios( + client: EiaClient, + aeo_year: int, + include_scenarios: list[str] | None = None, + exclude_aliases: list[str] | None = None, +) -> list[str]: + """ + Resolve beta-regression scenarios. + + If include_scenarios is provided, only those scenarios are used + (matching by scenario id or scenario name, case-insensitive). + Otherwise, all non-legacy scenarios are used except exclude_aliases. + """ + facets = client.get_facets(aeo_year, "scenario") + rows: list[dict[str, str]] = [] + for item in facets: + sid = str(item.get("id", "")).strip() + sname = str(item.get("name", "")).strip() + if not sid or _norm(sid).startswith("aeo"): + continue + rows.append({"scenario_id": sid, "scenario_name": sname}) + + if include_scenarios: + selected: list[str] = [] + missing: list[str] = [] + for raw in include_scenarios: + token = _norm(str(raw).replace("{aeo_year}", str(aeo_year))) + found = None + for row in rows: + sid = str(row["scenario_id"]) + sname = str(row["scenario_name"]) + if _norm(sid) == token or _norm(sname) == token: + found = sid + break + if found is None: + missing.append(str(raw)) + continue + if found not in selected: + selected.append(found) + if missing: + raise ValueError( + f"Configured beta_regression.include scenarios not found: {missing}" + ) + LOGGER.info("Regression scenarios from config (%d): %s", len(selected), selected) + if len(selected) < 3: + LOGGER.warning( + "Only %d scenarios in beta_regression.include - regression may be unreliable.", + len(selected), + ) + return selected + + excluded_tokens = {_norm(x) for x in (exclude_aliases or sorted(EXCLUDE_SCENARIOS))} + selected: list[str] = [] + excluded: list[str] = [] + for row in rows: + sid = str(row["scenario_id"]) + sname = str(row["scenario_name"]) + if _norm(sid) in excluded_tokens or _norm(sname) in excluded_tokens: + excluded.append(f"{sid} ({sname})") + continue + selected.append(sid) + LOGGER.info("Regression scenarios (%d): %s", len(selected), selected) + if excluded: + LOGGER.info("Excluded scenarios: %s", excluded) + if len(selected) < 3: + LOGGER.warning("Only %d scenarios - regression may be unreliable.", + len(selected)) + return selected + + +# ============================================================================ +# Data Fetching +# ============================================================================ + +def fetch_series(client: EiaClient, aeo_year: int, series_name: str, + scenario_ids: list[str], region_ids: list[str], + value_col: str, + raw_output_path: Path | None = None, + start_year: int | None = None, + end_year: int | None = None) -> pd.DataFrame: + """Fetch an AEO series for all scenarios/regions.""" + if (start_year is None) != (end_year is None): + raise ValueError( + "Both start_year and end_year must be provided together for fetch_series." + ) + if start_year is not None and end_year is not None and start_year > end_year: + raise ValueError(f"Invalid year window: start_year={start_year} > end_year={end_year}") + + series_facets = client.get_facets(aeo_year, "seriesId") + series_ids = [str(item["id"]) for item in series_facets + if _norm(item.get("name")) == _norm(series_name)] + if not series_ids: + raise ValueError(f"Series not found: {series_name}") + + params: list[tuple[str, str]] = [("data[]", "value")] + if start_year is not None and end_year is not None: + params.append(("start", str(start_year))) + params.append(("end", str(end_year))) + params.extend(("facets[scenario][]", s) for s in scenario_ids) + params.extend(("facets[regionId][]", r) for r in region_ids) + params.extend(("facets[seriesId][]", s) for s in series_ids) + + df = client.get_data(aeo_year, params) + if raw_output_path is not None: + raw_output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(raw_output_path, index=False, float_format="%.6f") + LOGGER.info("Wrote raw data for '%s' to %s (%d rows)", + series_name, raw_output_path, len(df)) + + df = df.rename(columns={"scenario": "scenario_id"}) + df["region"] = df["regionName"].map(_region_label) + df["year"] = pd.to_numeric(df["period"], errors="coerce").astype(int) + df[value_col] = pd.to_numeric(df["value"], errors="coerce") + df = df.dropna(subset=[value_col]) + df = df[df["scenario_id"].isin(scenario_ids)] + + out = (df.groupby(["scenario_id", "region", "year"], as_index=False) + [value_col].mean()) + LOGGER.info("Fetched '%s': %d rows (%d scenarios x %d regions x %d years)", + series_name, len(out), out["scenario_id"].nunique(), + out["region"].nunique(), out["year"].nunique()) + return out + + +# ============================================================================ +# Joint Beta Regression +# ============================================================================ + +def estimate_joint_betas( + price: pd.DataFrame, + demand: pd.DataFrame, + deflator: float, + regions: list[str], +) -> tuple[float, float, dict[str, float], dict[str, dict], pd.DataFrame, float]: + """ + Estimate all betas jointly from the full equation: + + Price_2004(r,t,s) = Alpha(r,t) + Beta_reg(r)*Q_reg(r,t,s) + Beta_nat*Q_nat(t,s) + + Demeaning across scenarios within each (region, year) removes Alpha. + OLS on the demeaned system estimates Beta_nat and all Beta_reg simultaneously. + + Returns: (beta_nat, beta_nat_r2, beta_reg_dict, regional_diagnostics, + regression_data, model_r2) + """ + # National demand = sum of regional demands per scenario-year + demand_nat = (demand.groupby(["scenario_id", "year"], as_index=False) + ["demand"].sum() + .rename(columns={"demand": "demand_nat"})) + + merged = price.merge(demand, on=["scenario_id", "region", "year"]) + merged = merged.merge(demand_nat, on=["scenario_id", "year"]) + merged["price_2004"] = merged["price"] * deflator + + # Demean across scenarios within each (region, year) group + for col, dcol in [("price_2004", "dp"), ("demand", "dq_reg"), + ("demand_nat", "dq_nat")]: + means = merged.groupby(["year", "region"])[col].transform("mean") + merged[dcol] = merged[col] - means + + # Build X matrix: [dq_nat, dq_reg(region1), dq_reg(region2), ...] + region_arr = merged["region"].to_numpy() + dq_reg_x = merged["dq_reg"].to_numpy() + x_cols = [merged["dq_nat"].to_numpy()] + for region in regions: + x_cols.append(np.where(region_arr == region, dq_reg_x, 0.0)) + X = np.column_stack(x_cols) + y = merged["dp"].to_numpy() + + # OLS + coeffs = np.linalg.lstsq(X, y, rcond=None)[0] + beta_nat = float(coeffs[0]) + beta_reg = {region: float(coeffs[i + 1]) for i, region in enumerate(regions)} + + # Diagnostics + y_hat = X @ coeffs + model_r2 = _calc_r2(y, y_hat) + + # Partial R2 for national beta (residualize out regional terms) + merged["dp_hat"] = y_hat + merged["beta_reg_val"] = merged["region"].map(beta_reg) + merged["dp_partial_nat"] = ( + merged["dp"] - merged["beta_reg_val"] * merged["dq_reg"] + ) + beta_nat_r2 = _calc_r2( + merged["dp_partial_nat"].to_numpy(), + beta_nat * merged["dq_nat"].to_numpy()) + + # Per-region partial R2 (residualize out national term) + merged["dp_partial_reg"] = merged["dp"] - beta_nat * merged["dq_nat"] + regional_diag: dict[str, dict] = {} + for region in regions: + reg = merged[merged["region"] == region] + if reg.empty: + continue + regional_diag[region] = { + "r2_partial": _calc_r2( + reg["dp_partial_reg"].to_numpy(), + beta_reg[region] * reg["dq_reg"].to_numpy()), + "r2_full": _calc_r2( + reg["dp"].to_numpy(), reg["dp_hat"].to_numpy()), + "n_obs": len(reg), + } + + # Log results + LOGGER.info("Joint model: beta_nat=%.10f (partial R2=%.4f, model R2=%.4f)", + beta_nat, beta_nat_r2, model_r2) + for region in regions: + diag = regional_diag.get(region, {}) + LOGGER.info(" %-25s beta=%.10f (partial R2=%.4f, n=%d)", + region, beta_reg[region], + diag.get("r2_partial", float("nan")), + diag.get("n_obs", 0)) + + return beta_nat, beta_nat_r2, beta_reg, regional_diag, merged, model_r2 + + +# ============================================================================ +# Output: CSVs +# ============================================================================ + +def save_outputs( + out_dir: Path, + beta_nat: float, + beta_nat_r2: float, + beta_reg: dict[str, float], + regional_diag: dict[str, dict], + reg_data: pd.DataFrame, + model_r2: float, + first_model_year: int, +) -> None: + """Write beta tables and regression diagnostics.""" + out_dir.mkdir(parents=True, exist_ok=True) + + # cd_beta0.csv + pd.DataFrame({ + "*cendiv": list(beta_reg.keys()), + "value": list(beta_reg.values()), + }).to_csv(out_dir / "cd_beta0.csv", index=False, float_format="%.6f") + + # national_beta.csv + pd.DataFrame([{ + "beta": beta_nat, + }]).to_csv(out_dir / "national_beta.csv", index=False, float_format="%.6f") + + # Summary table + rows = [{ + "scope": "national", "region": "ALL", "beta": beta_nat, + "r2": beta_nat_r2, "r2_full": model_r2, "n_obs": len(reg_data), + }] + for region, beta in beta_reg.items(): + diag = regional_diag.get(region, {}) + rows.append({ + "scope": "regional", "region": region, "beta": beta, + "r2": diag.get("r2_partial", float("nan")), + "r2_full": diag.get("r2_full", float("nan")), + "n_obs": diag.get("n_obs", 0), + }) + pd.DataFrame(rows).to_csv( + out_dir / "beta_regression_summary.csv", + index=False, float_format="%.6f") + + # Regression point data for review + point_cols = ["scenario_id", "year", "region", "demand", "demand_nat", + "price", "price_2004", "dp", "dq_reg", "dq_nat", + "dp_partial_reg", "dp_partial_nat", "dp_hat"] + available = [c for c in point_cols if c in reg_data.columns] + reg_data[available].to_csv( + out_dir / "regression_points.csv", index=False, float_format="%.6f") + + # Alpha from beta-step data: + # 1) Scenario residual alpha candidate: + # alpha_candidate(r,t,s) = price_2004 - beta_reg*regional_demand - beta_nat*national_demand + # except in the first modeled year, where ReEDS disables both beta terms + # and alpha must therefore equal the full converted price. + # 2) Model alpha indexed by (region, year): + # alpha_shared(r,t) = mean_s(alpha_candidate(r,t,s)) + alpha_need = {"scenario_id", "year", "region", "price_2004", "demand", "demand_nat"} + missing_alpha_cols = sorted(alpha_need - set(reg_data.columns)) + if missing_alpha_cols: + raise ValueError( + f"Regression data missing columns needed for alpha export: {missing_alpha_cols}" + ) + + alpha_long = reg_data[ + ["scenario_id", "year", "region", "price_2004", "demand", "demand_nat"] + ].copy() + alpha_long = alpha_long.rename( + columns={"demand": "demand_elec_quads", "demand_nat": "q_nat"} + ) + alpha_long["beta_reg"] = alpha_long["region"].map(beta_reg) + if alpha_long["beta_reg"].isna().any(): + bad = sorted(alpha_long.loc[alpha_long["beta_reg"].isna(), "region"].astype(str).unique().tolist()) + raise ValueError(f"Missing regional beta values while exporting alpha: {bad}") + alpha_long["beta_nat"] = beta_nat + alpha_long["alpha_candidate"] = ( + alpha_long["price_2004"] + - alpha_long["beta_reg"] * alpha_long["demand_elec_quads"] + - alpha_long["beta_nat"] * alpha_long["q_nat"] + ) + first_year_mask = alpha_long["year"] == first_model_year + alpha_long.loc[first_year_mask, "alpha_candidate"] = alpha_long.loc[first_year_mask, "price_2004"] + alpha_long = alpha_long.sort_values(["scenario_id", "year", "region"]).reset_index(drop=True) + alpha_long.to_csv( + out_dir / "alpha_from_beta_by_scenario.csv", + index=False, + float_format="%.6f", + ) + + alpha_shared = ( + alpha_long.groupby(["region", "year"], as_index=False)["alpha_candidate"] + .mean() + .rename(columns={"alpha_candidate": "alpha_2004"}) + ) + n_scenarios = ( + alpha_long.groupby(["region", "year"], as_index=False)["scenario_id"] + .nunique() + .rename(columns={"scenario_id": "n_scenarios_used"}) + ) + alpha_shared = alpha_shared.merge(n_scenarios, on=["region", "year"], how="left") + alpha_shared = alpha_shared.sort_values(["region", "year"]).reset_index(drop=True) + alpha_shared.to_csv( + out_dir / "alpha_from_beta_regression.csv", + index=False, + float_format="%.6f", + ) + + alpha_wide_dir = out_dir / "alpha from beta regression" + alpha_wide_dir.mkdir(parents=True, exist_ok=True) + # Remove stale per-scenario files from earlier output format. + for old in alpha_wide_dir.glob("alpha_from_beta_*.csv"): + old.unlink() + old_index = alpha_wide_dir / "alpha_from_beta_files.csv" + if old_index.exists(): + old_index.unlink() + + alpha_wide = ( + alpha_shared.pivot_table( + index="year", + columns="region", + values="alpha_2004", + aggfunc="mean", + ) + .sort_index() + .reset_index() + ) + alpha_wide.to_csv( + alpha_wide_dir / "alpha_from_beta_shared.csv", + index=False, + float_format="%.6f", + ) + + LOGGER.info("Wrote outputs to %s", out_dir) + + +# ============================================================================ +# Pipeline +# ============================================================================ + +def run(config: dict[str, Any], base_dir: Path) -> None: + ng_cfg = config["ng"] + aeo_year = int(config["aeo_year"]) + deflator = float(ng_cfg["price_deflator_to_2004"]) + regions_config = ng_cfg["regions"] + regions = [REGION_TO_LABEL[r] for r in regions_config] + + # API key + api_key = os.getenv( + config["api"].get("key_env_var", "EIA_API_KEY"), + config["api"].get("key_fallback", ""), + ).strip() + if not api_key: + raise ValueError( + "Missing EIA API key. Set EIA_API_KEY or api.key_fallback.") + client = EiaClient(config["api"], api_key) + + scenarios_cfg = config.get("scenarios", {}) + if scenarios_cfg is None: + scenarios_cfg = {} + if not isinstance(scenarios_cfg, dict): + raise ValueError("Config key 'scenarios' must be an object.") + beta_scen_cfg = scenarios_cfg.get("beta_regression", {}) + if beta_scen_cfg is None: + beta_scen_cfg = {} + if not isinstance(beta_scen_cfg, dict): + raise ValueError("Config key 'scenarios.beta_regression' must be an object.") + + include_cfg = beta_scen_cfg.get("include") + if include_cfg is not None and not isinstance(include_cfg, list): + raise ValueError("Config key 'scenarios.beta_regression.include' must be a list.") + include_scenarios = [str(x).strip() for x in (include_cfg or []) if str(x).strip()] + + exclude_cfg = beta_scen_cfg.get("exclude_aliases") + if exclude_cfg is not None and not isinstance(exclude_cfg, list): + raise ValueError("Config key 'scenarios.beta_regression.exclude_aliases' must be a list.") + exclude_aliases = [str(x).strip() for x in (exclude_cfg or []) if str(x).strip()] + + beta_start_year = _parse_optional_int( + beta_scen_cfg.get("start_year"), "scenarios.beta_regression.start_year" + ) + beta_end_year = _parse_optional_int( + beta_scen_cfg.get("end_year"), "scenarios.beta_regression.end_year" + ) + if (beta_start_year is None) != (beta_end_year is None): + raise ValueError( + "Config keys 'scenarios.beta_regression.start_year' and " + "'scenarios.beta_regression.end_year' must be provided together." + ) + if beta_start_year is not None and beta_end_year is not None and beta_start_year > beta_end_year: + raise ValueError( + "Config key 'scenarios.beta_regression': start_year cannot be greater than end_year." + ) + if beta_start_year is not None and beta_end_year is not None: + LOGGER.info( + "Beta regression year range from config: %s-%s", + beta_start_year, beta_end_year, + ) + else: + LOGGER.info("Beta regression year range: all available AEO years.") + + # Step 1: Discover scenarios + LOGGER.info("Step 1: Discovering AEO %d scenarios...", aeo_year) + scenario_ids = discover_regression_scenarios( + client, + aeo_year, + include_scenarios=include_scenarios or None, + exclude_aliases=exclude_aliases or None, + ) + + # Resolve region IDs + facets = client.get_facets(aeo_year, "regionId") + region_lookup = {_norm(item["name"]): str(item["id"]) for item in facets} + region_ids = [] + for name in regions_config: + key = _norm(name) + if key not in region_lookup: + raise ValueError(f"Region not in EIA facets: {name}") + region_ids.append(region_lookup[key]) + + # Save raw facets for reference + out_dir = base_dir / "outputs of beta regression" + raw_dir = out_dir / "raw_aeo_data" + raw_dir.mkdir(parents=True, exist_ok=True) + pd.DataFrame(client.get_facets(aeo_year, "scenario")).to_csv( + raw_dir / "scenario_facets.csv", index=False) + pd.DataFrame({"selected_scenario_id": scenario_ids}).to_csv( + raw_dir / "selected_scenarios.csv", index=False) + + # Step 2: Fetch data + LOGGER.info("Step 2: Fetching price and demand data...") + price_raw = fetch_series( + client, aeo_year, NG_SERIES_NAMES["price"], + scenario_ids, region_ids, "price", + raw_output_path=raw_dir / "raw_ng_price.csv", + start_year=beta_start_year, + end_year=beta_end_year, + ) + demand_raw = fetch_series( + client, aeo_year, NG_SERIES_NAMES["demand_elec"], + scenario_ids, region_ids, "demand", + raw_output_path=raw_dir / "raw_ng_demand_elec.csv", + start_year=beta_start_year, + end_year=beta_end_year, + ) + if price_raw.empty or demand_raw.empty: + raise ValueError( + "No data available for configured beta regression year range." + ) + LOGGER.info( + "Price years used: %s-%s", + int(price_raw["year"].min()), + int(price_raw["year"].max()), + ) + LOGGER.info( + "Demand years used: %s-%s", + int(demand_raw["year"].min()), + int(demand_raw["year"].max()), + ) + + # Step 3: Estimate betas + LOGGER.info("Step 3: Estimating betas (joint fixed-effects regression)...") + (beta_nat, beta_nat_r2, beta_reg, regional_diag, + reg_data, model_r2) = estimate_joint_betas( + price_raw, + demand_raw, + deflator, + regions, + ) + + # Step 4: Save outputs + LOGGER.info("Step 4: Saving outputs...") + save_outputs(out_dir, beta_nat, beta_nat_r2, beta_reg, + regional_diag, reg_data, model_r2, + first_model_year=int(config["start_year"])) + + # Print summary + print("\n" + "=" * 60) + print("RESULTS") + print("=" * 60) + print(f"\nNational beta: {beta_nat:.15f}") + print(f"Model R2: {model_r2:.6f}") + print(f"\nRegional betas:") + for region, beta in beta_reg.items(): + diag = regional_diag.get(region, {}) + print(f" {region:25s} {beta:.12f} " + f"(partial R2={diag.get('r2_partial', float('nan')):.4f})") + print(f"\nOutputs: {out_dir}") + print("\nNext commands:") + print(" python sync_beta_to_alpha_inputs.py --config aeo_pipeline_config.json") + print(" python aeo_alpha_regression.py --config aeo_pipeline_config.json") + print(" python results_visualization.py --config aeo_pipeline_config.json") + print("=" * 60) + + +# ============================================================================ +# Entry Point +# ============================================================================ + +def main() -> int: + parser = argparse.ArgumentParser( + description="Estimate NG beta coefficients from AEO scenarios") + parser.add_argument("--config", default="aeo_pipeline_config.json") + parser.add_argument("--log-level", default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR"]) + parser.add_argument("--aeo-year", type=int, default=None) + args = parser.parse_args() + + logging.basicConfig(level=getattr(logging, args.log_level), + format="%(asctime)s | %(levelname)s | %(message)s") + + script_dir = Path(__file__).resolve().parent + cfg_path = Path(args.config) + if not cfg_path.is_absolute(): + cfg_path = (Path.cwd() / cfg_path if (Path.cwd() / cfg_path).exists() + else script_dir / cfg_path) + + with cfg_path.open() as f: + config = json.load(f) + if args.aeo_year is not None: + config["aeo_year"] = args.aeo_year + + run(config, script_dir) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as exc: + LOGGER.exception("Failed: %s", exc) + sys.exit(1) diff --git a/aeo_updates/natural_gas_price_regression/aeo_pipeline_config.json b/aeo_updates/natural_gas_price_regression/aeo_pipeline_config.json new file mode 100644 index 0000000..c9ecc72 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/aeo_pipeline_config.json @@ -0,0 +1,74 @@ +{ + "aeo_year": 2026, + "start_year": 2010, + "end_year": 2050, + "paths": { + "output_dir": "outputs of alpha regression", + "input_dir": "inputs for alpha regression" + }, + "scenarios": { + "beta_regression": { + "start_year": 2025, + "end_year": 2050, + "include": [ + "hm2026", + "lm2026", + "highZTC", + "lowZTC", + "higheldmd", + "altelec", + "alttrnp", + "electrnp", + "cb2026" + ], + "exclude_aliases": [ + "highogs", + "highprice", + "lowprice", + "lowogs" + ] + }, + "alpha_regression": { + "fetch": [ + "cb{aeo_year}", + "highogs", + "lowogs" + ], + "outputs": { + "reference": [ + "cb{aeo_year}" + ], + "HOG": [ + "highogs" + ], + "LOG": [ + "lowogs" + ] + } + } + }, + "api": { + "base_url": "https://api.eia.gov/v2", + "key_env_var": "EIA_API_KEY", + "key_fallback": "", + "verify_ssl": false, + "timeout_seconds": 60, + "max_retries": 4, + "backoff_seconds": 1 + }, + "ng": { + "regions": [ + "New England", + "Middle Atlantic", + "East North Central", + "West North Central", + "South Atlantic", + "East South Central", + "West South Central", + "Mountain", + "Pacific" + ], + "price_deflator_to_2004": 0.586750, + "regional_beta_path": "inputs for alpha regression/cd_beta0.csv" + } +} diff --git a/aeo_updates/natural_gas_price_regression/inputs for alpha regression/cd_beta0.csv b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/cd_beta0.csv new file mode 100644 index 0000000..b5cfcb2 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/cd_beta0.csv @@ -0,0 +1,10 @@ +*cendiv,value +New_England,2.359082 +Mid_Atlantic,0.185420 +East_North_Central,0.013440 +West_North_Central,0.513493 +South_Atlantic,0.029744 +East_South_Central,-0.026099 +West_South_Central,-0.126897 +Mountain,0.049527 +Pacific,0.704760 diff --git a/aeo_updates/natural_gas_price_regression/inputs for alpha regression/national_beta.csv b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/national_beta.csv new file mode 100644 index 0000000..d285858 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/national_beta.csv @@ -0,0 +1,2 @@ +beta +0.078702 diff --git a/aeo_updates/natural_gas_price_regression/inputs for alpha regression/ng_AEO_historical.csv b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/ng_AEO_historical.csv new file mode 100644 index 0000000..afae51b --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/ng_AEO_historical.csv @@ -0,0 +1,17 @@ +year,East_North_Central,East_South_Central,Mid_Atlantic,Mountain,New_England,Pacific,South_Atlantic,West_North_Central,West_South_Central +2010,7.09099,6.86463,7.81636,7.21463,7.77043,6.92208,8.64559,7.73429,6.61385 +2011,6.41524,6.03580,7.18161,6.75904,6.97661,6.52309,7.61268,7.18783,6.02193 +2012,4.31840,4.17134,4.92095,4.81269,5.12569,4.98196,5.74414,4.83697,4.08624 +2013,5.64457,5.42566,6.22177,6.02671,8.06812,5.96659,6.48806,6.21740,5.31173 +2014,6.90157,6.19613,6.74979,6.53325,8.76920,6.65287,7.21768,7.39862,6.14191 +2015,3.72847,3.83186,3.85032,4.28349,5.66414,4.31823,5.20964,4.42822,3.70961 +2016,3.60433,3.34894,3.66107,3.54438,5.14830,3.66709,4.51428,3.50041,3.09747 +2017,4.25133,4.06024,4.15969,4.19525,5.35528,4.18916,4.87257,4.46950,3.81434 +2018,4.08306,4.02598,4.22836,4.31170,5.88125,4.31732,4.69319,4.33441,3.78456 +2019,3.07415,3.29214,3.56932,3.55569,5.47646,3.74884,4.09898,3.35261,2.85090 +2020,2.43360,2.77132,2.92226,3.11701,4.89456,3.30504,3.53561,2.85416,2.37295 +2021,5.21205,5.94087,5.32393,6.29594,7.94785,6.71271,6.63282,5.65821,5.44123 +2022,6.73681,7.91093,6.17517,7.81644,8.92891,8.04040,8.54369,7.62799,7.36052 +2023,2.57670,2.93427,3.09409,3.30029,5.18236,3.49938,3.74350,3.02199,2.51248 +2024,2.28508,2.64533,2.26953,2.84119,3.91474,3.17877,3.58168,2.60507,2.19948 +2025,3.26855,3.77881,3.37901,3.67871,5.32540,3.91963,4.78151,3.54143,3.29486 diff --git a/aeo_updates/natural_gas_price_regression/inputs for alpha regression/ng_demand_AEO_historical.csv b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/ng_demand_AEO_historical.csv new file mode 100644 index 0000000..de8133d --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/ng_demand_AEO_historical.csv @@ -0,0 +1,17 @@ +year,East_North_Central,East_South_Central,Mid_Atlantic,Mountain,New_England,Pacific,South_Atlantic,West_North_Central,West_South_Central +2010,0.32600,0.56700,0.89000,0.64100,0.42000,0.98900,1.54400,0.12400,2.04900 +2011,0.39500,0.63900,0.96500,0.56700,0.45600,0.76600,1.67100,0.11700,2.18800 +2012,0.65600,0.79700,1.15400,0.66300,0.44800,1.04500,2.04300,0.17100,2.33700 +2013,0.47500,0.62900,1.06600,0.65100,0.37200,1.06900,1.88300,0.14000,2.07500 +2014,0.48286,0.67191,1.13148,0.64891,0.34175,1.07373,1.89625,0.10807,2.02567 +2015,0.71331,0.87542,1.23645,0.75092,0.39761,1.08443,2.33010,0.14716,2.43411 +2016,0.91214,0.96190,1.34450,0.76576,0.39318,0.91791,2.46983,0.19192,2.35184 +2017,0.81026,0.91090,1.19616,0.69033,0.37141,0.84090,2.47589,0.17219,2.06317 +2018,1.00257,0.99460,1.30006,0.76026,0.40041,0.87174,2.53553,0.18961,2.66131 +2019,1.18572,1.02982,1.45155,0.98821,0.33865,0.83400,2.79026,0.22831,2.92826 +2020,1.26278,1.04720,1.68422,0.95476,0.36085,0.84768,2.76804,0.24731,2.94039 +2021,1.08225,0.91391,1.65478,0.94725,0.34459,0.88703,2.46873,0.19506,2.73224 +2022,1.47769,0.90048,1.99829,0.94774,0.36677,0.85018,2.60534,0.25751,2.79701 +2023,1.33703,1.10878,1.78325,1.01090,0.38207,0.89752,2.93080,0.26185,3.11328 +2024,1.80377,1.12075,1.94787,1.20213,0.45162,1.02698,2.77112,0.32823,3.27809 +2025,1.65962,1.02549,1.82135,1.21019,0.41596,0.85889,2.87384,0.32285,3.06582 diff --git a/aeo_updates/natural_gas_price_regression/inputs for alpha regression/ng_tot_demand_AEO_historical.csv b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/ng_tot_demand_AEO_historical.csv new file mode 100644 index 0000000..13dbe98 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/ng_tot_demand_AEO_historical.csv @@ -0,0 +1,17 @@ +year,East_North_Central,East_South_Central,Mid_Atlantic,Mountain,New_England,Pacific,South_Atlantic,West_North_Central,West_South_Central +2010,3.35827,1.35412,2.64274,4.75630,0.85818,2.70713,2.86466,1.54981,5.37928 +2011,3.52540,1.41626,2.71985,4.79351,0.91535,2.54856,2.93492,1.55552,5.52935 +2012,3.54768,1.54045,2.77095,4.90369,0.87908,2.80361,3.28012,1.49108,5.73093 +2013,3.81810,1.45590,2.89180,4.97218,0.85950,2.89684,3.26105,1.64592,5.57736 +2014,4.04516,1.54466,3.12045,5.00056,0.85773,2.78484,3.32657,1.68047,5.64674 +2015,3.88672,1.69639,3.09914,5.25547,0.90592,2.76397,3.66395,1.57498,5.95167 +2016,3.92096,1.74669,3.15614,1.58165,0.85664,2.77540,3.73514,1.63864,5.80345 +2017,3.99711,1.76773,3.11356,1.57646,0.87954,2.82770,3.88878,1.72960,5.92221 +2018,4.39524,1.91248,3.31636,1.67603,0.93098,2.95235,4.06765,1.84624,6.69761 +2019,4.66544,1.96209,3.50768,1.98524,0.88497,2.94931,4.32576,1.97242,7.13452 +2020,4.57850,1.96789,3.61531,1.90072,0.84980,2.84772,4.22153,1.91562,7.06612 +2021,4.41846,1.84455,3.66561,1.92141,0.87847,2.94745,3.97168,1.89320,6.86597 +2022,4.94008,1.89274,4.04472,1.91471,0.92315,2.85715,4.15986,1.99868,7.15639 +2023,4.64873,1.86139,3.70918,1.73492,0.91437,2.86945,4.15805,1.87774,6.91027 +2024,4.96334,2.09747,3.90312,2.15953,0.96217,2.98253,4.37537,2.04819,7.42142 +2025,5.01978,2.01659,3.82598,2.18927,0.94249,2.78771,4.54999,2.08700,7.43280 diff --git a/aeo_updates/natural_gas_price_regression/inputs for alpha regression/st_cendiv.csv b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/st_cendiv.csv new file mode 100644 index 0000000..ddd1f6c --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/inputs for alpha regression/st_cendiv.csv @@ -0,0 +1,49 @@ +st,cendiv +WA,Pacific +OR,Pacific +CA,Pacific +NV,Mountain +ID,Mountain +MT,Mountain +WY,Mountain +UT,Mountain +AZ,Mountain +NM,Mountain +CO,Mountain +ND,WestNorthCentral +SD,WestNorthCentral +NE,WestNorthCentral +MN,WestNorthCentral +IA,WestNorthCentral +WI,EastNorthCentral +TX,WestSouthCentral +OK,WestSouthCentral +KS,WestNorthCentral +MO,WestNorthCentral +AR,WestSouthCentral +LA,WestSouthCentral +MI,EastNorthCentral +IL,EastNorthCentral +MS,EastSouthCentral +AL,EastSouthCentral +FL,SouthAtlantic +TN,EastSouthCentral +KY,EastSouthCentral +GA,SouthAtlantic +SC,SouthAtlantic +NC,SouthAtlantic +VA,SouthAtlantic +IN,EastNorthCentral +OH,EastNorthCentral +PA,MiddleAtlantic +WV,SouthAtlantic +MD,SouthAtlantic +DE,SouthAtlantic +NJ,MiddleAtlantic +NY,MiddleAtlantic +VT,NewEngland +NH,NewEngland +MA,NewEngland +CT,NewEngland +RI,NewEngland +ME,NewEngland diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2025_HOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2025_HOG.csv new file mode 100644 index 0000000..bd15a24 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2025_HOG.csv @@ -0,0 +1,42 @@ +t,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,4.683875,4.711564,4.274324,4.662090,5.211405,4.137874,3.986708,4.348851,4.172502 +2011,1.816973,3.561499,3.232512,3.611184,3.674279,2.927836,3.128783,3.072885,2.283110 +2012,0.608249,2.046157,1.835802,2.023827,2.358645,1.657890,1.846136,1.711282,0.855822 +2013,2.752758,2.919934,2.717595,2.958825,2.911482,2.513759,2.646788,2.527453,1.493478 +2014,3.290932,3.226641,3.473380,3.696553,3.347306,2.970546,3.142848,2.832533,1.899154 +2015,1.075644,1.335855,1.425834,1.745415,1.933618,1.389862,1.571987,1.280897,0.350384 +2016,0.754709,1.178244,1.317182,1.120629,1.462783,1.059535,1.171321,0.798267,0.154122 +2017,1.026077,1.563513,1.772830,1.783921,1.739926,1.557646,1.649825,1.303398,0.634367 +2018,1.135938,1.494356,1.570148,1.592753,1.526519,1.430504,1.569901,1.231905,0.575444 +2019,1.046547,0.989432,0.871149,0.883062,1.038922,0.898527,0.937047,0.538670,0.198677 +2020,0.582664,0.537068,0.455374,0.539249,0.676152,0.555077,0.622532,0.269587,-0.114263 +2021,2.557157,2.060218,2.207226,2.345262,2.666628,2.555234,2.531615,2.261831,1.958028 +2022,2.984533,2.443258,3.035049,3.401068,3.716464,3.666597,3.614054,3.099996,2.729752 +2023,0.616925,0.568648,0.482150,0.570957,0.715910,0.587715,0.659137,0.285438,-0.120982 +2024,-0.468010,-0.031321,0.210737,0.184579,0.497425,0.345082,0.398380,-0.191626,-0.546024 +2025,-0.454252,0.112763,0.344876,0.320929,0.499549,0.410939,0.468171,-0.100039,-0.445625 +2026,-0.252073,0.019707,0.197131,0.206801,0.316351,0.224597,0.318176,-0.232231,-0.439900 +2027,-0.258877,-0.033279,0.157925,0.160226,0.353443,0.175124,0.277408,-0.229435,-0.439578 +2028,-0.289135,-0.037449,0.207416,0.252221,0.437009,0.262914,0.367318,-0.125625,-0.368992 +2029,-0.322310,-0.032144,0.242622,0.276503,0.477042,0.314815,0.415802,-0.066422,-0.365759 +2030,-0.251269,-0.033219,0.269754,0.334502,0.533248,0.378667,0.493271,0.001634,-0.312151 +2031,-0.182167,-0.023406,0.268865,0.299335,0.481149,0.424748,0.550534,0.069528,-0.296495 +2032,-0.176144,-0.010662,0.332092,0.340568,0.630270,0.490504,0.644607,0.153234,-0.183365 +2033,-0.082352,0.064029,0.371831,0.450130,0.590180,0.539430,0.707330,0.282865,-0.174894 +2034,-0.011003,0.122283,0.403914,0.500722,0.619309,0.565403,0.733782,0.370414,-0.123494 +2035,0.056341,0.171979,0.439723,0.566309,0.668940,0.607326,0.777152,0.464788,-0.103677 +2036,0.093461,0.217097,0.495611,0.691647,0.737113,0.667947,0.818150,0.464172,0.082731 +2037,0.092945,0.236787,0.497352,0.689038,0.757634,0.683358,0.805579,0.502848,0.041974 +2038,0.198916,0.280015,0.536568,0.770227,0.813723,0.734539,0.833305,0.692031,0.112107 +2039,0.222122,0.334648,0.576221,0.812897,0.867698,0.788379,0.877299,0.836469,0.184849 +2040,0.164746,0.307851,0.575600,0.855858,0.862221,0.801046,0.902691,0.861927,0.224821 +2041,0.221233,0.307528,0.575753,0.895945,0.870508,0.810443,0.930069,0.924016,0.204696 +2042,0.198786,0.317525,0.557631,0.881745,0.867713,0.813421,0.943432,0.923621,0.243462 +2043,0.097936,0.235768,0.489477,0.812074,0.819954,0.749603,0.912820,0.831754,0.237156 +2044,0.027747,0.142980,0.424122,0.753046,0.782351,0.713009,0.886083,0.730459,0.171369 +2045,0.070819,0.076273,0.370421,0.713242,0.758601,0.681674,0.876443,0.676002,0.217207 +2046,0.043637,0.029397,0.349860,0.715050,0.743219,0.669778,0.883537,0.695452,0.199327 +2047,-0.012972,-0.008888,0.310455,0.696182,0.833784,0.649055,0.856814,0.660313,0.230160 +2048,-0.033075,-0.029444,0.296592,0.695373,0.847091,0.657421,0.868570,0.662000,0.294965 +2049,-0.062187,-0.064672,0.245406,0.700461,0.791758,0.617461,0.841883,0.593020,0.265435 +2050,-0.061527,-0.095693,0.225173,0.701374,0.756995,0.585727,0.823963,0.525706,0.281430 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2025_LOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2025_LOG.csv new file mode 100644 index 0000000..2b3490f --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2025_LOG.csv @@ -0,0 +1,42 @@ +t,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,4.683875,4.711564,4.274324,4.662090,5.211405,4.137874,3.986708,4.348851,4.172502 +2011,1.816973,3.561499,3.232512,3.611184,3.674279,2.927836,3.128783,3.072885,2.283110 +2012,0.608249,2.046157,1.835802,2.023827,2.358645,1.657890,1.846136,1.711282,0.855822 +2013,2.752758,2.919934,2.717595,2.958825,2.911482,2.513759,2.646788,2.527453,1.493478 +2014,3.290932,3.226641,3.473380,3.696553,3.347306,2.970546,3.142848,2.832533,1.899154 +2015,1.075644,1.335855,1.425834,1.745415,1.933618,1.389862,1.571987,1.280897,0.350384 +2016,0.754709,1.178244,1.317182,1.120629,1.462783,1.059535,1.171321,0.798267,0.154122 +2017,1.026077,1.563513,1.772830,1.783921,1.739926,1.557646,1.649825,1.303398,0.634367 +2018,1.135938,1.494356,1.570148,1.592753,1.526519,1.430504,1.569901,1.231905,0.575444 +2019,1.046547,0.989432,0.871149,0.883062,1.038922,0.898527,0.937047,0.538670,0.198677 +2020,0.582664,0.537068,0.455374,0.539249,0.676152,0.555077,0.622532,0.269587,-0.114263 +2021,2.557157,2.060218,2.207226,2.345262,2.666628,2.555234,2.531615,2.261831,1.958028 +2022,2.984533,2.443258,3.035049,3.401068,3.716464,3.666597,3.614054,3.099996,2.729752 +2023,0.616925,0.568648,0.482150,0.570957,0.715910,0.587715,0.659137,0.285438,-0.120982 +2024,-0.484707,-0.029711,0.215051,0.187075,0.490094,0.339058,0.400083,-0.191824,-0.544057 +2025,0.320668,0.889994,1.169469,1.126793,1.370276,1.322634,1.365049,0.743120,0.468185 +2026,0.993648,1.277484,1.435401,1.432441,1.674898,1.591261,1.515472,1.117014,0.752048 +2027,0.980905,1.273043,1.502568,1.511037,1.757396,1.668365,1.596232,1.214643,0.838834 +2028,1.115336,1.426261,1.681291,1.729195,1.990057,1.917707,1.845684,1.497429,1.103785 +2029,1.380942,1.682775,1.977326,2.076482,2.297298,2.224647,2.187165,1.860526,1.387974 +2030,1.840662,2.120230,2.498073,2.597319,2.855960,2.814701,2.806373,2.476454,1.950263 +2031,2.351311,2.540952,2.922170,3.044480,3.339785,3.231337,3.234114,2.981482,2.424394 +2032,2.669670,2.992064,3.379454,3.481538,3.697091,3.625742,3.664041,3.420201,2.877220 +2033,3.059481,3.288055,3.661245,3.739282,3.982265,3.901682,3.892631,3.630120,3.098685 +2034,3.327007,3.470360,3.835762,3.868858,4.168600,4.056546,4.057258,3.855987,3.313662 +2035,3.407640,3.607970,3.940490,3.954335,4.302966,4.191552,4.110478,4.031437,3.491140 +2036,3.491305,3.658682,4.100178,4.105921,4.517574,4.374171,4.338926,4.314109,3.745742 +2037,3.563082,3.640128,4.073614,4.119122,4.676863,4.422630,4.451877,4.387943,3.863371 +2038,3.496389,3.579627,4.013947,4.087316,4.663879,4.282977,4.440391,4.456922,3.936299 +2039,3.442832,3.570535,3.965983,3.972211,4.789079,4.428681,4.607058,4.655245,4.161951 +2040,3.426911,3.551047,4.031987,4.088288,4.760876,4.429668,4.625246,4.705194,4.199913 +2041,3.355756,3.562317,4.022913,4.099254,4.802329,4.621356,4.622140,4.565469,4.239417 +2042,3.316872,3.460866,3.963444,4.055829,4.750108,4.518745,4.571450,4.480272,4.109517 +2043,3.212400,3.392686,3.895821,4.008246,4.675000,4.415660,4.488237,4.357078,4.036913 +2044,3.171646,3.403920,3.905402,4.001124,4.676325,4.547160,4.638910,4.514365,4.149064 +2045,3.150600,3.423385,3.930714,4.048498,4.583925,4.517120,4.686094,4.607649,4.239064 +2046,3.230419,3.472359,4.060075,4.183568,4.623913,4.547413,4.733054,4.655728,4.276129 +2047,3.330979,3.606305,4.195563,4.287413,4.773827,4.762990,4.918477,4.865448,4.463472 +2048,3.450908,3.740855,4.353308,4.443471,4.956059,4.887999,5.152434,5.097561,4.618085 +2049,3.627606,3.934228,4.524340,4.607130,5.109198,5.046874,5.350720,5.257141,4.741983 +2050,3.805277,4.124450,4.696611,4.764436,5.306124,5.230470,5.537398,5.452070,4.890810 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2025_reference.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2025_reference.csv new file mode 100644 index 0000000..bb30fc0 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2025_reference.csv @@ -0,0 +1,42 @@ +t,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,4.683875,4.711564,4.274324,4.662090,5.211405,4.137874,3.986708,4.348851,4.172502 +2011,1.816973,3.561499,3.232512,3.611184,3.674279,2.927836,3.128783,3.072885,2.283110 +2012,0.608249,2.046157,1.835802,2.023827,2.358645,1.657890,1.846136,1.711282,0.855822 +2013,2.752758,2.919934,2.717595,2.958825,2.911482,2.513759,2.646788,2.527453,1.493478 +2014,3.290932,3.226641,3.473380,3.696553,3.347306,2.970546,3.142848,2.832533,1.899154 +2015,1.075644,1.335855,1.425834,1.745415,1.933618,1.389862,1.571987,1.280897,0.350384 +2016,0.754709,1.178244,1.317182,1.120629,1.462783,1.059535,1.171321,0.798267,0.154122 +2017,1.026077,1.563513,1.772830,1.783921,1.739926,1.557646,1.649825,1.303398,0.634367 +2018,1.135938,1.494356,1.570148,1.592753,1.526519,1.430504,1.569901,1.231905,0.575444 +2019,1.046547,0.989432,0.871149,0.883062,1.038922,0.898527,0.937047,0.538670,0.198677 +2020,0.582664,0.537068,0.455374,0.539249,0.676152,0.555077,0.622532,0.269587,-0.114263 +2021,2.557157,2.060218,2.207226,2.345262,2.666628,2.555234,2.531615,2.261831,1.958028 +2022,2.984533,2.443258,3.035049,3.401068,3.716464,3.666597,3.614054,3.099996,2.729752 +2023,0.616925,0.568648,0.482150,0.570957,0.715910,0.587715,0.659137,0.285438,-0.120982 +2024,-0.505133,-0.041746,0.202311,0.174344,0.557962,0.323449,0.391092,-0.207917,-0.576337 +2025,-0.229539,0.347671,0.602813,0.572750,0.772238,0.692976,0.749541,0.205100,-0.115501 +2026,0.116280,0.431180,0.613034,0.607796,0.782696,0.694058,0.728853,0.230798,-0.054895 +2027,0.025300,0.312819,0.540627,0.519140,0.799770,0.615471,0.672565,0.178473,-0.086562 +2028,-0.005586,0.295013,0.581150,0.597140,0.796321,0.707458,0.769714,0.304707,-0.041584 +2029,0.067221,0.342341,0.669987,0.689225,1.001846,0.822394,0.892482,0.459622,0.070685 +2030,0.201912,0.407551,0.746032,0.790169,1.023289,0.956262,1.043240,0.616757,0.206840 +2031,0.270865,0.511255,0.851821,0.911957,1.103114,1.056561,1.155028,0.763157,0.321341 +2032,0.383170,0.615433,1.019251,1.080553,1.263653,1.236409,1.392116,1.014594,0.526482 +2033,0.667851,0.857661,1.250259,1.359765,1.550552,1.509990,1.683922,1.402480,0.878486 +2034,0.851366,1.009215,1.397943,1.597190,1.725658,1.689997,1.869690,1.640171,1.045899 +2035,0.910406,1.079863,1.492374,1.688441,1.819346,1.788715,1.911618,1.722907,1.135310 +2036,0.978942,1.117161,1.541326,1.757318,1.874950,1.811399,1.912304,1.746874,1.165524 +2037,0.991961,1.155582,1.536159,1.773991,1.942654,1.834123,1.941018,1.796241,1.212068 +2038,0.941332,1.140687,1.487182,1.777185,1.946024,1.834754,1.955180,1.836029,1.255656 +2039,0.903317,1.106120,1.483093,1.738510,1.942899,1.813399,1.955097,1.852664,1.292380 +2040,0.902075,1.076208,1.465823,1.783399,1.951433,1.816640,1.997464,1.929209,1.386383 +2041,0.940136,1.079738,1.485430,1.852105,1.952438,1.837854,2.062283,2.034826,1.503686 +2042,0.974734,1.120315,1.518709,1.936311,1.987258,1.858898,2.136794,2.110740,1.655205 +2043,1.000147,1.132037,1.529437,1.962805,2.038861,1.936040,2.223861,2.199057,1.697789 +2044,1.041327,1.123951,1.519776,1.987807,2.050695,1.964234,2.287435,2.219031,1.774132 +2045,1.086377,1.099421,1.520266,1.967755,2.027370,1.974527,2.312392,2.234710,1.797817 +2046,1.034779,1.143505,1.528193,2.025646,2.053118,2.019305,2.355159,2.302125,1.933259 +2047,1.059170,1.127624,1.514791,1.996091,2.015636,2.018503,2.409184,2.347354,1.939243 +2048,0.948902,1.098686,1.495751,1.948797,1.953655,1.965800,2.405515,2.275422,1.943456 +2049,0.957797,1.069420,1.466790,1.903059,1.896208,1.856363,2.375852,2.194083,1.949716 +2050,0.950400,1.008986,1.448063,1.910854,1.828244,1.769984,2.347436,2.211597,1.936905 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2026_HOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2026_HOG.csv new file mode 100644 index 0000000..4bc4878 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2026_HOG.csv @@ -0,0 +1,42 @@ +t,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,4.559300,4.586249,4.160638,4.538095,5.072800,4.027822,3.880676,4.233184,4.061530 +2011,2.406742,3.423837,3.147791,3.546338,3.805995,2.947141,3.199976,3.326743,2.676535 +2012,1.217599,1.940362,1.791974,2.017254,2.576577,1.735304,1.961129,2.057979,1.453660 +2013,3.198442,2.795017,2.647619,2.918222,3.092913,2.541974,2.722020,2.845981,2.089560 +2014,3.679539,3.091068,3.383434,3.626075,3.518999,2.993543,3.201245,3.141674,2.487277 +2015,1.600819,1.245292,1.393473,1.738072,2.202829,1.486571,1.700875,1.691527,0.984838 +2016,1.281884,1.087498,1.291244,1.143979,1.763954,1.178758,1.304545,1.230402,0.693421 +2017,1.515899,1.468781,1.733453,1.783935,2.035212,1.655994,1.749749,1.677248,1.115232 +2018,1.662846,1.396555,1.538883,1.602474,1.834935,1.544824,1.714925,1.648859,1.075442 +2019,1.487711,0.898453,0.861123,0.923210,1.395384,1.031842,1.117654,1.110659,0.685163 +2020,1.067273,0.449013,0.457608,0.594351,1.038851,0.700067,0.812120,0.828284,0.388486 +2021,2.966989,1.933491,2.160129,2.336297,2.934881,2.626162,2.655859,2.763732,2.430043 +2022,3.413554,2.292514,2.972719,3.383250,3.975273,3.704996,3.713473,3.579114,3.158288 +2023,1.130024,0.475416,0.484518,0.629304,1.099934,0.741230,0.859873,0.876987,0.411334 +2024,0.135202,-0.125890,0.220165,0.263618,0.922764,0.485035,0.610162,0.511167,0.045006 +2025,1.108914,0.601185,0.863123,0.880009,1.635120,1.213779,1.285697,1.064128,0.670486 +2026,0.671290,0.451134,0.715084,0.751348,1.418080,1.016711,1.183269,0.952131,0.546283 +2027,0.395931,0.292047,0.617925,0.666421,1.242231,0.875931,0.989540,0.862069,0.483612 +2028,0.318788,0.168433,0.567460,0.643537,1.183696,0.839861,0.953149,0.851125,0.480907 +2029,0.296394,0.143769,0.575616,0.660824,1.196084,0.878862,1.008313,0.877827,0.504958 +2030,0.269400,0.130782,0.634125,0.723273,1.420834,0.974161,1.113869,1.002017,0.626526 +2031,0.273626,0.134451,0.652880,0.754305,1.466179,1.023170,1.198958,1.091447,0.704086 +2032,0.219313,0.069968,0.666611,0.846099,1.520358,1.089121,1.321114,1.216806,0.797587 +2033,0.260556,0.096617,0.604105,0.834930,1.485130,1.038929,1.292949,1.241048,0.787401 +2034,0.187888,0.029316,0.489419,0.775984,1.216269,0.958522,1.232195,1.240482,0.744750 +2035,0.181927,-0.030959,0.430027,0.747584,1.263167,0.945117,1.235206,1.273895,0.737747 +2036,0.137024,-0.109223,0.381435,0.709425,1.172084,0.918964,1.242404,1.300273,0.731842 +2037,0.168563,-0.117597,0.320492,0.649962,1.130389,0.866084,1.216437,1.278962,0.646398 +2038,0.112350,-0.176348,0.243586,0.572945,1.046034,0.790218,1.149587,1.184959,0.500813 +2039,0.079598,-0.268235,0.148291,0.465093,0.950178,0.703505,1.064718,1.048065,0.346407 +2040,-0.059601,-0.392519,0.053479,0.376114,0.871614,0.624793,0.993441,0.962721,0.232990 +2041,-0.218848,-0.545699,-0.061608,0.257277,0.764985,0.523068,0.918969,0.842347,0.137274 +2042,-0.378413,-0.707355,-0.185466,0.124114,0.647987,0.416798,0.823727,0.708869,-0.003099 +2043,-0.512458,-0.825149,-0.286196,0.005823,0.550221,0.323207,0.746389,0.597777,-0.137443 +2044,-0.637612,-0.931948,-0.371072,-0.118332,0.450141,0.229335,0.668595,0.492979,-0.263946 +2045,-0.676116,-1.032834,-0.441924,-0.193050,0.378631,0.162548,0.613378,0.432171,-0.335872 +2046,-0.713449,-1.097480,-0.497549,-0.237493,0.314151,0.103904,0.574419,0.395583,-0.365369 +2047,-0.767542,-1.158374,-0.556369,-0.302063,0.253196,0.048921,0.540740,0.344603,-0.438947 +2048,-0.820429,-1.211544,-0.603926,-0.328176,0.195107,-0.006141,0.507863,0.289875,-0.507049 +2049,-0.865166,-1.276104,-0.665558,-0.403639,0.131508,-0.069103,0.468287,0.225292,-0.588833 +2050,-0.920714,-1.330371,-0.715123,-0.490857,0.074984,-0.120909,0.430522,0.164523,-0.682461 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2026_LOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2026_LOG.csv new file mode 100644 index 0000000..00f9f67 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2026_LOG.csv @@ -0,0 +1,42 @@ +t,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,4.559300,4.586249,4.160638,4.538095,5.072800,4.027822,3.880676,4.233184,4.061530 +2011,2.406742,3.423837,3.147791,3.546338,3.805995,2.947141,3.199976,3.326743,2.676535 +2012,1.217599,1.940362,1.791974,2.017254,2.576577,1.735304,1.961129,2.057979,1.453660 +2013,3.198442,2.795017,2.647619,2.918222,3.092913,2.541974,2.722020,2.845981,2.089560 +2014,3.679539,3.091068,3.383434,3.626075,3.518999,2.993543,3.201245,3.141674,2.487277 +2015,1.600819,1.245292,1.393473,1.738072,2.202829,1.486571,1.700875,1.691527,0.984838 +2016,1.281884,1.087498,1.291244,1.143979,1.763954,1.178758,1.304545,1.230402,0.693421 +2017,1.515899,1.468781,1.733453,1.783935,2.035212,1.655994,1.749749,1.677248,1.115232 +2018,1.662846,1.396555,1.538883,1.602474,1.834935,1.544824,1.714925,1.648859,1.075442 +2019,1.487711,0.898453,0.861123,0.923210,1.395384,1.031842,1.117654,1.110659,0.685163 +2020,1.067273,0.449013,0.457608,0.594351,1.038851,0.700067,0.812120,0.828284,0.388486 +2021,2.966989,1.933491,2.160129,2.336297,2.934881,2.626162,2.655859,2.763732,2.430043 +2022,3.413554,2.292514,2.972719,3.383250,3.975273,3.704996,3.713473,3.579114,3.158288 +2023,1.130024,0.475416,0.484518,0.629304,1.099934,0.741230,0.859873,0.876987,0.411334 +2024,0.135202,-0.125890,0.220165,0.263618,0.922764,0.485035,0.610162,0.511167,0.045006 +2025,1.082937,0.587947,0.849352,0.878566,1.705951,1.206806,1.272325,1.040695,0.647898 +2026,1.150798,0.902877,1.271322,1.283954,1.896542,1.604584,1.801210,1.466624,1.036835 +2027,1.422653,1.379729,1.778635,1.858564,2.469879,2.202551,2.202648,1.984390,1.564029 +2028,1.984482,1.864329,2.393737,2.506162,2.978362,2.801213,2.793915,2.641150,2.259690 +2029,2.468448,2.364850,2.917074,3.035462,3.365883,3.242736,3.163091,3.104513,2.763274 +2030,2.697164,2.602218,3.143560,3.326393,3.601954,3.483422,3.372951,3.394860,3.045638 +2031,2.864192,2.830859,3.361073,3.564746,3.843580,3.706062,3.655872,3.643465,3.281765 +2032,3.090680,3.030698,3.678378,3.926319,4.103140,3.972975,4.039783,4.061855,3.640292 +2033,3.320014,3.266007,3.854392,4.108464,4.298088,4.151818,4.159946,4.265576,3.868352 +2034,3.534281,3.449482,4.016388,4.252373,4.496548,4.326855,4.255850,4.434075,4.047557 +2035,3.799912,3.676480,4.197675,4.382622,4.737640,4.538563,4.397422,4.626949,4.248096 +2036,4.056435,3.960529,4.482252,4.669082,5.074204,4.808703,4.708541,4.946172,4.551676 +2037,4.373952,4.240087,4.766913,4.933817,5.387953,5.093013,5.008427,5.223659,4.850293 +2038,4.587928,4.453101,4.924970,5.068652,5.577611,5.252117,5.157384,5.356710,4.978115 +2039,4.787981,4.668341,5.186362,5.106292,5.766134,5.481660,5.380505,5.577910,5.155541 +2040,4.914851,4.846008,5.334884,5.176427,5.876168,5.593291,5.494554,5.787268,5.325949 +2041,5.331300,5.201336,5.721343,5.585408,6.149723,5.986905,5.919292,6.218070,5.784100 +2042,5.630000,5.503065,6.004181,5.860687,6.428461,6.257741,6.170805,6.510048,6.040577 +2043,5.815721,5.712630,6.190326,6.035113,6.619925,6.437477,6.336945,6.691883,6.192059 +2044,5.888823,5.895344,6.388643,6.222943,6.818350,6.611998,6.513248,6.876992,6.326224 +2045,6.153762,6.175327,6.694716,6.490861,7.112264,6.893358,6.791655,7.186384,6.608804 +2046,6.427409,6.414420,6.962092,6.758920,7.345939,7.120543,7.003431,7.505726,6.872441 +2047,6.592273,6.559214,7.128275,6.879106,7.515275,7.292545,7.168189,7.708294,7.024219 +2048,6.761227,6.683449,7.269464,7.012994,7.668647,7.457916,7.307539,7.877018,7.136751 +2049,6.911001,6.800232,7.378307,7.123527,7.788373,7.576050,7.415736,7.980815,7.215116 +2050,7.068169,6.927563,7.497430,7.212858,7.932874,7.694907,7.536294,8.055626,7.339126 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2026_reference.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2026_reference.csv new file mode 100644 index 0000000..13a200d --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/alpha_AEO_2026_reference.csv @@ -0,0 +1,42 @@ +t,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,4.559300,4.586249,4.160638,4.538095,5.072800,4.027822,3.880676,4.233184,4.061530 +2011,2.406742,3.423837,3.147791,3.546338,3.805995,2.947141,3.199976,3.326743,2.676535 +2012,1.217599,1.940362,1.791974,2.017254,2.576577,1.735304,1.961129,2.057979,1.453660 +2013,3.198442,2.795017,2.647619,2.918222,3.092913,2.541974,2.722020,2.845981,2.089560 +2014,3.679539,3.091068,3.383434,3.626075,3.518999,2.993543,3.201245,3.141674,2.487277 +2015,1.600819,1.245292,1.393473,1.738072,2.202829,1.486571,1.700875,1.691527,0.984838 +2016,1.281884,1.087498,1.291244,1.143979,1.763954,1.178758,1.304545,1.230402,0.693421 +2017,1.515899,1.468781,1.733453,1.783935,2.035212,1.655994,1.749749,1.677248,1.115232 +2018,1.662846,1.396555,1.538883,1.602474,1.834935,1.544824,1.714925,1.648859,1.075442 +2019,1.487711,0.898453,0.861123,0.923210,1.395384,1.031842,1.117654,1.110659,0.685163 +2020,1.067273,0.449013,0.457608,0.594351,1.038851,0.700067,0.812120,0.828284,0.388486 +2021,2.966989,1.933491,2.160129,2.336297,2.934881,2.626162,2.655859,2.763732,2.430043 +2022,3.413554,2.292514,2.972719,3.383250,3.975273,3.704996,3.713473,3.579114,3.158288 +2023,1.130024,0.475416,0.484518,0.629304,1.099934,0.741230,0.859873,0.876987,0.411334 +2024,0.135202,-0.125890,0.220165,0.263618,0.922764,0.485035,0.610162,0.511167,0.045006 +2025,1.100276,0.601804,0.852401,0.869033,1.676951,1.200867,1.279185,1.055427,0.651410 +2026,0.937584,0.679842,1.003042,1.022661,1.813726,1.372998,1.539204,1.226692,0.823660 +2027,0.803180,0.716659,1.069620,1.156444,1.879705,1.391995,1.459920,1.317888,0.939826 +2028,0.864588,0.719824,1.121800,1.229853,1.690256,1.434658,1.509028,1.420702,1.072397 +2029,0.935078,0.796138,1.239455,1.337114,1.814477,1.563456,1.600793,1.556980,1.209064 +2030,0.983331,0.890630,1.449430,1.616648,2.216904,1.932912,1.960196,1.882389,1.541132 +2031,1.089024,0.960544,1.569582,1.794783,2.340246,2.142311,2.205757,2.146476,1.795804 +2032,1.092690,0.929750,1.701904,1.990277,2.567253,2.265472,2.424819,2.331625,1.951962 +2033,1.146902,0.983800,1.628033,2.021673,2.416365,2.249207,2.403020,2.379130,1.974265 +2034,1.132332,0.943273,1.505045,1.938153,2.308355,2.141583,2.290749,2.329366,1.808836 +2035,1.080194,0.895384,1.427770,1.885064,2.287886,2.128535,2.291685,2.319018,1.781731 +2036,1.083786,0.845248,1.399536,1.871273,2.504856,2.148171,2.361992,2.385127,1.773895 +2037,1.144225,0.886161,1.421311,1.898142,2.315196,2.163694,2.399047,2.419176,1.783674 +2038,1.152753,0.929833,1.445013,1.915447,2.306061,2.152901,2.392285,2.399261,1.781693 +2039,1.186466,0.948910,1.483199,1.888120,2.260834,2.136147,2.390423,2.338296,1.755008 +2040,1.192576,0.931508,1.483271,1.895158,2.278851,2.149574,2.383420,2.322414,1.754340 +2041,1.207616,0.956656,1.454831,1.878740,2.245404,2.093792,2.371799,2.341682,1.745758 +2042,1.191591,0.944770,1.417710,1.805271,2.343205,1.996432,2.286511,2.270628,1.653727 +2043,1.128883,0.892979,1.351047,1.693067,2.232732,1.892081,2.179623,2.148028,1.525319 +2044,0.985737,0.784756,1.272927,1.594616,2.115356,1.790872,2.102605,2.051051,1.422989 +2045,0.906896,0.666874,1.193022,1.527544,1.856207,1.682490,2.046563,1.986795,1.359777 +2046,0.853422,0.569800,1.135599,1.513098,1.848290,1.615208,2.041603,1.988558,1.382786 +2047,0.772307,0.497443,1.073197,1.443065,1.715850,1.551866,1.967115,1.934622,1.292597 +2048,0.699996,0.437942,1.008579,1.376723,1.649481,1.489174,1.892655,1.868772,1.217253 +2049,0.653817,0.384605,0.949038,1.312196,1.583388,1.429534,1.823080,1.820173,1.154207 +2050,0.563710,0.340709,0.897455,1.250993,1.528480,1.375311,1.768898,1.764423,1.110525 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/cd_beta0.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/cd_beta0.csv new file mode 100644 index 0000000..b5cfcb2 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/cd_beta0.csv @@ -0,0 +1,10 @@ +*cendiv,value +New_England,2.359082 +Mid_Atlantic,0.185420 +East_North_Central,0.013440 +West_North_Central,0.513493 +South_Atlantic,0.029744 +East_South_Central,-0.026099 +West_South_Central,-0.126897 +Mountain,0.049527 +Pacific,0.704760 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/national_beta.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/national_beta.csv new file mode 100644 index 0000000..d285858 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/national_beta.csv @@ -0,0 +1,2 @@ +beta +0.078702 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2025_HOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2025_HOG.csv new file mode 100644 index 0000000..fc1046e --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2025_HOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,7.770429,7.816364,7.090995,7.734288,8.645587,6.864627,6.613847,7.214633,6.922075 +2011,6.976606,7.181610,6.415244,7.187830,7.612683,6.035803,6.021934,6.759040,6.523092 +2012,5.125688,4.920955,4.318396,4.836968,5.744142,4.171337,4.086242,4.812692,4.981963 +2013,8.068125,6.221772,5.644572,6.217397,6.488062,5.425661,5.311725,6.026714,5.966594 +2014,8.769202,6.749786,6.901574,7.398615,7.217680,6.196134,6.141908,6.533248,6.652865 +2015,5.664139,3.850320,3.728469,4.428217,5.209638,3.831858,3.709608,4.283486,4.318228 +2016,5.148299,3.661068,3.604329,3.500407,4.514280,3.348942,3.097470,3.544384,3.667086 +2017,5.355285,4.159690,4.251332,4.469497,4.872571,4.060241,3.814341,4.195250,4.189158 +2018,5.881254,4.228362,4.083061,4.334410,4.693188,4.025985,3.784559,4.311702,4.317322 +2019,5.476462,3.569320,3.074155,3.352608,4.098979,3.292144,2.850897,3.555691,3.748837 +2020,4.894557,2.922261,2.433601,2.854164,3.535609,2.771318,2.372946,3.117011,3.305045 +2021,7.947848,5.323930,5.212050,5.658208,6.632816,5.940867,5.441225,6.295938,6.712710 +2022,8.928909,6.175171,6.736810,7.627995,8.543692,7.910931,7.360517,7.816437,8.040398 +2023,5.182357,3.094090,2.576697,3.021989,3.743503,2.934271,2.512475,3.300291,3.499382 +2024,3.942886,2.287270,2.298708,2.621773,3.481087,2.680677,2.210615,2.863694,3.208913 +2025,3.545907,2.313296,2.394202,2.708847,3.371045,2.663157,2.218614,2.777358,2.887983 +2026,3.424357,2.298251,2.279395,2.664633,3.222955,2.498695,2.059296,2.712186,2.803907 +2027,3.179948,2.255369,2.267412,2.702573,3.352517,2.484879,2.033904,2.717369,2.824805 +2028,2.963628,2.219732,2.342423,2.822133,3.471617,2.607332,2.178029,2.860942,2.993797 +2029,2.897579,2.237025,2.422638,2.905383,3.536583,2.709477,2.279984,2.964342,3.117639 +2030,2.761189,2.309547,2.509715,3.080562,3.637365,2.835151,2.440423,3.228207,3.405689 +2031,2.747252,2.337410,2.525936,3.122771,3.510499,2.905147,2.544189,3.393098,3.586969 +2032,3.035183,2.531026,2.816504,3.501460,3.964278,3.177044,2.843096,3.814344,4.013958 +2033,3.119033,2.633990,2.860132,3.516799,3.848393,3.242102,2.926303,3.950235,4.208856 +2034,3.235024,2.714677,2.896689,3.576314,3.874689,3.271004,2.959214,4.049577,4.334771 +2035,3.332014,2.718911,2.911631,3.632553,3.886527,3.285653,2.990757,4.106940,4.363559 +2036,3.345162,2.679176,2.902459,3.651698,3.871522,3.277782,2.995825,4.154849,4.288165 +2037,3.331612,2.655793,2.849265,3.584962,3.826073,3.248419,2.955194,4.159271,4.251697 +2038,3.345557,2.610942,2.787631,3.559825,3.806650,3.233404,2.903788,4.197432,4.234464 +2039,3.217742,2.571926,2.754783,3.507277,3.787173,3.219203,2.895023,4.173839,4.281111 +2040,3.172366,2.542018,2.765788,3.599415,3.826042,3.249156,2.954316,4.291714,4.376058 +2041,3.339386,2.589555,2.809392,3.717389,3.902796,3.310269,3.031373,4.404143,4.470148 +2042,3.391913,2.603526,2.803964,3.731760,3.909537,3.326160,3.068110,4.446998,4.516488 +2043,3.330580,2.536290,2.752379,3.703559,3.898544,3.289931,3.063311,4.391260,4.504367 +2044,3.113896,2.437677,2.683260,3.646818,3.864220,3.263479,3.049417,4.328261,4.424861 +2045,2.985338,2.366950,2.627810,3.651798,3.847422,3.248250,3.060245,4.364906,4.511312 +2046,2.984034,2.338613,2.628173,3.720915,3.857837,3.277446,3.101784,4.417610,4.539052 +2047,2.934076,2.304000,2.562113,3.699291,4.006942,3.260696,3.061580,4.407011,4.557307 +2048,2.892148,2.251554,2.512380,3.689516,4.003499,3.256391,3.065140,4.434324,4.586201 +2049,2.865312,2.203854,2.461685,3.655248,3.966560,3.213834,3.041644,4.389663,4.565211 +2050,2.790408,2.150464,2.421813,3.591890,3.907777,3.162359,3.010422,4.376517,4.556719 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2025_LOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2025_LOG.csv new file mode 100644 index 0000000..ebfea64 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2025_LOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,7.770429,7.816364,7.090995,7.734288,8.645587,6.864627,6.613847,7.214633,6.922075 +2011,6.976606,7.181610,6.415244,7.187830,7.612683,6.035803,6.021934,6.759040,6.523092 +2012,5.125688,4.920955,4.318396,4.836968,5.744142,4.171337,4.086242,4.812692,4.981963 +2013,8.068125,6.221772,5.644572,6.217397,6.488062,5.425661,5.311725,6.026714,5.966594 +2014,8.769202,6.749786,6.901574,7.398615,7.217680,6.196134,6.141908,6.533248,6.652865 +2015,5.664139,3.850320,3.728469,4.428217,5.209638,3.831858,3.709608,4.283486,4.318228 +2016,5.148299,3.661068,3.604329,3.500407,4.514280,3.348942,3.097470,3.544384,3.667086 +2017,5.355285,4.159690,4.251332,4.469497,4.872571,4.060241,3.814341,4.195250,4.189158 +2018,5.881254,4.228362,4.083061,4.334410,4.693188,4.025985,3.784559,4.311702,4.317322 +2019,5.476462,3.569320,3.074155,3.352608,4.098979,3.292144,2.850897,3.555691,3.748837 +2020,4.894557,2.922261,2.433601,2.854164,3.535609,2.771318,2.372946,3.117011,3.305045 +2021,7.947848,5.323930,5.212050,5.658208,6.632816,5.940867,5.441225,6.295938,6.712710 +2022,8.928909,6.175171,6.736810,7.627995,8.543692,7.910931,7.360517,7.816437,8.040398 +2023,5.182357,3.094090,2.576697,3.021989,3.743503,2.934271,2.512475,3.300291,3.499382 +2024,3.951021,2.291188,2.306225,2.628691,3.470091,2.671443,2.214949,2.867855,3.213850 +2025,4.825856,3.629599,3.757886,4.032818,4.788136,4.158859,3.699224,4.146041,4.247904 +2026,4.605783,3.834864,3.800100,4.000161,4.790429,4.182667,3.667104,4.116380,4.119196 +2027,4.343827,3.835330,3.930441,4.200141,4.924905,4.337811,3.808350,4.269623,4.225954 +2028,4.395217,4.009411,4.168205,4.513605,5.188810,4.686580,4.175693,4.650997,4.694008 +2029,4.783993,4.403663,4.620580,5.094242,5.605813,5.149542,4.716477,5.207363,5.210474 +2030,5.429995,5.116742,5.483608,6.184416,6.492614,6.110940,5.740700,6.219917,6.208527 +2031,6.023727,5.623803,6.032155,6.668324,7.070742,6.622880,6.323299,6.823698,6.887148 +2032,6.769484,6.455877,6.900975,7.624558,7.802534,7.392770,7.123114,7.823699,7.795189 +2033,7.239462,6.816369,7.243104,7.822190,8.118889,7.728008,7.409045,8.029579,8.010993 +2034,7.573184,7.007079,7.448060,7.872682,8.326830,7.892791,7.609113,8.258191,8.196181 +2035,7.670228,7.163031,7.551923,7.904687,8.486915,8.054188,7.651300,8.444173,8.278070 +2036,7.708939,7.137417,7.728963,8.052999,8.726404,8.253443,7.956063,8.753573,8.491802 +2037,7.558604,6.949639,7.568828,7.982419,8.848077,8.212115,8.047225,8.739521,8.407489 +2038,7.343988,6.762915,7.392748,7.834982,8.740216,7.899144,7.963167,8.673278,8.322899 +2039,7.251116,6.737899,7.297922,7.645207,8.910620,8.110153,8.226623,9.006240,8.591912 +2040,7.218706,6.678354,7.382362,7.822319,8.836534,8.071926,8.232173,9.026676,8.661890 +2041,7.065958,6.644736,7.322939,7.732145,8.862980,8.334088,8.189538,8.673353,8.644938 +2042,6.963392,6.447856,7.199334,7.611486,8.753935,8.141396,8.084666,8.462022,8.381509 +2043,6.808531,6.334739,7.084161,7.540718,8.612486,7.972393,7.942930,8.241702,8.252924 +2044,6.503371,6.344906,7.090244,7.552183,8.590377,8.179617,8.183352,8.507860,8.446995 +2045,6.349429,6.351938,7.115554,7.610549,8.392526,8.105153,8.243489,8.658021,8.574844 +2046,6.509525,6.429235,7.327775,7.833263,8.438897,8.147901,8.316871,8.684699,8.631374 +2047,6.689818,6.661896,7.564465,8.053114,8.687522,8.513703,8.630199,9.064015,8.964581 +2048,6.918985,6.915947,7.853740,8.339838,9.020991,8.746612,9.039744,9.510916,9.287422 +2049,7.237922,7.254817,8.156782,8.634782,9.287102,9.029250,9.384081,9.833487,9.528601 +2050,7.553951,7.582260,8.458136,8.921481,9.627280,9.348893,9.705161,10.207071,9.821965 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2025_reference.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2025_reference.csv new file mode 100644 index 0000000..f3e1eec --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2025_reference.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,7.770429,7.816364,7.090995,7.734288,8.645587,6.864627,6.613847,7.214633,6.922075 +2011,6.976606,7.181610,6.415244,7.187830,7.612683,6.035803,6.021934,6.759040,6.523092 +2012,5.125688,4.920955,4.318396,4.836968,5.744142,4.171337,4.086242,4.812692,4.981963 +2013,8.068125,6.221772,5.644572,6.217397,6.488062,5.425661,5.311725,6.026714,5.966594 +2014,8.769202,6.749786,6.901574,7.398615,7.217680,6.196134,6.141908,6.533248,6.652865 +2015,5.664139,3.850320,3.728469,4.428217,5.209638,3.831858,3.709608,4.283486,4.318228 +2016,5.148299,3.661068,3.604329,3.500407,4.514280,3.348942,3.097470,3.544384,3.667086 +2017,5.355285,4.159690,4.251332,4.469497,4.872571,4.060241,3.814341,4.195250,4.189158 +2018,5.881254,4.228362,4.083061,4.334410,4.693188,4.025985,3.784559,4.311702,4.317322 +2019,5.476462,3.569320,3.074155,3.352608,4.098979,3.292144,2.850897,3.555691,3.748837 +2020,4.894557,2.922261,2.433601,2.854164,3.535609,2.771318,2.372946,3.117011,3.305045 +2021,7.947848,5.323930,5.212050,5.658208,6.632816,5.940867,5.441225,6.295938,6.712710 +2022,8.928909,6.175171,6.736810,7.627995,8.543692,7.910931,7.360517,7.816437,8.040398 +2023,5.182357,3.094090,2.576697,3.021989,3.743503,2.934271,2.512475,3.300291,3.499382 +2024,3.914745,2.269530,2.285080,2.605072,3.581676,2.645327,2.199480,2.841191,3.178771 +2025,3.945919,2.724665,2.837277,3.162053,3.846201,3.142428,2.702884,3.296411,3.449765 +2026,3.819109,2.780215,2.763278,3.076137,3.765639,3.055122,2.602545,3.156343,3.232441 +2027,3.435360,2.636923,2.703812,3.054238,3.837702,2.988694,2.549224,3.122164,3.154894 +2028,3.240752,2.576812,2.762565,3.195382,3.785440,3.127218,2.701754,3.269873,3.344125 +2029,3.229812,2.643674,2.918250,3.398271,4.085795,3.306478,2.910596,3.513918,3.634970 +2030,3.129285,2.773706,3.049576,3.646155,4.087484,3.512392,3.163967,3.861304,4.019860 +2031,3.326129,2.866469,3.170081,3.762861,4.101537,3.613755,3.303848,4.079747,4.274564 +2032,3.779099,3.222987,3.635774,4.423059,4.559186,4.085257,3.843493,4.733683,4.891140 +2033,4.071227,3.483385,3.881344,4.704285,4.842482,4.388517,4.214373,5.040173,5.191451 +2034,4.168971,3.598001,3.987692,4.772688,4.962396,4.544195,4.409548,5.230925,5.384827 +2035,4.199443,3.631619,4.045470,4.784186,5.016607,4.609845,4.425838,5.303466,5.412060 +2036,4.265835,3.610814,4.036805,4.780307,5.044314,4.556694,4.374448,5.251937,5.282920 +2037,4.120316,3.518157,3.899876,4.672058,4.979167,4.464017,4.317363,5.176685,5.104523 +2038,3.993610,3.376865,3.733063,4.566430,4.909653,4.376263,4.273121,5.154803,5.010477 +2039,3.816716,3.256926,3.681409,4.408439,4.850672,4.286094,4.233821,5.160571,4.923649 +2040,3.872509,3.215503,3.662889,4.491251,4.854046,4.299133,4.305593,5.306404,5.055543 +2041,3.966687,3.252421,3.723782,4.649086,4.880734,4.361627,4.432408,5.505489,5.287903 +2042,4.091134,3.298153,3.773970,4.751801,4.925858,4.390012,4.548378,5.683669,5.408764 +2043,4.169929,3.344024,3.805754,4.807343,5.008918,4.522407,4.704851,5.836872,5.563621 +2044,4.087124,3.347215,3.800041,4.817755,5.012309,4.578125,4.813431,5.905303,5.645613 +2045,3.891919,3.329523,3.798050,4.839172,4.970164,4.596054,4.858135,5.945200,5.636967 +2046,3.902629,3.393556,3.810532,4.945189,4.994895,4.665256,4.923351,6.013248,5.758965 +2047,3.945969,3.383668,3.770669,4.909540,4.917217,4.652761,5.001916,6.069484,5.808951 +2048,3.787596,3.349662,3.740844,4.859097,4.802450,4.571362,5.000763,6.010077,5.813332 +2049,3.870886,3.274235,3.685088,4.774216,4.690213,4.388964,4.946911,5.912456,5.796697 +2050,3.788044,3.161351,3.642668,4.791852,4.578797,4.235358,4.904408,5.972120,5.745820 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2026_HOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2026_HOG.csv new file mode 100644 index 0000000..2ba1142 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2026_HOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,7.770430,7.816360,7.090990,7.734290,8.645590,6.864630,6.613850,7.214630,6.922080 +2011,6.976610,7.181610,6.415240,7.187830,7.612680,6.035800,6.021930,6.759040,6.523090 +2012,5.125690,4.920950,4.318400,4.836970,5.744140,4.171340,4.086240,4.812690,4.981960 +2013,8.068120,6.221770,5.644570,6.217400,6.488060,5.425660,5.311730,6.026710,5.966590 +2014,8.769200,6.749790,6.901570,7.398620,7.217680,6.196130,6.141910,6.533250,6.652870 +2015,5.664140,3.850320,3.728470,4.428220,5.209640,3.831860,3.709610,4.283490,4.318230 +2016,5.148300,3.661070,3.604330,3.500410,4.514280,3.348940,3.097470,3.544380,3.667090 +2017,5.355280,4.159690,4.251330,4.469500,4.872570,4.060240,3.814340,4.195250,4.189160 +2018,5.881250,4.228360,4.083060,4.334410,4.693190,4.025980,3.784560,4.311700,4.317320 +2019,5.476460,3.569320,3.074150,3.352610,4.098980,3.292140,2.850900,3.555690,3.748840 +2020,4.894560,2.922260,2.433600,2.854160,3.535610,2.771320,2.372950,3.117010,3.305040 +2021,7.947850,5.323930,5.212050,5.658210,6.632820,5.940870,5.441230,6.295940,6.712710 +2022,8.928910,6.175170,6.736810,7.627990,8.543690,7.910930,7.360520,7.816440,8.040400 +2023,5.182360,3.094090,2.576700,3.021990,3.743500,2.934270,2.512480,3.300290,3.499380 +2024,3.914740,2.269530,2.285080,2.605070,3.581680,2.645330,2.199480,2.841190,3.178770 +2025,5.345607,3.387378,3.286574,3.555411,4.709843,3.800634,3.311579,3.695329,3.943778 +2026,4.852463,3.147799,3.084828,3.362976,4.388223,3.513833,3.102779,3.556494,3.751729 +2027,3.666937,2.785328,2.862534,3.210815,4.032940,3.208148,2.776315,3.328036,3.450739 +2028,3.465783,2.605392,2.797957,3.184686,3.949574,3.162952,2.734897,3.330281,3.458980 +2029,3.167606,2.580803,2.850465,3.304868,4.017421,3.264244,2.837093,3.407689,3.559944 +2030,2.926042,2.662569,3.002350,3.534045,4.451139,3.475777,3.069016,3.680808,3.842189 +2031,2.915775,2.699370,3.094841,3.667346,4.593555,3.617326,3.234858,3.904559,4.088018 +2032,3.220855,2.858877,3.380701,4.172229,4.939392,3.970807,3.627450,4.376479,4.569868 +2033,3.376226,2.926083,3.330557,4.209935,4.935080,3.937970,3.608490,4.499977,4.363488 +2034,3.326463,2.870562,3.208234,4.190933,4.550412,3.868895,3.556247,4.575318,4.426440 +2035,3.366169,2.851336,3.182476,4.240469,4.708346,3.917942,3.616472,4.714975,4.544697 +2036,3.393479,2.811184,3.181453,4.298808,4.637809,3.951176,3.696667,4.848455,4.679333 +2037,3.464981,2.863233,3.152076,4.302949,4.647636,3.932922,3.704575,4.887789,4.710261 +2038,3.410531,2.828988,3.081966,4.243705,4.570607,3.861789,3.629383,4.774141,4.764500 +2039,3.453385,2.790657,3.007529,4.165158,4.504010,3.800644,3.543797,4.619734,4.775273 +2040,3.260410,2.642806,2.915007,4.079655,4.440323,3.731605,3.470656,4.545332,4.720599 +2041,3.059327,2.468462,2.794668,3.977866,4.338802,3.631639,3.384425,4.422882,4.656564 +2042,2.890143,2.317640,2.674123,3.853020,4.229695,3.534928,3.284886,4.290660,4.540991 +2043,2.752141,2.249035,2.589398,3.749722,4.150131,3.457415,3.211062,4.191004,4.440181 +2044,2.547546,2.198134,2.543918,3.686228,4.079580,3.390016,3.147907,4.115810,4.342209 +2045,2.426736,2.131695,2.486152,3.641717,4.022791,3.335948,3.098754,4.076365,4.308940 +2046,2.440843,2.101854,2.454142,3.623240,3.974510,3.294115,3.066741,4.076965,4.314435 +2047,2.399387,2.082169,2.419697,3.587930,3.935534,3.262509,3.043042,4.058603,4.266572 +2048,2.361834,2.054138,2.394982,3.563347,3.893207,3.220711,3.014104,4.025033,4.200194 +2049,2.339232,2.025916,2.352619,3.518968,3.848231,3.173899,2.969493,3.980294,4.140861 +2050,2.374989,2.000027,2.313998,3.463644,3.799017,3.130726,2.930648,3.923027,4.091122 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2026_LOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2026_LOG.csv new file mode 100644 index 0000000..88af7cd --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2026_LOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,7.770430,7.816360,7.090990,7.734290,8.645590,6.864630,6.613850,7.214630,6.922080 +2011,6.976610,7.181610,6.415240,7.187830,7.612680,6.035800,6.021930,6.759040,6.523090 +2012,5.125690,4.920950,4.318400,4.836970,5.744140,4.171340,4.086240,4.812690,4.981960 +2013,8.068120,6.221770,5.644570,6.217400,6.488060,5.425660,5.311730,6.026710,5.966590 +2014,8.769200,6.749790,6.901570,7.398620,7.217680,6.196130,6.141910,6.533250,6.652870 +2015,5.664140,3.850320,3.728470,4.428220,5.209640,3.831860,3.709610,4.283490,4.318230 +2016,5.148300,3.661070,3.604330,3.500410,4.514280,3.348940,3.097470,3.544380,3.667090 +2017,5.355280,4.159690,4.251330,4.469500,4.872570,4.060240,3.814340,4.195250,4.189160 +2018,5.881250,4.228360,4.083060,4.334410,4.693190,4.025980,3.784560,4.311700,4.317320 +2019,5.476460,3.569320,3.074150,3.352610,4.098980,3.292140,2.850900,3.555690,3.748840 +2020,4.894560,2.922260,2.433600,2.854160,3.535610,2.771320,2.372950,3.117010,3.305040 +2021,7.947850,5.323930,5.212050,5.658210,6.632820,5.940870,5.441230,6.295940,6.712710 +2022,8.928910,6.175170,6.736810,7.627990,8.543690,7.910930,7.360520,7.816440,8.040400 +2023,5.182360,3.094090,2.576700,3.021990,3.743500,2.934270,2.512480,3.300290,3.499380 +2024,3.914740,2.269530,2.285080,2.605070,3.581680,2.645330,2.199480,2.841190,3.178770 +2025,5.294798,3.353367,3.263801,3.540977,4.832328,3.789199,3.288816,3.655058,3.905813 +2026,5.685656,3.963801,4.033495,4.260452,5.198661,4.519078,4.150634,4.433659,4.561513 +2027,5.053397,4.249576,4.479586,4.782177,5.751545,5.134305,4.657691,4.871719,4.903999 +2028,5.736206,4.996347,5.468980,5.827187,6.548896,6.094103,5.634982,5.929786,6.002690 +2029,6.381851,5.790377,6.310399,6.769516,7.144698,6.794102,6.258209,6.662073,6.815924 +2030,6.649747,6.203698,6.713041,7.371433,7.568275,7.222132,6.632887,7.181173,7.346393 +2031,6.826379,6.481824,7.004788,7.679960,7.898701,7.529885,7.050432,7.526796,7.611009 +2032,7.462828,7.052512,7.753469,8.613667,8.551950,8.184606,7.788092,8.462646,8.514933 +2033,7.706581,7.312173,7.940298,8.760403,8.767888,8.380210,7.947466,8.690731,8.761770 +2034,7.939864,7.490126,8.109896,8.820741,9.000998,8.575316,8.073168,8.864508,8.877319 +2035,8.260612,7.736515,8.306912,8.889380,9.300612,8.830894,8.261165,9.076791,9.041146 +2036,8.656876,8.154497,8.737354,9.314214,9.819419,9.240022,8.776074,9.564580,9.496066 +2037,9.160049,8.585943,9.195817,9.722696,10.329938,9.701205,9.260747,10.010752,9.970448 +2038,9.502454,8.932805,9.448765,9.927127,10.634950,9.956623,9.497434,10.219106,10.117539 +2039,9.768356,9.263495,9.869632,9.853006,10.923554,10.316548,9.844378,10.563908,10.335305 +2040,9.921204,9.538232,10.104809,9.953640,11.088107,10.489601,10.011313,10.902516,10.608404 +2041,10.627369,10.137972,10.753163,10.661174,11.537051,11.151180,10.722422,11.630302,11.381525 +2042,11.117891,10.656480,11.243264,11.140112,12.021381,11.619881,11.159154,12.137983,11.829603 +2043,11.486072,11.023234,11.569989,11.447901,12.357872,11.936073,11.444778,12.457313,12.094296 +2044,11.489775,11.338474,11.909888,11.744664,12.697484,12.234546,11.745035,12.775576,12.299760 +2045,11.881731,11.800955,12.426816,12.195352,13.193504,12.709871,12.218987,13.299936,12.771711 +2046,12.337296,12.201417,12.876985,12.648473,13.586978,13.091917,12.583334,13.838241,13.206833 +2047,12.625209,12.439937,13.157214,12.880009,13.874180,13.383212,12.864495,14.182432,13.455883 +2048,12.923044,12.661606,13.406144,13.117165,14.144248,13.672522,13.108910,14.479213,13.656775 +2049,13.178946,12.862795,13.595934,13.306535,14.350445,13.877510,13.296640,14.660414,13.802397 +2050,13.449883,13.066490,13.788420,13.447649,14.584373,14.070285,13.498145,14.776094,14.035362 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2026_reference.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2026_reference.csv new file mode 100644 index 0000000..aff9e03 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_AEO_2026_reference.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,7.770430,7.816360,7.090990,7.734290,8.645590,6.864630,6.613850,7.214630,6.922080 +2011,6.976610,7.181610,6.415240,7.187830,7.612680,6.035800,6.021930,6.759040,6.523090 +2012,5.125690,4.920950,4.318400,4.836970,5.744140,4.171340,4.086240,4.812690,4.981960 +2013,8.068120,6.221770,5.644570,6.217400,6.488060,5.425660,5.311730,6.026710,5.966590 +2014,8.769200,6.749790,6.901570,7.398620,7.217680,6.196130,6.141910,6.533250,6.652870 +2015,5.664140,3.850320,3.728470,4.428220,5.209640,3.831860,3.709610,4.283490,4.318230 +2016,5.148300,3.661070,3.604330,3.500410,4.514280,3.348940,3.097470,3.544380,3.667090 +2017,5.355280,4.159690,4.251330,4.469500,4.872570,4.060240,3.814340,4.195250,4.189160 +2018,5.881250,4.228360,4.083060,4.334410,4.693190,4.025980,3.784560,4.311700,4.317320 +2019,5.476460,3.569320,3.074150,3.352610,4.098980,3.292140,2.850900,3.555690,3.748840 +2020,4.894560,2.922260,2.433600,2.854160,3.535610,2.771320,2.372950,3.117010,3.305040 +2021,7.947850,5.323930,5.212050,5.658210,6.632820,5.940870,5.441230,6.295940,6.712710 +2022,8.928910,6.175170,6.736810,7.627990,8.543690,7.910930,7.360520,7.816440,8.040400 +2023,5.182360,3.094090,2.576700,3.021990,3.743500,2.934270,2.512480,3.300290,3.499380 +2024,3.914740,2.269530,2.285080,2.605070,3.581680,2.645330,2.199480,2.841190,3.178770 +2025,5.325405,3.379013,3.268552,3.541426,4.781505,3.778815,3.294860,3.678707,3.919628 +2026,5.324889,3.579604,3.595711,3.873691,5.080299,4.140339,3.737669,4.047685,4.242898 +2027,4.231030,3.347171,3.484283,3.847872,4.968219,3.950409,3.512904,3.955493,4.092942 +2028,4.155579,3.315939,3.544278,3.934699,4.602939,3.991779,3.576442,4.099092,4.283302 +2029,4.123753,3.428625,3.747785,4.209109,4.818919,4.209246,3.767626,4.329078,4.532804 +2030,3.902078,3.655290,4.146273,4.832505,5.549663,4.875952,4.442943,4.929817,5.166997 +2031,4.108847,3.748328,4.348245,5.122258,5.758215,5.231403,4.857481,5.380268,5.607684 +2032,4.358751,3.915184,4.762383,5.663197,6.329670,5.617828,5.341123,5.885259,6.052363 +2033,4.339409,3.894549,4.535047,5.558170,5.962244,5.491001,5.265240,5.859532,6.025182 +2034,4.286699,3.800765,4.305512,5.381374,5.758488,5.286448,5.074927,5.756997,5.569988 +2035,4.220026,3.741856,4.187365,5.300879,5.739176,5.277354,5.081973,5.745817,5.531476 +2036,4.300572,3.728936,4.195457,5.328556,6.168354,5.364456,5.247342,5.915677,5.644624 +2037,4.436268,3.877474,4.298908,5.447619,5.916055,5.454193,5.365159,6.044068,5.804154 +2038,4.501762,3.969138,4.356161,5.486612,5.918227,5.450463,5.373397,6.023650,5.857410 +2039,4.565640,4.036901,4.453778,5.425759,5.873913,5.453105,5.386227,5.945096,5.782586 +2040,4.540153,4.024423,4.465064,5.435948,5.914954,5.484130,5.406227,5.929826,5.834826 +2041,4.596998,4.111586,4.457271,5.452645,5.902722,5.426873,5.420537,6.007441,5.918538 +2042,4.617490,4.149686,4.454994,5.401155,6.136202,5.319212,5.317644,5.951848,5.858219 +2043,4.589023,4.125682,4.400375,5.292042,6.009053,5.196852,5.179297,5.804568,5.733068 +2044,4.330436,4.025949,4.332788,5.210352,5.878681,5.086615,5.093683,5.709803,5.622851 +2045,4.219499,3.884732,4.241616,5.142129,5.486709,4.944200,5.035359,5.648788,5.588075 +2046,4.196424,3.792771,4.197471,5.176457,5.530549,4.880015,5.060451,5.711275,5.670421 +2047,4.119616,3.717444,4.132073,5.109780,5.348881,4.810425,4.958267,5.661644,5.581164 +2048,4.044715,3.663227,4.062656,5.041185,5.279084,4.740374,4.862533,5.595249,5.488992 +2049,3.981374,3.620282,4.000563,4.967881,5.208039,4.675830,4.769575,5.556533,5.420349 +2050,4.201423,3.592049,3.939520,4.903520,5.144186,4.611669,4.701788,5.484926,5.438996 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2025_HOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2025_HOG.csv new file mode 100644 index 0000000..20a30c6 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2025_HOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.420000,0.890000,0.326000,0.124000,1.544000,0.567000,2.049000,0.641000,0.989000 +2011,0.456000,0.965000,0.395000,0.117000,1.671000,0.639000,2.188000,0.567000,0.766000 +2012,0.448000,1.154000,0.656000,0.171000,2.043000,0.797000,2.337000,0.663000,1.045000 +2013,0.372000,1.066000,0.475000,0.140000,1.883000,0.629000,2.075000,0.651000,1.069000 +2014,0.341750,1.131480,0.482860,0.108070,1.896250,0.671910,2.025670,0.648910,1.073730 +2015,0.397610,1.236450,0.713310,0.147160,2.330100,0.875420,2.434110,0.750920,1.084430 +2016,0.393180,1.344500,0.912140,0.191920,2.469830,0.961900,2.351840,0.765760,0.917910 +2017,0.371410,1.196160,0.810260,0.172190,2.475890,0.910900,2.063170,0.690330,0.840900 +2018,0.400407,1.300064,1.002570,0.189613,2.535531,0.994602,2.661307,0.760259,0.871735 +2019,0.338651,1.451554,1.185716,0.228314,2.790259,1.029824,2.928261,0.988215,0.834003 +2020,0.360853,1.684218,1.262778,0.247312,2.768037,1.047204,2.940385,0.954761,0.847681 +2021,0.344592,1.654776,1.082247,0.195056,2.468727,0.913914,2.732240,0.947247,0.887034 +2022,0.366773,1.998293,1.477695,0.257507,2.605338,0.900480,2.797009,0.947744,0.850175 +2023,0.382071,1.783250,1.337029,0.261854,2.930798,1.108780,3.113280,1.010901,0.897525 +2024,0.446506,1.951789,1.807217,0.328414,2.772431,1.120767,3.282638,1.198553,1.018170 +2025,0.400232,1.590906,1.721987,0.316580,2.802444,1.101454,3.123186,1.094271,0.854088 +2026,0.311662,1.697816,2.042994,0.339554,2.951403,1.240576,3.375084,1.131739,0.761514 +2027,0.268560,1.701938,2.206338,0.389654,3.033981,1.345413,3.394222,1.092678,0.750901 +2028,0.244934,1.644293,2.334784,0.376410,3.015770,1.306726,3.293242,1.079716,0.780610 +2029,0.241361,1.630497,2.512054,0.398223,2.969723,1.324362,3.188257,1.073624,0.828471 +2030,0.196057,1.778863,2.602928,0.427727,2.868471,1.249770,3.244220,1.176084,0.901304 +2031,0.174589,1.787377,2.751464,0.491814,2.697150,1.181122,3.251625,1.214582,0.966842 +2032,0.193283,1.845946,3.253971,0.599109,2.864202,1.202948,3.415374,1.326644,1.003485 +2033,0.185470,1.848220,3.239930,0.497179,2.768513,1.226310,3.408022,1.274809,1.094409 +2034,0.187396,1.843454,3.205562,0.490385,2.743542,1.228781,3.366460,1.246807,1.119302 +2035,0.192078,1.710656,3.212188,0.484944,2.655394,1.185304,3.317858,1.197920,1.137609 +2036,0.199700,1.642419,3.132051,0.420684,2.550056,1.137773,2.939176,1.329532,1.008565 +2037,0.206521,1.644400,3.140409,0.416561,2.471244,1.144755,2.535467,1.326660,1.047966 +2038,0.197384,1.590891,2.716094,0.376668,2.439109,1.158549,2.472733,1.171483,1.034278 +2039,0.185696,1.437783,2.581804,0.354041,2.377795,1.105213,2.369297,1.016782,1.041979 +2040,0.191300,1.440836,2.550396,0.359027,2.495180,1.084304,2.332093,1.072407,1.048653 +2041,0.196422,1.471445,2.603838,0.366828,2.572892,1.106517,2.428086,1.044572,1.087787 +2042,0.208031,1.400841,2.774909,0.382864,2.560002,1.088461,2.434259,1.070091,1.072913 +2043,0.215927,1.453301,2.887747,0.405494,2.604740,1.150788,2.535568,1.107367,1.047360 +2044,0.195102,1.539554,2.989638,0.410800,2.585567,1.153446,2.570477,1.171576,1.045559 +2045,0.159384,1.580606,3.061960,0.439917,2.561460,1.184258,2.601221,1.259128,1.036926 +2046,0.161159,1.648451,3.099662,0.463651,2.573675,1.255747,2.629858,1.248391,1.048170 +2047,0.167061,1.739527,2.979110,0.466230,2.549114,1.305926,2.650514,1.285874,1.030689 +2048,0.168816,1.745558,2.855375,0.474269,2.529676,1.313169,2.607504,1.325776,1.004289 +2049,0.167950,1.680766,2.984837,0.425213,2.625262,1.298944,2.676126,1.364214,1.004726 +2050,0.156610,1.685399,2.927017,0.381755,2.632061,1.317363,2.658570,1.455590,0.990377 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2025_LOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2025_LOG.csv new file mode 100644 index 0000000..c848a10 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2025_LOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.420000,0.890000,0.326000,0.124000,1.544000,0.567000,2.049000,0.641000,0.989000 +2011,0.456000,0.965000,0.395000,0.117000,1.671000,0.639000,2.188000,0.567000,0.766000 +2012,0.448000,1.154000,0.656000,0.171000,2.043000,0.797000,2.337000,0.663000,1.045000 +2013,0.372000,1.066000,0.475000,0.140000,1.883000,0.629000,2.075000,0.651000,1.069000 +2014,0.341750,1.131480,0.482860,0.108070,1.896250,0.671910,2.025670,0.648910,1.073730 +2015,0.397610,1.236450,0.713310,0.147160,2.330100,0.875420,2.434110,0.750920,1.084430 +2016,0.393180,1.344500,0.912140,0.191920,2.469830,0.961900,2.351840,0.765760,0.917910 +2017,0.371410,1.196160,0.810260,0.172190,2.475890,0.910900,2.063170,0.690330,0.840900 +2018,0.400407,1.300064,1.002570,0.189613,2.535531,0.994602,2.661307,0.760259,0.871735 +2019,0.338651,1.451554,1.185716,0.228314,2.790259,1.029824,2.928261,0.988215,0.834003 +2020,0.360853,1.684218,1.262778,0.247312,2.768037,1.047204,2.940385,0.954761,0.847681 +2021,0.344592,1.654776,1.082247,0.195056,2.468727,0.913914,2.732240,0.947247,0.887034 +2022,0.366773,1.998293,1.477695,0.257507,2.605338,0.900480,2.797009,0.947744,0.850175 +2023,0.382071,1.783250,1.337029,0.261854,2.930798,1.108780,3.113280,1.010901,0.897525 +2024,0.452001,1.954733,1.804559,0.330010,2.774692,1.121850,3.271640,1.202133,1.018694 +2025,0.400253,1.720835,1.748320,0.311531,2.728033,1.053865,3.139995,1.072385,0.786553 +2026,0.251434,1.490830,1.402841,0.199013,2.304525,0.872166,2.062922,0.829442,0.687765 +2027,0.211858,1.467800,1.498055,0.238384,2.249545,0.930938,2.132700,0.809352,0.664922 +2028,0.194398,1.385326,1.509668,0.246067,2.034196,0.907154,1.991213,0.783828,0.704409 +2029,0.192247,1.412108,1.496857,0.275980,1.846410,0.871673,1.856138,0.776658,0.741716 +2030,0.174559,1.373499,1.534067,0.437111,1.719123,0.808875,1.857033,0.770984,0.772348 +2031,0.157962,1.197684,1.384653,0.358888,1.451879,0.668381,1.640671,0.692458,0.789767 +2032,0.176228,1.132102,1.565061,0.451031,1.585336,0.734924,1.789956,0.847123,0.815216 +2033,0.166235,1.060135,1.333007,0.368381,1.433254,0.691406,1.579497,0.820230,0.797631 +2034,0.161400,0.929961,1.233128,0.308266,1.357102,0.634744,1.513954,0.759908,0.756096 +2035,0.165505,0.893260,1.111427,0.275428,1.358099,0.637080,1.327169,0.722170,0.688957 +2036,0.162913,0.788534,1.031229,0.261540,1.244350,0.549481,1.226203,0.653720,0.632692 +2037,0.137538,0.586714,0.854965,0.271243,1.119637,0.482770,1.115833,0.626503,0.554879 +2038,0.132298,0.522783,0.732474,0.254090,1.064564,0.435245,1.057815,0.527240,0.494092 +2039,0.134267,0.549630,0.758101,0.266654,0.993744,0.374653,1.017851,0.545790,0.454257 +2040,0.136717,0.527879,0.698537,0.271079,0.974679,0.296413,1.049425,0.509211,0.467174 +2041,0.137923,0.488661,0.664124,0.224717,0.975145,0.238602,0.994796,0.438115,0.449188 +2042,0.135460,0.463385,0.615075,0.206150,0.974492,0.237566,0.978935,0.395162,0.437638 +2043,0.138723,0.473387,0.606697,0.213672,0.925296,0.257154,0.991719,0.383094,0.435116 +2044,0.103130,0.474389,0.587082,0.236184,0.871767,0.247615,1.000719,0.395633,0.442636 +2045,0.087387,0.444759,0.607441,0.234486,0.779892,0.218179,1.004305,0.407519,0.440998 +2046,0.092547,0.450912,0.663438,0.237393,0.730188,0.209494,0.993623,0.364727,0.441239 +2047,0.092992,0.450285,0.687064,0.263399,0.693493,0.198955,1.047300,0.383422,0.446543 +2048,0.093810,0.473709,0.734405,0.265423,0.714790,0.201429,1.086109,0.416587,0.465141 +2049,0.095297,0.480987,0.789578,0.270487,0.700422,0.213645,1.095479,0.453987,0.473856 +2050,0.096417,0.472205,0.816287,0.278686,0.698386,0.217900,1.125022,0.486470,0.488408 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2025_reference.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2025_reference.csv new file mode 100644 index 0000000..1f7f0bd --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2025_reference.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.420000,0.890000,0.326000,0.124000,1.544000,0.567000,2.049000,0.641000,0.989000 +2011,0.456000,0.965000,0.395000,0.117000,1.671000,0.639000,2.188000,0.567000,0.766000 +2012,0.448000,1.154000,0.656000,0.171000,2.043000,0.797000,2.337000,0.663000,1.045000 +2013,0.372000,1.066000,0.475000,0.140000,1.883000,0.629000,2.075000,0.651000,1.069000 +2014,0.341750,1.131480,0.482860,0.108070,1.896250,0.671910,2.025670,0.648910,1.073730 +2015,0.397610,1.236450,0.713310,0.147160,2.330100,0.875420,2.434110,0.750920,1.084430 +2016,0.393180,1.344500,0.912140,0.191920,2.469830,0.961900,2.351840,0.765760,0.917910 +2017,0.371410,1.196160,0.810260,0.172190,2.475890,0.910900,2.063170,0.690330,0.840900 +2018,0.400407,1.300064,1.002570,0.189613,2.535531,0.994602,2.661307,0.760259,0.871735 +2019,0.338651,1.451554,1.185716,0.228314,2.790259,1.029824,2.928261,0.988215,0.834003 +2020,0.360853,1.684218,1.262778,0.247312,2.768037,1.047204,2.940385,0.954761,0.847681 +2021,0.344592,1.654776,1.082247,0.195056,2.468727,0.913914,2.732240,0.947247,0.887034 +2022,0.366773,1.998293,1.477695,0.257507,2.605338,0.900480,2.797009,0.947744,0.850175 +2023,0.382071,1.783250,1.337029,0.261854,2.930798,1.108780,3.113280,1.010901,0.897525 +2024,0.451624,1.947869,1.803768,0.328229,2.771119,1.120752,3.278091,1.202134,1.026979 +2025,0.402601,1.629397,1.779150,0.333078,2.839481,1.098819,3.062861,1.095031,0.855043 +2026,0.307184,1.645651,1.709301,0.292994,2.802256,1.090066,2.831672,1.009045,0.751345 +2027,0.263452,1.664455,1.930932,0.347046,2.786452,1.157464,2.922580,1.012900,0.718387 +2028,0.243434,1.601191,2.042475,0.365627,2.676992,1.160019,2.856328,0.970603,0.776476 +2029,0.223449,1.567711,2.250298,0.403164,2.548651,1.124709,2.763967,0.961866,0.824927 +2030,0.172437,1.638802,2.271644,0.456933,2.418587,1.037678,2.771827,1.036249,0.894584 +2031,0.193628,1.537363,2.249849,0.434738,2.196743,0.989105,2.681658,1.062644,0.947843 +2032,0.210223,1.631743,2.753094,0.589867,2.304419,1.045986,2.831487,1.131738,1.000090 +2033,0.202380,1.585095,2.610210,0.553112,2.088257,0.955824,2.650223,0.945356,0.930929 +2034,0.189251,1.527352,2.315766,0.410186,1.922410,0.866626,2.548273,0.872272,0.947975 +2035,0.192515,1.546738,2.128180,0.374010,1.878592,0.824893,2.161885,0.893621,0.933309 +2036,0.197367,1.530112,1.923508,0.346125,1.926551,0.771852,1.876698,0.882123,0.888036 +2037,0.189569,1.373293,1.718799,0.332811,1.718812,0.714015,1.742069,0.846004,0.825748 +2038,0.194608,1.208284,1.530699,0.307617,1.718832,0.654886,1.661109,0.834513,0.784750 +2039,0.183550,1.129269,1.492246,0.271675,1.680835,0.603034,1.621889,0.853311,0.737564 +2040,0.191371,1.131776,1.538328,0.272217,1.617794,0.603428,1.688687,0.863450,0.723354 +2041,0.192375,1.157560,1.606427,0.285775,1.619307,0.615171,1.746875,0.862811,0.729275 +2042,0.203848,1.098378,1.636847,0.264481,1.598931,0.615970,1.757097,0.915820,0.673652 +2043,0.207948,1.162631,1.703603,0.265405,1.554882,0.590668,1.737841,0.912592,0.706872 +2044,0.183532,1.205421,1.783943,0.239437,1.479005,0.604031,1.767898,0.939069,0.684177 +2045,0.141631,1.299167,1.743539,0.278295,1.469072,0.610125,1.725534,0.952057,0.662889 +2046,0.157335,1.281461,1.815236,0.288586,1.423282,0.608258,1.737410,0.916862,0.618836 +2047,0.158988,1.377790,1.637457,0.303452,1.396680,0.593526,1.773730,0.906929,0.640318 +2048,0.162292,1.420297,1.609448,0.320825,1.344266,0.605255,1.755996,0.957831,0.637655 +2049,0.174212,1.345600,1.616478,0.320191,1.313927,0.636592,1.718013,0.998673,0.629154 +2050,0.165085,1.342685,1.626209,0.331866,1.359366,0.643452,1.536503,1.036890,0.621188 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2026_HOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2026_HOG.csv new file mode 100644 index 0000000..46b65a9 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2026_HOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.420000,0.890000,0.326000,0.124000,1.544000,0.567000,2.049000,0.641000,0.989000 +2011,0.456000,0.965000,0.395000,0.117000,1.671000,0.639000,2.188000,0.567000,0.766000 +2012,0.448000,1.154000,0.656000,0.171000,2.043000,0.797000,2.337000,0.663000,1.045000 +2013,0.372000,1.066000,0.475000,0.140000,1.883000,0.629000,2.075000,0.651000,1.069000 +2014,0.341750,1.131480,0.482860,0.108070,1.896250,0.671910,2.025670,0.648910,1.073730 +2015,0.397610,1.236450,0.713310,0.147160,2.330100,0.875420,2.434110,0.750920,1.084430 +2016,0.393180,1.344500,0.912140,0.191920,2.469830,0.961900,2.351840,0.765760,0.917910 +2017,0.371410,1.196160,0.810260,0.172190,2.475890,0.910900,2.063170,0.690330,0.840900 +2018,0.400410,1.300060,1.002570,0.189610,2.535530,0.994600,2.661310,0.760260,0.871740 +2019,0.338650,1.451550,1.185720,0.228310,2.790260,1.029820,2.928260,0.988210,0.834000 +2020,0.360850,1.684220,1.262780,0.247310,2.768040,1.047200,2.940390,0.954760,0.847680 +2021,0.344590,1.654780,1.082250,0.195060,2.468730,0.913910,2.732240,0.947250,0.887030 +2022,0.366770,1.998290,1.477690,0.257510,2.605340,0.900480,2.797010,0.947740,0.850180 +2023,0.382070,1.783250,1.337030,0.261850,2.930800,1.108780,3.113280,1.010900,0.897520 +2024,0.451620,1.947870,1.803770,0.328230,2.771120,1.120750,3.278090,1.202130,1.026980 +2025,0.417325,1.851161,1.648597,0.317455,2.866583,1.029691,3.039832,1.231434,0.851933 +2026,0.467969,1.746961,1.713044,0.292045,2.850809,1.030120,3.425049,1.266515,0.827414 +2027,0.304663,1.646666,1.841328,0.351720,2.930920,1.167425,3.132115,1.085011,0.715407 +2028,0.282638,1.684212,1.952928,0.344860,2.882026,1.225841,3.124128,1.108953,0.710391 +2029,0.208550,1.619591,1.985248,0.405253,3.056920,1.294265,3.261393,1.038270,0.728797 +2030,0.147087,1.785222,2.011865,0.486595,3.039507,1.349217,3.259249,1.155577,0.748359 +2031,0.127850,1.692420,2.040323,0.510064,3.143278,1.390848,3.439810,1.291346,0.793124 +2032,0.166115,1.773454,2.854984,0.629632,3.334630,1.451952,3.714446,1.462801,0.858645 +2033,0.174169,1.675639,3.015952,0.634252,3.394392,1.451795,3.823832,1.812109,0.657401 +2034,0.175485,1.645214,3.205416,0.648723,3.488226,1.470505,3.904728,1.901085,0.713055 +2035,0.169778,1.679040,3.321302,0.677436,3.590709,1.491139,3.986648,2.018383,0.760828 +2036,0.175985,1.724456,3.448392,0.728275,3.705425,1.515289,4.037294,2.132609,0.815626 +2037,0.162578,1.707575,3.572106,0.766930,3.887541,1.510466,4.127420,2.179991,0.902957 +2038,0.158196,1.729452,3.658869,0.781836,4.040754,1.528602,4.220967,2.032956,1.105590 +2039,0.161481,1.833264,3.768442,0.804460,4.263766,1.502221,4.343083,1.955187,1.262494 +2040,0.156339,1.830200,3.948766,0.805841,4.367780,1.498289,4.419831,2.027627,1.323843 +2041,0.155380,1.869890,4.019770,0.836198,4.486732,1.515667,4.574669,2.128645,1.344587 +2042,0.159258,1.988616,4.167163,0.853261,4.548312,1.577872,4.687437,2.224538,1.374970 +2043,0.160852,2.140853,4.294076,0.869560,4.607835,1.623893,4.807927,2.291301,1.411695 +2044,0.139318,2.254237,4.463818,0.929913,4.701151,1.684566,4.927483,2.387595,1.430295 +2045,0.110367,2.394387,4.541317,0.954616,4.777538,1.717293,5.002665,2.422866,1.453691 +2046,0.114923,2.460590,4.688744,0.952163,4.820875,1.746517,5.118394,2.464838,1.450654 +2047,0.111870,2.527314,4.810600,0.965562,4.858366,1.766934,5.253918,2.530149,1.462748 +2048,0.111630,2.555921,4.932360,0.927140,4.920039,1.800730,5.376229,2.603085,1.459536 +2049,0.109753,2.621164,4.997245,0.953469,4.963941,1.816386,5.553566,2.652108,1.475239 +2050,0.130816,2.687166,5.002142,1.007838,4.991158,1.830329,5.647074,2.658764,1.528597 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2026_LOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2026_LOG.csv new file mode 100644 index 0000000..b75d262 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2026_LOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.420000,0.890000,0.326000,0.124000,1.544000,0.567000,2.049000,0.641000,0.989000 +2011,0.456000,0.965000,0.395000,0.117000,1.671000,0.639000,2.188000,0.567000,0.766000 +2012,0.448000,1.154000,0.656000,0.171000,2.043000,0.797000,2.337000,0.663000,1.045000 +2013,0.372000,1.066000,0.475000,0.140000,1.883000,0.629000,2.075000,0.651000,1.069000 +2014,0.341750,1.131480,0.482860,0.108070,1.896250,0.671910,2.025670,0.648910,1.073730 +2015,0.397610,1.236450,0.713310,0.147160,2.330100,0.875420,2.434110,0.750920,1.084430 +2016,0.393180,1.344500,0.912140,0.191920,2.469830,0.961900,2.351840,0.765760,0.917910 +2017,0.371410,1.196160,0.810260,0.172190,2.475890,0.910900,2.063170,0.690330,0.840900 +2018,0.400410,1.300060,1.002570,0.189610,2.535530,0.994600,2.661310,0.760260,0.871740 +2019,0.338650,1.451550,1.185720,0.228310,2.790260,1.029820,2.928260,0.988210,0.834000 +2020,0.360850,1.684220,1.262780,0.247310,2.768040,1.047200,2.940390,0.954760,0.847680 +2021,0.344590,1.654780,1.082250,0.195060,2.468730,0.913910,2.732240,0.947250,0.887030 +2022,0.366770,1.998290,1.477690,0.257510,2.605340,0.900480,2.797010,0.947740,0.850180 +2023,0.382070,1.783250,1.337030,0.261850,2.930800,1.108780,3.113280,1.010900,0.897520 +2024,0.451620,1.947870,1.803770,0.328230,2.771120,1.120750,3.278090,1.202130,1.026980 +2025,0.415699,1.814928,1.679000,0.303772,2.901429,1.019626,3.039703,1.227479,0.852376 +2026,0.471940,1.892826,1.742216,0.280337,2.752043,0.955223,3.449575,1.270218,0.805547 +2027,0.301155,1.519425,1.323182,0.224736,2.449615,0.842020,2.377727,0.850297,0.683090 +2028,0.247602,1.456988,1.344027,0.225546,2.255227,0.866748,2.243650,0.828409,0.660176 +2029,0.215992,1.435084,1.413110,0.331048,2.007415,0.875769,2.030363,0.765199,0.666055 +2030,0.181131,1.405076,1.342575,0.431404,2.066261,0.885410,2.036111,0.836330,0.691846 +2031,0.173324,1.294632,1.241520,0.407349,1.972942,0.774044,1.980588,0.819401,0.640920 +2032,0.185051,1.379459,1.442365,0.537819,2.122517,0.852192,2.535162,1.050292,0.715507 +2033,0.176080,1.283438,1.349913,0.477644,2.018384,0.810948,2.231772,0.954536,0.689854 +2034,0.169384,1.189227,1.283549,0.386284,2.015351,0.771330,1.920939,0.854735,0.619171 +2035,0.163692,1.089855,1.158178,0.335705,1.971972,0.684919,1.662948,0.767628,0.561833 +2036,0.166770,1.049285,1.103241,0.324191,1.942473,0.639220,1.487373,0.732576,0.554200 +2037,0.163613,0.986870,1.041893,0.304279,1.963684,0.596215,1.492653,0.715167,0.546481 +2038,0.162177,0.987886,1.044960,0.294146,1.929736,0.579216,1.495804,0.692604,0.501309 +2039,0.150060,0.956840,1.119514,0.166236,1.804188,0.690582,1.528119,0.623204,0.452780 +2040,0.138701,0.924067,1.108948,0.164876,1.700179,0.679190,1.573129,0.617334,0.453095 +2041,0.140308,0.937378,1.098532,0.188367,1.557672,0.661338,1.585708,0.660225,0.455047 +2042,0.133711,0.925667,1.102409,0.190433,1.583407,0.680625,1.585218,0.684994,0.457538 +2043,0.144300,0.927282,1.119741,0.192070,1.605054,0.662954,1.615774,0.689120,0.455408 +2044,0.113996,0.936408,1.161269,0.164290,1.614498,0.661131,1.621199,0.710736,0.435302 +2045,0.100468,0.906334,1.182112,0.163450,1.620273,0.638799,1.599687,0.737419,0.431589 +2046,0.099194,0.902110,1.189454,0.165687,1.638262,0.626578,1.557580,0.734336,0.424507 +2047,0.101461,0.882888,1.153745,0.198688,1.653636,0.619159,1.545822,0.747760,0.418307 +2048,0.101902,0.888655,1.161926,0.199668,1.664755,0.633662,1.551339,0.760913,0.419132 +2049,0.101241,0.885032,1.205107,0.197028,1.642012,0.625737,1.551197,0.772766,0.426429 +2050,0.104621,0.876168,1.204226,0.196322,1.605915,0.609529,1.520899,0.757297,0.453179 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2026_reference.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2026_reference.csv new file mode 100644 index 0000000..d378767 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_demand_AEO_2026_reference.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.420000,0.890000,0.326000,0.124000,1.544000,0.567000,2.049000,0.641000,0.989000 +2011,0.456000,0.965000,0.395000,0.117000,1.671000,0.639000,2.188000,0.567000,0.766000 +2012,0.448000,1.154000,0.656000,0.171000,2.043000,0.797000,2.337000,0.663000,1.045000 +2013,0.372000,1.066000,0.475000,0.140000,1.883000,0.629000,2.075000,0.651000,1.069000 +2014,0.341750,1.131480,0.482860,0.108070,1.896250,0.671910,2.025670,0.648910,1.073730 +2015,0.397610,1.236450,0.713310,0.147160,2.330100,0.875420,2.434110,0.750920,1.084430 +2016,0.393180,1.344500,0.912140,0.191920,2.469830,0.961900,2.351840,0.765760,0.917910 +2017,0.371410,1.196160,0.810260,0.172190,2.475890,0.910900,2.063170,0.690330,0.840900 +2018,0.400410,1.300060,1.002570,0.189610,2.535530,0.994600,2.661310,0.760260,0.871740 +2019,0.338650,1.451550,1.185720,0.228310,2.790260,1.029820,2.928260,0.988210,0.834000 +2020,0.360850,1.684220,1.262780,0.247310,2.768040,1.047200,2.940390,0.954760,0.847680 +2021,0.344590,1.654780,1.082250,0.195060,2.468730,0.913910,2.732240,0.947250,0.887030 +2022,0.366770,1.998290,1.477690,0.257510,2.605340,0.900480,2.797010,0.947740,0.850180 +2023,0.382070,1.783250,1.337030,0.261850,2.930800,1.108780,3.113280,1.010900,0.897520 +2024,0.451620,1.947870,1.803770,0.328230,2.771120,1.120750,3.278090,1.202130,1.026980 +2025,0.415962,1.821352,1.659616,0.322851,2.873841,1.025486,3.065815,1.210192,0.858895 +2026,0.467906,1.820325,1.769046,0.325733,2.829953,1.019964,3.381424,1.318901,0.827082 +2027,0.307878,1.586809,1.615740,0.288666,2.767963,1.040631,2.772255,1.008156,0.721722 +2028,0.270876,1.570074,1.720648,0.280728,2.549712,1.040693,2.720559,1.004709,0.718189 +2029,0.232678,1.509975,1.780526,0.383555,2.602069,1.122983,2.567170,0.958634,0.730647 +2030,0.147137,1.591015,1.807315,0.505788,2.698203,1.189790,2.461878,1.031264,0.754155 +2031,0.154523,1.518058,1.818539,0.493469,2.725821,1.153110,2.466089,1.071877,0.762239 +2032,0.171140,1.652519,2.332649,0.528790,2.878121,1.160405,2.773816,1.221084,0.763647 +2033,0.168154,1.611316,2.258104,0.461596,2.670293,1.146366,2.491760,1.138652,0.792403 +2034,0.166387,1.598844,2.295054,0.445965,2.691954,1.154389,2.390950,1.174780,0.665455 +2035,0.168846,1.631779,2.349791,0.443320,2.756552,1.135390,2.422616,1.105567,0.661612 +2036,0.173900,1.690080,2.441845,0.439980,2.860936,1.145885,2.462188,1.142129,0.721888 +2037,0.166064,1.736329,2.535429,0.450350,2.993963,1.166464,2.506264,1.215184,0.787384 +2038,0.174862,1.741541,2.590947,0.443396,3.036464,1.187185,2.486950,1.190728,0.821557 +2039,0.169670,1.766711,2.819755,0.395888,3.144396,1.099472,2.539166,1.167671,0.774401 +2040,0.158342,1.790556,2.885969,0.382803,3.157984,1.133317,2.436097,1.193164,0.810810 +2041,0.156059,1.802938,2.898490,0.387704,3.244412,1.191263,2.465115,1.245131,0.859055 +2042,0.153217,1.800179,2.975342,0.404266,3.393687,1.212832,2.542635,1.319474,0.890109 +2043,0.158617,1.824126,3.075706,0.433313,3.481175,1.240058,2.602159,1.378363,0.920913 +2044,0.139163,1.890954,3.161263,0.459030,3.601084,1.270154,2.685139,1.460349,0.921405 +2045,0.133956,1.939437,3.189294,0.460997,3.706220,1.316607,2.718325,1.509268,0.945207 +2046,0.137652,2.003619,3.211974,0.467564,3.787705,1.377722,2.809215,1.583674,0.936823 +2047,0.142889,2.027699,3.237010,0.481618,3.860097,1.423071,2.881419,1.606334,0.956864 +2048,0.144934,2.050089,3.263034,0.486595,3.923238,1.497705,2.922786,1.674062,0.953635 +2049,0.139210,2.080405,3.307015,0.484645,3.986758,1.526371,2.981777,1.742009,0.953994 +2050,0.225399,2.142084,3.297439,0.499338,4.038809,1.500245,2.993482,1.698395,1.008947 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2025_HOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2025_HOG.csv new file mode 100644 index 0000000..c50b4aa --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2025_HOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.858180,2.642740,3.358270,1.549812,2.864664,1.354117,5.379282,4.756302,2.707125 +2011,0.915348,2.719853,3.525401,1.555518,2.934925,1.416257,5.529355,4.793510,2.548560 +2012,0.879083,2.770951,3.547677,1.491081,3.280124,1.540446,5.730931,4.903691,2.803605 +2013,0.859505,2.891799,3.818099,1.645916,3.261053,1.455897,5.577358,4.972179,2.896845 +2014,0.857731,3.120449,4.045156,1.680470,3.326574,1.544663,5.646739,5.000559,2.784841 +2015,0.905922,3.099138,3.886718,1.574979,3.663951,1.696394,5.951674,5.255469,2.763970 +2016,0.856641,3.156144,3.920965,1.638642,3.735140,1.746690,5.803449,1.581652,2.775395 +2017,0.879541,3.113560,3.997107,1.729600,3.888779,1.767733,5.922212,1.576464,2.827698 +2018,0.930977,3.316355,4.395236,1.846236,4.067651,1.912479,6.697608,1.676031,2.952353 +2019,0.884965,3.507681,4.665439,1.972424,4.325762,1.962087,7.134515,1.985243,2.949311 +2020,0.849804,3.615312,4.578500,1.915619,4.221534,1.967894,7.066121,1.900725,2.847719 +2021,0.878472,3.665614,4.418462,1.893204,3.971677,1.844552,6.865973,1.921409,2.947445 +2022,0.923152,4.044715,4.940085,1.998683,4.159855,1.892741,7.156394,1.914708,2.857152 +2023,0.914369,3.709178,4.648726,1.877739,4.158048,1.861391,6.910273,1.734919,2.869450 +2024,0.957036,3.906965,4.966578,2.048244,4.376607,2.097415,7.426676,2.155910,2.973608 +2025,0.937966,3.654048,5.179120,2.077749,4.469378,2.111273,7.503117,2.100823,2.843258 +2026,0.860033,3.759808,5.488719,2.125013,4.646142,2.230853,7.636079,2.131084,2.739021 +2027,0.818995,3.774266,5.671466,2.190878,4.738679,2.335734,7.659956,2.100113,2.732933 +2028,0.795159,3.716908,5.789507,2.173376,4.717777,2.291502,7.557886,2.087887,2.772114 +2029,0.790301,3.712479,5.954311,2.195643,4.672873,2.306812,7.466064,2.086488,2.829799 +2030,0.744455,3.873317,6.039167,2.226819,4.578413,2.233391,7.538048,2.194306,2.921344 +2031,0.722228,3.884992,6.186546,2.296895,4.417633,2.170599,7.592106,2.241075,2.999857 +2032,0.741138,3.944881,6.693543,2.414316,4.601673,2.200978,7.769639,2.365347,3.048926 +2033,0.733747,3.943127,6.668369,2.318362,4.486273,2.200232,7.632854,2.325531,3.145908 +2034,0.735347,3.933705,6.625295,2.316933,4.505866,2.237109,7.758178,2.308635,3.176948 +2035,0.739885,3.800751,6.627602,2.316072,4.430108,2.198426,7.729087,2.271489,3.205086 +2036,0.747582,3.739552,6.545062,2.257230,4.338734,2.156050,7.367331,2.397209,3.013469 +2037,0.757997,3.743082,6.556759,2.263292,4.274085,2.167767,6.954051,2.404191,3.038831 +2038,0.749867,3.694221,6.133348,2.231797,4.256666,2.189104,6.922351,2.272732,3.078229 +2039,0.738831,3.544951,6.001094,2.216016,4.208876,2.141802,6.825901,2.137090,3.124984 +2040,0.745195,3.550549,5.973637,2.230641,4.339911,2.126414,6.780476,2.205422,3.147804 +2041,0.750859,3.586475,6.025984,2.242017,4.432344,2.154529,6.878246,2.189955,3.201372 +2042,0.762990,3.519784,6.201727,2.267177,4.433652,2.144847,6.934600,2.228513,3.202829 +2043,0.771564,3.577761,6.316656,2.304414,4.492250,2.215720,7.085598,2.279858,3.196146 +2044,0.751563,3.673051,6.423592,2.318251,4.486453,2.223858,7.117551,2.359413,3.216992 +2045,0.716803,3.723683,6.500850,2.359171,4.477860,2.262689,7.170787,2.461398,3.226556 +2046,0.718965,3.795890,6.540277,2.392810,4.501936,2.342593,7.244801,2.465548,3.255445 +2047,0.724840,3.895167,6.419639,2.406723,4.490632,2.399555,7.288305,2.517160,3.255843 +2048,0.726701,3.907527,6.308069,2.422065,4.457742,2.390973,7.159330,2.572379,3.249705 +2049,0.725793,3.848761,6.439577,2.387223,4.593865,2.408647,7.386389,2.626165,3.269576 +2050,0.713332,3.860977,6.387714,2.353932,4.613367,2.433508,7.393847,2.731677,3.273041 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2025_LOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2025_LOG.csv new file mode 100644 index 0000000..3ac2e3a --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2025_LOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.858180,2.642740,3.358270,1.549812,2.864664,1.354117,5.379282,4.756302,2.707125 +2011,0.915348,2.719853,3.525401,1.555518,2.934925,1.416257,5.529355,4.793510,2.548560 +2012,0.879083,2.770951,3.547677,1.491081,3.280124,1.540446,5.730931,4.903691,2.803605 +2013,0.859505,2.891799,3.818099,1.645916,3.261053,1.455897,5.577358,4.972179,2.896845 +2014,0.857731,3.120449,4.045156,1.680470,3.326574,1.544663,5.646739,5.000559,2.784841 +2015,0.905922,3.099138,3.886718,1.574979,3.663951,1.696394,5.951674,5.255469,2.763970 +2016,0.856641,3.156144,3.920965,1.638642,3.735140,1.746690,5.803449,1.581652,2.775395 +2017,0.879541,3.113560,3.997107,1.729600,3.888779,1.767733,5.922212,1.576464,2.827698 +2018,0.930977,3.316355,4.395236,1.846236,4.067651,1.912479,6.697608,1.676031,2.952353 +2019,0.884965,3.507681,4.665439,1.972424,4.325762,1.962087,7.134515,1.985243,2.949311 +2020,0.849804,3.615312,4.578500,1.915619,4.221534,1.967894,7.066121,1.900725,2.847719 +2021,0.878472,3.665614,4.418462,1.893204,3.971677,1.844552,6.865973,1.921409,2.947445 +2022,0.923152,4.044715,4.940085,1.998683,4.159855,1.892741,7.156394,1.914708,2.857152 +2023,0.914369,3.709178,4.648726,1.877739,4.158048,1.861391,6.910273,1.734919,2.869450 +2024,0.962553,3.909995,4.964130,2.049968,4.378995,2.098575,7.414999,2.159559,2.974449 +2025,0.930546,3.749795,5.087374,2.005418,4.343349,2.031063,7.370770,2.058202,2.728772 +2026,0.781939,3.475330,4.620926,1.842845,3.891988,1.784934,5.958117,1.781902,2.557914 +2027,0.742323,3.444869,4.707423,1.880801,3.828328,1.835414,6.003426,1.754856,2.526092 +2028,0.725009,3.364414,4.696098,1.881701,3.591180,1.783213,5.845942,1.713990,2.556307 +2029,0.720008,3.386134,4.647316,1.897199,3.361863,1.719558,5.699160,1.696235,2.585802 +2030,0.699351,3.335698,4.641392,2.038857,3.190711,1.629336,5.668550,1.677637,2.614153 +2031,0.679255,3.142804,4.456717,1.944253,2.885270,1.470770,5.419774,1.608598,2.628891 +2032,0.694852,3.056186,4.594091,2.025953,2.987373,1.521027,5.548148,1.763594,2.637849 +2033,0.682768,2.965106,4.325788,1.940535,2.804255,1.469214,5.332105,1.746244,2.634555 +2034,0.676451,2.821101,4.196589,1.874597,2.729338,1.399418,5.265339,1.690408,2.577221 +2035,0.679203,2.774165,4.056654,1.840279,2.722006,1.387416,5.066918,1.656034,2.517057 +2036,0.675569,2.661531,3.955245,1.820973,2.598980,1.295392,4.959162,1.584508,2.446003 +2037,0.649520,2.454186,3.762507,1.831730,2.459701,1.225727,4.837954,1.567165,2.364758 +2038,0.645287,2.387821,3.630200,1.815282,2.406118,1.169368,4.794869,1.473797,2.291434 +2039,0.653401,2.414410,3.649924,1.826088,2.401801,1.127985,4.765383,1.487857,2.252511 +2040,0.655786,2.393730,3.580663,1.829875,2.404084,1.060108,4.802187,1.457779,2.281605 +2041,0.656713,2.350435,3.536101,1.785332,2.420143,1.008874,4.750147,1.393410,2.238770 +2042,0.654128,2.321419,3.480375,1.763587,2.433650,1.021078,4.739210,1.361234,2.244678 +2043,0.657176,2.332030,3.467469,1.775419,2.402768,1.043947,4.753344,1.362778,2.247817 +2044,0.621498,2.330870,3.439527,1.802037,2.371214,1.045289,4.766046,1.392945,2.278843 +2045,0.605406,2.309155,3.448897,1.802026,2.302510,1.024428,4.775317,1.404418,2.285449 +2046,0.609633,2.310539,3.494394,1.805467,2.268174,1.023384,4.763526,1.375992,2.290656 +2047,0.609062,2.304937,3.507108,1.829551,2.242798,1.011038,4.806602,1.401801,2.294402 +2048,0.608704,2.319615,3.543561,1.830128,2.267759,1.015026,4.840192,1.456912,2.322879 +2049,0.608873,2.317298,3.585941,1.837048,2.260479,1.036841,4.846149,1.501403,2.333663 +2050,0.608951,2.297522,3.602013,1.850036,2.244589,1.030486,4.884940,1.540658,2.361397 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2025_reference.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2025_reference.csv new file mode 100644 index 0000000..da86224 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2025_reference.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.858180,2.642740,3.358270,1.549812,2.864664,1.354117,5.379282,4.756302,2.707125 +2011,0.915348,2.719853,3.525401,1.555518,2.934925,1.416257,5.529355,4.793510,2.548560 +2012,0.879083,2.770951,3.547677,1.491081,3.280124,1.540446,5.730931,4.903691,2.803605 +2013,0.859505,2.891799,3.818099,1.645916,3.261053,1.455897,5.577358,4.972179,2.896845 +2014,0.857731,3.120449,4.045156,1.680470,3.326574,1.544663,5.646739,5.000559,2.784841 +2015,0.905922,3.099138,3.886718,1.574979,3.663951,1.696394,5.951674,5.255469,2.763970 +2016,0.856641,3.156144,3.920965,1.638642,3.735140,1.746690,5.803449,1.581652,2.775395 +2017,0.879541,3.113560,3.997107,1.729600,3.888779,1.767733,5.922212,1.576464,2.827698 +2018,0.930977,3.316355,4.395236,1.846236,4.067651,1.912479,6.697608,1.676031,2.952353 +2019,0.884965,3.507681,4.665439,1.972424,4.325762,1.962087,7.134515,1.985243,2.949311 +2020,0.849804,3.615312,4.578500,1.915619,4.221534,1.967894,7.066121,1.900725,2.847719 +2021,0.878472,3.665614,4.418462,1.893204,3.971677,1.844552,6.865973,1.921409,2.947445 +2022,0.923152,4.044715,4.940085,1.998683,4.159855,1.892741,7.156394,1.914708,2.857152 +2023,0.914369,3.709178,4.648726,1.877739,4.158048,1.861391,6.910273,1.734919,2.869450 +2024,0.962173,3.903117,4.963340,2.048189,4.375371,2.097466,7.421416,2.159530,2.982530 +2025,0.935287,3.661196,5.156418,2.129834,4.476222,2.089903,7.237640,2.086673,2.807245 +2026,0.848885,3.692177,5.073037,2.030126,4.465045,2.052864,6.971332,1.988881,2.694449 +2027,0.807334,3.721632,5.310193,2.096073,4.458486,2.121335,7.088854,1.999269,2.667008 +2028,0.788003,3.668039,5.422043,2.115744,4.351885,2.120489,7.020803,1.959295,2.730486 +2029,0.767319,3.644482,5.623929,2.153275,4.227018,2.085303,6.950130,1.954141,2.789246 +2030,0.715819,3.723123,5.640369,2.207593,4.102344,2.000149,6.981818,2.035256,2.874931 +2031,0.736064,3.624680,5.612779,2.187774,3.890681,1.956317,6.930852,2.067181,2.935515 +2032,0.752671,3.716164,6.106751,2.346345,4.012863,2.020668,7.100438,2.145240,2.998962 +2033,0.746767,3.659340,5.944864,2.309086,3.806603,1.933828,6.931971,1.967468,2.933176 +2034,0.733941,3.596563,5.637051,2.168169,3.651626,1.848481,6.835082,1.903266,2.945720 +2035,0.737544,3.614042,5.440754,2.135197,3.610039,1.810854,6.454259,1.936649,2.938173 +2036,0.742872,3.597095,5.228507,2.114645,3.659100,1.762085,6.180867,1.936809,2.895602 +2037,0.736183,3.445315,5.027087,2.105235,3.464379,1.709159,6.054377,1.914083,2.844030 +2038,0.742570,3.288879,4.835952,2.094151,3.479662,1.657892,6.006368,1.916980,2.813577 +2039,0.732698,3.218215,4.804575,2.066591,3.457128,1.614115,5.997215,1.950361,2.782817 +2040,0.741344,3.226690,4.854189,2.076052,3.408876,1.621953,6.087286,1.974375,2.780822 +2041,0.742691,3.255468,4.917473,2.094103,3.422421,1.639145,6.161439,1.985551,2.793409 +2042,0.754417,3.196219,4.942109,2.077646,3.412856,1.644759,6.188160,2.049288,2.748004 +2043,0.758587,3.258941,5.006926,2.081888,3.378010,1.624161,6.185266,2.057952,2.793068 +2044,0.734295,3.302742,5.082698,2.062417,3.311311,1.641373,6.232446,2.095899,2.783741 +2045,0.692461,3.395932,5.035407,2.105197,3.316263,1.651322,6.209975,2.119609,2.772924 +2046,0.707880,3.377838,5.099153,2.118033,3.285728,1.651759,6.231880,2.093664,2.733654 +2047,0.709487,3.474578,4.917622,2.137938,3.267011,1.640283,6.275684,2.094466,2.770767 +2048,0.713195,3.520399,4.895315,2.163854,3.225357,1.657869,6.278858,2.157986,2.782060 +2049,0.725422,3.449674,4.902303,2.172264,3.207612,1.695295,6.269968,2.213768,2.792679 +2050,0.716690,3.453480,4.926717,2.197461,3.263909,1.705988,6.099909,2.264314,2.801707 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2026_HOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2026_HOG.csv new file mode 100644 index 0000000..f10f4be --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2026_HOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.858180,2.642740,3.358270,1.549810,2.864660,1.354120,5.379280,4.756300,2.707130 +2011,0.915350,2.719850,3.525400,1.555520,2.934920,1.416260,5.529350,4.793510,2.548560 +2012,0.879080,2.770950,3.547680,1.491080,3.280120,1.540450,5.730930,4.903690,2.803610 +2013,0.859500,2.891800,3.818100,1.645920,3.261050,1.455900,5.577360,4.972180,2.896840 +2014,0.857730,3.120450,4.045160,1.680470,3.326570,1.544660,5.646740,5.000560,2.784840 +2015,0.905920,3.099140,3.886720,1.574980,3.663950,1.696390,5.951670,5.255470,2.763970 +2016,0.856640,3.156140,3.920960,1.638640,3.735140,1.746690,5.803450,1.581650,2.775400 +2017,0.879540,3.113560,3.997110,1.729600,3.888780,1.767730,5.922210,1.576460,2.827700 +2018,0.930980,3.316360,4.395240,1.846240,4.067650,1.912480,6.697610,1.676030,2.952350 +2019,0.884970,3.507680,4.665440,1.972420,4.325760,1.962090,7.134520,1.985240,2.949310 +2020,0.849800,3.615310,4.578500,1.915620,4.221530,1.967890,7.066120,1.900720,2.847720 +2021,0.878470,3.665610,4.418460,1.893200,3.971680,1.844550,6.865970,1.921410,2.947450 +2022,0.923150,4.044720,4.940080,1.998680,4.159860,1.892740,7.156390,1.914710,2.857150 +2023,0.914370,3.709180,4.648730,1.877740,4.158050,1.861390,6.910270,1.734920,2.869450 +2024,0.962170,3.903120,4.963340,2.048190,4.375370,2.097470,7.421420,2.159530,2.982530 +2025,0.945037,3.860492,5.015822,2.087011,4.549070,2.021427,7.369015,2.215069,2.788580 +2026,0.987475,3.733586,5.023307,1.980979,4.487000,2.000728,7.778195,2.263465,2.761178 +2027,0.827551,3.635760,5.207797,2.081153,4.596195,2.127968,7.426451,2.089241,2.638196 +2028,0.809603,3.672941,5.313888,2.068848,4.547182,2.180254,7.392116,2.123260,2.642632 +2029,0.732051,3.591873,5.322313,2.121058,4.711706,2.242150,7.534577,2.057393,2.655006 +2030,0.667574,3.739352,5.314254,2.196064,4.689672,2.293315,7.542428,2.180421,2.678366 +2031,0.643146,3.626257,5.313427,2.215526,4.790040,2.332252,7.705997,2.328898,2.728465 +2032,0.679556,3.701978,6.114575,2.340974,4.989939,2.397843,8.011409,2.515427,2.803385 +2033,0.687615,3.600985,6.259030,2.350158,5.059957,2.406489,8.150441,2.877159,2.536253 +2034,0.686607,3.578408,6.443136,2.374649,5.164662,2.438355,8.261511,2.978614,2.583217 +2035,0.678884,3.610814,6.553623,2.410021,5.289208,2.456965,8.369002,3.110021,2.628823 +2036,0.682520,3.654597,6.680509,2.465123,5.416865,2.487286,8.444514,3.235569,2.683835 +2037,0.667253,3.636546,6.782361,2.515822,5.612709,2.489349,8.572031,3.293550,2.769227 +2038,0.661932,3.652292,6.861833,2.539302,5.780676,2.516107,8.704844,3.167639,3.038968 +2039,0.664658,3.756619,6.976791,2.568685,6.020574,2.498929,8.866220,3.112101,3.227316 +2040,0.658783,3.752446,7.152576,2.576104,6.141085,2.504207,8.994414,3.200019,3.312811 +2041,0.657174,3.792520,7.226554,2.616238,6.275909,2.529565,9.176105,3.317863,3.352399 +2042,0.660726,3.911083,7.385400,2.647443,6.353165,2.600203,9.328657,3.430503,3.406197 +2043,0.661555,4.061972,7.520255,2.672293,6.427284,2.654492,9.500743,3.513082,3.458118 +2044,0.639036,4.171165,7.689994,2.736487,6.526492,2.724593,9.640671,3.620002,3.483273 +2045,0.609310,4.309261,7.767219,2.771669,6.608011,2.770861,9.747934,3.671841,3.528038 +2046,0.613073,4.371390,7.915614,2.780104,6.675318,2.796036,9.891739,3.728738,3.546356 +2047,0.609351,4.436017,8.041780,2.806571,6.729046,2.823490,10.062180,3.811924,3.576517 +2048,0.608406,4.464117,8.167671,2.778922,6.805774,2.864776,10.210494,3.900021,3.588124 +2049,0.605763,4.526435,8.235657,2.816069,6.863463,2.887931,10.420832,3.965691,3.624958 +2050,0.626196,4.589968,8.242544,2.879888,6.905205,2.910258,10.557559,3.989760,3.704296 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2026_LOG.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2026_LOG.csv new file mode 100644 index 0000000..9e34fa7 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2026_LOG.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.858180,2.642740,3.358270,1.549810,2.864660,1.354120,5.379280,4.756300,2.707130 +2011,0.915350,2.719850,3.525400,1.555520,2.934920,1.416260,5.529350,4.793510,2.548560 +2012,0.879080,2.770950,3.547680,1.491080,3.280120,1.540450,5.730930,4.903690,2.803610 +2013,0.859500,2.891800,3.818100,1.645920,3.261050,1.455900,5.577360,4.972180,2.896840 +2014,0.857730,3.120450,4.045160,1.680470,3.326570,1.544660,5.646740,5.000560,2.784840 +2015,0.905920,3.099140,3.886720,1.574980,3.663950,1.696390,5.951670,5.255470,2.763970 +2016,0.856640,3.156140,3.920960,1.638640,3.735140,1.746690,5.803450,1.581650,2.775400 +2017,0.879540,3.113560,3.997110,1.729600,3.888780,1.767730,5.922210,1.576460,2.827700 +2018,0.930980,3.316360,4.395240,1.846240,4.067650,1.912480,6.697610,1.676030,2.952350 +2019,0.884970,3.507680,4.665440,1.972420,4.325760,1.962090,7.134520,1.985240,2.949310 +2020,0.849800,3.615310,4.578500,1.915620,4.221530,1.967890,7.066120,1.900720,2.847720 +2021,0.878470,3.665610,4.418460,1.893200,3.971680,1.844550,6.865970,1.921410,2.947450 +2022,0.923150,4.044720,4.940080,1.998680,4.159860,1.892740,7.156390,1.914710,2.857150 +2023,0.914370,3.709180,4.648730,1.877740,4.158050,1.861390,6.910270,1.734920,2.869450 +2024,0.962170,3.903120,4.963340,2.048190,4.375370,2.097470,7.421420,2.159530,2.982530 +2025,0.943428,3.824324,5.052043,2.075537,4.583968,2.012200,7.365664,2.212374,2.781979 +2026,0.987331,3.864871,5.001111,1.936648,4.356728,1.904128,7.699248,2.255174,2.714072 +2027,0.814699,3.473172,4.545505,1.869176,4.050457,1.754747,6.422260,1.823242,2.567224 +2028,0.759689,3.392891,4.503676,1.834690,3.833844,1.757903,6.196378,1.793492,2.510289 +2029,0.720305,3.338682,4.507474,1.909920,3.558805,1.746712,5.907505,1.726727,2.488916 +2030,0.679239,3.280586,4.395971,1.993817,3.602709,1.746778,5.891112,1.791341,2.504655 +2031,0.667293,3.154895,4.266676,1.966177,3.507431,1.634011,5.840665,1.779657,2.452621 +2032,0.675957,3.204367,4.443736,2.092644,3.652616,1.713040,6.407189,2.021989,2.521406 +2033,0.664833,3.096838,4.330495,2.035296,3.554395,1.673788,6.102612,1.928428,2.498123 +2034,0.656634,2.994922,4.251393,1.947029,3.560668,1.641038,5.847789,1.840954,2.424235 +2035,0.649742,2.888081,4.115732,1.896545,3.526124,1.558864,5.600486,1.766208,2.364536 +2036,0.651234,2.839342,4.014419,1.870778,3.504481,1.505583,5.264624,1.737461,2.353991 +2037,0.645901,2.761872,3.925659,1.845425,3.522559,1.468723,5.248013,1.728174,2.315224 +2038,0.642371,2.749322,3.908282,1.830415,3.493361,1.453365,5.284578,1.712330,2.268514 +2039,0.628123,2.708414,3.960386,1.696099,3.373119,1.566121,5.338129,1.649599,2.220009 +2040,0.614494,2.664435,3.929336,1.689776,3.274434,1.555270,5.387284,1.650336,2.218868 +2041,0.612308,2.663845,3.886926,1.700919,3.135031,1.530226,5.381207,1.695383,2.196539 +2042,0.601879,2.636851,3.862782,1.693417,3.158982,1.547435,5.385444,1.721956,2.191618 +2043,0.608784,2.607196,3.856205,1.686449,3.173237,1.530275,5.427279,1.728730,2.193652 +2044,0.576423,2.602728,3.879140,1.654073,3.184794,1.526655,5.452078,1.754698,2.172728 +2045,0.559948,2.560731,3.879535,1.648236,3.191702,1.504468,5.449376,1.785650,2.170413 +2046,0.555836,2.544347,3.866001,1.646746,3.211336,1.493087,5.431397,1.786631,2.163434 +2047,0.555495,2.514506,3.814792,1.677195,3.229958,1.486906,5.431805,1.804871,2.167439 +2048,0.553564,2.510631,3.808950,1.676222,3.245131,1.502543,5.441922,1.823724,2.165981 +2049,0.550647,2.497353,3.839051,1.672237,3.226012,1.495376,5.444039,1.842031,2.185901 +2050,0.551988,2.479117,3.827497,1.670714,3.194472,1.481388,5.433867,1.833009,2.212266 diff --git a/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2026_reference.csv b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2026_reference.csv new file mode 100644 index 0000000..54e63a9 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of alpha regression/ng_tot_demand_AEO_2026_reference.csv @@ -0,0 +1,42 @@ +year,New_England,Mid_Atlantic,East_North_Central,West_North_Central,South_Atlantic,East_South_Central,West_South_Central,Mountain,Pacific +2010,0.858180,2.642740,3.358270,1.549810,2.864660,1.354120,5.379280,4.756300,2.707130 +2011,0.915350,2.719850,3.525400,1.555520,2.934920,1.416260,5.529350,4.793510,2.548560 +2012,0.879080,2.770950,3.547680,1.491080,3.280120,1.540450,5.730930,4.903690,2.803610 +2013,0.859500,2.891800,3.818100,1.645920,3.261050,1.455900,5.577360,4.972180,2.896840 +2014,0.857730,3.120450,4.045160,1.680470,3.326570,1.544660,5.646740,5.000560,2.784840 +2015,0.905920,3.099140,3.886720,1.574980,3.663950,1.696390,5.951670,5.255470,2.763970 +2016,0.856640,3.156140,3.920960,1.638640,3.735140,1.746690,5.803450,1.581650,2.775400 +2017,0.879540,3.113560,3.997110,1.729600,3.888780,1.767730,5.922210,1.576460,2.827700 +2018,0.930980,3.316360,4.395240,1.846240,4.067650,1.912480,6.697610,1.676030,2.952350 +2019,0.884970,3.507680,4.665440,1.972420,4.325760,1.962090,7.134520,1.985240,2.949310 +2020,0.849800,3.615310,4.578500,1.915620,4.221530,1.967890,7.066120,1.900720,2.847720 +2021,0.878470,3.665610,4.418460,1.893200,3.971680,1.844550,6.865970,1.921410,2.947450 +2022,0.923150,4.044720,4.940080,1.998680,4.159860,1.892740,7.156390,1.914710,2.857150 +2023,0.914370,3.709180,4.648730,1.877740,4.158050,1.861390,6.910270,1.734920,2.869450 +2024,0.962170,3.903120,4.963340,2.048190,4.375370,2.097470,7.421420,2.159530,2.982530 +2025,0.942486,3.825981,5.019777,2.086996,4.549990,2.016593,7.432802,2.189275,2.787714 +2026,0.984773,3.793366,5.029572,2.060658,4.449345,1.982398,7.588451,2.305218,2.753546 +2027,0.826823,3.576899,4.916392,1.982203,4.414279,1.985681,6.998650,2.005402,2.641933 +2028,0.793062,3.555542,5.010098,1.969896,4.194464,1.979965,6.926657,2.006204,2.632389 +2029,0.750938,3.476859,5.038648,2.060655,4.235593,2.053308,6.759314,1.963060,2.642720 +2030,0.661602,3.538993,5.035486,2.174834,4.323945,2.113672,6.625940,2.036885,2.657496 +2031,0.663432,3.454977,5.033191,2.159881,4.352014,2.074529,6.634375,2.084137,2.671026 +2032,0.677784,3.581752,5.534689,2.194160,4.510006,2.083152,6.939042,2.245369,2.675305 +2033,0.675912,3.534634,5.447337,2.134476,4.314651,2.075462,6.685588,2.177674,2.709949 +2034,0.673442,3.524130,5.487921,2.136691,4.340604,2.110237,6.656223,2.225151,2.519226 +2035,0.677331,3.555830,5.551868,2.140519,4.433356,2.085604,6.737303,2.171232,2.508650 +2036,0.683761,3.613431,5.633051,2.145258,4.552836,2.102115,6.793973,2.220229,2.559445 +2037,0.671905,3.658090,5.719880,2.160131,4.697305,2.127887,6.878952,2.304298,2.619508 +2038,0.678061,3.661387,5.763537,2.154860,4.751060,2.153105,6.876564,2.302169,2.704062 +2039,0.670787,3.683881,5.992211,2.114554,4.872325,2.072491,6.957989,2.293129,2.674567 +2040,0.657359,3.706461,6.058501,2.103828,4.898774,2.112561,6.875210,2.333061,2.730716 +2041,0.653394,3.714217,6.060242,2.111508,4.996305,2.176871,6.933425,2.397303,2.795197 +2042,0.649386,3.706437,6.133780,2.133687,5.157597,2.203106,7.031544,2.483388,2.841049 +2043,0.653150,3.724521,6.226460,2.169468,5.253417,2.238532,7.124736,2.555906,2.891721 +2044,0.632348,3.786657,6.304413,2.205308,5.377156,2.279501,7.246978,2.653509,2.912265 +2045,0.626222,3.836870,6.340540,2.220046,5.494728,2.333329,7.336374,2.715578,2.928550 +2046,0.629178,3.899887,6.362288,2.234760,5.596853,2.394007,7.472200,2.804054,2.934182 +2047,0.633655,3.923208,6.389232,2.259722,5.683524,2.448742,7.605679,2.841660,2.969567 +2048,0.634865,3.943225,6.415991,2.275084,5.760139,2.530869,7.685361,2.924933,2.989456 +2049,0.628185,3.970002,6.462479,2.284726,5.835090,2.566808,7.797091,3.008272,2.999893 +2050,0.712875,4.027762,6.452061,2.307911,5.898229,2.547479,7.860336,2.979851,3.069551 diff --git a/aeo_updates/natural_gas_price_regression/outputs of beta regression/beta_regression_summary.csv b/aeo_updates/natural_gas_price_regression/outputs of beta regression/beta_regression_summary.csv new file mode 100644 index 0000000..7e72ca1 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of beta regression/beta_regression_summary.csv @@ -0,0 +1,11 @@ +scope,region,beta,r2,r2_full,n_obs +national,ALL,0.078702,0.620493,0.696840,2106 +regional,New_England,2.359082,0.208966,0.699953,234 +regional,Mid_Atlantic,0.185420,0.229007,0.769863,234 +regional,East_North_Central,0.013440,0.002511,0.753149,234 +regional,West_North_Central,0.513493,0.215771,0.790110,234 +regional,South_Atlantic,0.029744,0.009323,0.489832,234 +regional,East_South_Central,-0.026099,0.002906,0.742087,234 +regional,West_South_Central,-0.126897,0.193671,0.541494,234 +regional,Mountain,0.049527,0.008427,0.670618,234 +regional,Pacific,0.704760,0.448534,0.774171,234 diff --git a/aeo_updates/natural_gas_price_regression/outputs of beta regression/cd_beta0.csv b/aeo_updates/natural_gas_price_regression/outputs of beta regression/cd_beta0.csv new file mode 100644 index 0000000..b5cfcb2 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of beta regression/cd_beta0.csv @@ -0,0 +1,10 @@ +*cendiv,value +New_England,2.359082 +Mid_Atlantic,0.185420 +East_North_Central,0.013440 +West_North_Central,0.513493 +South_Atlantic,0.029744 +East_South_Central,-0.026099 +West_South_Central,-0.126897 +Mountain,0.049527 +Pacific,0.704760 diff --git a/aeo_updates/natural_gas_price_regression/outputs of beta regression/national_beta.csv b/aeo_updates/natural_gas_price_regression/outputs of beta regression/national_beta.csv new file mode 100644 index 0000000..d285858 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/outputs of beta regression/national_beta.csv @@ -0,0 +1,2 @@ +beta +0.078702 diff --git a/aeo_updates/natural_gas_price_regression/run_ng_pipeline.bat b/aeo_updates/natural_gas_price_regression/run_ng_pipeline.bat new file mode 100644 index 0000000..3413f4e --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/run_ng_pipeline.bat @@ -0,0 +1,39 @@ +@echo off +setlocal + +cd /d "%~dp0" + +set "CONFIG=%~1" +if "%CONFIG%"=="" set "CONFIG=aeo_pipeline_config.json" + +python --version >nul 2>&1 +if errorlevel 1 ( + echo ERROR: Python is not available on PATH. + exit /b 9009 +) + +echo [1/4] Running beta regression... +python aeo_beta_regression.py --config "%CONFIG%" +if errorlevel 1 goto :fail + +echo [2/4] Syncing beta outputs to alpha inputs... +python sync_beta_to_alpha_inputs.py --config "%CONFIG%" +if errorlevel 1 goto :fail + +echo [3/4] Running alpha regression... +python aeo_alpha_regression.py --config "%CONFIG%" +if errorlevel 1 goto :fail + +echo [4/4] Generating visualization and validation... +python visualization.py --config "%CONFIG%" +if errorlevel 1 goto :fail + +echo. +echo NG pipeline finished successfully. +exit /b 0 + +:fail +set "CODE=%errorlevel%" +echo. +echo NG pipeline failed with exit code %CODE%. +exit /b %CODE% diff --git a/aeo_updates/natural_gas_price_regression/sync_beta_to_alpha_inputs.py b/aeo_updates/natural_gas_price_regression/sync_beta_to_alpha_inputs.py new file mode 100644 index 0000000..300aed4 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/sync_beta_to_alpha_inputs.py @@ -0,0 +1,122 @@ +""" +Copy beta regression outputs into alpha regression inputs. + +This removes the manual handoff step between: +1) aeo_beta_regression.py +2) aeo_alpha_regression.py + +Copied files: +- outputs of beta regression/cd_beta0.csv -> inputs for alpha regression/cd_beta0.csv +- outputs of beta regression/national_beta.csv -> inputs for alpha regression/national_beta.csv +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from pathlib import Path +from typing import Any + + +def require(condition: bool, message: str) -> None: + """Raise ValueError when condition is false.""" + if not condition: + raise ValueError(message) + + +def resolve_case_insensitive(path: Path) -> Path: + """Resolve a path with case-insensitive matching on each component.""" + if path.exists(): + return path + path = path.resolve() + current = Path(path.anchor) + for part in path.parts[1:]: + if not current.exists(): + return path + try: + matches = [p for p in current.iterdir() if p.name.lower() == part.lower()] + except PermissionError: + return path + if not matches: + return path + current = matches[0] + return current + + +def resolve_path(base_dir: Path, configured_path: str) -> Path: + """Resolve a configured path relative to base_dir.""" + p = Path(configured_path) + if not p.is_absolute(): + p = base_dir / p + return resolve_case_insensitive(p) + + +def load_config(config_path: Path) -> dict[str, Any]: + """Load JSON config file.""" + require(config_path.exists(), f"Config not found: {config_path}") + with config_path.open("r", encoding="utf-8") as f: + cfg = json.load(f) + require(isinstance(cfg, dict), f"Config root must be an object: {config_path}") + return cfg + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Copy beta outputs into alpha input directory.") + parser.add_argument("--config", default="aeo_pipeline_config.json") + parser.add_argument("--beta-output-dir", default="outputs of beta regression") + parser.add_argument( + "--alpha-input-dir", + default=None, + help="Override alpha input directory (default: config paths.input_dir).", + ) + return parser.parse_args() + + +def copy_required_files(beta_out_dir: Path, alpha_input_dir: Path) -> None: + """Copy cd_beta0.csv and national_beta.csv from beta outputs to alpha inputs.""" + copies = [ + ("cd_beta0.csv", "cd_beta0.csv"), + ("national_beta.csv", "national_beta.csv"), + ] + alpha_input_dir.mkdir(parents=True, exist_ok=True) + + for src_name, dst_name in copies: + src = beta_out_dir / src_name + dst = alpha_input_dir / dst_name + require(src.exists(), f"Missing beta output file: {src}") + shutil.copy2(src, dst) + print(f"Copied: {src} -> {dst}") + + +def main() -> int: + args = parse_args() + + script_dir = Path(__file__).resolve().parent + cfg_path = Path(args.config) + if not cfg_path.is_absolute(): + cwd_candidate = resolve_case_insensitive((Path.cwd() / cfg_path).resolve()) + script_candidate = resolve_case_insensitive((script_dir / cfg_path).resolve()) + cfg_path = cwd_candidate if cwd_candidate.exists() else script_candidate + + config = load_config(cfg_path) + base_dir = cfg_path.parent + + input_dir_cfg = str(config.get("paths", {}).get("input_dir", "inputs for alpha regression")) + alpha_input_dir = resolve_path(base_dir, args.alpha_input_dir or input_dir_cfg) + beta_out_dir = resolve_path(base_dir, args.beta_output_dir) + require(beta_out_dir.exists(), f"Beta output directory not found: {beta_out_dir}") + + copy_required_files(beta_out_dir, alpha_input_dir) + print("Beta-to-alpha input sync completed.") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/aeo_updates/natural_gas_price_regression/visualization.py b/aeo_updates/natural_gas_price_regression/visualization.py new file mode 100644 index 0000000..48bbaf5 --- /dev/null +++ b/aeo_updates/natural_gas_price_regression/visualization.py @@ -0,0 +1,1191 @@ +""" +Unified visualization and validation for the NG regression pipeline. + +Consolidates three prior scripts into one entry point: + - Beta raw-data scatter grid (was beta_raw_data_visualization.py) + - Beta/alpha diagnostic plots (was results_visualization.py) + - Validation: actual vs predicted, parity, alpha comparison (was results_validation.py) + +Usage +----- + python visualization.py --config aeo_pipeline_config.json + +Run after: + 1) aeo_beta_regression.py + 2) sync_beta_to_alpha_inputs.py + 3) aeo_alpha_regression.py +""" + +from __future__ import annotations + +import argparse +import json +import logging +import re +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.lines import Line2D +except Exception as exc: # pragma: no cover + raise RuntimeError("matplotlib is required for visualization.py") from exc + +LOGGER = logging.getLogger("visualization") + +# ============================================================================ +# Constants +# ============================================================================ + +CENDIV_CANONICAL = { + "newengland": "NewEngland", + "middleatlantic": "MiddleAtlantic", + "eastnorthcentral": "EastNorthCentral", + "westnorthcentral": "WestNorthCentral", + "southatlantic": "SouthAtlantic", + "eastsouthcentral": "EastSouthCentral", + "westsouthcentral": "WestSouthCentral", + "mountain": "Mountain", + "pacific": "Pacific", +} + +CENDIV_OUTPUT = { + "NewEngland": "New_England", + "MiddleAtlantic": "Mid_Atlantic", + "EastNorthCentral": "East_North_Central", + "WestNorthCentral": "West_North_Central", + "SouthAtlantic": "South_Atlantic", + "EastSouthCentral": "East_South_Central", + "WestSouthCentral": "West_South_Central", + "Mountain": "Mountain", + "Pacific": "Pacific", +} + +SCENARIO_COLOR = { + "reference": "#1f77b4", + "HOG": "#2ca02c", + "LOG": "#d62728", +} + + +# ============================================================================ +# Shared helpers +# ============================================================================ + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def normalize_token(value: Any) -> str: + return re.sub( + r"[^a-z0-9]+", "", + str(value).replace("\xa0", " ").strip().lower(), + ) + + +def any_to_cendiv(value: str) -> str: + token = normalize_token(value) + if token in CENDIV_CANONICAL: + return CENDIV_CANONICAL[token] + for cendiv, out_label in CENDIV_OUTPUT.items(): + if token in {normalize_token(cendiv), normalize_token(out_label)}: + return cendiv + raise ValueError(f"Unknown region label: {value}") + + +def cendiv_output_label(cendiv: str) -> str: + require(cendiv in CENDIV_OUTPUT, f"Unknown cendiv key: {cendiv}") + return CENDIV_OUTPUT[cendiv] + + +def resolve_case_insensitive(path: Path) -> Path: + if path.exists(): + return path + path = path.resolve() + current = Path(path.anchor) + for part in path.parts[1:]: + if not current.exists(): + return path + try: + matches = [p for p in current.iterdir() if p.name.lower() == part.lower()] + except PermissionError: + return path + if not matches: + return path + current = matches[0] + return current + + +def resolve_path(base_dir: Path, configured_path: str) -> Path: + p = Path(configured_path) + if not p.is_absolute(): + p = base_dir / p + return resolve_case_insensitive(p) + + +def load_config(config_path: Path) -> dict[str, Any]: + require(config_path.exists(), f"Config not found: {config_path}") + with config_path.open("r", encoding="utf-8") as f: + cfg = json.load(f) + require(isinstance(cfg, dict), f"Config root must be an object: {config_path}") + return cfg + + +def region_order_from_config(config: dict[str, Any]) -> list[str]: + configured = config["ng"]["regions"] + return [cendiv_output_label(any_to_cendiv(x)) for x in configured] + + +def zero_ng_betas_in_first_model_year( + frame: pd.DataFrame, + first_model_year: int, + beta_reg_col: str = "beta_reg", + beta_nat_col: str = "beta_nat", +) -> None: + first_year_mask = frame["year"] == first_model_year + frame.loc[first_year_mask, beta_reg_col] = 0.0 + frame.loc[first_year_mask, beta_nat_col] = 0.0 + + +def zero_ng_terms_in_first_model_year( + frame: pd.DataFrame, + first_model_year: int, + regional_term_col: str = "term_reg", + national_term_col: str = "term_nat", +) -> None: + first_year_mask = frame["year"] == first_model_year + frame.loc[first_year_mask, regional_term_col] = 0.0 + frame.loc[first_year_mask, national_term_col] = 0.0 + + +def sanitize_file_component(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "_", value) + + +def compute_r2(actual: pd.Series | np.ndarray, predicted: pd.Series | np.ndarray) -> float: + a = np.asarray(actual, dtype=float) + p = np.asarray(predicted, dtype=float) + mask = np.isfinite(a) & np.isfinite(p) + if not np.any(mask): + return float("nan") + a, p = a[mask], p[mask] + ss_res = float(np.sum(np.square(a - p))) + ss_tot = float(np.sum(np.square(a - np.mean(a)))) + if ss_tot <= 0: + return float("nan") + return 1.0 - (ss_res / ss_tot) + + +def compute_fit_metrics(actual: pd.Series | np.ndarray, predicted: pd.Series | np.ndarray) -> dict[str, float]: + a = np.asarray(actual, dtype=float) + p = np.asarray(predicted, dtype=float) + mask = np.isfinite(a) & np.isfinite(p) + if not np.any(mask): + return {"n_obs": 0.0, "mae": float("nan"), "rmse": float("nan"), + "max_abs": float("nan"), "r2": float("nan")} + a, p = a[mask], p[mask] + err = a - p + return { + "n_obs": float(len(a)), + "mae": float(np.mean(np.abs(err))), + "rmse": float(np.sqrt(np.mean(np.square(err)))), + "max_abs": float(np.max(np.abs(err))), + "r2": compute_r2(a, p), + } + + +def summarize_fit( + frame: pd.DataFrame, + group_cols: list[str], + actual_col: str, + predicted_col: str, + mae_col: str, + rmse_col: str, + max_abs_col: str, + r2_col: str, +) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + for keys, grp in frame.groupby(group_cols, sort=True): + if not isinstance(keys, tuple): + keys = (keys,) + row = {group_cols[i]: keys[i] for i in range(len(group_cols))} + m = compute_fit_metrics(grp[actual_col], grp[predicted_col]) + row["n_obs"] = int(m["n_obs"]) + row[mae_col] = m["mae"] + row[rmse_col] = m["rmse"] + row[max_abs_col] = m["max_abs"] + row[r2_col] = m["r2"] + rows.append(row) + out = pd.DataFrame(rows) + if out.empty: + return out + return out.sort_values(group_cols).reset_index(drop=True) + + +# ============================================================================ +# Beta / alpha CSV loaders +# ============================================================================ + +def load_regional_beta_from_csv(source_path: Path) -> dict[str, float]: + require(source_path.exists(), f"Missing regional beta file: {source_path}") + df = pd.read_csv(source_path) + cendiv_col = next( + (c for c in df.columns if normalize_token(c).endswith("cendiv")), None) + value_col = next( + (c for c in df.columns if normalize_token(c) == "value"), None) + require(cendiv_col is not None and value_col is not None, + f"Missing cendiv/value columns in {source_path}") + beta_map: dict[str, float] = {} + for raw_label, raw_beta in zip(df[cendiv_col], df[value_col]): + beta_val = pd.to_numeric([raw_beta], errors="coerce")[0] + if pd.isna(beta_val): + continue + region_label = cendiv_output_label(any_to_cendiv(str(raw_label))) + beta_map[region_label] = float(beta_val) + return beta_map + + +def load_national_beta_from_csv(source_path: Path) -> float: + require(source_path.exists(), f"Missing national beta file: {source_path}") + df = pd.read_csv(source_path) + require("beta" in df.columns, f"Missing 'beta' column in {source_path}") + beta_vals = pd.to_numeric(df["beta"], errors="coerce").dropna() + require(not beta_vals.empty, f"No numeric beta value found in {source_path}") + return float(beta_vals.iloc[0]) + + +def load_regional_beta_map(alpha_out_dir: Path, alpha_input_dir: Path) -> tuple[dict[str, float], Path]: + candidates = [alpha_out_dir / "cd_beta0.csv", alpha_input_dir / "cd_beta0.csv"] + source_path = next((p for p in candidates if p.exists()), None) + require(source_path is not None, f"Could not find cd_beta0.csv in: {candidates}") + return load_regional_beta_from_csv(source_path), source_path + + +def load_national_beta(alpha_out_dir: Path, alpha_input_dir: Path, beta_out_dir: Path) -> tuple[float, Path]: + candidates = [ + alpha_out_dir / "national_beta.csv", + alpha_input_dir / "national_beta.csv", + beta_out_dir / "national_beta.csv", + ] + source_path = next((p for p in candidates if p.exists()), None) + require(source_path is not None, f"Could not find national_beta.csv in: {candidates}") + return load_national_beta_from_csv(source_path), source_path + + +def load_alpha_from_beta_step(beta_out_dir: Path) -> tuple[pd.DataFrame, Path, list[str]]: + source_path = beta_out_dir / "alpha_from_beta_regression.csv" + require(source_path.exists(), + "Missing alpha_from_beta_regression.csv. Re-run aeo_beta_regression.py first.") + df = pd.read_csv(source_path) + if "scenario_id" in df.columns: + need = {"scenario_id", "region", "year", "alpha_2004"} + require(not (need - set(df.columns)), + f"Missing columns in {source_path}: {sorted(need - set(df.columns))}") + out = df[["scenario_id", "region", "year", "alpha_2004"]].copy() + out["scenario_id"] = out["scenario_id"].astype(str) + merge_cols = ["scenario_id", "region", "year"] + else: + need = {"region", "year", "alpha_2004"} + require(not (need - set(df.columns)), + f"Missing columns in {source_path}: {sorted(need - set(df.columns))}") + out = df[["region", "year", "alpha_2004"]].copy() + merge_cols = ["region", "year"] + + out["region"] = out["region"].astype(str).map( + lambda x: cendiv_output_label(any_to_cendiv(x))) + out["year"] = pd.to_numeric(out["year"], errors="coerce") + out["alpha_2004"] = pd.to_numeric(out["alpha_2004"], errors="coerce") + require(not out["year"].isna().any(), f"Non-numeric year in {source_path}") + require(not out["alpha_2004"].isna().any(), f"Non-numeric alpha_2004 in {source_path}") + out["year"] = out["year"].astype(int) + out = out.rename(columns={"alpha_2004": "alpha1"}) + return out[merge_cols + ["alpha1"]], source_path, merge_cols + + +# ============================================================================ +# Scenario discovery helpers +# ============================================================================ + +def scenario_display_label(suffix: str, scenario_id: str) -> str: + t = normalize_token(suffix) + if t.startswith("ref"): + return "reference" + if t in {"hog", "highogs"}: + return "HOG" + if t in {"log", "lowogs"}: + return "LOG" + return scenario_id + + +def discover_output_scenarios_with_labels( + alpha_out_dir: Path, aeo_year: int, +) -> list[tuple[str, str, str]]: + """Returns list of (suffix, scenario_id, display_label).""" + mapping_path = alpha_out_dir / "raw_aeo_data" / "selected_scenarios_outputs.csv" + suffix_to_scenario: dict[str, str] = {} + if mapping_path.exists(): + map_df = pd.read_csv(mapping_path) + if {"file_suffix", "scenario_id"}.issubset(map_df.columns): + for row in map_df.itertuples(index=False): + suffix_to_scenario[str(row.file_suffix)] = str(row.scenario_id) + + pattern = re.compile(rf"^alpha_AEO_{aeo_year}_(.+)\.csv$") + rows: list[tuple[str, str, str]] = [] + for path in sorted(alpha_out_dir.glob(f"alpha_AEO_{aeo_year}_*.csv")): + match = pattern.match(path.name) + if not match: + continue + suffix = match.group(1) + scenario_id = suffix_to_scenario.get(suffix, suffix) + label = scenario_display_label(suffix, scenario_id) + rows.append((suffix, scenario_id, label)) + require(rows, f"No alpha scenario files found in {alpha_out_dir}") + order_index = {"reference": 0, "HOG": 1, "LOG": 2} + rows.sort(key=lambda x: (order_index.get(x[2], 99), x[2], x[1])) + return rows + + +def discover_output_scenarios(alpha_out_dir: Path, aeo_year: int) -> list[tuple[str, str]]: + """Returns list of (suffix, scenario_id).""" + return [(s, sid) for s, sid, _ in discover_output_scenarios_with_labels(alpha_out_dir, aeo_year)] + + +# ============================================================================ +# Wide-series loader +# ============================================================================ + +def load_wide_series( + csv_path: Path, + value_col: str, + scenario_id: str, + region_order: list[str], + scenario_label: str | None = None, +) -> pd.DataFrame: + require(csv_path.exists(), f"Missing file: {csv_path}") + wide = pd.read_csv(csv_path) + year_col = "t" if "t" in wide.columns else ("year" if "year" in wide.columns else None) + require(year_col is not None, f"Missing year column (t/year): {csv_path}") + missing_regions = [c for c in region_order if c not in wide.columns] + require(not missing_regions, f"Missing region columns in {csv_path}: {missing_regions}") + long = wide[[year_col, *region_order]].melt( + id_vars=[year_col], var_name="region", value_name=value_col) + long["year"] = pd.to_numeric(long[year_col], errors="coerce") + long[value_col] = pd.to_numeric(long[value_col], errors="coerce") + require(not long["year"].isna().any(), f"Non-numeric year in {csv_path}") + require(not long[value_col].isna().any(), f"Non-numeric values in {csv_path}") + long["year"] = long["year"].astype(int) + long["scenario_id"] = scenario_id + cols = ["scenario_id", "region", "year", value_col] + if scenario_label is not None: + long["scenario_label"] = scenario_label + cols = ["scenario_id", "scenario_label", "region", "year", value_col] + return long[cols] + + +# ============================================================================ +# Part 1: Beta raw-data scatter grid +# ============================================================================ + +_RAW_KEY_COLS = ["period", "scenario", "regionId"] +_RAW_READ_COLS = ["period", "scenario", "scenarioDescription", "regionId", + "regionName", "value"] + + +def _coerce_numeric(df: pd.DataFrame, col: str) -> pd.DataFrame: + df[col] = pd.to_numeric(df[col], errors="coerce") + return df.dropna(subset=[col]) + + +def _load_beta_include_scenarios(config: dict[str, Any]) -> list[str]: + aeo_year = config.get("aeo_year") + include = (config.get("scenarios", {}) + .get("beta_regression", {}) + .get("include", [])) + if not isinstance(include, list): + raise ValueError("Expected list at scenarios.beta_regression.include") + resolved: list[str] = [] + for item in include: + token = str(item) + if aeo_year is not None: + token = token.replace("{aeo_year}", str(aeo_year)) + resolved.append(token) + return resolved + + +def _filter_to_beta_include(df: pd.DataFrame, include_tokens: list[str]) -> pd.DataFrame: + if not include_tokens: + return df + include_norm = {normalize_token(t) for t in include_tokens} + scenario_norm = df["scenario"].map(normalize_token) + desc_norm = df["scenarioDescription"].fillna("").map(normalize_token) + keep_mask = scenario_norm.isin(include_norm) | desc_norm.isin(include_norm) + filtered = df[keep_mask].copy() + if filtered.empty: + available = sorted(df["scenario"].unique()) + raise ValueError( + f"No rows matched beta_regression.include. " + f"Configured: {include_tokens}. Available: {available}") + return filtered + + +def _load_and_merge_raw(demand_csv: Path, price_csv: Path) -> pd.DataFrame: + demand = pd.read_csv(demand_csv, usecols=_RAW_READ_COLS).rename( + columns={"value": "demand"}) + price = pd.read_csv(price_csv, usecols=_RAW_READ_COLS).rename( + columns={"value": "price"}) + demand = _coerce_numeric(demand, "demand") + price = _coerce_numeric(price, "price") + for df in [demand, price]: + df["period"] = pd.to_numeric(df["period"], errors="coerce") + df.dropna(subset=["period"], inplace=True) + df["period"] = df["period"].astype(int) + demand = demand.groupby(_RAW_KEY_COLS, as_index=False).agg( + {"scenarioDescription": "first", "regionName": "first", "demand": "mean"}) + price = price.groupby(_RAW_KEY_COLS, as_index=False).agg({"price": "mean"}) + merged = demand.merge(price, on=_RAW_KEY_COLS, how="inner", validate="one_to_one") + return merged.sort_values(["period", "scenario", "regionId"]).reset_index(drop=True) + + +def _fit_line(panel_df: pd.DataFrame) -> tuple[np.ndarray, np.ndarray, float] | None: + x = panel_df["demand"].to_numpy() + y = panel_df["price"].to_numpy() + if x.size < 2 or np.unique(x).size < 2: + return None + slope, intercept = np.polyfit(x, y, deg=1) + x_line = np.linspace(x.min(), x.max(), 50) + y_line = slope * x_line + intercept + return x_line, y_line, float(slope) + + +def generate_raw_scatter_grid( + beta_out_dir: Path, + config: dict[str, Any], + output_dir: Path, + dpi: int = 150, +) -> None: + """Generate beta raw-data scatter grid (demand vs price by year/region).""" + raw_dir = beta_out_dir / "raw_aeo_data" + demand_csv = raw_dir / "raw_ng_demand_elec.csv" + price_csv = raw_dir / "raw_ng_price.csv" + if not demand_csv.exists() or not price_csv.exists(): + LOGGER.warning("Skipping raw scatter grid: raw CSVs not found in %s", raw_dir) + return + + merged = _load_and_merge_raw(demand_csv, price_csv) + include_scenarios = _load_beta_include_scenarios(config) + merged = _filter_to_beta_include(merged, include_scenarios) + + years = sorted(merged["period"].unique()) + scenarios = sorted(merged["scenario"].unique()) + region_table = (merged[["regionId", "regionName"]].drop_duplicates() + .sort_values(["regionId", "regionName"]).reset_index(drop=True)) + regions = region_table.to_dict("records") + scenario_colors = {s: plt.cm.tab10(i % 10) for i, s in enumerate(scenarios)} + + n_rows, n_cols = len(years), len(regions) + fig_w = max(15, n_cols * 2.6) + fig_h = max(12, n_rows * 1.8) + fig, axes = plt.subplots(nrows=n_rows, ncols=n_cols, + figsize=(fig_w, fig_h), sharex=True, sharey=True, squeeze=False) + + x_min, x_max = merged["demand"].min(), merged["demand"].max() + y_min, y_max = merged["price"].min(), merged["price"].max() + x_pad = 0.05 * (x_max - x_min) if x_max > x_min else 0.1 + y_pad = 0.05 * (y_max - y_min) if y_max > y_min else 0.1 + + for row_idx, year in enumerate(years): + for col_idx, region in enumerate(regions): + ax = axes[row_idx, col_idx] + panel = merged[(merged["period"] == year) & (merged["regionId"] == region["regionId"])] + ax.scatter(panel["demand"], panel["price"], s=20, alpha=0.95, + c=panel["scenario"].map(scenario_colors), edgecolors="none") + fit = _fit_line(panel) + if fit is not None: + x_line, y_line, slope = fit + is_negative = slope < 0 + line_color = "#c62828" if is_negative else "#4d4d4d" + ax.plot(x_line, y_line, color=line_color, linewidth=1.1) + slope_text = f"slope={slope:.3f}" + if is_negative: + slope_text += " (NEG)" + ax.text(0.03, 0.97, slope_text, transform=ax.transAxes, + ha="left", va="top", fontsize=6, + color="#8b0000" if is_negative else "#303030", + bbox={"boxstyle": "round,pad=0.2", + "facecolor": "#ffe6e6" if is_negative else "white", + "alpha": 0.85, + "edgecolor": "#c62828" if is_negative else "none", + "linewidth": 0.6 if is_negative else 0.0}) + if is_negative: + ax.set_facecolor("#fff5f5") + for spine in ax.spines.values(): + spine.set_edgecolor("#c62828") + spine.set_linewidth(0.9) + ax.set_xlim(x_min - x_pad, x_max + x_pad) + ax.set_ylim(y_min - y_pad, y_max + y_pad) + ax.grid(True, alpha=0.25, linewidth=0.5) + ax.tick_params(axis="both", labelsize=6, length=2) + if row_idx == 0: + ax.set_title(str(region["regionName"]), fontsize=8, pad=3) + if col_idx == 0: + ax.set_ylabel(str(year), fontsize=8, rotation=0, labelpad=18, va="center") + + legend_handles = [ + Line2D([0], [0], marker="o", linestyle="", markersize=5, + markerfacecolor=scenario_colors[s], markeredgecolor="none", label=s) + for s in scenarios + ] + fig.supxlabel("Natural Gas Demand (quads)", fontsize=11) + fig.supylabel("Natural Gas Price (2024 $/MMBtu)", fontsize=11) + fig.suptitle( + "Raw AEO NG Data: Demand vs Price\n" + "(Scatter + Linear Regression by Year and Region; negative slopes highlighted)", + fontsize=13, y=0.995) + fig.legend(handles=legend_handles, loc="upper center", + bbox_to_anchor=(0.5, 0.978), ncol=min(len(scenarios), 5), + fontsize=8, frameon=False, title="Scenario", title_fontsize=8) + fig.tight_layout(rect=[0.04, 0.02, 1, 0.95]) + + output_dir.mkdir(parents=True, exist_ok=True) + fig.savefig(output_dir / "beta_raw_data_scatter_linear_grid.png", dpi=dpi) + plt.close(fig) + LOGGER.info("Wrote raw scatter grid to %s", output_dir) + + +# ============================================================================ +# Part 2: Beta / alpha diagnostic plots +# ============================================================================ + +def generate_beta_plots(beta_out_dir: Path, region_order: list[str], plots_dir: Path) -> None: + plots_dir.mkdir(parents=True, exist_ok=True) + summary_path = beta_out_dir / "beta_regression_summary.csv" + points_path = beta_out_dir / "regression_points.csv" + require(summary_path.exists(), f"Missing file: {summary_path}") + require(points_path.exists(), f"Missing file: {points_path}") + + summary = pd.read_csv(summary_path) + points = pd.read_csv(points_path) + nat_y_col = "dp_partial_nat" + reg_x_col = "dq_reg" + for col in ["scope", "region", "beta"]: + require(col in summary.columns, f"Missing '{col}' in {summary_path}") + for col in ["region", "dq_nat", nat_y_col, reg_x_col, "dp_partial_reg"]: + require(col in points.columns, f"Missing '{col}' in {points_path}") + + national_rows = summary[summary["scope"].astype(str).str.lower() == "national"] + require(not national_rows.empty, f"No national row found in {summary_path}") + nrow = national_rows.iloc[0] + beta_nat = float(pd.to_numeric([nrow["beta"]], errors="coerce")[0]) + beta_nat_r2 = (float(pd.to_numeric([nrow["r2"]], errors="coerce")[0]) + if "r2" in summary.columns else float("nan")) + model_r2 = (float(pd.to_numeric([nrow["r2_full"]], errors="coerce")[0]) + if "r2_full" in summary.columns else float("nan")) + + # National beta plot + nat_points = points[["dq_nat", nat_y_col]].dropna() + require(not nat_points.empty, + f"No valid national plotting rows in {points_path}") + fig, ax = plt.subplots(figsize=(8, 6)) + ax.scatter(nat_points["dq_nat"], nat_points[nat_y_col], s=12, alpha=0.5, color="#1f77b4") + x_min, x_max = float(nat_points["dq_nat"].min()), float(nat_points["dq_nat"].max()) + x_range = np.linspace(x_min, x_max, 200) + ax.plot(x_range, beta_nat * x_range, color="#d62728", linewidth=2) + ax.axhline(0, color="#888888", linewidth=0.8) + ax.axvline(0, color="#888888", linewidth=0.8) + ax.set_xlabel("Demeaned national demand (partial)") + ax.set_ylabel("Price residual net of regional term (partial)") + ax.set_title(f"National beta | beta={beta_nat:.6f}, partial R2={beta_nat_r2:.3f}, model R2={model_r2:.3f}") + fig.tight_layout() + fig.savefig(plots_dir / "national_beta_regression.png", dpi=220) + plt.close(fig) + + # Regional beta plot + regional = summary[summary["scope"].astype(str).str.lower() == "regional"].copy() + regional["region"] = regional["region"].astype(str) + beta_map: dict[str, float] = {} + r2_map: dict[str, float] = {} + for row in regional.itertuples(index=False): + beta_val = pd.to_numeric([getattr(row, "beta")], errors="coerce")[0] + r2_val = (pd.to_numeric([getattr(row, "r2")], errors="coerce")[0] + if "r2" in regional.columns else float("nan")) + if pd.notna(beta_val): + beta_map[str(getattr(row, "region"))] = float(beta_val) + if pd.notna(r2_val): + r2_map[str(getattr(row, "region"))] = float(r2_val) + + regions = [r for r in region_order if r in beta_map] + if not regions: + regions = sorted(points["region"].dropna().astype(str).unique().tolist()) + + ncols = 3 + nrows = int(np.ceil(len(regions) / ncols)) + fig, axes = plt.subplots(nrows, ncols, figsize=(5.2 * ncols, 4.2 * nrows)) + axes = np.atleast_1d(axes).ravel() + for i, region in enumerate(regions): + ax = axes[i] + reg_r = points[points["region"] == region][[reg_x_col, "dp_partial_reg"]].dropna() + beta = beta_map.get(region, float("nan")) + r2 = r2_map.get(region, float("nan")) + ax.scatter(reg_r[reg_x_col], reg_r["dp_partial_reg"], s=10, alpha=0.65, color="#1f77b4") + if not reg_r.empty: + x_line = np.linspace(float(reg_r[reg_x_col].min()), float(reg_r[reg_x_col].max()), 120) + ax.plot(x_line, beta * x_line, color="#d62728", linewidth=1.6) + ax.axhline(0, color="#999999", linewidth=0.6) + ax.axvline(0, color="#999999", linewidth=0.6) + ax.set_title(f"{region}\nbeta={beta:.4f}, partial R2={r2:.3f}", fontsize=10) + ax.set_xlabel(f"{reg_x_col} (partial)") + ax.set_ylabel("dp_reg (partial)") + for i in range(len(regions), len(axes)): + axes[i].axis("off") + fig.suptitle("Regional betas (joint fixed-effects regression)", fontsize=14) + fig.tight_layout(rect=[0, 0, 1, 0.97]) + fig.savefig(plots_dir / "regional_beta_regression.png", dpi=220) + plt.close(fig) + LOGGER.info("Wrote beta plots to %s", plots_dir) + + +def generate_alpha_plots( + alpha_out_dir: Path, + alpha_input_dir: Path, + beta_out_dir: Path, + plots_dir: Path, + region_order: list[str], + aeo_year: int, + deflator_to_2004: float, + first_model_year: int, +) -> None: + plots_dir.mkdir(parents=True, exist_ok=True) + beta_regional, _ = load_regional_beta_map(alpha_out_dir, alpha_input_dir) + beta_national, _ = load_national_beta(alpha_out_dir, alpha_input_dir, beta_out_dir) + missing_regions = [r for r in region_order if r not in beta_regional] + require(not missing_regions, f"Missing regional beta values: {missing_regions}") + + scenarios = discover_output_scenarios(alpha_out_dir, aeo_year) + + for suffix, scenario_id in scenarios: + alpha_path = alpha_out_dir / f"alpha_AEO_{aeo_year}_{suffix}.csv" + price_path = alpha_out_dir / f"ng_AEO_{aeo_year}_{suffix}.csv" + demand_path = alpha_out_dir / f"ng_demand_AEO_{aeo_year}_{suffix}.csv" + if not (alpha_path.exists() and price_path.exists() and demand_path.exists()): + LOGGER.warning("Skipping suffix '%s' due to missing files.", suffix) + continue + + alpha_df = load_wide_series(alpha_path, "alpha_2004", scenario_id, region_order) + price_df = load_wide_series(price_path, "ng_price", scenario_id, region_order) + demand_df = load_wide_series(demand_path, "demand_elec_quads", scenario_id, region_order) + + merged = alpha_df.merge(price_df, on=["scenario_id", "region", "year"], how="inner") + merged = merged.merge(demand_df, on=["scenario_id", "region", "year"], how="inner") + require(not merged.empty, f"No merged data for scenario suffix '{suffix}'") + + q_nat = (demand_df.groupby(["scenario_id", "year"], as_index=False)["demand_elec_quads"] + .sum().rename(columns={"demand_elec_quads": "q_nat"})) + merged = merged.merge(q_nat, on=["scenario_id", "year"], how="left") + merged["price_2004"] = merged["ng_price"] * deflator_to_2004 + merged["beta_reg"] = merged["region"].map(beta_regional) + merged["term_reg"] = merged["beta_reg"] * merged["demand_elec_quads"] + merged["term_nat"] = beta_national * merged["q_nat"] + zero_ng_terms_in_first_model_year(merged, first_model_year) + + ncols = 3 + nrows = int(np.ceil(len(region_order) / ncols)) + fig, axes = plt.subplots(nrows, ncols, figsize=(5.5 * ncols, 4.0 * nrows), sharex=True) + axes = np.atleast_1d(axes).ravel() + for i, region in enumerate(region_order): + ax = axes[i] + reg = merged[merged["region"] == region].sort_values("year") + if reg.empty: + continue + years = reg["year"].to_numpy() + ax.fill_between(years, 0, reg["alpha_2004"].to_numpy(), + alpha=0.5, color="#2ca02c", label="Alpha") + ax.fill_between(years, reg["alpha_2004"].to_numpy(), + reg["alpha_2004"].to_numpy() + reg["term_reg"].to_numpy(), + alpha=0.5, color="#1f77b4", label="Beta_reg * Q_reg") + ax.fill_between(years, + reg["alpha_2004"].to_numpy() + reg["term_reg"].to_numpy(), + reg["alpha_2004"].to_numpy() + reg["term_reg"].to_numpy() + reg["term_nat"].to_numpy(), + alpha=0.5, color="#ff7f0e", label="Beta_nat * Q_nat") + ax.plot(years, reg["price_2004"].to_numpy(), color="#d62728", + linewidth=1.5, linestyle="--", label="Actual price") + ax.set_title(region, fontsize=12, pad=6) + ax.set_ylabel("2004$/MMBtu", fontsize=11) + ax.grid(True, alpha=0.3) + for i in range(len(region_order), len(axes)): + axes[i].axis("off") + handles, labels = axes[0].get_legend_handles_labels() + fig.legend(handles, labels, loc="lower center", ncol=4, fontsize=10, + bbox_to_anchor=(0.5, -0.02)) + fig.suptitle(f"Price decomposition: {scenario_id}\n" + "Price = Alpha + Beta_reg*Q_reg + Beta_nat*Q_nat", fontsize=15) + fig.tight_layout(rect=[0, 0.04, 1, 0.95]) + safe_id = sanitize_file_component(scenario_id) + fig.savefig(plots_dir / f"alpha_price_decomposition_{safe_id}.png", + dpi=220, bbox_inches="tight") + plt.close(fig) + LOGGER.info("Wrote alpha plot for %s", scenario_id) + + +# ============================================================================ +# Part 3: Validation +# ============================================================================ + +def build_validation_frame( + alpha_out_dir: Path, + aeo_year: int, + region_order: list[str], +) -> pd.DataFrame: + LOGGER.info("Building validation frame from alpha outputs in %s", alpha_out_dir) + frames: list[pd.DataFrame] = [] + for suffix, scenario_id, scenario_label in discover_output_scenarios_with_labels(alpha_out_dir, aeo_year): + alpha_path = alpha_out_dir / f"alpha_AEO_{aeo_year}_{suffix}.csv" + price_path = alpha_out_dir / f"ng_AEO_{aeo_year}_{suffix}.csv" + demand_path = alpha_out_dir / f"ng_demand_AEO_{aeo_year}_{suffix}.csv" + if not (alpha_path.exists() and price_path.exists() and demand_path.exists()): + LOGGER.warning("Skipping suffix '%s' due to missing files.", suffix) + continue + alpha_df = load_wide_series(alpha_path, "alpha_2004", scenario_id, region_order, scenario_label) + price_df = load_wide_series(price_path, "ng_price", scenario_id, region_order, scenario_label) + demand_df = load_wide_series(demand_path, "demand_elec_quads", scenario_id, region_order, scenario_label) + merged = alpha_df.merge(price_df, on=["scenario_id", "scenario_label", "region", "year"], how="inner") + merged = merged.merge(demand_df, on=["scenario_id", "scenario_label", "region", "year"], how="inner") + require(not merged.empty, f"No merged rows for scenario '{scenario_id}' ({suffix})") + frames.append(merged) + require(frames, "No scenario frames available for validation.") + out = pd.concat(frames, ignore_index=True) + q_nat = (out.groupby(["scenario_id", "scenario_label", "year"], as_index=False)["demand_elec_quads"] + .sum().rename(columns={"demand_elec_quads": "q_nat"})) + return out.merge(q_nat, on=["scenario_id", "scenario_label", "year"], how="left") + + +def load_beta_regression_points(beta_out_dir: Path) -> tuple[pd.DataFrame, Path]: + path = beta_out_dir / "regression_points.csv" + require(path.exists(), f"Missing regression_points.csv: {path}") + df = pd.read_csv(path) + need = {"scenario_id", "year", "region", "demand", "demand_nat", "price_2004", "dp", "dp_hat"} + require(not (need - set(df.columns)), f"Missing columns in {path}: {sorted(need - set(df.columns))}") + out = df[list(need)].copy() + out["scenario_id"] = out["scenario_id"].astype(str) + out["region"] = out["region"].astype(str).map(lambda x: cendiv_output_label(any_to_cendiv(x))) + for c in ["year", "demand", "demand_nat", "price_2004", "dp", "dp_hat"]: + out[c] = pd.to_numeric(out[c], errors="coerce") + require(not out[["year", "demand", "demand_nat", "price_2004", "dp", "dp_hat"]].isna().any().any(), + f"Non-numeric value(s) in {path}") + out["year"] = out["year"].astype(int) + return out, path + + +def build_step1_beta_validation_frame( + beta_out_dir: Path, + beta_reg_step_map: dict[str, float], + beta_nat_step: float, + alpha1_frame: pd.DataFrame, + alpha1_merge_cols: list[str], + first_model_year: int, +) -> tuple[pd.DataFrame, Path]: + points, points_path = load_beta_regression_points(beta_out_dir) + frame = points.rename(columns={ + "demand": "demand_elec_quads", "demand_nat": "q_nat", + "price_2004": "actual_2004", "dp": "actual_dprice", "dp_hat": "predicted_dprice", + }).copy() + frame["scenario_label"] = frame["scenario_id"] + frame["beta_reg"] = frame["region"].map(beta_reg_step_map) + require(not frame["beta_reg"].isna().any(), + "Missing regional beta mapping in step-1 validation.") + frame["beta_nat"] = beta_nat_step + frame = frame.merge(alpha1_frame, on=alpha1_merge_cols, how="left") + require(not frame["alpha1"].isna().any(), + "Missing alpha1 rows in step-1 validation.") + zero_ng_betas_in_first_model_year(frame, first_model_year) + frame["predicted_2004"] = (frame["alpha1"] + + frame["beta_reg"] * frame["demand_elec_quads"] + + frame["beta_nat"] * frame["q_nat"]) + frame["error"] = frame["actual_2004"] - frame["predicted_2004"] + frame["error_dprice"] = frame["actual_dprice"] - frame["predicted_dprice"] + return frame, points_path + + +def _parity_panel(ax, frame, title, actual_col, predicted_col, xlabel, ylabel): + m = compute_fit_metrics(frame[actual_col], frame[predicted_col]) + x = frame[actual_col].to_numpy(dtype=float) + y = frame[predicted_col].to_numpy(dtype=float) + mask = np.isfinite(x) & np.isfinite(y) + x, y = x[mask], y[mask] + ax.scatter(x, y, s=8, alpha=0.35, color="#1f77b4", edgecolors="none") + if len(x) > 0: + lo = float(min(np.min(x), np.min(y))) + hi = float(max(np.max(x), np.max(y))) + if hi <= lo: + hi = lo + 1.0 + pad = 0.05 * (hi - lo) + ax.plot([lo - pad, hi + pad], [lo - pad, hi + pad], color="#333333", + linestyle="--", linewidth=1.2) + ax.set_xlim(lo - pad, hi + pad) + ax.set_ylim(lo - pad, hi + pad) + ax.set_title(title, fontsize=11, pad=4) + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + ax.grid(True, alpha=0.25) + ax.text(0.02, 0.98, + f"N={int(m['n_obs'])}\nR2={m['r2']:.4f}\nRMSE={m['rmse']:.4f}\nMAE={m['mae']:.4f}", + transform=ax.transAxes, va="top", ha="left", fontsize=9, + bbox={"boxstyle": "round,pad=0.2", "facecolor": "white", + "alpha": 0.75, "edgecolor": "#bbbbbb"}) + return m + + +def plot_overall_parity(step1_frame, step2_frame, out_path): + fig, axes = plt.subplots(1, 2, figsize=(12.8, 5.6)) + step1_m = _parity_panel(axes[0], step1_frame, + "Step 1: beta-regression fit\n(demeaned price: dp vs dp_hat)", + "actual_dprice", "predicted_dprice", + "Actual demeaned price (2004$/MMBtu)", + "Predicted demeaned price (2004$/MMBtu)") + step2_m = _parity_panel(axes[1], step2_frame, + "Step 2: alpha-regression scenarios\n(level price: alpha2 + beta x demand)", + "actual_2004", "predicted_2004", + "Actual (2004$/MMBtu)", "Predicted (2004$/MMBtu)") + fig.suptitle("Results validation parity: step1 demeaned fit vs step2 level fit", + fontsize=14, y=0.98) + fig.tight_layout(rect=[0, 0, 1, 0.95]) + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=220, bbox_inches="tight") + plt.close(fig) + return step1_m, step2_m + + +def plot_actual_vs_predicted(frame, region_order, out_path): + scenarios = list(dict.fromkeys(frame["scenario_label"].astype(str).tolist())) + ncols = 3 + nrows = int(np.ceil(len(region_order) / ncols)) + fig, axes = plt.subplots(nrows, ncols, figsize=(5.3 * ncols, 4.0 * nrows), sharex=True) + axes = np.atleast_1d(axes).ravel() + for i, region in enumerate(region_order): + ax = axes[i] + reg = frame[frame["region"] == region].copy() + if reg.empty: + ax.set_title(region, fontsize=10) + ax.grid(True, alpha=0.25) + continue + for scen in scenarios: + sdf = reg[reg["scenario_label"] == scen].sort_values("year") + if sdf.empty: + continue + color = SCENARIO_COLOR.get(scen) + ax.plot(sdf["year"], sdf["actual_2004"], color=color, linewidth=1.6) + ax.plot(sdf["year"], sdf["predicted_2004"], color=color, + linewidth=3.0, linestyle="--", alpha=0.98, zorder=4) + ax.set_title(region, fontsize=10, pad=4) + ax.set_ylabel("2004$/MMBtu") + ax.grid(True, alpha=0.25) + for i in range(len(region_order), len(axes)): + axes[i].axis("off") + legend_handles = [] + for scen in scenarios: + color = SCENARIO_COLOR.get(scen, "#333333") + legend_handles.append(Line2D([0], [0], color=color, lw=2.0, linestyle="-", + label=f"{scen} actual")) + legend_handles.append(Line2D([0], [0], color=color, lw=2.0, linestyle="--", + alpha=0.7, label=f"{scen} predicted")) + fig.legend(handles=legend_handles, loc="lower center", + ncol=max(2, min(6, len(legend_handles))), fontsize=8, + bbox_to_anchor=(0.5, -0.005)) + fig.suptitle("Results validation: actual price vs predicted\n" + "(predicted = alpha(region,year,scenario) + beta x scenario demand)", + fontsize=14, y=0.98) + fig.tight_layout(rect=[0, 0.04, 1, 0.95]) + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=220, bbox_inches="tight") + plt.close(fig) + + +def plot_alpha_vs_alpha1(frame, region_order, out_path): + scenarios = list(dict.fromkeys(frame["scenario_label"].astype(str).tolist())) + alpha1_shared = bool( + frame.groupby(["region", "year"], as_index=False)["alpha1"].nunique()["alpha1"].max() <= 1) + ncols = 3 + nrows = int(np.ceil(len(region_order) / ncols)) + fig, axes = plt.subplots(nrows, ncols, figsize=(5.3 * ncols, 4.0 * nrows), sharex=True) + axes = np.atleast_1d(axes).ravel() + for i, region in enumerate(region_order): + ax = axes[i] + reg = frame[frame["region"] == region].copy() + if reg.empty: + ax.set_title(region, fontsize=10) + ax.grid(True, alpha=0.25) + continue + for scen in scenarios: + sdf = reg[reg["scenario_label"] == scen].sort_values("year") + if sdf.empty: + continue + color = SCENARIO_COLOR.get(scen) + ax.plot(sdf["year"], sdf["alpha_2004"], color=color, linewidth=1.7) + if not alpha1_shared: + ax.plot(sdf["year"], sdf["alpha1"], color=color, linewidth=2.6, + linestyle="--", alpha=0.98, zorder=4) + if alpha1_shared: + a1 = reg[["year", "alpha1"]].drop_duplicates(subset=["year"]).sort_values("year") + ax.plot(a1["year"], a1["alpha1"], color="#222222", linewidth=2.8, + linestyle="--", alpha=0.98, zorder=5, label="alpha1 shared") + ax.set_title(region, fontsize=10, pad=4) + ax.set_ylabel("alpha") + ax.grid(True, alpha=0.25) + for i in range(len(region_order), len(axes)): + axes[i].axis("off") + legend_handles = [] + for scen in scenarios: + color = SCENARIO_COLOR.get(scen, "#333333") + legend_handles.append(Line2D([0], [0], color=color, lw=2.2, linestyle="-", + label=f"{scen} alpha2")) + if not alpha1_shared: + legend_handles.append(Line2D([0], [0], color=color, lw=2.6, linestyle="--", + alpha=0.98, label=f"{scen} alpha1")) + if alpha1_shared: + legend_handles.append(Line2D([0], [0], color="#222222", lw=2.8, linestyle="--", + alpha=0.98, label="alpha1 shared")) + fig.legend(handles=legend_handles, loc="lower center", + ncol=max(2, min(6, len(legend_handles))), fontsize=8, + bbox_to_anchor=(0.5, -0.005)) + fig.suptitle("Results validation: alpha2 vs alpha1\n" + "(alpha2 = alpha regression output, alpha1 = beta regression output)", + fontsize=14, y=0.98) + fig.tight_layout(rect=[0, 0.04, 1, 0.95]) + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=220, bbox_inches="tight") + plt.close(fig) + + +def run_validation( + config: dict[str, Any], + base_dir: Path, + beta_out_dir: Path, + alpha_out_dir: Path, + alpha_input_dir: Path, + validation_dir: Path, + region_order: list[str], +) -> None: + """Run full validation: CSV metrics + plots.""" + aeo_year = int(config["aeo_year"]) + first_model_year = int(config["start_year"]) + deflator = float(config["ng"]["price_deflator_to_2004"]) + validation_dir.mkdir(parents=True, exist_ok=True) + + # Load betas + beta_reg_map, beta_reg_src = load_regional_beta_map(alpha_out_dir, alpha_input_dir) + beta_nat, beta_nat_src = load_national_beta(alpha_out_dir, alpha_input_dir, beta_out_dir) + missing_regions = [r for r in region_order if r not in beta_reg_map] + require(not missing_regions, f"Missing regional beta: {missing_regions}") + + beta_reg_step_map = load_regional_beta_from_csv(beta_out_dir / "cd_beta0.csv") + beta_nat_step = load_national_beta_from_csv(beta_out_dir / "national_beta.csv") + + # Beta comparison CSV + beta_compare_rows = [{ + "scope": "national", "region": "ALL", + "beta1_from_beta_step": beta_nat_step, + "beta2_used_in_alpha_step": beta_nat, + "diff_beta1_minus_beta2": beta_nat_step - beta_nat, + }] + for region in region_order: + b1, b2 = beta_reg_step_map[region], beta_reg_map[region] + beta_compare_rows.append({ + "scope": "regional", "region": region, + "beta1_from_beta_step": b1, + "beta2_used_in_alpha_step": b2, + "diff_beta1_minus_beta2": b1 - b2, + }) + pd.DataFrame(beta_compare_rows).to_csv( + validation_dir / "results_validation_beta_comparison.csv", + index=False, float_format="%.6f") + + # Build step-2 validation frame + frame = build_validation_frame(alpha_out_dir, aeo_year, region_order) + frame["actual_2004"] = frame["ng_price"] * deflator + frame["beta_reg"] = frame["region"].map(beta_reg_map) + frame["beta_nat"] = beta_nat + zero_ng_betas_in_first_model_year(frame, first_model_year) + frame["predicted_2004"] = (frame["alpha_2004"] + + frame["beta_reg"] * frame["demand_elec_quads"] + + frame["beta_nat"] * frame["q_nat"]) + frame["error"] = frame["actual_2004"] - frame["predicted_2004"] + + # Alpha1 from beta step + alpha1_frame, _, alpha1_merge_cols = load_alpha_from_beta_step(beta_out_dir) + step1_frame, _ = build_step1_beta_validation_frame( + beta_out_dir, beta_reg_step_map, beta_nat_step, + alpha1_frame, alpha1_merge_cols, first_model_year) + + frame = frame.merge(alpha1_frame, on=alpha1_merge_cols, how="left") + frame["alpha_vs_alpha1_error"] = frame["alpha_2004"] - frame["alpha1"] + alpha_cmp_frame = frame[~frame["alpha1"].isna()].copy() + + # Write detail/summary CSVs + step1_frame[[ + "scenario_id", "scenario_label", "region", "year", + "actual_2004", "predicted_2004", "error", + "actual_dprice", "predicted_dprice", "error_dprice", + "alpha1", "beta_reg", "beta_nat", "demand_elec_quads", "q_nat", + ]].to_csv(validation_dir / "results_validation_step1_beta_actual_vs_predicted_detail.csv", + index=False, float_format="%.6f") + + summarize_fit(step1_frame, ["scenario_label", "region"], + "actual_dprice", "predicted_dprice", + "mae_error", "rmse_error", "max_abs_error", "r2").to_csv( + validation_dir / "results_validation_step1_beta_actual_vs_predicted_summary.csv", + index=False, float_format="%.6f") + + frame[[ + "scenario_id", "scenario_label", "region", "year", + "actual_2004", "predicted_2004", "error", + "alpha_2004", "beta_reg", "beta_nat", "demand_elec_quads", "q_nat", + "alpha1", "alpha_vs_alpha1_error", + ]].to_csv(validation_dir / "results_validation_actual_vs_predicted_detail.csv", + index=False, float_format="%.6f") + + summarize_fit(frame, ["scenario_label", "region"], + "actual_2004", "predicted_2004", + "mae_error", "rmse_error", "max_abs_error", "r2").to_csv( + validation_dir / "results_validation_actual_vs_predicted_summary.csv", + index=False, float_format="%.6f") + + if not alpha_cmp_frame.empty: + summarize_fit(alpha_cmp_frame, ["scenario_label", "region"], + "alpha_2004", "alpha1", + "mae_alpha1_error", "rmse_alpha1_error", + "max_abs_alpha1_error", "r2_alpha1").to_csv( + validation_dir / "results_validation_alpha_vs_alpha1_summary.csv", + index=False, float_format="%.6f") + + alpha_spread = (alpha_cmp_frame.groupby(["region", "year"], as_index=False)["alpha_2004"] + .agg(alpha_min="min", alpha_max="max")) + alpha_spread["alpha_spread"] = alpha_spread["alpha_max"] - alpha_spread["alpha_min"] + alpha_spread.to_csv(validation_dir / "results_validation_alpha_vs_alpha1_spread.csv", + index=False, float_format="%.6f") + + alpha_cmp_frame[[ + "scenario_id", "scenario_label", "region", "year", + "alpha_2004", "alpha1", "alpha_vs_alpha1_error", + ]].to_csv(validation_dir / "results_validation_alpha_vs_alpha1_detail.csv", + index=False, float_format="%.6f") + + # Plots + plot_actual_vs_predicted(frame, region_order, + validation_dir / "results_validation_actual_vs_predicted.png") + if not alpha_cmp_frame.empty: + plot_alpha_vs_alpha1(alpha_cmp_frame, region_order, + validation_dir / "results_validation_alpha_vs_alpha1.png") + step1_m, step2_m = plot_overall_parity(step1_frame, frame, + validation_dir / "results_validation_parity_overall.png") + + pd.DataFrame([ + {"step": "step1_beta_regression_scenarios", "metric_basis": "demeaned_price_dp", + "n_obs": int(step1_m["n_obs"]), "mae_error": step1_m["mae"], + "rmse_error": step1_m["rmse"], "max_abs_error": step1_m["max_abs"], "r2": step1_m["r2"]}, + {"step": "step2_alpha_regression_scenarios", "metric_basis": "level_price_2004_per_mmbtu", + "n_obs": int(step2_m["n_obs"]), "mae_error": step2_m["mae"], + "rmse_error": step2_m["rmse"], "max_abs_error": step2_m["max_abs"], "r2": step2_m["r2"]}, + ]).to_csv(validation_dir / "results_validation_overall_metrics.csv", + index=False, float_format="%.6f") + + LOGGER.info("Validation complete. Outputs in %s", validation_dir) + + +# ============================================================================ +# CLI +# ============================================================================ + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Unified visualization and validation for NG regression pipeline.") + parser.add_argument("--config", default="aeo_pipeline_config.json") + parser.add_argument("--beta-output-dir", default="outputs of beta regression") + parser.add_argument("--alpha-output-dir", default=None) + parser.add_argument("--alpha-input-dir", default=None) + parser.add_argument("--output-dir", default="results validation", + help="Directory for all plots and validation CSVs.") + parser.add_argument("--skip-raw-scatter", action="store_true", + help="Skip beta raw-data scatter grid.") + parser.add_argument("--skip-beta", action="store_true", + help="Skip beta diagnostic plots.") + parser.add_argument("--skip-alpha", action="store_true", + help="Skip alpha decomposition plots.") + parser.add_argument("--skip-validation", action="store_true", + help="Skip validation plots and CSVs.") + parser.add_argument("--log-level", default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR"]) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + logging.basicConfig(level=getattr(logging, args.log_level), + format="%(asctime)s | %(levelname)s | %(message)s") + + script_dir = Path(__file__).resolve().parent + cfg_path = Path(args.config) + if not cfg_path.is_absolute(): + cwd_candidate = resolve_case_insensitive((Path.cwd() / cfg_path).resolve()) + script_candidate = resolve_case_insensitive((script_dir / cfg_path).resolve()) + cfg_path = cwd_candidate if cwd_candidate.exists() else script_candidate + + config = load_config(cfg_path) + base_dir = cfg_path.parent + + beta_out_dir = resolve_path(base_dir, args.beta_output_dir) + alpha_out_dir = resolve_path(base_dir, + args.alpha_output_dir or config["paths"]["output_dir"]) + alpha_input_dir = resolve_path(base_dir, + args.alpha_input_dir or config["paths"].get("input_dir", "inputs for alpha regression")) + output_dir = resolve_path(base_dir, args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + region_order = region_order_from_config(config) + aeo_year = int(config["aeo_year"]) + first_model_year = int(config["start_year"]) + deflator = float(config["ng"]["price_deflator_to_2004"]) + + if not args.skip_raw_scatter: + generate_raw_scatter_grid(beta_out_dir, config, output_dir) + + if not args.skip_beta: + generate_beta_plots(beta_out_dir, region_order, output_dir) + + if not args.skip_alpha: + generate_alpha_plots( + alpha_out_dir=alpha_out_dir, + alpha_input_dir=alpha_input_dir, + beta_out_dir=beta_out_dir, + plots_dir=output_dir, + region_order=region_order, + aeo_year=aeo_year, + deflator_to_2004=deflator, + first_model_year=first_model_year, + ) + + if not args.skip_validation: + run_validation( + config=config, + base_dir=base_dir, + beta_out_dir=beta_out_dir, + alpha_out_dir=alpha_out_dir, + alpha_input_dir=alpha_input_dir, + validation_dir=output_dir, + region_order=region_order, + ) + + LOGGER.info("All visualization and validation complete.") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as exc: + LOGGER.exception("Failed: %s", exc) + sys.exit(1)