diff --git a/aeo_updates/AEO_scraper.ipynb b/aeo_updates/AEO_scraper.ipynb index 99c4137..3daf326 100644 --- a/aeo_updates/AEO_scraper.ipynb +++ b/aeo_updates/AEO_scraper.ipynb @@ -7,19 +7,20 @@ "metadata": {}, "outputs": [], "source": [ - "# Script to pull AEO 2023 data using the API\n", + "# Script to pull AEO 2026 data using the API\n", "\n", "import requests\n", - "import json\n", "import pandas as pd\n", "from time import sleep\n", "import os\n", "\n", "# Set AEO_year\n", - "AEO_year = 2025\n", + "AEO_year = 2026\n", "\n", "# Get EIA API key\n", - "api_key = os.getenv('EIA_API_KEY')" + "api_key = os.getenv('EIA_API_KEY')\n", + "if not api_key:\n", + " raise ValueError(\"Missing EIA_API_KEY environment variable. Set EIA_API_KEY before running this notebook.\")" ] }, { @@ -52,16 +53,16 @@ " 'Energy Prices : Electric Power : Steam Coal',\n", " 'Energy Prices : Electric Power : Natural Gas',\n", "# 'Energy Prices : Nominal : Electric Power : Natural Gas',\n", - " 'Energy Prices : Electric Power : Uranium',\n", + " 'Energy Prices : Electric Power : Nuclear Fuel',\n", "# 'Energy Prices : Nominal : Electric Power : Uranium',\n", " 'Energy Use : Electric Power : Natural Gas',\n", " 'Energy Use : Total : Natural Gas',\n", " 'Energy Use : Delivered : All Sectors : Electricity'\n", - "]\n", + " ]\n", "\n", "# This API call will get us a list of dictionaries that associate each seriesId to a series name.\n", "path = f'https://api.eia.gov/v2/aeo/{AEO_year}/facet/seriesId?api_key={api_key}'\n", - "resp = json.loads(requests.get(path).text)\n", + "resp = requests.get(path).json()\n", "series_maps = resp['response']['facets']\n", "\n", "# Next we construct the series_lists dictionary, filtered to the desired series in series_names.\n", @@ -95,9 +96,9 @@ " 'Mountain',\n", " 'Pacific',\n", " 'United States'\n", - "]\n", + " ]\n", "path = f'https://api.eia.gov/v2/aeo/{AEO_year}/facet/regionId?api_key={api_key}'\n", - "resp = json.loads(requests.get(path).text)\n", + "resp = requests.get(path).json()\n", "region_maps = resp['response']['facets']\n", "regionIds = {item['id'] for item in region_maps if str(item['name']) in region_names}\n", "#n = [(item['id'], item['name']) for item in x if item['name'] in region_names]\n", @@ -105,6 +106,22 @@ "regionIds" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0b3db81", + "metadata": {}, + "outputs": [], + "source": [ + "# Inspect available scenario IDs before building the scenario filter in Cell 4\n", + "path = f'https://api.eia.gov/v2/aeo/{AEO_year}/facet/scenario?api_key={api_key}'\n", + "resp = requests.get(path).json()\n", + "scenario_maps = resp['response']['facets']\n", + "scenario_df = pd.DataFrame(scenario_maps)[['id', 'name']]\n", + "scenario_df = scenario_df.sort_values('id').reset_index(drop=True)\n", + "scenario_df" + ] + }, { "cell_type": "code", "execution_count": null, @@ -113,8 +130,18 @@ "outputs": [], "source": [ "# Scenario filter\n", - "# hm2025 = high economic growth, lm2025 = low economic growth, nocaa111 = alternative electricity scenario\n", - "scenarios = [f'ref{AEO_year}', 'highogs', 'lowogs', 'hm2025', 'lm2025', 'nocaa111']\n", + "# Use dynamic IDs for high/low economic growth so this updates with AEO_year\n", + "scenarios = [f'cb{AEO_year}', 'highogs', 'lowogs', f'hm{AEO_year}', f'lm{AEO_year}', 'altelec']\n", + "\n", + "# Validate scenario IDs so naming changes are caught early\n", + "path = f'https://api.eia.gov/v2/aeo/{AEO_year}/facet/scenario?api_key={api_key}'\n", + "resp = requests.get(path).json()\n", + "available_scenarios = {item['id'] for item in resp['response']['facets']}\n", + "missing_scenarios = [s for s in scenarios if s not in available_scenarios]\n", + "if missing_scenarios:\n", + " print('Scenario IDs not found for this AEO year:', missing_scenarios)\n", + " print('Run the scenario lookup cell above and update scenarios.')\n", + " raise ValueError('Invalid scenario IDs in scenarios list')\n", "f1 = '&'.join(['facets[scenario][]=' + scenario for scenario in scenarios])\n", "\n", "# RegionId filter\n", @@ -129,7 +156,7 @@ "\n", " # Construct API call\n", " path = f'https://api.eia.gov/v2/aeo/{AEO_year}/data?api_key={api_key}&data[]=value&{f1}&{f2}&{f3}'\n", - " resp = json.loads(requests.get(path).text)\n", + " resp = requests.get(path).json()\n", "\n", " # The API throttling is very aggressive. We have to slow things way down or our api_key will\n", " # be temporarily locked. This makes it important for us to do as few api calls as possible, or\n", @@ -154,9 +181,7 @@ "# Additional processing\n", "df = df.drop_duplicates()\n", "df = df.drop(columns=['history'])\n", - "df = df.rename(columns={'period': 'year', 'unit': 'units'})\n", - "\n", - "\n" + "df = df.rename(columns={'period': 'year', 'unit': 'units'})" ] }, { @@ -203,7 +228,7 @@ "file_name_suffix_list = {\n", " 'Energy Prices : Electric Power : Steam Coal': 'coal_prices',\n", " 'Energy Prices : Electric Power : Natural Gas': 'ng_prices',\n", - " 'Energy Prices : Electric Power : Uranium': 'uranium_prices',\n", + " 'Energy Prices : Electric Power : Nuclear Fuel': 'uranium_prices',\n", " 'Energy Use : Electric Power : Natural Gas': 'ng_demand_electricity',\n", " 'Energy Use : Total : Natural Gas': 'ng_tot_demand',\n", " 'Energy Use : Delivered : All Sectors : Electricity': 'electricity_consumption'\n", @@ -241,6 +266,10 @@ "\n", "#df_new = {}\n", "\n", + "# Specify outputs location\n", + "output_dir = 'outputs'\n", + "os.makedirs(output_dir, exist_ok=True)\n", + "\n", "for x in series_names:\n", " for j in scenario_filter:\n", " df_filter = pd.DataFrame(df2.loc[(df2['seriesName'] == x) & (df2['scenario'] == j)])\n", @@ -265,9 +294,8 @@ " \n", " fileseries_suffix = file_name_suffix_list.get(x, 'Unknown')\n", " \n", - " #Please alter file path if needed for the generated files to go\n", - " \n", - " filename = f\"AEO_{j}_{AEO_year}_{fileseries_suffix}.csv\"\n", + " # Output files\n", + " filename = os.path.join(output_dir, f\"AEO_{j}_{AEO_year}_{fileseries_suffix}.csv\")\n", " \n", " # Only output coal and uranium prices for REF scenario\n", " # if x == 'Energy Prices : Electric Power : Steam Coal' or x == 'Energy Prices : Electric Power : Uranium':\n", @@ -279,24 +307,8 @@ " df_pivot_filtered.to_csv(filename, index=False)\n", " \n", "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", "print('Scrape complete! Files generated!')" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "925f73c4", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -315,7 +327,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.14" + "version": "3.11.15" } }, "nbformat": 4, diff --git a/aeo_updates/Compare AEO 2025 to AEO 2023.xlsx b/aeo_updates/Compare AEO 2025 to AEO 2023.xlsx deleted file mode 100644 index c5ab839..0000000 Binary files a/aeo_updates/Compare AEO 2025 to AEO 2023.xlsx and /dev/null differ diff --git a/aeo_updates/Compare AEO 2026 to AEO 2025.xlsx b/aeo_updates/Compare AEO 2026 to AEO 2025.xlsx new file mode 100644 index 0000000..1aae3ec Binary files /dev/null and b/aeo_updates/Compare AEO 2026 to AEO 2025.xlsx differ diff --git a/aeo_updates/README.md b/aeo_updates/README.md index f02f09f..b0f986c 100644 --- a/aeo_updates/README.md +++ b/aeo_updates/README.md @@ -9,19 +9,18 @@ This script creates the demand projection files for AEO scenarios. It uses histo ### AEO_scraper.ipynb The scraper will grab the following data from EIA's API: -* Electricity growth by region for high, low, and reference economic growth. +* Electricity growth by region for high, low, and Counterfactual Baseline economic growth. * Natural gas prices by region for high, low, and reference oil and gas growth. * Total natural gas use by region for high, low, and reference oil and gas resource * Natural gas use for electricity by region for high, low, and reference oil and gas resource. -* Coal prices by region for the reference scenario. -* Uranium prices for the US for the reference scenario. +* Coal prices by region for the Counterfactual Baseline scenario. +* Uranium/Nuclear Fuel prices for the US for the Counterfactual Baseline scenario. This can be run as is. You will want to adjust the default AEO year. Sometimes the scenario names change from one year to the next, so if a scenario is not populated, its name has likely changed. +In the AEO 2026, "Uranium" became "Nuclear Fuel", but only for real prices (nominal prices still use Uranium)--so you can check for similar name changes if you run into an error. -These results will be written out as csv files. -Note that the write directory is not specified and hence the files will be output to the directory from where the file is run. -The output directory may be altered as required. +These results will be written out as csv files to the outputs directory (it will be created if it does not already exist). If more data sets are desired, search for the API url here: https://www.eia.gov/opendata/qb.php?category=371