Fix issue #1: Download US states data with Python and save as JSON.

Author: yaph

.gitignore

@@ -4,6 +4,8 @@ data/*.json
 data/*.txt
 data/*.zip
 
+geonamescache/data/us_states.json
+
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

bin/us_states.js

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+from pathlib import Path
+import json
+import re
+
+from bs4 import BeautifulSoup
+import httpx
+
+
+def extract_us_states_data():
+    """
+    Downloads HTML from GeoNames US administrative divisions page
+    and extracts state information into a dictionary structure.
+    """
+
+    # Download the HTML content
+    url = 'http://www.geonames.org/US/administrative-division-united-states.html'
+
+    try:
+        response = httpx.get(url)
+        response.raise_for_status()  # Raise an exception for bad status codes
+        html_content = response.text
+    except httpx.RequestError as e:
+        print(f'Error downloading the page: {e}')
+        return {}
+
+    # Parse the HTML
+    soup = BeautifulSoup(html_content, 'html.parser')
+
+    # Find the table with id 'subdivtable1'
+    table = soup.find('table', id='subdivtable1')
+    if not table:
+        print("Table with id 'subdivtable1' not found")
+        return {}
+
+    # Dictionary to store the results
+    states_data = {}
+
+    # Find all table rows
+    rows = table.find_all('tr')
+
+    for row in rows:
+        # Find span with id containing "isoSpan" for the state code
+        iso_span = row.find('span', id=re.compile(r'.*isoSpan.*'))
+        if iso_span and iso_span.get_text(strip=True):
+            code = iso_span.get_text(strip=True)
+
+            # Find span with id containing "fipsSpan" for FIPS code
+            fips_span = row.find('span', id=re.compile(r'.*fipsSpan.*'))
+            fips = fips_span.get_text(strip=True) if fips_span else ""
+
+            # Find span with id containing "nameSpan" and get the link inside it
+            name_span = row.find('span', id=re.compile(r'.*nameSpan.*'))
+            if name_span:
+                link = name_span.find('a')
+                if link and link.get('href'):
+                    href = link.get('href')
+                    name = link.get_text(strip=True)
+
+                    # Extract geoname ID from href (assuming format like "/path/path/ID/name")
+                    href_parts = href.split('/')
+                    if len(href_parts) > 3:
+                        try:
+                            geoname_id = int(href_parts[3])
+
+                            # Create the data structure matching the JavaScript output
+                            states_data[code] = {
+                                'code': code,
+                                'name': name,
+                                'fips': fips,
+                                'geonameid': geoname_id
+                            }
+
+                        except (ValueError, IndexError):
+                            # Skip if geoname ID extraction fails
+                            continue
+
+    return states_data
+
+
+if __name__ == '__main__':
+    states_data = extract_us_states_data()
+    if states_data:
+        print(f'Total states extracted: {len(states_data)}')
+        Path('geonamescache/data/us_states.json').write_text(json.dumps(states_data))
+    else:
+        print('No data extracted. Please check the URL and HTML structure.')

bin/us_states.py

@@ -1,5 +1,5 @@
 __title__ = 'geonamescache'
-__version__ = '2.0.0'
+__version__ = '3.0.0'
 __author__ = 'Ramiro Gómez'
 __license__ = 'MIT'
 
@@ -9,7 +9,6 @@
 from collections.abc import Mapping
 from typing import Any, ClassVar, TypeVar
 
-from geonamescache import geonamesdata
 from geonamescache.types import (
     City,
     CitySearchAttribute,
@@ -28,13 +27,13 @@
 
 
 class GeonamesCache:
-    us_states: dict[USStateCode, USState] = geonamesdata.us_states
     continents: dict[ContinentCode, Continent] | None = None
     countries: dict[ISOStr, Country] | None = None
     cities: dict[GeoNameIdStr, City] | None = None
     cities_items: list[tuple[GeoNameIdStr, City]] | None = None
     cities_by_names: ClassVar[dict[str, list[dict[GeoNameIdStr, City]]]] = {}
     us_counties: list[USCounty] | None = None
+    us_states: dict[USStateCode, USState] | None = None
 
     def __init__(self, min_city_population: int = 15000):
         self.min_city_population = min_city_population
@@ -43,17 +42,13 @@ def get_dataset_by_key(self, dataset: dict[Any, TDict], key: str) -> dict[Any, T
         return {d[key]: d for c, d in list(dataset.items())}
 
     def get_continents(self) -> dict[ContinentCode, Continent]:
-        if self.continents is None:
-            self.continents = self._load_data(self.continents, 'continents.json')
-        return self.continents
+        return self._load_data(self.continents, 'continents.json')
 
     def get_countries(self) -> dict[ISOStr, Country]:
-        if self.countries is None:
-            self.countries = self._load_data(self.countries, 'countries.json')
-        return self.countries
+        return self._load_data(self.countries, 'countries.json')
 
     def get_us_states(self) -> dict[USStateCode, USState]:
-        return self.us_states
+        return self._load_data(self.us_states, 'us_states.json')
 
     def get_countries_by_names(self) -> dict[str, Country]:
         return self.get_dataset_by_key(self.get_countries(), 'name')
@@ -63,10 +58,7 @@ def get_us_states_by_names(self) -> dict[USStateName, USState]:
 
     def get_cities(self) -> dict[GeoNameIdStr, City]:
         """Get a dictionary of cities keyed by geonameid."""
-
-        if self.cities is None:
-            self.cities = self._load_data(self.cities, f'cities{self.min_city_population}.json')
-        return self.cities
+        return self._load_data(self.cities, f'cities{self.min_city_population}.json')
 
     def get_cities_by_name(self, name: str) -> list[dict[GeoNameIdStr, City]]:
         """Get a list of city dictionaries with the given name.
@@ -81,9 +73,7 @@ def get_cities_by_name(self, name: str) -> list[dict[GeoNameIdStr, City]]:
         return self.cities_by_names[name]
 
     def get_us_counties(self):
-        if self.us_counties is None:
-            self.us_counties = self._load_data(self.us_counties, 'us_counties.json')
-        return self.us_counties
+        return self._load_data(self.us_counties, 'us_counties.json')
 
     def search_cities(
         self,