Add validation config, goldens, counters, and existing statvar mcf for statistics_poland

statvar_imports/statistics_poland/README.md

@@ -17,7 +17,7 @@ type of place: State.
 
 statvars: Demographics
 
-years: 2003 to 2024.
+years: 2003 to 2025.
 
 ## Processing Instructions
 To process the Poland Census data and generate statistical variables, use the following command from the "data" directory:
@@ -33,6 +33,7 @@ python3 tools/statvar_importer/stat_var_processor.py \
   --pv_map=statvar_imports/statistics_poland/StatisticsPoland_pvmap.csv \
   --output_path=statvar_imports/statistics_poland/test/StatisticsPoland_output \
   --config_file=statvar_imports/statistics_poland/StatisticsPoland_metadata.csv \
+  --output_counters=statvar_imports/statistics_poland/test/StatisticsPoland_output_counters.csv \
   --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf
 ```
 **For Main data run**
@@ -42,5 +43,6 @@ python3 tools/statvar_importer/stat_var_processor.py \
   --pv_map=statvar_imports/statistics_poland/StatisticsPoland_pvmap.csv \
   --output_path=statvar_imports/statistics_poland/StatisticsPoland_output \
   --config_file=statvar_imports/statistics_poland/StatisticsPoland_metadata.csv \
+  --output_counters=statvar_imports/statistics_poland/counters/StatisticsPoland_output_counters.csv \
   --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf
 ```

statvar_imports/statistics_poland/StatisticsPoland_metadata.csv

@@ -1,11 +1,12 @@
-config,value
-provenance_url,https://bdl.stat.gov.pl/bdl/dane/podgrup/tablica
-output_columns,"observationDate,observationAbout,value,variableMeasured"
-places_within,country/POL
-#place_types,"AdministrativeArea,AdministrativeArea1,AdministrativeArea2,State"
-#debug,1
-#input_rows,100
-#word_delimiter,''
-#skip_rows,1
-header_rows,5
-mapped_columns,2
+config,value
+provenance_url,https://bdl.stat.gov.pl/bdl/dane/podgrup/tablica
+output_columns,"observationDate,observationAbout,value,variableMeasured"
+places_within,country/POL
+recon_property,"variableMeasured,measurementMethod"
+#place_types,"AdministrativeArea,AdministrativeArea1,AdministrativeArea2,State"
+#debug,1
+#input_rows,100
+#word_delimiter,''
+#skip_rows,1
+header_rows,5
+mapped_columns,2

statvar_imports/statistics_poland/download_input_data.py

@@ -3,6 +3,8 @@
 import logging
 import requests
 import time
+import sys
+import traceback
 from datetime import datetime
 from google.cloud import storage
 import io
@@ -13,12 +15,11 @@
 # --- CONFIGURATION ---
 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
 
-# GCS Template Path
-GCS_TEMPLATE_PATH = "gs://datcom-prod-imports/statvar_imports/statistics_poland/poland_data_sample/StatisticsPoland_input.csv"
-
-# Local Output Directory
+# Outputs exactly to source_files to match your manifest.json
 OUTPUT_DIR = os.path.join(BASE_PATH, "source_files")
 
+GCS_TEMPLATE_PATH = "gs://datcom-prod-imports/statvar_imports/statistics_poland/poland_data_sample/StatisticsPoland_input.csv"
+
 API_BASE_URL = "https://bdl.stat.gov.pl/api/v1"
 API_KEY = "c9a9da02-47ab-4391-dff1-08de66e5ba7b"
 HEADERS = {'X-ClientId': API_KEY}
@@ -47,14 +48,7 @@ def load_template_from_gcs(gcs_path):
     """Loads the template CSV directly from GCS."""
     try:
         logging.info(f"Reading template from {gcs_path}...")
-
-        # Method 1: Direct Pandas Read (requires gcsfs)
-        # return pd.read_csv(gcs_path, header=[0,1,2,3], index_col=[0,1])
-
-        # Method 2: Google Cloud Storage Client (More robust if gcsfs isn't configured)
         storage_client = storage.Client()
-
-        # Parse bucket and blob
         path_parts = gcs_path.replace("gs://", "").split("/", 1)
         bucket_name = path_parts[0]
         blob_name = path_parts[1]
@@ -64,7 +58,6 @@ def load_template_from_gcs(gcs_path):
         content = blob.download_as_text()
 
         return pd.read_csv(io.StringIO(content), header=[0,1,2,3], index_col=[0,1])
-
     except Exception as e:
         logging.error(f"Failed to load template from GCS: {e}")
         return None
@@ -82,7 +75,6 @@ def fetch_variables():
     logging.info(f"Downloading variable list for Subject {SUBJECT_ID}...")
     v_map = {}
 
-    # Check pages 0-10 to ensure we get ALL variables
     for page in range(10): 
         url = f"{API_BASE_URL}/variables?subject-id={SUBJECT_ID}&page-size=100&lang=pl&page={page}"
         try:
@@ -106,29 +98,28 @@ def fetch_variables():
     return v_map
 
 def download_and_process():
-    if not os.path.exists(OUTPUT_DIR): os.makedirs(OUTPUT_DIR)
+    # Safely create output directory
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
 
-    # 1. LOAD TEMPLATE FROM GCS
     template_df = load_template_from_gcs(GCS_TEMPLATE_PATH)
-    if template_df is None: return
+    if template_df is None: 
+        raise ValueError("Template DataFrame failed to load from GCS.")
 
-    # Force index to strings
     template_df.index = template_df.index.set_levels([
         template_df.index.levels[0].astype(str), 
         template_df.index.levels[1].astype(str)
     ])
 
     region_map = get_template_map(template_df)
     v_metadata = fetch_variables()
-    if not v_metadata: return
+    if not v_metadata: 
+        raise ValueError("Variable metadata failed to download.")
 
     master_data = []
     unique_cols = template_df.columns.droplevel('Year').unique()
     current_year = datetime.now().year
 
-    # 2. MATCH & DOWNLOAD (Specific Ages Only)
     for age, sex, loc in unique_cols:
-        # SKIP TOTALS HERE -> We will calculate them later!
         if pd.isna(age) or str(age).strip() == '' or str(age).lower() == 'total':
             continue
 
@@ -161,7 +152,6 @@ def download_and_process():
 
         logging.info(f"MATCH: {age}|{sex}|{loc} -> ID {var_id}")
 
-        # Download Loop
         for lv in ["0", "2"]:
             api_url = f"{API_BASE_URL}/data/by-variable/{var_id}"
             params = [('unit-level', lv), ('page-size', '100')]
@@ -206,26 +196,23 @@ def download_and_process():
         time.sleep(0.05)
 
     if not master_data:
-        logging.error("No data collected.")
-        return
+        raise ValueError("No data collected during the download loop.")
 
-    # 3. PROCESS & CALCULATE TOTALS
     full_df = pd.DataFrame(master_data)
 
     for year in sorted(full_df['Year'].unique()):
         year_df = full_df[full_df['Year'] == year]
 
-        # Pivot specific ages
         pivot_df = year_df.pivot_table(
             index=['Code', 'Name'],
             columns=['Age', 'Sex', 'Location', 'Year'],
             values='Value'
         )
 
-        # Sum Age columns to create "Total" Age column
-        totals = pivot_df.groupby(level=['Sex', 'Location', 'Year'], axis=1).sum()
+        # CLOUD FIX: Replaced 'axis=1' grouping which crashes in modern Pandas environments.
+        # Transposing before and after groupby achieves the exact same result safely.
+        totals = pivot_df.T.groupby(level=['Sex', 'Location', 'Year']).sum().T
 
-        # Map calculated totals to the 'Age' level (using 'total' label for now)
         new_columns = pd.MultiIndex.from_tuples(
             [('total', s, l, y) for s, l, y in totals.columns],
             names=['Age', 'Sex', 'Location', 'Year']
@@ -234,35 +221,22 @@ def download_and_process():
 
         combined_df = pd.concat([pivot_df, totals], axis=1)
 
-        # 4. REINDEX AGAINST TEMPLATE
-        # Construct expected columns based on template structure for THIS year
         target_columns = []
         for col in template_df.columns:
             t_age, t_sex, t_loc, _ = col
-
-            # Map Template Age to Our Age
-            if pd.isna(t_age) or str(t_age).strip() == '':
-                lookup_age = 'total'
-            else:
-                lookup_age = t_age
-
+            lookup_age = 'total' if pd.isna(t_age) or str(t_age).strip() == '' else t_age
             target_columns.append((lookup_age, t_sex, t_loc, str(year)))
 
-        # Reindex rows (Code/Name)
         final_df = combined_df.reindex(template_df.index)
 
-        # Reindex columns to match template order
         try:
             final_df = final_df[target_columns]
-
-            # Restore original headers (e.g. putting back empty strings for Total Age)
             final_headers = []
             for col in template_df.columns:
                  t_age, t_sex, t_loc, _ = col
                  final_headers.append((t_age, t_sex, t_loc, str(year)))
 
             final_df.columns = pd.MultiIndex.from_tuples(final_headers, names=['Age', 'Sex', 'Location', 'Year'])
-
         except KeyError as e:
             logging.warning(f"Column alignment warning for {year}: {e}")
             pass
@@ -272,4 +246,11 @@ def download_and_process():
         logging.info(f"Generated: {out_path}")
 
 if __name__ == "__main__":
-    download_and_process()
+    try:
+        download_and_process()
+    except Exception as e:
+        # CLOUD FIX: Catch any unhandled exceptions to prevent silent exit 1 failures.
+        # This pushes the exact stack trace directly into Cloud Logging.
+        logging.critical(f"FATAL SCRIPT ERROR: {e}")
+        logging.critical(traceback.format_exc())
+        sys.exit(1)

statvar_imports/statistics_poland/golden_data/golden_observations.csv

@@ -0,0 +1,18 @@
+"observationAbout"
+"country/POL"
+"nuts/PL51"
+"nuts/PL61"
+"nuts/PL81"
+"nuts/PL43"
+"nuts/PL71"
+"nuts/PL21"
+"nuts/PL9"
+"nuts/PL52"
+"nuts/PL82"
+"nuts/PL84"
+"nuts/PL63"
+"nuts/PL22"
+"nuts/PL72"
+"nuts/PL62"
+"nuts/PL41"
+"nuts/PL42"

statvar_imports/statistics_poland/golden_data/golden_summary_report.csv

@@ -0,0 +1,109 @@
+"MinDate","StatVar","Units","ScalingFactors","NumPlaces","observationPeriods","MeasurementMethods"
+"2003","Count_Person_Years3To6_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_7To12Years_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Years0To2_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years7To12_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years0To2_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years3To6_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_16To19Years","[]","[]","17","[]","[]"
+"2003","Count_Person_20To24Years_Urban_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_20To24Years_Rural_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Years65Onwards_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years0To2_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years0To2_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_20To24Years_Rural_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_Rural_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_Years0To2_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Years55To64_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years35To44_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years25To34_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_65OrMoreYears_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_45To54Years_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_Years0To2_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years3To6_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years13To15_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years55To64_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years35To44_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years0To2_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years25To34_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_16To19Years_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Years16To19_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_7To12Years_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_Years45To54_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years0To2","[]","[]","17","[]","[]"
+"2003","Count_Person_Years13To15_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years45To54_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_25To34Years_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_Urban_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_25To34Years_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_20To24Years_Urban_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Years7To12_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years35To44_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years13To15","[]","[]","17","[]","[]"
+"2003","Count_Person_Years65Onwards_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_65OrMoreYears","[]","[]","17","[]","[]"
+"2003","Count_Person_55To64Years_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years55To64_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_20To24Years_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years7To12_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years16To19_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years35To44_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years3To6_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_Years25To34_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years3To6","[]","[]","17","[]","[]"
+"2003","Count_Person_Years3To6_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years3To6_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years7To12_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years0To2_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_Years45To54_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_55To64Years_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_55To64Years","[]","[]","17","[]","[]"
+"2003","Count_Person_35To44Years_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_20To24Years_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_35To44Years_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_65OrMoreYears_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Years13To15_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_7To12Years","[]","[]","17","[]","[]"
+"2003","Count_Person_65OrMoreYears_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years13To15_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years45To54_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years65Onwards_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years35To44_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_16To19Years_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_Rural_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_55To64Years_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_20To24Years_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_65OrMoreYears_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_25To34Years_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person","[]","[]","17","[]","[]"
+"2003","Count_Person_Years13To15_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_45To54Years","[]","[]","17","[]","[]"
+"2003","Count_Person_Years13To15_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years16To19_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years16To19_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years13To15_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Years3To6_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years16To19_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Urban_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Years25To34_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years16To19_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_55To64Years_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_35To44Years","[]","[]","17","[]","[]"
+"2003","Count_Person_Years65Onwards_Male_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_20To24Years_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years13To15_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years7To12_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years45To54_Female_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_20To24Years","[]","[]","17","[]","[]"
+"2003","Count_Person_25To34Years","[]","[]","17","[]","[]"
+"2003","Count_Person_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Years45To54_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_Years35To44_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years55To64_Female_Urban","[]","[]","17","[]","[]"
+"2003","Count_Person_Years3To6_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_45To54Years_Male","[]","[]","17","[]","[]"
+"2003","Count_Person_Female","[]","[]","17","[]","[]"
+"2003","Count_Person_Years7To12_Male_Rural","[]","[]","17","[]","[]"
+"2003","Count_Person_25To34Years_Male","[]","[]","17","[]","[]"

statvar_imports/statistics_poland/manifest.json

@@ -9,10 +9,12 @@
             "provenance_description": "Population data for demographic variables such as population counts, age distributions, and other census-related metrics in Poland",
             "scripts": [
                 "download_input_data.py",
-                "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.csv --pv_map=StatisticsPoland_pvmap.csv --config_file=StatisticsPoland_metadata.csv --output_path=StatisticsPoland_output"
+                "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.csv --pv_map=StatisticsPoland_pvmap.csv --config_file=StatisticsPoland_metadata.csv --output_path=StatisticsPoland_output --output_counters=counters/StatisticsPoland_output_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf"
             ],
             "source_files": [
-                "source_files/*.csv"
+                "source_files/*.csv",
+                "counters/*.csv",
+                "golden_data/*.csv"
             ],
             "import_inputs": [
                 {
@@ -21,7 +23,22 @@
                     "stat_var_mcf": "StatisticsPoland_output_stat_vars.mcf"
                 }
             ],
-            "cron_schedule": "0 0 1 1,4,7,10 *"
+            "cron_schedule": "0 0 1 1,4,7,10 *",
+            "validation_config_file": "validation_config.json",
+            "resource_limits": {
+                "cpu": 8,
+                "memory": 32,
+                "disk": 100
+            }
         }
-    ]
+    ],
+    "config_override": {
+        "invoke_import_validation": true,
+        "invoke_import_tool": true,
+        "invoke_differ_tool": true,
+        "use_autopush_dc_api": false,
+        "skip_input_upload": false,
+        "skip_gcs_upload": false,
+        "cleanup_gcs_volume_mount": false
+    }
 }