diff --git a/statvar_imports/statistics_poland/README.md b/statvar_imports/statistics_poland/README.md index 50c70ef17a..f396a06bde 100644 --- a/statvar_imports/statistics_poland/README.md +++ b/statvar_imports/statistics_poland/README.md @@ -17,7 +17,7 @@ type of place: State. statvars: Demographics -years: 2003 to 2024. +years: 2003 to 2025. ## Processing Instructions To process the Poland Census data and generate statistical variables, use the following command from the "data" directory: @@ -33,6 +33,7 @@ python3 tools/statvar_importer/stat_var_processor.py \ --pv_map=statvar_imports/statistics_poland/StatisticsPoland_pvmap.csv \ --output_path=statvar_imports/statistics_poland/test/StatisticsPoland_output \ --config_file=statvar_imports/statistics_poland/StatisticsPoland_metadata.csv \ + --output_counters=statvar_imports/statistics_poland/test/StatisticsPoland_output_counters.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ``` **For Main data run** @@ -42,5 +43,6 @@ python3 tools/statvar_importer/stat_var_processor.py \ --pv_map=statvar_imports/statistics_poland/StatisticsPoland_pvmap.csv \ --output_path=statvar_imports/statistics_poland/StatisticsPoland_output \ --config_file=statvar_imports/statistics_poland/StatisticsPoland_metadata.csv \ + --output_counters=statvar_imports/statistics_poland/counters/StatisticsPoland_output_counters.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ``` diff --git a/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv b/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv index a3a30ec1c2..25635e5ce1 100644 --- a/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv +++ b/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv @@ -1,11 +1,12 @@ -config,value -provenance_url,https://bdl.stat.gov.pl/bdl/dane/podgrup/tablica -output_columns,"observationDate,observationAbout,value,variableMeasured" -places_within,country/POL -#place_types,"AdministrativeArea,AdministrativeArea1,AdministrativeArea2,State" -#debug,1 -#input_rows,100 -#word_delimiter,'' -#skip_rows,1 -header_rows,5 -mapped_columns,2 +config,value +provenance_url,https://bdl.stat.gov.pl/bdl/dane/podgrup/tablica +output_columns,"observationDate,observationAbout,value,variableMeasured" +places_within,country/POL +recon_property,"variableMeasured,measurementMethod" +#place_types,"AdministrativeArea,AdministrativeArea1,AdministrativeArea2,State" +#debug,1 +#input_rows,100 +#word_delimiter,'' +#skip_rows,1 +header_rows,5 +mapped_columns,2 diff --git a/statvar_imports/statistics_poland/download_input_data.py b/statvar_imports/statistics_poland/download_input_data.py index 147335540a..4069186c33 100644 --- a/statvar_imports/statistics_poland/download_input_data.py +++ b/statvar_imports/statistics_poland/download_input_data.py @@ -3,6 +3,8 @@ import logging import requests import time +import sys +import traceback from datetime import datetime from google.cloud import storage import io @@ -13,12 +15,11 @@ # --- CONFIGURATION --- BASE_PATH = os.path.dirname(os.path.abspath(__file__)) -# GCS Template Path -GCS_TEMPLATE_PATH = "gs://datcom-prod-imports/statvar_imports/statistics_poland/poland_data_sample/StatisticsPoland_input.csv" - -# Local Output Directory +# Outputs exactly to source_files to match your manifest.json OUTPUT_DIR = os.path.join(BASE_PATH, "source_files") +GCS_TEMPLATE_PATH = "gs://datcom-prod-imports/statvar_imports/statistics_poland/poland_data_sample/StatisticsPoland_input.csv" + API_BASE_URL = "https://bdl.stat.gov.pl/api/v1" API_KEY = "c9a9da02-47ab-4391-dff1-08de66e5ba7b" HEADERS = {'X-ClientId': API_KEY} @@ -47,14 +48,7 @@ def load_template_from_gcs(gcs_path): """Loads the template CSV directly from GCS.""" try: logging.info(f"Reading template from {gcs_path}...") - - # Method 1: Direct Pandas Read (requires gcsfs) - # return pd.read_csv(gcs_path, header=[0,1,2,3], index_col=[0,1]) - - # Method 2: Google Cloud Storage Client (More robust if gcsfs isn't configured) storage_client = storage.Client() - - # Parse bucket and blob path_parts = gcs_path.replace("gs://", "").split("/", 1) bucket_name = path_parts[0] blob_name = path_parts[1] @@ -64,7 +58,6 @@ def load_template_from_gcs(gcs_path): content = blob.download_as_text() return pd.read_csv(io.StringIO(content), header=[0,1,2,3], index_col=[0,1]) - except Exception as e: logging.error(f"Failed to load template from GCS: {e}") return None @@ -82,7 +75,6 @@ def fetch_variables(): logging.info(f"Downloading variable list for Subject {SUBJECT_ID}...") v_map = {} - # Check pages 0-10 to ensure we get ALL variables for page in range(10): url = f"{API_BASE_URL}/variables?subject-id={SUBJECT_ID}&page-size=100&lang=pl&page={page}" try: @@ -106,13 +98,13 @@ def fetch_variables(): return v_map def download_and_process(): - if not os.path.exists(OUTPUT_DIR): os.makedirs(OUTPUT_DIR) + # Safely create output directory + os.makedirs(OUTPUT_DIR, exist_ok=True) - # 1. LOAD TEMPLATE FROM GCS template_df = load_template_from_gcs(GCS_TEMPLATE_PATH) - if template_df is None: return + if template_df is None: + raise ValueError("Template DataFrame failed to load from GCS.") - # Force index to strings template_df.index = template_df.index.set_levels([ template_df.index.levels[0].astype(str), template_df.index.levels[1].astype(str) @@ -120,15 +112,14 @@ def download_and_process(): region_map = get_template_map(template_df) v_metadata = fetch_variables() - if not v_metadata: return + if not v_metadata: + raise ValueError("Variable metadata failed to download.") master_data = [] unique_cols = template_df.columns.droplevel('Year').unique() current_year = datetime.now().year - # 2. MATCH & DOWNLOAD (Specific Ages Only) for age, sex, loc in unique_cols: - # SKIP TOTALS HERE -> We will calculate them later! if pd.isna(age) or str(age).strip() == '' or str(age).lower() == 'total': continue @@ -161,7 +152,6 @@ def download_and_process(): logging.info(f"MATCH: {age}|{sex}|{loc} -> ID {var_id}") - # Download Loop for lv in ["0", "2"]: api_url = f"{API_BASE_URL}/data/by-variable/{var_id}" params = [('unit-level', lv), ('page-size', '100')] @@ -206,26 +196,23 @@ def download_and_process(): time.sleep(0.05) if not master_data: - logging.error("No data collected.") - return + raise ValueError("No data collected during the download loop.") - # 3. PROCESS & CALCULATE TOTALS full_df = pd.DataFrame(master_data) for year in sorted(full_df['Year'].unique()): year_df = full_df[full_df['Year'] == year] - # Pivot specific ages pivot_df = year_df.pivot_table( index=['Code', 'Name'], columns=['Age', 'Sex', 'Location', 'Year'], values='Value' ) - # Sum Age columns to create "Total" Age column - totals = pivot_df.groupby(level=['Sex', 'Location', 'Year'], axis=1).sum() + # CLOUD FIX: Replaced 'axis=1' grouping which crashes in modern Pandas environments. + # Transposing before and after groupby achieves the exact same result safely. + totals = pivot_df.T.groupby(level=['Sex', 'Location', 'Year']).sum().T - # Map calculated totals to the 'Age' level (using 'total' label for now) new_columns = pd.MultiIndex.from_tuples( [('total', s, l, y) for s, l, y in totals.columns], names=['Age', 'Sex', 'Location', 'Year'] @@ -234,35 +221,22 @@ def download_and_process(): combined_df = pd.concat([pivot_df, totals], axis=1) - # 4. REINDEX AGAINST TEMPLATE - # Construct expected columns based on template structure for THIS year target_columns = [] for col in template_df.columns: t_age, t_sex, t_loc, _ = col - - # Map Template Age to Our Age - if pd.isna(t_age) or str(t_age).strip() == '': - lookup_age = 'total' - else: - lookup_age = t_age - + lookup_age = 'total' if pd.isna(t_age) or str(t_age).strip() == '' else t_age target_columns.append((lookup_age, t_sex, t_loc, str(year))) - # Reindex rows (Code/Name) final_df = combined_df.reindex(template_df.index) - # Reindex columns to match template order try: final_df = final_df[target_columns] - - # Restore original headers (e.g. putting back empty strings for Total Age) final_headers = [] for col in template_df.columns: t_age, t_sex, t_loc, _ = col final_headers.append((t_age, t_sex, t_loc, str(year))) final_df.columns = pd.MultiIndex.from_tuples(final_headers, names=['Age', 'Sex', 'Location', 'Year']) - except KeyError as e: logging.warning(f"Column alignment warning for {year}: {e}") pass @@ -272,4 +246,11 @@ def download_and_process(): logging.info(f"Generated: {out_path}") if __name__ == "__main__": - download_and_process() \ No newline at end of file + try: + download_and_process() + except Exception as e: + # CLOUD FIX: Catch any unhandled exceptions to prevent silent exit 1 failures. + # This pushes the exact stack trace directly into Cloud Logging. + logging.critical(f"FATAL SCRIPT ERROR: {e}") + logging.critical(traceback.format_exc()) + sys.exit(1) \ No newline at end of file diff --git a/statvar_imports/statistics_poland/golden_data/golden_observations.csv b/statvar_imports/statistics_poland/golden_data/golden_observations.csv new file mode 100644 index 0000000000..5a9a6bc718 --- /dev/null +++ b/statvar_imports/statistics_poland/golden_data/golden_observations.csv @@ -0,0 +1,18 @@ +"observationAbout" +"country/POL" +"nuts/PL51" +"nuts/PL61" +"nuts/PL81" +"nuts/PL43" +"nuts/PL71" +"nuts/PL21" +"nuts/PL9" +"nuts/PL52" +"nuts/PL82" +"nuts/PL84" +"nuts/PL63" +"nuts/PL22" +"nuts/PL72" +"nuts/PL62" +"nuts/PL41" +"nuts/PL42" diff --git a/statvar_imports/statistics_poland/golden_data/golden_summary_report.csv b/statvar_imports/statistics_poland/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..281859e678 --- /dev/null +++ b/statvar_imports/statistics_poland/golden_data/golden_summary_report.csv @@ -0,0 +1,109 @@ +"MinDate","StatVar","Units","ScalingFactors","NumPlaces","observationPeriods","MeasurementMethods" +"2003","Count_Person_Years3To6_Male","[]","[]","17","[]","[]" +"2003","Count_Person_7To12Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_16To19Years","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Urban_Female","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Rural_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years65Onwards_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Rural_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Rural_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years55To64_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years25To34_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears_Female","[]","[]","17","[]","[]" +"2003","Count_Person_45To54Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years55To64_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years25To34_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_16To19Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_7To12Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Urban_Female","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Urban_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15","[]","[]","17","[]","[]" +"2003","Count_Person_Years65Onwards_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years55To64_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years25To34_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years","[]","[]","17","[]","[]" +"2003","Count_Person_35To44Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_35To44Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Female","[]","[]","17","[]","[]" +"2003","Count_Person_7To12Years","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years65Onwards_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_16To19Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Rural_Male","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years_Rural","[]","[]","17","[]","[]" +"2003","Count_Person","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_45To54Years","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Urban_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years25To34_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_35To44Years","[]","[]","17","[]","[]" +"2003","Count_Person_Years65Onwards_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years","[]","[]","17","[]","[]" +"2003","Count_Person_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years55To64_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_45To54Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years_Male","[]","[]","17","[]","[]" diff --git a/statvar_imports/statistics_poland/manifest.json b/statvar_imports/statistics_poland/manifest.json index 4d2b33ca93..5419d0efaf 100644 --- a/statvar_imports/statistics_poland/manifest.json +++ b/statvar_imports/statistics_poland/manifest.json @@ -9,10 +9,12 @@ "provenance_description": "Population data for demographic variables such as population counts, age distributions, and other census-related metrics in Poland", "scripts": [ "download_input_data.py", - "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.csv --pv_map=StatisticsPoland_pvmap.csv --config_file=StatisticsPoland_metadata.csv --output_path=StatisticsPoland_output" + "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.csv --pv_map=StatisticsPoland_pvmap.csv --config_file=StatisticsPoland_metadata.csv --output_path=StatisticsPoland_output --output_counters=counters/StatisticsPoland_output_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" ], "source_files": [ - "source_files/*.csv" + "source_files/*.csv", + "counters/*.csv", + "golden_data/*.csv" ], "import_inputs": [ { @@ -21,7 +23,22 @@ "stat_var_mcf": "StatisticsPoland_output_stat_vars.mcf" } ], - "cron_schedule": "0 0 1 1,4,7,10 *" + "cron_schedule": "0 0 1 1,4,7,10 *", + "validation_config_file": "validation_config.json", + "resource_limits": { + "cpu": 8, + "memory": 32, + "disk": 100 + } } - ] + ], + "config_override": { + "invoke_import_validation": true, + "invoke_import_tool": true, + "invoke_differ_tool": true, + "use_autopush_dc_api": false, + "skip_input_upload": false, + "skip_gcs_upload": false, + "cleanup_gcs_volume_mount": false + } } diff --git a/statvar_imports/statistics_poland/validation_config.json b/statvar_imports/statistics_poland/validation_config.json new file mode 100644 index 0000000000..f4eb0d2919 --- /dev/null +++ b/statvar_imports/statistics_poland/validation_config.json @@ -0,0 +1,31 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted records for the entire import is within threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.5 + } + }, + { + "rule_id": "check_goldens_summary_report", + "description": "Validates statistics poland data against its golden summary report.", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv", + "input_files": "../../input0/genmcf/summary_report.csv" + } + }, + { + "rule_id": "check_goldens_observations", + "description": "Verifies the generated output CSV data matches established critical golden records for Statistics Poland", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../StatisticsPoland_output.csv" + } + } + ] +}