From 34b147e365595e4b65fc0311097e8a1d61d7250a Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Tue, 4 Aug 2026 08:52:04 +0000 Subject: [PATCH 1/4] Add validation config, goldens, counters, and existing statvar mcf for statistics_poland --- statvar_imports/statistics_poland/README.md | 4 +- .../StatisticsPoland_metadata.csv | 23 ++-- .../statistics_poland/download_input_data.py | 74 +++++------- .../golden_data/golden_observations.csv | 18 +++ .../golden_data/golden_summary_report.csv | 109 ++++++++++++++++++ .../statistics_poland/manifest.json | 24 +++- .../statistics_poland/validation_config.json | 31 +++++ .../schema/schema_reconciler.py | 23 +++- 8 files changed, 243 insertions(+), 63 deletions(-) create mode 100644 statvar_imports/statistics_poland/golden_data/golden_observations.csv create mode 100644 statvar_imports/statistics_poland/golden_data/golden_summary_report.csv create mode 100644 statvar_imports/statistics_poland/validation_config.json diff --git a/statvar_imports/statistics_poland/README.md b/statvar_imports/statistics_poland/README.md index 50c70ef17a..f396a06bde 100644 --- a/statvar_imports/statistics_poland/README.md +++ b/statvar_imports/statistics_poland/README.md @@ -17,7 +17,7 @@ type of place: State. statvars: Demographics -years: 2003 to 2024. +years: 2003 to 2025. ## Processing Instructions To process the Poland Census data and generate statistical variables, use the following command from the "data" directory: @@ -33,6 +33,7 @@ python3 tools/statvar_importer/stat_var_processor.py \ --pv_map=statvar_imports/statistics_poland/StatisticsPoland_pvmap.csv \ --output_path=statvar_imports/statistics_poland/test/StatisticsPoland_output \ --config_file=statvar_imports/statistics_poland/StatisticsPoland_metadata.csv \ + --output_counters=statvar_imports/statistics_poland/test/StatisticsPoland_output_counters.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ``` **For Main data run** @@ -42,5 +43,6 @@ python3 tools/statvar_importer/stat_var_processor.py \ --pv_map=statvar_imports/statistics_poland/StatisticsPoland_pvmap.csv \ --output_path=statvar_imports/statistics_poland/StatisticsPoland_output \ --config_file=statvar_imports/statistics_poland/StatisticsPoland_metadata.csv \ + --output_counters=statvar_imports/statistics_poland/counters/StatisticsPoland_output_counters.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ``` diff --git a/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv b/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv index a3a30ec1c2..ef2e639519 100644 --- a/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv +++ b/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv @@ -1,11 +1,12 @@ -config,value -provenance_url,https://bdl.stat.gov.pl/bdl/dane/podgrup/tablica -output_columns,"observationDate,observationAbout,value,variableMeasured" -places_within,country/POL -#place_types,"AdministrativeArea,AdministrativeArea1,AdministrativeArea2,State" -#debug,1 -#input_rows,100 -#word_delimiter,'' -#skip_rows,1 -header_rows,5 -mapped_columns,2 +config,value +provenance_url,https://bdl.stat.gov.pl/bdl/dane/podgrup/tablica +output_columns,"observationDate,observationAbout,value,variableMeasured" +places_within,country/POL +recon_lookup_api,False +#place_types,"AdministrativeArea,AdministrativeArea1,AdministrativeArea2,State" +#debug,1 +#input_rows,100 +#word_delimiter,'' +#skip_rows,1 +header_rows,5 +mapped_columns,2 diff --git a/statvar_imports/statistics_poland/download_input_data.py b/statvar_imports/statistics_poland/download_input_data.py index 147335540a..0e7a634402 100644 --- a/statvar_imports/statistics_poland/download_input_data.py +++ b/statvar_imports/statistics_poland/download_input_data.py @@ -3,6 +3,8 @@ import logging import requests import time +import sys +import traceback from datetime import datetime from google.cloud import storage import io @@ -13,12 +15,11 @@ # --- CONFIGURATION --- BASE_PATH = os.path.dirname(os.path.abspath(__file__)) -# GCS Template Path -GCS_TEMPLATE_PATH = "gs://datcom-prod-imports/statvar_imports/statistics_poland/poland_data_sample/StatisticsPoland_input.csv" - -# Local Output Directory +# Outputs exactly to source_files to match your manifest.json OUTPUT_DIR = os.path.join(BASE_PATH, "source_files") +GCS_TEMPLATE_PATH = "gs://datcom-prod-imports/statvar_imports/statistics_poland/poland_data_sample/StatisticsPoland_input.csv" + API_BASE_URL = "https://bdl.stat.gov.pl/api/v1" API_KEY = "c9a9da02-47ab-4391-dff1-08de66e5ba7b" HEADERS = {'X-ClientId': API_KEY} @@ -47,14 +48,7 @@ def load_template_from_gcs(gcs_path): """Loads the template CSV directly from GCS.""" try: logging.info(f"Reading template from {gcs_path}...") - - # Method 1: Direct Pandas Read (requires gcsfs) - # return pd.read_csv(gcs_path, header=[0,1,2,3], index_col=[0,1]) - - # Method 2: Google Cloud Storage Client (More robust if gcsfs isn't configured) storage_client = storage.Client() - - # Parse bucket and blob path_parts = gcs_path.replace("gs://", "").split("/", 1) bucket_name = path_parts[0] blob_name = path_parts[1] @@ -64,7 +58,6 @@ def load_template_from_gcs(gcs_path): content = blob.download_as_text() return pd.read_csv(io.StringIO(content), header=[0,1,2,3], index_col=[0,1]) - except Exception as e: logging.error(f"Failed to load template from GCS: {e}") return None @@ -82,7 +75,6 @@ def fetch_variables(): logging.info(f"Downloading variable list for Subject {SUBJECT_ID}...") v_map = {} - # Check pages 0-10 to ensure we get ALL variables for page in range(10): url = f"{API_BASE_URL}/variables?subject-id={SUBJECT_ID}&page-size=100&lang=pl&page={page}" try: @@ -106,13 +98,20 @@ def fetch_variables(): return v_map def download_and_process(): - if not os.path.exists(OUTPUT_DIR): os.makedirs(OUTPUT_DIR) + global OUTPUT_DIR + + # Safely create output directory + try: + os.makedirs(OUTPUT_DIR, exist_ok=True) + except PermissionError: + logging.warning(f"Permission denied for {OUTPUT_DIR}. Falling back to /tmp/source_files") + OUTPUT_DIR = "/tmp/source_files" + os.makedirs(OUTPUT_DIR, exist_ok=True) - # 1. LOAD TEMPLATE FROM GCS template_df = load_template_from_gcs(GCS_TEMPLATE_PATH) - if template_df is None: return + if template_df is None: + raise ValueError("Template DataFrame failed to load from GCS.") - # Force index to strings template_df.index = template_df.index.set_levels([ template_df.index.levels[0].astype(str), template_df.index.levels[1].astype(str) @@ -120,15 +119,14 @@ def download_and_process(): region_map = get_template_map(template_df) v_metadata = fetch_variables() - if not v_metadata: return + if not v_metadata: + raise ValueError("Variable metadata failed to download.") master_data = [] unique_cols = template_df.columns.droplevel('Year').unique() current_year = datetime.now().year - # 2. MATCH & DOWNLOAD (Specific Ages Only) for age, sex, loc in unique_cols: - # SKIP TOTALS HERE -> We will calculate them later! if pd.isna(age) or str(age).strip() == '' or str(age).lower() == 'total': continue @@ -161,7 +159,6 @@ def download_and_process(): logging.info(f"MATCH: {age}|{sex}|{loc} -> ID {var_id}") - # Download Loop for lv in ["0", "2"]: api_url = f"{API_BASE_URL}/data/by-variable/{var_id}" params = [('unit-level', lv), ('page-size', '100')] @@ -206,26 +203,23 @@ def download_and_process(): time.sleep(0.05) if not master_data: - logging.error("No data collected.") - return + raise ValueError("No data collected during the download loop.") - # 3. PROCESS & CALCULATE TOTALS full_df = pd.DataFrame(master_data) for year in sorted(full_df['Year'].unique()): year_df = full_df[full_df['Year'] == year] - # Pivot specific ages pivot_df = year_df.pivot_table( index=['Code', 'Name'], columns=['Age', 'Sex', 'Location', 'Year'], values='Value' ) - # Sum Age columns to create "Total" Age column - totals = pivot_df.groupby(level=['Sex', 'Location', 'Year'], axis=1).sum() + # CLOUD FIX: Replaced 'axis=1' grouping which crashes in modern Pandas environments. + # Transposing before and after groupby achieves the exact same result safely. + totals = pivot_df.T.groupby(level=['Sex', 'Location', 'Year']).sum().T - # Map calculated totals to the 'Age' level (using 'total' label for now) new_columns = pd.MultiIndex.from_tuples( [('total', s, l, y) for s, l, y in totals.columns], names=['Age', 'Sex', 'Location', 'Year'] @@ -234,35 +228,22 @@ def download_and_process(): combined_df = pd.concat([pivot_df, totals], axis=1) - # 4. REINDEX AGAINST TEMPLATE - # Construct expected columns based on template structure for THIS year target_columns = [] for col in template_df.columns: t_age, t_sex, t_loc, _ = col - - # Map Template Age to Our Age - if pd.isna(t_age) or str(t_age).strip() == '': - lookup_age = 'total' - else: - lookup_age = t_age - + lookup_age = 'total' if pd.isna(t_age) or str(t_age).strip() == '' else t_age target_columns.append((lookup_age, t_sex, t_loc, str(year))) - # Reindex rows (Code/Name) final_df = combined_df.reindex(template_df.index) - # Reindex columns to match template order try: final_df = final_df[target_columns] - - # Restore original headers (e.g. putting back empty strings for Total Age) final_headers = [] for col in template_df.columns: t_age, t_sex, t_loc, _ = col final_headers.append((t_age, t_sex, t_loc, str(year))) final_df.columns = pd.MultiIndex.from_tuples(final_headers, names=['Age', 'Sex', 'Location', 'Year']) - except KeyError as e: logging.warning(f"Column alignment warning for {year}: {e}") pass @@ -272,4 +253,11 @@ def download_and_process(): logging.info(f"Generated: {out_path}") if __name__ == "__main__": - download_and_process() \ No newline at end of file + try: + download_and_process() + except Exception as e: + # CLOUD FIX: Catch any unhandled exceptions to prevent silent exit 1 failures. + # This pushes the exact stack trace directly into Cloud Logging. + logging.critical(f"FATAL SCRIPT ERROR: {e}") + logging.critical(traceback.format_exc()) + sys.exit(1) \ No newline at end of file diff --git a/statvar_imports/statistics_poland/golden_data/golden_observations.csv b/statvar_imports/statistics_poland/golden_data/golden_observations.csv new file mode 100644 index 0000000000..5a9a6bc718 --- /dev/null +++ b/statvar_imports/statistics_poland/golden_data/golden_observations.csv @@ -0,0 +1,18 @@ +"observationAbout" +"country/POL" +"nuts/PL51" +"nuts/PL61" +"nuts/PL81" +"nuts/PL43" +"nuts/PL71" +"nuts/PL21" +"nuts/PL9" +"nuts/PL52" +"nuts/PL82" +"nuts/PL84" +"nuts/PL63" +"nuts/PL22" +"nuts/PL72" +"nuts/PL62" +"nuts/PL41" +"nuts/PL42" diff --git a/statvar_imports/statistics_poland/golden_data/golden_summary_report.csv b/statvar_imports/statistics_poland/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..281859e678 --- /dev/null +++ b/statvar_imports/statistics_poland/golden_data/golden_summary_report.csv @@ -0,0 +1,109 @@ +"MinDate","StatVar","Units","ScalingFactors","NumPlaces","observationPeriods","MeasurementMethods" +"2003","Count_Person_Years3To6_Male","[]","[]","17","[]","[]" +"2003","Count_Person_7To12Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_16To19Years","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Urban_Female","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Rural_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years65Onwards_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Rural_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Rural_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years55To64_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years25To34_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears_Female","[]","[]","17","[]","[]" +"2003","Count_Person_45To54Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years55To64_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years25To34_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_16To19Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_7To12Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Urban_Female","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Urban_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15","[]","[]","17","[]","[]" +"2003","Count_Person_Years65Onwards_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years55To64_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years25To34_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years0To2_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years","[]","[]","17","[]","[]" +"2003","Count_Person_35To44Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_35To44Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Female","[]","[]","17","[]","[]" +"2003","Count_Person_7To12Years","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years65Onwards_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_16To19Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Rural_Male","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_65OrMoreYears_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years_Rural","[]","[]","17","[]","[]" +"2003","Count_Person","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_45To54Years","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Urban_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years25To34_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years16To19_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_55To64Years_Female","[]","[]","17","[]","[]" +"2003","Count_Person_35To44Years","[]","[]","17","[]","[]" +"2003","Count_Person_Years65Onwards_Male_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years13To15_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Female_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_20To24Years","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years","[]","[]","17","[]","[]" +"2003","Count_Person_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Years45To54_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_Years35To44_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years55To64_Female_Urban","[]","[]","17","[]","[]" +"2003","Count_Person_Years3To6_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_45To54Years_Male","[]","[]","17","[]","[]" +"2003","Count_Person_Female","[]","[]","17","[]","[]" +"2003","Count_Person_Years7To12_Male_Rural","[]","[]","17","[]","[]" +"2003","Count_Person_25To34Years_Male","[]","[]","17","[]","[]" diff --git a/statvar_imports/statistics_poland/manifest.json b/statvar_imports/statistics_poland/manifest.json index 4d2b33ca93..0a7496baf9 100644 --- a/statvar_imports/statistics_poland/manifest.json +++ b/statvar_imports/statistics_poland/manifest.json @@ -9,10 +9,12 @@ "provenance_description": "Population data for demographic variables such as population counts, age distributions, and other census-related metrics in Poland", "scripts": [ "download_input_data.py", - "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.csv --pv_map=StatisticsPoland_pvmap.csv --config_file=StatisticsPoland_metadata.csv --output_path=StatisticsPoland_output" + "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.csv --pv_map=StatisticsPoland_pvmap.csv --config_file=StatisticsPoland_metadata.csv --output_path=StatisticsPoland_output --output_counters=counters/StatisticsPoland_output_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" ], "source_files": [ - "source_files/*.csv" + "source_files/*.csv", + "counters/*.csv", + "golden_data/*.csv" ], "import_inputs": [ { @@ -21,7 +23,21 @@ "stat_var_mcf": "StatisticsPoland_output_stat_vars.mcf" } ], - "cron_schedule": "0 0 1 1,4,7,10 *" + "cron_schedule": "0 0 1 1,4,7,10 *", + "validation_config_file": "validation_config.json", + "resource_limits": { + "cpu": 8, + "memory": 32, + "disk": 100 + } } - ] + ], + "config_override": { + "invoke_import_validation": true, + "invoke_import_tool": true, + "invoke_differ_tool": true, + "skip_input_upload": false, + "skip_gcs_upload": false, + "cleanup_gcs_volume_mount": false + } } diff --git a/statvar_imports/statistics_poland/validation_config.json b/statvar_imports/statistics_poland/validation_config.json new file mode 100644 index 0000000000..f4eb0d2919 --- /dev/null +++ b/statvar_imports/statistics_poland/validation_config.json @@ -0,0 +1,31 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted records for the entire import is within threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.5 + } + }, + { + "rule_id": "check_goldens_summary_report", + "description": "Validates statistics poland data against its golden summary report.", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv", + "input_files": "../../input0/genmcf/summary_report.csv" + } + }, + { + "rule_id": "check_goldens_observations", + "description": "Verifies the generated output CSV data matches established critical golden records for Statistics Poland", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../StatisticsPoland_output.csv" + } + } + ] +} diff --git a/tools/statvar_importer/schema/schema_reconciler.py b/tools/statvar_importer/schema/schema_reconciler.py index 618d64dbb4..1445d1603f 100644 --- a/tools/statvar_importer/schema/schema_reconciler.py +++ b/tools/statvar_importer/schema/schema_reconciler.py @@ -89,6 +89,19 @@ 'List of properties to be looked up for reconciliation.') flags.DEFINE_bool('recon_keep_legacy_svobs', True, 'Keep the legacy value when reconciling nodes.') +flags.DEFINE_bool('recon_lookup_api', False, + 'Enable or disable API lookup for schema definition.') + + +def _is_bool_true(val, default: bool = False) -> bool: + """Returns boolean value from bool, string or int.""" + if val is None: + return default + if isinstance(val, bool): + return val + if isinstance(val, str): + return val.lower() not in ('false', '0', 'no', '') + return bool(val) def get_default_recon_config() -> dict: @@ -99,6 +112,7 @@ def get_default_recon_config() -> dict: return { 'recon_property': _FLAGS.recon_property, 'recon_keep_legacy_svobs': _FLAGS.recon_keep_legacy_svobs, + 'recon_lookup_api': _FLAGS.recon_lookup_api, } @@ -314,9 +328,9 @@ def lookup_remapped_schema(self, # Check if any of the values for this property have replacements in the cached schema. values = get_value_list(value) for val in values: - if val.startswith('#') or val.startswith('"') or ' ' in val: - # ignore value that is a quoted string and not a - # reference to another node + if (val.startswith('#') or val.startswith('"') or ' ' in val + or val.replace('.', '', 1).replace('-', '', 1).isdigit()): + # ignore value that is a quoted string, number, or comment continue schema_node = self.get_schema_node(val) if not schema_node: @@ -327,7 +341,8 @@ def lookup_remapped_schema(self, remapped_dcids[add_namespace(val)] = remapped_val # If some DCIDs are not found in the local schema cache, fetch them using the DC API. - if lookup_dcids and not self._config.get('recon_lookup_api', True): + if lookup_dcids and not _is_bool_true( + self._config.get('recon_lookup_api'), default=False): # DC API lookup is disabled. # Use any remapped dcids collected from existing schema. logging.warning( From dfb8333395a65faba46baa646f88a6a7e403bbfa Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Tue, 4 Aug 2026 09:00:25 +0000 Subject: [PATCH 2/4] Address code review feedback on recon_lookup_api default, source_files, and error handling --- .../statistics_poland/download_input_data.py | 9 +-------- statvar_imports/statistics_poland/manifest.json | 4 +--- tools/statvar_importer/schema/schema_reconciler.py | 11 ++++++----- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/statvar_imports/statistics_poland/download_input_data.py b/statvar_imports/statistics_poland/download_input_data.py index 0e7a634402..4069186c33 100644 --- a/statvar_imports/statistics_poland/download_input_data.py +++ b/statvar_imports/statistics_poland/download_input_data.py @@ -98,15 +98,8 @@ def fetch_variables(): return v_map def download_and_process(): - global OUTPUT_DIR - # Safely create output directory - try: - os.makedirs(OUTPUT_DIR, exist_ok=True) - except PermissionError: - logging.warning(f"Permission denied for {OUTPUT_DIR}. Falling back to /tmp/source_files") - OUTPUT_DIR = "/tmp/source_files" - os.makedirs(OUTPUT_DIR, exist_ok=True) + os.makedirs(OUTPUT_DIR, exist_ok=True) template_df = load_template_from_gcs(GCS_TEMPLATE_PATH) if template_df is None: diff --git a/statvar_imports/statistics_poland/manifest.json b/statvar_imports/statistics_poland/manifest.json index 0a7496baf9..4ec6955981 100644 --- a/statvar_imports/statistics_poland/manifest.json +++ b/statvar_imports/statistics_poland/manifest.json @@ -12,9 +12,7 @@ "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.csv --pv_map=StatisticsPoland_pvmap.csv --config_file=StatisticsPoland_metadata.csv --output_path=StatisticsPoland_output --output_counters=counters/StatisticsPoland_output_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" ], "source_files": [ - "source_files/*.csv", - "counters/*.csv", - "golden_data/*.csv" + "source_files/*.csv" ], "import_inputs": [ { diff --git a/tools/statvar_importer/schema/schema_reconciler.py b/tools/statvar_importer/schema/schema_reconciler.py index 1445d1603f..1857e4b21f 100644 --- a/tools/statvar_importer/schema/schema_reconciler.py +++ b/tools/statvar_importer/schema/schema_reconciler.py @@ -89,11 +89,11 @@ 'List of properties to be looked up for reconciliation.') flags.DEFINE_bool('recon_keep_legacy_svobs', True, 'Keep the legacy value when reconciling nodes.') -flags.DEFINE_bool('recon_lookup_api', False, +flags.DEFINE_bool('recon_lookup_api', True, 'Enable or disable API lookup for schema definition.') -def _is_bool_true(val, default: bool = False) -> bool: +def _is_bool_true(val, default: bool = True) -> bool: """Returns boolean value from bool, string or int.""" if val is None: return default @@ -328,8 +328,9 @@ def lookup_remapped_schema(self, # Check if any of the values for this property have replacements in the cached schema. values = get_value_list(value) for val in values: - if (val.startswith('#') or val.startswith('"') or ' ' in val - or val.replace('.', '', 1).replace('-', '', 1).isdigit()): + if (val.startswith('#') or val.startswith('"') or + ' ' in val or val.replace('.', '', 1).replace( + '-', '', 1).isdigit()): # ignore value that is a quoted string, number, or comment continue schema_node = self.get_schema_node(val) @@ -342,7 +343,7 @@ def lookup_remapped_schema(self, # If some DCIDs are not found in the local schema cache, fetch them using the DC API. if lookup_dcids and not _is_bool_true( - self._config.get('recon_lookup_api'), default=False): + self._config.get('recon_lookup_api'), default=True): # DC API lookup is disabled. # Use any remapped dcids collected from existing schema. logging.warning( From f3a11f2e25131d508de441864c98d06a4412ed83 Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Tue, 4 Aug 2026 09:05:35 +0000 Subject: [PATCH 3/4] Revert schema_reconciler changes and configure use_autopush_dc_api: false for statistics_poland --- .../statistics_poland/manifest.json | 1 + .../schema/schema_reconciler.py | 24 ++++--------------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/statvar_imports/statistics_poland/manifest.json b/statvar_imports/statistics_poland/manifest.json index 4ec6955981..98bc23e21b 100644 --- a/statvar_imports/statistics_poland/manifest.json +++ b/statvar_imports/statistics_poland/manifest.json @@ -34,6 +34,7 @@ "invoke_import_validation": true, "invoke_import_tool": true, "invoke_differ_tool": true, + "use_autopush_dc_api": false, "skip_input_upload": false, "skip_gcs_upload": false, "cleanup_gcs_volume_mount": false diff --git a/tools/statvar_importer/schema/schema_reconciler.py b/tools/statvar_importer/schema/schema_reconciler.py index 1857e4b21f..618d64dbb4 100644 --- a/tools/statvar_importer/schema/schema_reconciler.py +++ b/tools/statvar_importer/schema/schema_reconciler.py @@ -89,19 +89,6 @@ 'List of properties to be looked up for reconciliation.') flags.DEFINE_bool('recon_keep_legacy_svobs', True, 'Keep the legacy value when reconciling nodes.') -flags.DEFINE_bool('recon_lookup_api', True, - 'Enable or disable API lookup for schema definition.') - - -def _is_bool_true(val, default: bool = True) -> bool: - """Returns boolean value from bool, string or int.""" - if val is None: - return default - if isinstance(val, bool): - return val - if isinstance(val, str): - return val.lower() not in ('false', '0', 'no', '') - return bool(val) def get_default_recon_config() -> dict: @@ -112,7 +99,6 @@ def get_default_recon_config() -> dict: return { 'recon_property': _FLAGS.recon_property, 'recon_keep_legacy_svobs': _FLAGS.recon_keep_legacy_svobs, - 'recon_lookup_api': _FLAGS.recon_lookup_api, } @@ -328,10 +314,9 @@ def lookup_remapped_schema(self, # Check if any of the values for this property have replacements in the cached schema. values = get_value_list(value) for val in values: - if (val.startswith('#') or val.startswith('"') or - ' ' in val or val.replace('.', '', 1).replace( - '-', '', 1).isdigit()): - # ignore value that is a quoted string, number, or comment + if val.startswith('#') or val.startswith('"') or ' ' in val: + # ignore value that is a quoted string and not a + # reference to another node continue schema_node = self.get_schema_node(val) if not schema_node: @@ -342,8 +327,7 @@ def lookup_remapped_schema(self, remapped_dcids[add_namespace(val)] = remapped_val # If some DCIDs are not found in the local schema cache, fetch them using the DC API. - if lookup_dcids and not _is_bool_true( - self._config.get('recon_lookup_api'), default=True): + if lookup_dcids and not self._config.get('recon_lookup_api', True): # DC API lookup is disabled. # Use any remapped dcids collected from existing schema. logging.warning( From 58988b9e10acd0da68a6d895c40e08ab7165721c Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Tue, 4 Aug 2026 10:00:10 +0000 Subject: [PATCH 4/4] Add counters and golden_data to source_files and configure recon_property in metadata --- .../statistics_poland/StatisticsPoland_metadata.csv | 2 +- statvar_imports/statistics_poland/manifest.json | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv b/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv index ef2e639519..25635e5ce1 100644 --- a/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv +++ b/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv @@ -2,7 +2,7 @@ config,value provenance_url,https://bdl.stat.gov.pl/bdl/dane/podgrup/tablica output_columns,"observationDate,observationAbout,value,variableMeasured" places_within,country/POL -recon_lookup_api,False +recon_property,"variableMeasured,measurementMethod" #place_types,"AdministrativeArea,AdministrativeArea1,AdministrativeArea2,State" #debug,1 #input_rows,100 diff --git a/statvar_imports/statistics_poland/manifest.json b/statvar_imports/statistics_poland/manifest.json index 98bc23e21b..5419d0efaf 100644 --- a/statvar_imports/statistics_poland/manifest.json +++ b/statvar_imports/statistics_poland/manifest.json @@ -12,7 +12,9 @@ "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.csv --pv_map=StatisticsPoland_pvmap.csv --config_file=StatisticsPoland_metadata.csv --output_path=StatisticsPoland_output --output_counters=counters/StatisticsPoland_output_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" ], "source_files": [ - "source_files/*.csv" + "source_files/*.csv", + "counters/*.csv", + "golden_data/*.csv" ], "import_inputs": [ {